diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index ecb8aad6..601d350a 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -3,6 +3,10 @@ name: BackFiller CodeQL configuration queries: - uses: ./.github/codeql/csharp-with-alert-suppression.qls +# paths-ignore documents intent for non-traced languages / documentation. +# For traced C# builds, compiled obj/bin (including source-generators) still appear in +# SARIF; tools/Filter-CodeQlSarif.ps1 and .github/workflows/codeql.yml strip those +# results before upload. See tools/Invoke-CodeQlAnalysis.ps1. paths-ignore: - '**/bin/**' - '**/obj/**' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffffe185..3f206470 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,13 +235,16 @@ jobs: # Same Native AOT invocation as tools/Publish-Release.ps1, plus the # x64 platform contract. Publish restores the ILCompiler pack and the # StuDev.AotAnywhere Zig toolchain for this runtime identifier. - # Both services share one flat artifact directory (identical RabbitMQ JSON). + # Each host publishes into its own package directory (independent deploy). run: | set -euo pipefail - for proj in \ - src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj \ - src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj - do + root=./.artifacts/linux-x64 + mkdir -p "$root" + publish_one() { + local package="$1" + local proj="$2" + local out="$root/$package" + mkdir -p "$out" dotnet publish "$proj" \ --configuration Release \ -p:Platform=x64 \ @@ -249,41 +252,61 @@ jobs: --self-contained true \ -p:PublishAot=true \ -p:SelfContained=true \ - --output ./.artifacts/linux-x64 - done + --output "$out" + } + publish_one backfiller src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj + publish_one storage src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj + publish_one reader src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.csproj + publish_one transit src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.csproj - name: Add Linux deployment files shell: bash run: | set -euo pipefail - dir=./.artifacts/linux-x64 + root=./.artifacts/linux-x64 test -f deploy/systemd/vectornntp-backfiller.service test -f deploy/systemd/backfiller.env.example test -f deploy/systemd/vectornntp-storageservice.service test -f deploy/systemd/storageservice.env.example + test -f deploy/systemd/vectornntp-reader.service + test -f deploy/systemd/reader.env.example + test -f deploy/systemd/vectornntp-transit.service + test -f deploy/systemd/transit.env.example test -f README.md - cp deploy/systemd/vectornntp-backfiller.service "$dir/vectornntp-backfiller.service" - cp deploy/systemd/backfiller.env.example "$dir/backfiller.env.example" - cp deploy/systemd/vectornntp-storageservice.service "$dir/vectornntp-storageservice.service" - cp deploy/systemd/storageservice.env.example "$dir/storageservice.env.example" - cp README.md "$dir/README.md" - test -f "$dir/VectorNNTPD.BackFillerService.json" - test -f "$dir/VectorNNTPD.StorageService.json" - test -f "$dir/VectorNNTPD.RabbitMQ.json" - test ! -e "$dir/VectorNNTP.RabbitMQ.json" - test ! -e "$dir/VectorNNTP.StorageServer.json" - test ! -e "$dir/appsettings.json" - test ! -e "$dir/RabbitMq.json" - - - name: Validate linux-x64 Native AOT executables + cp deploy/systemd/vectornntp-backfiller.service "$root/backfiller/vectornntp-backfiller.service" + cp deploy/systemd/backfiller.env.example "$root/backfiller/backfiller.env.example" + cp README.md "$root/backfiller/README.md" + cp deploy/systemd/vectornntp-storageservice.service "$root/storage/vectornntp-storageservice.service" + cp deploy/systemd/storageservice.env.example "$root/storage/storageservice.env.example" + cp README.md "$root/storage/README.md" + cp deploy/systemd/vectornntp-reader.service "$root/reader/vectornntp-reader.service" + cp deploy/systemd/reader.env.example "$root/reader/reader.env.example" + cp README.md "$root/reader/README.md" + cp deploy/systemd/vectornntp-transit.service "$root/transit/vectornntp-transit.service" + cp deploy/systemd/transit.env.example "$root/transit/transit.env.example" + cp README.md "$root/transit/README.md" + + - name: Validate linux-x64 Native AOT packages shell: bash run: | set -euo pipefail - dir=./.artifacts/linux-x64 - for exe_name in VectorNNTPD.BackFillerService VectorNNTPD.StorageService; do - exe="$dir/$exe_name" + root=./.artifacts/linux-x64 + validate_pkg() { + local package="$1" + local exe_name="$2" + local config_name="$3" + local unit_name="$4" + local env_name="$5" + local dir="$root/$package" + local exe="$dir/$exe_name" + test -d "$dir" test -f "$exe" test -x "$exe" + test -f "$dir/$config_name" + test -f "$dir/VectorNNTPD.RabbitMQ.json" + test -f "$dir/$unit_name" + test -f "$dir/$env_name" + test -f "$dir/README.md" file "$exe" readelf -h "$exe" objdump -f "$exe" @@ -299,26 +322,48 @@ jobs: ldd "$exe" exit 1 fi - done - test -f "$dir/vectornntp-backfiller.service" - test -f "$dir/backfiller.env.example" - test -f "$dir/vectornntp-storageservice.service" - test -f "$dir/storageservice.env.example" - test -f "$dir/README.md" - test -f "$dir/VectorNNTPD.BackFillerService.json" - test -f "$dir/VectorNNTPD.StorageService.json" - test -f "$dir/VectorNNTPD.RabbitMQ.json" - test ! -e "$dir/VectorNNTP.RabbitMQ.json" - test ! -e "$dir/VectorNNTP.StorageServer.json" - test ! -e "$dir/appsettings.json" - test ! -e "$dir/RabbitMq.json" - for forbidden in \ - VectorNNTPD.BackFillerService.dll \ - VectorNNTPD.StorageService.dll \ - libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll - do - if [ -e "$dir/$forbidden" ]; then - echo "Native AOT output contains $forbidden." + for forbidden in \ + "$exe_name.dll" \ + libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll \ + appsettings.json RabbitMq.json VectorNNTP.RabbitMQ.json VectorNNTP.StorageServer.json \ + VectorNNTPD.NntpServer.dll VectorNNTPD.NntpServer.exe libVectorNNTPD.NntpServer.so + do + if [ -e "$dir/$forbidden" ]; then + echo "Package $package contains forbidden file $forbidden." + exit 1 + fi + done + if find "$dir" -maxdepth 1 -type f -iname '*NntpServer*' | grep -q .; then + echo "Package $package contains an unexpected NntpServer artifact." + find "$dir" -maxdepth 1 -type f -iname '*NntpServer*' -print + exit 1 + fi + # Role isolation: foreign host JSON/exe must not appear in this package. + for foreign in \ + VectorNNTPD.BackFillerService \ + VectorNNTPD.StorageService \ + VectorNNTPD.ReaderService \ + VectorNNTPD.TransitService + do + if [ "$foreign" = "$exe_name" ]; then + continue + fi + if [ -e "$dir/$foreign" ] || [ -e "$dir/$foreign.json" ]; then + echo "Package $package unexpectedly contains foreign host artifact $foreign." + exit 1 + fi + done + } + validate_pkg backfiller VectorNNTPD.BackFillerService VectorNNTPD.BackFillerService.json vectornntp-backfiller.service backfiller.env.example + validate_pkg storage VectorNNTPD.StorageService VectorNNTPD.StorageService.json vectornntp-storageservice.service storageservice.env.example + validate_pkg reader VectorNNTPD.ReaderService VectorNNTPD.ReaderService.json vectornntp-reader.service reader.env.example + validate_pkg transit VectorNNTPD.TransitService VectorNNTPD.TransitService.json vectornntp-transit.service transit.env.example + # Platform root must contain only the four package directories. + for unexpected in VectorNNTPD.BackFillerService VectorNNTPD.StorageService VectorNNTPD.ReaderService VectorNNTPD.TransitService \ + VectorNNTPD.BackFillerService.json VectorNNTPD.StorageService.json VectorNNTPD.ReaderService.json VectorNNTPD.TransitService.json \ + VectorNNTPD.RabbitMQ.json README.md; do + if [ -e "$root/$unexpected" ]; then + echo "Loose file at platform root is not allowed: $root/$unexpected" exit 1 fi done @@ -327,7 +372,7 @@ jobs: shell: bash run: | set -euo pipefail - exe=./.artifacts/linux-x64/VectorNNTPD.BackFillerService + exe=./.artifacts/linux-x64/backfiller/VectorNNTPD.BackFillerService config="$PWD/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/startup-smoke.json" test -f "$exe" test -f "$config" @@ -356,7 +401,7 @@ jobs: shell: bash run: | set -euo pipefail - exe=./.artifacts/linux-x64/VectorNNTPD.StorageService + exe=./.artifacts/linux-x64/storage/VectorNNTPD.StorageService config="$PWD/tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json" test -f "$exe" test -f "$config" @@ -383,6 +428,66 @@ jobs: grep -q "Application logging initialized" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt grep -q "VectorNNTPD.StorageService host starting" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + - name: Smoke-test linux-x64 Native AOT startup (ReaderService) + shell: bash + run: | + set -euo pipefail + exe=./.artifacts/linux-x64/reader/VectorNNTPD.ReaderService + config="$PWD/tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json" + test -f "$exe" + test -f "$config" + export VECTORNNTP_READER_STARTUP_SMOKE=1 + export VECTORNNTP_READER_STARTUP_SMOKE_CONFIG="$config" + set +e + timeout 30 "$exe" >./.artifacts/linux-x64-reader-smoke-stdout.txt 2>./.artifacts/linux-x64-reader-smoke-stderr.txt + code=$? + set -e + echo "linux-x64 ReaderService startup smoke exit=$code" + echo "----- stdout -----" + cat ./.artifacts/linux-x64-reader-smoke-stdout.txt + echo "----- stderr -----" + cat ./.artifacts/linux-x64-reader-smoke-stderr.txt + if [ "$code" -eq 124 ]; then + echo "linux-x64 ReaderService startup smoke timed out." + exit 1 + fi + if [ "$code" -ne 0 ]; then + echo "linux-x64 ReaderService startup smoke failed." + exit 1 + fi + grep -q "ReaderService startup smoke: configuration serverId=1 fqdn=reader01.usenet.ninja" ./.artifacts/linux-x64-reader-smoke-stdout.txt + grep -q "VectorNNTPD.ReaderService host starting" ./.artifacts/linux-x64-reader-smoke-stdout.txt + + - name: Smoke-test linux-x64 Native AOT startup (TransitService) + shell: bash + run: | + set -euo pipefail + exe=./.artifacts/linux-x64/transit/VectorNNTPD.TransitService + config="$PWD/tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json" + test -f "$exe" + test -f "$config" + export VECTORNNTP_TRANSIT_STARTUP_SMOKE=1 + export VECTORNNTP_TRANSIT_STARTUP_SMOKE_CONFIG="$config" + set +e + timeout 30 "$exe" >./.artifacts/linux-x64-transit-smoke-stdout.txt 2>./.artifacts/linux-x64-transit-smoke-stderr.txt + code=$? + set -e + echo "linux-x64 TransitService startup smoke exit=$code" + echo "----- stdout -----" + cat ./.artifacts/linux-x64-transit-smoke-stdout.txt + echo "----- stderr -----" + cat ./.artifacts/linux-x64-transit-smoke-stderr.txt + if [ "$code" -eq 124 ]; then + echo "linux-x64 TransitService startup smoke timed out." + exit 1 + fi + if [ "$code" -ne 0 ]; then + echo "linux-x64 TransitService startup smoke failed." + exit 1 + fi + grep -q "TransitService startup smoke: configuration serverId=1 fqdn=transit01.usenet.ninja" ./.artifacts/linux-x64-transit-smoke-stdout.txt + grep -q "VectorNNTPD.TransitService host starting" ./.artifacts/linux-x64-transit-smoke-stdout.txt + - name: Upload Linux build artifact uses: actions/upload-artifact@v7 with: @@ -421,32 +526,38 @@ jobs: # The solution restore above does not pass a runtime identifier, so this # publish restores itself. --no-restore here would publish a self-contained # managed app and the RequireNativeAotCompiler target would fail. - # Both services share one flat artifact directory (identical RabbitMQ JSON). + # Each host publishes into its own package directory (independent deploy). shell: pwsh run: | $ErrorActionPreference = 'Stop' - foreach ($proj in @( - 'src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj', - 'src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj' - )) { - dotnet publish $proj ` + $root = './.artifacts/win-x64' + New-Item -ItemType Directory -Path $root -Force | Out-Null + $packages = @( + @{ Package = 'backfiller'; Project = 'src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj' }, + @{ Package = 'storage'; Project = 'src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj' }, + @{ Package = 'reader'; Project = 'src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.csproj' }, + @{ Package = 'transit'; Project = 'src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.csproj' } + ) + foreach ($item in $packages) { + $out = Join-Path $root $item.Package + New-Item -ItemType Directory -Path $out -Force | Out-Null + dotnet publish $item.Project ` --configuration Release ` -p:Platform=x64 ` --runtime win-x64 ` --self-contained true ` -p:PublishAot=true ` -p:SelfContained=true ` - --output ./.artifacts/win-x64 - if ($LASTEXITCODE -ne 0) { throw "dotnet publish failed for $proj" } + --output $out + if ($LASTEXITCODE -ne 0) { throw "dotnet publish failed for $($item.Project)" } } - - name: Validate win-x64 Native AOT executables + - name: Validate win-x64 Native AOT packages shell: pwsh run: | $ErrorActionPreference = 'Stop' - $dir = './.artifacts/win-x64' - foreach ($exeName in @('VectorNNTPD.BackFillerService.exe', 'VectorNNTPD.StorageService.exe')) { - $exe = Join-Path $dir $exeName + $root = './.artifacts/win-x64' + function Test-WinNativeAotExe([string] $exe) { if (-not (Test-Path -LiteralPath $exe)) { throw "Missing $exe" } $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path $exe)) if ($bytes.Length -lt 512 -or $bytes[0] -ne 0x4D -or $bytes[1] -ne 0x5A) { throw "Expected a Windows PE executable: $exe" } @@ -460,28 +571,66 @@ jobs: $clrSize = [System.BitConverter]::ToUInt32($bytes, $optional + 112 + (14 * 8) + 4) if ($clrSize -ne 0) { throw "Executable still has a CLR runtime header, so it is not Native AOT: $exe" } } - foreach ($required in @('VectorNNTPD.BackFillerService.json', 'VectorNNTPD.StorageService.json', 'VectorNNTPD.RabbitMQ.json')) { - $path = Join-Path $dir $required - if (-not (Test-Path -LiteralPath $path)) { throw "Missing $path" } + function Test-WinPackage([string] $package, [string] $exeBase, [string] $configName) { + $dir = Join-Path $root $package + if (-not (Test-Path -LiteralPath $dir)) { throw "Missing package directory $dir" } + $exe = Join-Path $dir ($exeBase + '.exe') + Test-WinNativeAotExe $exe + foreach ($required in @($configName, 'VectorNNTPD.RabbitMQ.json')) { + $path = Join-Path $dir $required + if (-not (Test-Path -LiteralPath $path)) { throw "Missing $path" } + } + foreach ($forbidden in @( + ($exeBase + '.dll'), + 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', + 'vectornntp-backfiller.service', 'backfiller.env.example', + 'vectornntp-storageservice.service', 'storageservice.env.example', + 'vectornntp-reader.service', 'reader.env.example', + 'vectornntp-transit.service', 'transit.env.example', + 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json', 'VectorNNTP.StorageServer.json', + 'VectorNNTPD.StorageService.Development.json', 'VectorNNTPD.BackFillerService.Development.json', + 'VectorNNTPD.ReaderService.Development.json', 'VectorNNTPD.TransitService.Development.json', + 'VectorNNTPD.NntpServer.dll', 'VectorNNTPD.NntpServer.exe' + )) { + $path = Join-Path $dir $forbidden + if (Test-Path -LiteralPath $path) { throw "Package $package contains forbidden file $forbidden." } + } + $nntpHits = @(Get-ChildItem -LiteralPath $dir -File -Force | Where-Object { $_.Name -like '*NntpServer*' }) + if ($nntpHits.Count -gt 0) { + throw "Package $package contains unexpected NntpServer artifacts: $($nntpHits.Name -join ', ')." + } + foreach ($foreign in @( + 'VectorNNTPD.BackFillerService', + 'VectorNNTPD.StorageService', + 'VectorNNTPD.ReaderService', + 'VectorNNTPD.TransitService' + )) { + if ($foreign -eq $exeBase) { continue } + if ((Test-Path -LiteralPath (Join-Path $dir ($foreign + '.exe'))) -or (Test-Path -LiteralPath (Join-Path $dir ($foreign + '.json')))) { + throw "Package $package unexpectedly contains foreign host artifact $foreign." + } + } } - foreach ($forbidden in @( - 'VectorNNTPD.BackFillerService.dll', - 'VectorNNTPD.StorageService.dll', - 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', - 'vectornntp-backfiller.service', 'backfiller.env.example', - 'vectornntp-storageservice.service', 'storageservice.env.example', - 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json', 'VectorNNTP.StorageServer.json', - 'VectorNNTPD.StorageService.Development.json', 'VectorNNTPD.BackFillerService.Development.json' + Test-WinPackage 'backfiller' 'VectorNNTPD.BackFillerService' 'VectorNNTPD.BackFillerService.json' + Test-WinPackage 'storage' 'VectorNNTPD.StorageService' 'VectorNNTPD.StorageService.json' + Test-WinPackage 'reader' 'VectorNNTPD.ReaderService' 'VectorNNTPD.ReaderService.json' + Test-WinPackage 'transit' 'VectorNNTPD.TransitService' 'VectorNNTPD.TransitService.json' + foreach ($loose in @( + 'VectorNNTPD.BackFillerService.exe', 'VectorNNTPD.StorageService.exe', + 'VectorNNTPD.ReaderService.exe', 'VectorNNTPD.TransitService.exe', + 'VectorNNTPD.BackFillerService.json', 'VectorNNTPD.StorageService.json', + 'VectorNNTPD.ReaderService.json', 'VectorNNTPD.TransitService.json', + 'VectorNNTPD.RabbitMQ.json' )) { - $path = Join-Path $dir $forbidden - if (Test-Path -LiteralPath $path) { throw "Native AOT output contains $forbidden." } + $path = Join-Path $root $loose + if (Test-Path -LiteralPath $path) { throw "Loose file at platform root is not allowed: $path" } } - name: Smoke-test win-x64 Native AOT startup (BackFiller) shell: pwsh run: | $ErrorActionPreference = 'Stop' - $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.BackFillerService.exe' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64/backfiller') 'VectorNNTPD.BackFillerService.exe' $config = (Resolve-Path './tests/VectorNNTPD.BackFillerService.Tests/Fixtures/startup-smoke.json').Path $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stdout.txt' $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stderr.txt' @@ -519,7 +668,7 @@ jobs: shell: pwsh run: | $ErrorActionPreference = 'Stop' - $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.StorageService.exe' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64/storage') 'VectorNNTPD.StorageService.exe' $config = (Resolve-Path './tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json').Path $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stdout.txt' $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stderr.txt' @@ -559,6 +708,88 @@ jobs: throw 'win-x64 StorageService startup smoke did not log host starting.' } + - name: Smoke-test win-x64 Native AOT startup (ReaderService) + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64/reader') 'VectorNNTPD.ReaderService.exe' + $config = (Resolve-Path './tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json').Path + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-reader-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-reader-smoke-stderr.txt' + $process = [System.Diagnostics.Process]::new() + $process.StartInfo.FileName = $exe + $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe + $process.StartInfo.UseShellExecute = $false + $process.StartInfo.RedirectStandardOutput = $true + $process.StartInfo.RedirectStandardError = $true + $process.StartInfo.Environment['VECTORNNTP_READER_STARTUP_SMOKE'] = '1' + $process.StartInfo.Environment['VECTORNNTP_READER_STARTUP_SMOKE_CONFIG'] = $config + if (-not $process.Start()) { throw 'Failed to start the win-x64 ReaderService startup smoke process.' } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(30000)) { + try { $process.Kill($true) } catch { } + throw 'win-x64 ReaderService startup smoke timed out.' + } + $process.WaitForExit() + $stdout = $stdoutTask.GetAwaiter().GetResult() + $stderr = $stderrTask.GetAwaiter().GetResult() + Set-Content -LiteralPath $stdoutPath -Value $stdout + Set-Content -LiteralPath $stderrPath -Value $stderr + Write-Host "win-x64 ReaderService startup smoke exit=$($process.ExitCode)" + Write-Host '----- stdout -----' + Write-Host $stdout + Write-Host '----- stderr -----' + Write-Host $stderr + if ($process.ExitCode -ne 0) { throw 'win-x64 ReaderService startup smoke failed.' } + if ($stdout -notmatch 'ReaderService startup smoke: configuration serverId=1 fqdn=reader01\.usenet\.ninja') { + throw 'win-x64 ReaderService startup smoke did not reach the configuration checkpoint.' + } + if ($stdout -notmatch 'VectorNNTPD\.ReaderService host starting') { + throw 'win-x64 ReaderService startup smoke did not log host starting.' + } + + - name: Smoke-test win-x64 Native AOT startup (TransitService) + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64/transit') 'VectorNNTPD.TransitService.exe' + $config = (Resolve-Path './tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json').Path + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-transit-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-transit-smoke-stderr.txt' + $process = [System.Diagnostics.Process]::new() + $process.StartInfo.FileName = $exe + $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe + $process.StartInfo.UseShellExecute = $false + $process.StartInfo.RedirectStandardOutput = $true + $process.StartInfo.RedirectStandardError = $true + $process.StartInfo.Environment['VECTORNNTP_TRANSIT_STARTUP_SMOKE'] = '1' + $process.StartInfo.Environment['VECTORNNTP_TRANSIT_STARTUP_SMOKE_CONFIG'] = $config + if (-not $process.Start()) { throw 'Failed to start the win-x64 TransitService startup smoke process.' } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(30000)) { + try { $process.Kill($true) } catch { } + throw 'win-x64 TransitService startup smoke timed out.' + } + $process.WaitForExit() + $stdout = $stdoutTask.GetAwaiter().GetResult() + $stderr = $stderrTask.GetAwaiter().GetResult() + Set-Content -LiteralPath $stdoutPath -Value $stdout + Set-Content -LiteralPath $stderrPath -Value $stderr + Write-Host "win-x64 TransitService startup smoke exit=$($process.ExitCode)" + Write-Host '----- stdout -----' + Write-Host $stdout + Write-Host '----- stderr -----' + Write-Host $stderr + if ($process.ExitCode -ne 0) { throw 'win-x64 TransitService startup smoke failed.' } + if ($stdout -notmatch 'TransitService startup smoke: configuration serverId=1 fqdn=transit01\.usenet\.ninja') { + throw 'win-x64 TransitService startup smoke did not reach the configuration checkpoint.' + } + if ($stdout -notmatch 'VectorNNTPD\.TransitService host starting') { + throw 'win-x64 TransitService startup smoke did not log host starting.' + } + - name: Upload Windows build artifact uses: actions/upload-artifact@v7 with: @@ -566,3 +797,4 @@ jobs: path: .artifacts/win-x64/ if-no-files-found: error retention-days: 7 + diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8cdf3169..40f45c00 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,8 +45,40 @@ jobs: - name: Restore solution run: dotnet restore VectorNNTPD.slnx -p:Platform=x64 + # --no-incremental ensures CoreCompile/csc runs under the CodeQL tracer for every + # project. An up-to-date incremental skip can leave only .csproj entries in the DB. - name: Build solution - run: dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore + run: dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore --no-incremental + # upload: never — traced C# SARIF includes **/obj/** source-generator noise that + # paths-ignore cannot drop. Filter first, then upload-sarif. - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 + with: + upload: never + output: codeql-sarif + + - name: Filter generated bin/obj findings from SARIF + shell: pwsh + run: | + $inputs = Get-ChildItem -Path codeql-sarif -Filter *.sarif -File -ErrorAction Stop + if ($inputs.Count -lt 1) { + throw "No SARIF files found under codeql-sarif/" + } + New-Item -ItemType Directory -Force -Path codeql-sarif-filtered | Out-Null + foreach ($sarif in $inputs) { + $out = Join-Path codeql-sarif-filtered $sarif.Name + # StripAcceptedSuppressions: in-source // codeql[...] findings stay justified in + # the repo; uploading them as results still produced "new alerts" on PR #256 + # because CodeQL's {state:accepted} was not treated as GitHub dismissals. + & ./tools/Filter-CodeQlSarif.ps1 -InputPath $sarif.FullName -OutputPath $out -StripAcceptedSuppressions + } + + # Keep the historical analysis category so GitHub's required CodeQL check continues + # to match refs/heads/master (category "csharp" yielded NEUTRAL + # "1 configuration not found" on PR #256 check run 123524079410). + - name: Upload filtered CodeQL SARIF + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: codeql-sarif-filtered + category: ".github/workflows/codeql.yml:analyze" diff --git a/AGENTS.md b/AGENTS.md index 7956c3e0..709108ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Concise routing for Cursor agents (including JetBrains Rider ACP). This file is an entry point; it does not prove that Rider ACP automatically loads it. -This repository is standalone **VectorNNTPD.BackFillerService**. It was extracted from VectorNNTP.NNTPD. NNTPD names that remain in code comments or shared configuration are provenance or shared-domain context. The solution, projects, and verification commands are VectorNNTPD's. +This repository is **VectorNNTPD**: BackFillerService, StorageService, NNTP ReaderService, NNTP TransitService, and `VectorNNTPD.Common` (Reader/Transit embed duplicated NNTP server code; there is no `VectorNNTPD.NntpServer` library). It was extracted from VectorNNTP.NNTPD. NNTPD names that remain in code comments or shared configuration are provenance or shared-domain context. Reader/Transit hosting is documented in [`docs/nntpd-reader-transit.md`](docs/nntpd-reader-transit.md). ## Before substantive work diff --git a/Directory.Build.targets b/Directory.Build.targets index 849cc76d..16cc2cd8 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -15,14 +15,22 @@ ©2026 Chris Knipe <cknipe@opticnetworks.net> + + © 2026 Chris Knipe <cknipe@opticnetworks.net> + + + + <_IsProductionExecutable Condition="'$(MSBuildProjectName)' == 'VectorNNTPD.BackFillerService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.StorageService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.ReaderService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.TransitService'">true + + + Condition="'$(DesignTimeBuild)' != 'true' AND '$(_IsProductionExecutable)' == 'true'"> <_ApplicationVersionNow>$([System.DateTime]::Now.Ticks) <_ApplicationVersionBuild>$([System.DateTime]::new($([System.Int64]::Parse($(_ApplicationVersionNow)))).Subtract($([System.DateTime]::new(2000,1,1))).Days) @@ -38,23 +46,23 @@ Production publish is self-contained Native AOT for win-x64 and linux-x64 only. A normal build does not set RuntimeIdentifier, so libraries, tests, and tools stay portable. --> - + win-x64;linux-x64 true true - - $(NoWarn);IL2104;IL3050;IL3000;IL3002 - $(WarningsNotAsErrors);IL2104;IL3050 + + $(NoWarn);IL2026;IL2104;IL3050;IL3000;IL3002 + $(WarningsNotAsErrors);IL2026;IL2104;IL3050 false false @@ -67,7 +75,7 @@ + Condition="'$(_IsProductionExecutable)' == 'true' AND '$(RuntimeIdentifier)' != '' AND '$(RuntimeIdentifier)' != 'win-x64' AND '$(RuntimeIdentifier)' != 'linux-x64'"> diff --git a/README.md b/README.md index a73b419a..47d09b24 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,18 @@ # VectorNNTPD -Standalone VectorNNTPD hosts extracted from VectorNNTP.NNTPD. This repository currently ships BackFiller and StorageService, plus shared `VectorNNTPD.Common`. +Standalone VectorNNTPD hosts extracted from VectorNNTP.NNTPD. This repository ships BackFiller, StorageService, NNTP ReaderService, NNTP TransitService, and shared `VectorNNTPD.Common`. Reader and Transit each embed their own NNTP server implementation (no shared NntpServer library). ## Structure -- `src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj` - BackFiller worker app (net10.0) -- `src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj` - StorageService / NNTP TAKETHIS feed host (net10.0) -- `src/VectorNNTPD.Common/VectorNNTPD.Common.csproj` - shared library -- `tests/VectorNNTPD.BackFillerService.Tests/` / `tests/VectorNNTPD.StorageService.Tests/` / `tests/VectorNNTPD.Common.Tests/` - xUnit suites +- `src/VectorNNTPD.BackFillerService/` - BackFiller worker app (net10.0) +- `src/VectorNNTPD.StorageService/` - StorageService / NNTP TAKETHIS feed host (net10.0) +- `src/VectorNNTPD.ReaderService/` - NNTP reader-server host (`AddNntpReaderHosting`) +- `src/VectorNNTPD.TransitService/` - NNTP transit-server host (`AddNntpTransitHosting`) +- `src/VectorNNTPD.Common/` - shared library +- `tests/VectorNNTPD.BackFillerService.Tests/` / `tests/VectorNNTPD.StorageService.Tests/` / `tests/VectorNNTPD.ReaderService.Tests/` / `tests/VectorNNTPD.TransitService.Tests/` / `tests/VectorNNTPD.Common.Tests/` - xUnit suites +- `docs/nntpd-reader-transit.md` - Reader/Transit architecture and deployment notes - `VectorNNTPD.slnx` - solution including apps + tests -- `tools/Publish-Release.ps1` - Native AOT release publish for both production hosts +- `tools/Publish-Release.ps1` - Native AOT release publish for BackFiller, StorageService, ReaderService, and TransitService - `tools/Invoke-CodeQlAnalysis.ps1` - local traced C# CodeQL analysis ## Build @@ -26,13 +29,24 @@ dotnet test .\tests\VectorNNTPD.BackFillerService.Tests\VectorNNTPD.BackFillerSe ## Publish -Preferred release path (both hosts into one flat directory): +Preferred release path (one independent package directory per service): ```powershell .\tools\Publish-Release.ps1 -Platform win-64 .\tools\Publish-Release.ps1 -Platform linux-64 ``` +Output layout: + +```text +release//backfiller/ +release//storage/ +release//reader/ +release//transit/ +``` + +Each package contains that host’s Native AOT executable, its primary JSON, and `VectorNNTPD.RabbitMQ.json`. Linux packages also include that role’s systemd unit, environment example, and `README.md`. Reader and Transit never share a package directory. CI writes the same package names under `.artifacts/win-x64/` and `.artifacts/linux-x64/`. + Per-project Native AOT publish still works. Example for StorageService: ```powershell @@ -45,6 +59,8 @@ Native AOT publish keeps trim and AOT analysis enabled. CI runs each published e - BackFiller: `VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1` / `VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG` → checkpoint `BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja` - StorageService: `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE=1` / `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG` → checkpoint `StorageService startup smoke: configuration serverId=1 fqdn=cache01.usenet.ninja` +- ReaderService: `VECTORNNTP_READER_STARTUP_SMOKE=1` / `VECTORNNTP_READER_STARTUP_SMOKE_CONFIG` → checkpoint `ReaderService startup smoke: configuration serverId=1 fqdn=reader01.usenet.ninja` +- TransitService: `VECTORNNTP_TRANSIT_STARTUP_SMOKE=1` / `VECTORNNTP_TRANSIT_STARTUP_SMOKE_CONFIG` → checkpoint `TransitService startup smoke: configuration serverId=1 fqdn=transit01.usenet.ninja` Those checkpoints prove configuration binding and host construction (and StorageService logging initialization). They do not open a listener or contact a database, broker, DNS provider, or certificate authority. StorageService AOT publish still scopes Serilog.Settings.Configuration IL suppressions (`IL2104`/`IL3050`/`IL3000`/`IL3002`) because MinimumLevel is applied via `ReadFrom.Configuration` while sinks are registered explicitly. @@ -56,13 +72,15 @@ BackFiller references `VectorNNTPD.Common` for `IApplicationService`, NntpDB, AC ## Linux service -`tools/Publish-Release.ps1 -Platform linux-64` writes `release/linux-64/`. The Linux CI job writes `.artifacts/linux-x64/` and uploads that directory. Both contain the flat Native AOT outputs for `VectorNNTPD.BackFillerService` and `VectorNNTPD.StorageService`, their primary JSON files, shared `VectorNNTPD.RabbitMQ.json`, `README.md`, and the BackFiller and StorageService systemd units plus environment-file examples. +`tools/Publish-Release.ps1 -Platform linux-64` writes per-service packages under `release/linux-64/{backfiller,storage,reader,transit}/`. The Linux CI job writes the same package names under `.artifacts/linux-x64/` and uploads that tree. Each package is independently installable (executable, host JSON, RabbitMQ JSON, that role’s systemd unit and environment example, and `README.md`). + +Install BackFiller from `backfiller/` at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. -Install BackFiller at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. +Install StorageService from `storage/` at `/opt/vectornntp/storageservice` using `VectorNNTPD.StorageService`, `VectorNNTPD.StorageService.json`, `VectorNNTPD.RabbitMQ.json`, `vectornntp-storageservice.service`, and `storageservice.env.example`. That unit’s `ReadWritePaths` also include `db/` and `spool/` for the article index and segment-pool roots. -Install StorageService at `/opt/vectornntp/storageservice` using `VectorNNTPD.StorageService`, `VectorNNTPD.StorageService.json`, `VectorNNTPD.RabbitMQ.json`, `vectornntp-storageservice.service`, and `storageservice.env.example` from the same release directory. That unit’s `ReadWritePaths` also include `db/` and `spool/` for the article index and segment-pool roots. +Install ReaderService from `reader/` at `/opt/vectornntp/reader` and TransitService from `transit/` at `/opt/vectornntp/transit`. See `docs/nntpd-reader-transit.md` for the Reader/Transit deployment handoff (environment files, schema, peers, rollback). -Create the account and install the release as root-owned files. This step copies the placeholder environment file and reloads systemd. It does not start the service. +Create the account and install the release as root-owned files. Run the following from the `backfiller/` package directory. This step copies the placeholder environment file and reloads systemd. It does not start the service. ```bash sudo groupadd --system vectornntp || true diff --git a/VectorNNTPD.slnx b/VectorNNTPD.slnx index a10628c1..056fe7c6 100644 --- a/VectorNNTPD.slnx +++ b/VectorNNTPD.slnx @@ -9,9 +9,15 @@ + + + + + + @@ -20,8 +26,14 @@ + + + + + + diff --git a/deploy/systemd/reader.env.example b/deploy/systemd/reader.env.example new file mode 100644 index 00000000..b430e1cf --- /dev/null +++ b/deploy/systemd/reader.env.example @@ -0,0 +1,13 @@ +# Installed as /etc/vectornntp/reader.env (root:root mode 0600). +# KEY=VALUE only. Unquoted backslash is an escape. +# +# Required database connection (no VECTOR__ prefix): +ConnectionStrings__NntpDB=REPLACE_WITH_MYSQL_CONNECTION_STRING +# +# Required AES-256 XTrace key (64 hex or Base64). Never commit the real value. +NNTPD__XTRACEKEY=REPLACE_WITH_32_BYTE_KEY_AS_64_HEX_OR_BASE64 +# +# Site-specific Nntpd overrides use Nntpd__* (section name Nntpd is preserved). +# Do not invent bind addresses or zone ids here; set them for the deployment. +# Example: Nntpd__ServerId=1 +# Example: Nntpd__DnsSuffix=example.test diff --git a/deploy/systemd/transit.env.example b/deploy/systemd/transit.env.example new file mode 100644 index 00000000..36196a18 --- /dev/null +++ b/deploy/systemd/transit.env.example @@ -0,0 +1,13 @@ +# Installed as /etc/vectornntp/transit.env (root:root mode 0600). +# KEY=VALUE only. Unquoted backslash is an escape. +# +# Required database connection (no VECTOR__ prefix): +ConnectionStrings__NntpDB=REPLACE_WITH_MYSQL_CONNECTION_STRING +# +# Required AES-256 XTrace key (64 hex or Base64). Never commit the real value. +NNTPD__XTRACEKEY=REPLACE_WITH_32_BYTE_KEY_AS_64_HEX_OR_BASE64 +# +# Site-specific Nntpd overrides use Nntpd__* (section name Nntpd is preserved). +# Do not invent bind addresses, zone ids, or peer catalogues here; set them for the deployment. +# Example: Nntpd__ServerId=1 +# Example: Nntpd__DnsSuffix=example.test diff --git a/deploy/systemd/vectornntp-reader.service b/deploy/systemd/vectornntp-reader.service new file mode 100644 index 00000000..fc331d74 --- /dev/null +++ b/deploy/systemd/vectornntp-reader.service @@ -0,0 +1,48 @@ +[Unit] +Description=VectorNNTP ReaderService +# Install VectorNNTPD.ReaderService, VectorNNTPD.ReaderService.json, +# VectorNNTPD.RabbitMQ.json, and this unit. Create user/group vectornntp, +# create logs/ and certs/ owned by that account, and install +# /etc/vectornntp/reader.env before enable. +Documentation=file:///opt/vectornntp/reader/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=vectornntp +Group=vectornntp + +# Configuration root is the executable directory (not the process CWD alone). +WorkingDirectory=/opt/vectornntp/reader +ExecStart=/opt/vectornntp/reader/VectorNNTPD.ReaderService + +# Align TimeoutStopSec above Nntpd graceful shutdown (shipped default 30s). +TimeoutStartSec=180 +TimeoutStopSec=60 + +Restart=on-failure +RestartSec=5 + +# Requires ConnectionStrings__NntpDB. RabbitMQ/ACME/Cloudflare secrets come from +# nntpsharedconfig when configured that way; this unit does not set them. +EnvironmentFile=-/etc/vectornntp/reader.env +Environment=DOTNET_ENVIRONMENT=Production + +# WatchdogSec example (enable only after readiness is verified): +# WatchdogSec=30 +# Restart=on-watchdog + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/vectornntp/reader/logs /opt/vectornntp/reader/certs + +KillSignal=SIGTERM +SendSIGKILL=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/vectornntp-transit.service b/deploy/systemd/vectornntp-transit.service new file mode 100644 index 00000000..9516303b --- /dev/null +++ b/deploy/systemd/vectornntp-transit.service @@ -0,0 +1,44 @@ +[Unit] +Description=VectorNNTP TransitService +# Install VectorNNTPD.TransitService, VectorNNTPD.TransitService.json, +# VectorNNTPD.RabbitMQ.json, and this unit. Create user/group vectornntp, +# create logs/ and certs/ owned by that account, and install +# /etc/vectornntp/transit.env before enable. +Documentation=file:///opt/vectornntp/transit/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=vectornntp +Group=vectornntp + +WorkingDirectory=/opt/vectornntp/transit +ExecStart=/opt/vectornntp/transit/VectorNNTPD.TransitService + +TimeoutStartSec=180 +TimeoutStopSec=60 + +Restart=on-failure +RestartSec=5 + +EnvironmentFile=-/etc/vectornntp/transit.env +Environment=DOTNET_ENVIRONMENT=Production + +# WatchdogSec example (enable only after readiness is verified): +# WatchdogSec=30 +# Restart=on-watchdog + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/vectornntp/transit/logs /opt/vectornntp/transit/certs + +KillSignal=SIGTERM +SendSIGKILL=yes + +[Install] +WantedBy=multi-user.target diff --git a/docs/nntpd-reader-transit.md b/docs/nntpd-reader-transit.md new file mode 100644 index 00000000..3ae0f35d --- /dev/null +++ b/docs/nntpd-reader-transit.md @@ -0,0 +1,203 @@ +# NNTP Reader and Transit hosts + +This document describes the migrated NNTP server split in VectorNNTPD. It does not replace BackFiller or StorageService architecture docs. + +## Projects + +| Project | Role | +|---------|------| +| `VectorNNTPD.Common` | Shared connectivity and domain primitives (ACME, Cloudflare, NntpDB, articles, VATP, RabbitMQ connection lifecycle). Not an NNTP server library. | +| `VectorNNTPD.ReaderService` | Reader executable **and** Reader NNTP implementation (session/commands, listeners, auth, PostFilter, moderation, email, Redis session state, ingestion, RabbitMQ topology) | +| `VectorNNTPD.TransitService` | Transit executable **and** Transit NNTP implementation (peer DNS/auth, CHECK/TAKETHIS/IHAVE, transit peer state, shared plumbing duplicated for independence) | +| `tests/VectorNNTPD.ReaderService.Tests` | Reader protocol, hosting, PostFilter, session-state, and related tests | +| `tests/VectorNNTPD.TransitService.Tests` | Transit protocol, peer, ingestion, and related tests | + +There is **no** `VectorNNTPD.NntpServer` assembly. ReaderService → Common and TransitService → Common only. Shared *server* plumbing required by both hosts is **duplicated** into each executable project so neither depends on the other and Common does not become a relocated monolith. + +Configuration section name remains `Nntpd`. Primary JSON files are `{entry assembly}.json` plus shared `VectorNNTPD.RabbitMQ.json`, loaded from `AppContext.BaseDirectory` (the executable directory). + +## Role isolation + +- Each host registers only its role via `AddNntpReaderHosting` / `AddNntpTransitHosting`. Opposite-role registration APIs throw `NotSupportedException`. +- FQDN host-label prefixes are `reader` and `transit`. +- Combined / dual-role hosting is not supported in production or tests. + +## Protocol gating + +Command catalog flags and **CAPABILITIES** advertising follow the host role. MODE READER / MODE STREAM behaviour is role-gated. + +### HELP vs CAPABILITIES + +**HELP** returns one static multiline syntax inventory for every session (see `Help` remarks and `NntpResponses.HelpComplete`). It lists both reader and transit command forms. That matches the source monolith HELP contract and RFC 3977’s view of HELP as human-readable syntax reference, not an authorization advertisement. + +**CAPABILITIES** is the authoritative, role-filtered advertisement. A Reader host omits STREAMING/IHAVE/SPEEDTEST; a Transit host omits READER/MODE-READER/POST. Invoking a command that HELP mentions but CAPABILITIES does not advertise still yields the normal protocol rejection for that role. + +## Canonical article bytes + +Ingress destuffs transport wire once, then `ArticleRecordFactory` / `NntpArticleCanonicalMaterializer` materializes CanonicalV1 `ArtData` with Path/Date rewrite and **one storage body stuffing pass** (dest Common contract). Source monolith Common copied destuffed bodies without that pass; VectorNNTPD Common does not. Migrated tests assert the dest CanonicalV1 stuffed-body contract. + +## Schema DDL + +Operator SQL for PostFilter and `nntpusers` account-art-type lives under `docs/schema/` (`postfilter.sql`, `nntpusers-account-art-type.sql`). Apply these on the site MySQL database before enabling features that depend on those tables/columns. Do not invent production DDL beyond the checked-in scripts. + +## Release package layout + +`tools/Publish-Release.ps1` and `.github/workflows/build.yml` publish BackFiller, StorageService, ReaderService, and TransitService as self-contained Native AOT for `win-x64` and `linux-x64`. Each service is an **independent package directory**. Published outputs must not contain a `VectorNNTPD.NntpServer` assembly. + +Local release script output: + +```text +release/win-64/ + backfiller/ VectorNNTPD.BackFillerService.exe + BackFiller JSON + RabbitMQ JSON + storage/ VectorNNTPD.StorageService.exe + Storage JSON + RabbitMQ JSON + reader/ VectorNNTPD.ReaderService.exe + Reader JSON + RabbitMQ JSON + transit/ VectorNNTPD.TransitService.exe + Transit JSON + RabbitMQ JSON + +release/linux-64/ + backfiller/ executable + JSON + RabbitMQ JSON + vectornntp-backfiller.service + backfiller.env.example + README.md + storage/ executable + JSON + RabbitMQ JSON + vectornntp-storageservice.service + storageservice.env.example + README.md + reader/ executable + JSON + RabbitMQ JSON + vectornntp-reader.service + reader.env.example + README.md + transit/ executable + JSON + RabbitMQ JSON + vectornntp-transit.service + transit.env.example + README.md +``` + +CI artifact roots use the same package names under `.artifacts/win-x64/` and `.artifacts/linux-x64/`. + +Role isolation in the package: + +- Reader package contains only Reader executable and `VectorNNTPD.ReaderService.json` (not Transit). +- Transit package contains only Transit executable and `VectorNNTPD.TransitService.json` (not Reader). +- Each package carries its own copy of `VectorNNTPD.RabbitMQ.json`. +- Linux packages include only that role’s systemd unit and environment example. +- Secrets and site credentials are never packaged; use `/etc/vectornntp/*.env` and site-edited JSON / `nntpsharedconfig`. + +Startup smoke (configuration bind only; no listeners or brokers): + +| Host | Env | Config path env | Checkpoint FQDN (ServerId=1) | +|------|-----|-----------------|------------------------------| +| Reader | `VECTORNNTP_READER_STARTUP_SMOKE=1` | `VECTORNNTP_READER_STARTUP_SMOKE_CONFIG` | `reader01.usenet.ninja` | +| Transit | `VECTORNNTP_TRANSIT_STARTUP_SMOKE=1` | `VECTORNNTP_TRANSIT_STARTUP_SMOKE_CONFIG` | `transit01.usenet.ninja` | + +Smoke fixtures: `tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json` and `tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json`. + +## Deployment handoff + +Install each role from its package directory alone. Do not merge Reader and Transit into one install tree. + +### 1. Install roots + +| Role | Package directory | Install root | Executable | +|------|-------------------|--------------|------------| +| Reader | `reader/` | `/opt/vectornntp/reader` | `VectorNNTPD.ReaderService` | +| Transit | `transit/` | `/opt/vectornntp/transit` | `VectorNNTPD.TransitService` | + +BackFiller and StorageService remain `/opt/vectornntp/backfiller` and `/opt/vectornntp/storageservice` respectively (packages `backfiller/` and `storage/`). + +### 2. Files that must accompany each executable + +In the install root (same directory as the executable): + +| File | Reader | Transit | +|------|--------|---------| +| Host executable | required | required | +| `VectorNNTPD.ReaderService.json` | required | absent | +| `VectorNNTPD.TransitService.json` | absent | required | +| `VectorNNTPD.RabbitMQ.json` | required | required | +| `README.md` (linux package) | optional documentation | optional documentation | + +Configuration is loaded from `AppContext.BaseDirectory` (the executable directory), not from a shared sibling role. + +### 3. Environment files and permissions + +| Role | Example in package | Installed path | Owner / mode | +|------|--------------------|----------------|--------------| +| Reader | `reader.env.example` | `/etc/vectornntp/reader.env` | `root:root` `0600` | +| Transit | `transit.env.example` | `/etc/vectornntp/transit.env` | `root:root` `0600` | + +Unit files install to `/etc/systemd/system/vectornntp-reader.service` and `/etc/systemd/system/vectornntp-transit.service`. Units use `EnvironmentFile=-/etc/vectornntp/.env` (leading `-` means a missing or unreadable file is ignored by systemd; missing keys still fail host validation). + +Create writable runtime directories owned by the service account before first start: + +```bash +sudo install -d -o vectornntp -g vectornntp -m 0750 /opt/vectornntp/reader/logs /opt/vectornntp/reader/certs +sudo install -d -o vectornntp -g vectornntp -m 0750 /opt/vectornntp/transit/logs /opt/vectornntp/transit/certs +``` + +Do not `chown -R` the install tree to `vectornntp`. Keep executables and JSON root-owned and not writable by the service account. + +### 4. Site-specific binds, ports, FQDNs, and Transit peers + +Mandatory site editing (do not invent production values here): + +- `Nntpd:ServerId`, DNS suffix / FQDN host label conventions (`reader` / `transit` prefixes). +- Listener bind addresses and ports in the host JSON (and optional `Nntpd__*` environment overrides). +- Transit peer catalogue and related peer DNS/auth settings for Transit hosts only. +- Cloudflare zone identifiers and related DNS publication settings when ACME/DNS features are enabled for the site. + +Edit JSON as root after install, or supply overrides in the role environment file. Never commit real site addresses, peer lists, or zone ids. + +### 5. Redis, RabbitMQ, and MySQL + +| Dependency | Where configured | Notes | +|------------|------------------|-------| +| MySQL (`ConnectionStrings:NntpDB`) | **Mandatory** in `/etc/vectornntp/.env` as `ConnectionStrings__NntpDB` | Placeholder in the example fails at NntpDB startup check; a missing key fails validation earlier. | +| `NNTPD__XTRACEKEY` | **Mandatory** in the role environment file | 32-byte key as 64 hex or Base64. Never commit the real value. | +| Redis (`Redis` section) | Host JSON / site overrides | Session state, transit peer state, PostFilter quotas/history as enabled for the role. | +| RabbitMQ | `VectorNNTPD.RabbitMQ.json` in the install root; credentials typically from `nntpsharedconfig` | Topology and Article Work. Do not put broker passwords in Git. | +| Cloudflare / ACME | Site policy via shared config / JSON | Certificate issuance and DNS publication when enabled. | + +### 6. Schema preparation + +Before enabling PostFilter or account art-type features that depend on the checked-in DDL: + +1. Apply `docs/schema/postfilter.sql` on the site database when PostFilter tables are required. +2. Apply `docs/schema/nntpusers-account-art-type.sql` when that column/feature is required. +3. Confirm `nntpsharedconfig` and other existing operational tables match the site’s current VectorNNTPD expectations. + +### 7. Service startup and health verification + +Example Reader install (from the `reader/` package contents; adjust paths to the extracted artifact): + +```bash +sudo groupadd --system vectornntp || true +sudo useradd --system --gid vectornntp --home-dir /opt/vectornntp --shell /usr/sbin/nologin vectornntp || true +sudo install -d -o root -g root -m 0755 /opt/vectornntp /opt/vectornntp/reader +sudo install -o root -g root -m 0755 VectorNNTPD.ReaderService /opt/vectornntp/reader/VectorNNTPD.ReaderService +sudo install -o root -g root -m 0644 VectorNNTPD.ReaderService.json VectorNNTPD.RabbitMQ.json README.md /opt/vectornntp/reader/ +sudo install -d -o vectornntp -g vectornntp -m 0750 /opt/vectornntp/reader/logs /opt/vectornntp/reader/certs +sudo install -d -o root -g root -m 0755 /etc/vectornntp +sudo install -o root -g root -m 0600 reader.env.example /etc/vectornntp/reader.env +sudo install -o root -g root -m 0644 vectornntp-reader.service /etc/systemd/system/vectornntp-reader.service +sudoedit /etc/vectornntp/reader.env +sudo systemctl daemon-reload +sudo systemd-analyze verify vectornntp-reader.service +sudo stat -c '%U:%G %a' /etc/vectornntp/reader.env # expect root:root 600 +sudo systemctl enable --now vectornntp-reader.service +sudo systemctl status vectornntp-reader.service --no-pager +sudo journalctl -u vectornntp-reader.service -e --no-pager +``` + +Repeat the same pattern for Transit using the `transit/` package, `/opt/vectornntp/transit`, `transit.env.example` → `/etc/vectornntp/transit.env`, and `vectornntp-transit.service`. + +Health checks that do **not** print secrets: + +- `systemctl is-active vectornntp-reader.service` / `vectornntp-transit.service` +- journal lines showing host starting and ready/notify as configured +- Listener reachability from an allowed client using site-known ports + +Do not run `systemctl show -p Environment` or paste environment files into tickets. + +Optional integrations (not required for a minimal start): + +- **Spamd**: configure only when PostFilter SpamAssassin path is intentionally enabled for the site. +- **systemd watchdog**: leave `WatchdogSec` commented in the unit until readiness notify behaviour is verified on the host; then enable deliberately. + +### 8. Safe rollback + +1. Stop the unit: `sudo systemctl stop vectornntp-reader.service` (or transit). +2. Keep the previous release tree (for example rename `/opt/vectornntp/reader` to `/opt/vectornntp/reader.prev` before installing the new package). +3. Restore the previous executable + JSON + RabbitMQ JSON into the install root (or move the previous tree back). +4. Keep `/etc/vectornntp/.env` unless the rollback also requires restoring a prior environment file from a secure backup (never from Git). +5. `sudo systemctl daemon-reload` if the unit file changed, then `sudo systemctl start vectornntp-.service`. +6. Verify status/journal as above. + +Rollback does not require reinstalling the opposite role. Reader and Transit packages are independent. diff --git a/docs/schema/nntpusers-account-art-type.sql b/docs/schema/nntpusers-account-art-type.sql new file mode 100644 index 00000000..9737d1e6 --- /dev/null +++ b/docs/schema/nntpusers-account-art-type.sql @@ -0,0 +1,27 @@ +-- Additive nntpusers.account_art_type for AUTHINFO ArticleType capability. +-- Apply to the existing account database before deploying an NNTPD that +-- SELECTs account_art_type. Idempotent. Does not drop nntpusers. +-- Default 65535 is ArticleTypeCapabilities.All (unrestricted for existing users). +-- NNTPD does not run this script. + +DROP PROCEDURE IF EXISTS nntpusers_add_account_art_type; + +DELIMITER $$ +CREATE PROCEDURE nntpusers_add_account_art_type() +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'nntpusers' + AND column_name = 'account_art_type' + ) THEN + ALTER TABLE nntpusers + ADD COLUMN account_art_type INT UNSIGNED NOT NULL DEFAULT 65535 + COMMENT 'ArticleType flags; 65535=ArticleTypeCapabilities.All (unrestricted)'; + END IF; +END$$ +DELIMITER ; + +CALL nntpusers_add_account_art_type(); +DROP PROCEDURE nntpusers_add_account_art_type; diff --git a/docs/schema/postfilter.sql b/docs/schema/postfilter.sql new file mode 100644 index 00000000..12a881c0 --- /dev/null +++ b/docs/schema/postfilter.sql @@ -0,0 +1,200 @@ +-- NntpDB PostFilter control-plane schema. +-- Apply this script to an empty NntpDB before VectorNNTP.NNTPD starts. +-- PostFilter policy is authoritative in NntpDB, not appsettings. +-- Policy rows are versioned and append-only. +-- nntppostfiltercurrent identifies the published revision. +-- Collection rows are keyed by revision so a load cannot mix revisions. +-- Seed revision 1 is Gate=Disabled (no filtering until an operator publishes). +-- Change policy by inserting a complete new revision, then publishing it. +-- Do not modify a published revision in place. +-- NNTPD only SELECTs. There is no application migration runner. +-- Conventions match existing NntpDB catalogues: InnoDB, snake_case, +-- CHAR(1) Y/N, DATETIME(3), utf8mb4_unicode_ci. + +CREATE TABLE nntppostfilterpolicy ( + revision BIGINT UNSIGNED NOT NULL, + updated_utc DATETIME(3) NOT NULL, + gate VARCHAR(16) NOT NULL, + long_window_ms BIGINT UNSIGNED NOT NULL, + short_window_ms BIGINT UNSIGNED NOT NULL, + max_messages_long BIGINT UNSIGNED NOT NULL, + max_bytes_long BIGINT UNSIGNED NOT NULL, + max_identical_long BIGINT UNSIGNED NOT NULL, + max_messages_short BIGINT UNSIGNED NOT NULL, + max_bytes_short BIGINT UNSIGNED NOT NULL, + max_identical_short BIGINT UNSIGNED NOT NULL, + sa_enabled CHAR(1) NOT NULL, + sa_on_failure VARCHAR(16) NULL, + sa_max_article_size INT NOT NULL, + sa_port INT UNSIGNED NOT NULL, + sa_protocol_version VARCHAR(16) NOT NULL, + sa_max_connections INT UNSIGNED NOT NULL, + sa_host_selection VARCHAR(16) NOT NULL, + sa_connect_timeout_ms INT UNSIGNED NOT NULL, + sa_operation_timeout_ms INT UNSIGNED NOT NULL, + PRIMARY KEY (revision), + CONSTRAINT chk_nntppostfilterpolicy_revision CHECK (revision >= 1), + CONSTRAINT chk_nntppostfilterpolicy_gate CHECK (gate IN ('Disabled', 'Active', 'Closed')), + CONSTRAINT chk_nntppostfilterpolicy_windows CHECK (long_window_ms >= 1 AND short_window_ms >= 1), + CONSTRAINT chk_nntppostfilterpolicy_sa_enabled CHECK (sa_enabled IN ('Y', 'N')), + CONSTRAINT chk_nntppostfilterpolicy_sa_on_failure CHECK ( + sa_on_failure IS NULL OR sa_on_failure IN ('Reject', 'Accept') + ), + CONSTRAINT chk_nntppostfilterpolicy_sa_required CHECK ( + sa_enabled <> 'Y' OR sa_on_failure IN ('Reject', 'Accept') + ), + CONSTRAINT chk_nntppostfilterpolicy_sa_size CHECK (sa_max_article_size >= 0), + CONSTRAINT chk_nntppostfilterpolicy_sa_port CHECK (sa_port BETWEEN 1 AND 65535), + CONSTRAINT chk_nntppostfilterpolicy_sa_pool CHECK (sa_max_connections BETWEEN 1 AND 32), + CONSTRAINT chk_nntppostfilterpolicy_sa_hosts CHECK (sa_host_selection IN ('RoundRobin', 'Failover')), + CONSTRAINT chk_nntppostfilterpolicy_sa_timeouts CHECK ( + sa_connect_timeout_ms >= 1 + AND sa_operation_timeout_ms >= 1 + AND sa_connect_timeout_ms <= sa_operation_timeout_ms + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='Append-only PostFilter scalar revisions'; + +CREATE TABLE nntppostfiltercurrent ( + policy_id TINYINT UNSIGNED NOT NULL, + revision BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (policy_id), + CONSTRAINT chk_nntppostfiltercurrent_policy_id CHECK (policy_id = 1), + CONSTRAINT fk_nntppostfiltercurrent_revision + FOREIGN KEY (revision) REFERENCES nntppostfilterpolicy (revision) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='Singleton published PostFilter revision (policy_id=1)'; + +CREATE TABLE nntppostfilteraccounts ( + revision BIGINT UNSIGNED NOT NULL, + list_kind VARCHAR(8) NOT NULL, + account_name CHAR(32) NOT NULL, + PRIMARY KEY (revision, list_kind, account_name), + CONSTRAINT chk_nntppostfilteraccounts_kind CHECK (list_kind IN ('deny', 'allow')), + CONSTRAINT chk_nntppostfilteraccounts_name CHECK (account_name REGEXP '^[0-9a-f]{32}$'), + CONSTRAINT fk_nntppostfilteraccounts_revision + FOREIGN KEY (revision) REFERENCES nntppostfilterpolicy (revision) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='PostFilter deny/allow MD5(hex) AUTH identities; real usernames are not stored'; + +CREATE TABLE nntppostfiltercidrs ( + revision BIGINT UNSIGNED NOT NULL, + list_kind VARCHAR(8) NOT NULL, + cidr VARCHAR(64) NOT NULL, + PRIMARY KEY (revision, list_kind, cidr), + CONSTRAINT chk_nntppostfiltercidrs_kind CHECK (list_kind IN ('deny', 'allow')), + CONSTRAINT chk_nntppostfiltercidrs_value CHECK (CHAR_LENGTH(TRIM(cidr)) > 0), + CONSTRAINT fk_nntppostfiltercidrs_revision + FOREIGN KEY (revision) REFERENCES nntppostfilterpolicy (revision) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='PostFilter deny/allow client CIDRs for one revision'; + +CREATE TABLE nntppostfilterarttypes ( + revision BIGINT UNSIGNED NOT NULL, + list_kind VARCHAR(16) NOT NULL, + art_type ENUM( + 'Default', + 'Control', + 'Cancel', + 'Mime', + 'Binary', + 'UuEncode', + 'Base64', + 'YEncoded', + 'BommaNews', + 'UniData', + 'Multipart', + 'Html', + 'PostScript', + 'BinHex', + 'Partial', + 'PgpMessage' + ) NOT NULL, + PRIMARY KEY (revision, list_kind, art_type), + CONSTRAINT chk_nntppostfilterarttypes_kind CHECK (list_kind IN ('reject', 'sa_exclude')), + CONSTRAINT fk_nntppostfilterarttypes_revision + FOREIGN KEY (revision) REFERENCES nntppostfilterpolicy (revision) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='PostFilter reject and SA-exclude ArtTypes; one classifier name per row'; + +CREATE TABLE nntppostfiltersahosts ( + revision BIGINT UNSIGNED NOT NULL, + host_order INT UNSIGNED NOT NULL, + host VARCHAR(255) NOT NULL, + PRIMARY KEY (revision, host_order), + UNIQUE KEY uq_nntppostfiltersahosts_host (revision, host), + CONSTRAINT chk_nntppostfiltersahosts_host CHECK (CHAR_LENGTH(TRIM(host)) > 0), + CONSTRAINT fk_nntppostfiltersahosts_revision + FOREIGN KEY (revision) REFERENCES nntppostfilterpolicy (revision) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='PostFilter SPAMD hosts in compiled order for one revision'; + +CREATE TABLE nntppostfilterrejections ( + rejection_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + rejected_utc DATETIME(3) NOT NULL, + revision BIGINT UNSIGNED NOT NULL, + account_name CHAR(32) NULL, + source_ip VARBINARY(16) NOT NULL, + art_type INT UNSIGNED NOT NULL, + message_id VARCHAR(250) NULL, + article_size INT UNSIGNED NOT NULL, + stage VARCHAR(32) NOT NULL, + reason VARCHAR(64) NOT NULL, + sa_status VARCHAR(16) NULL, + sa_score DECIMAL(8,3) NULL, + sa_threshold DECIMAL(8,3) NULL, + article_payload LONGBLOB NULL, + PRIMARY KEY (rejection_id), + KEY ix_nntppostfilterrejections_rejected_utc (rejected_utc), + KEY ix_nntppostfilterrejections_account (account_name), + KEY ix_nntppostfilterrejections_revision (revision), + CONSTRAINT chk_nntppostfilterrejections_source_ip CHECK (OCTET_LENGTH(source_ip) IN (4, 16)), + CONSTRAINT chk_nntppostfilterrejections_account CHECK ( + account_name IS NULL OR account_name REGEXP '^[0-9a-f]{32}$' + ), + CONSTRAINT chk_nntppostfilterrejections_stage CHECK (CHAR_LENGTH(TRIM(stage)) > 0), + CONSTRAINT chk_nntppostfilterrejections_reason CHECK (CHAR_LENGTH(TRIM(reason)) > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + COMMENT='Append-only PostFilter rejection evidence; payload is unstuffed article bytes or NULL'; + +DELIMITER $$ +CREATE TRIGGER trg_nntppostfiltercurrent_revision_forward +BEFORE UPDATE ON nntppostfiltercurrent +FOR EACH ROW +BEGIN + IF NEW.revision <= OLD.revision THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'nntppostfiltercurrent.revision must increase'; + END IF; +END$$ +DELIMITER ; + +INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms +) VALUES ( + 1, UTC_TIMESTAMP(3), 'Disabled', + 86400000, 600000, + 0, 0, 0, 0, 0, 0, + 'N', NULL, 131072, 783, + '1.5', 4, 'RoundRobin', + 5000, 30000 +); + +INSERT INTO nntppostfilterarttypes (revision, list_kind, art_type) +VALUES (1, 'sa_exclude', 'YEncoded'); + +INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, 1); + +-- nntppostfilterrejections is append-oriented evidence. This script does not +-- delete rows. Retention is an explicit operator decision; NNTPD has no +-- PostFilter evidence sweeper. +-- +-- Authenticated account ArtType capability is not created here. +-- Apply docs/schema/nntpusers-account-art-type.sql to the existing account +-- database before deploying the AUTHINFO SELECT that reads account_art_type. diff --git a/src/VectorNNTPD.Common/Acme/AcmeTransactionJournal.cs b/src/VectorNNTPD.Common/Acme/AcmeTransactionJournal.cs index 2a11ce33..13252943 100644 --- a/src/VectorNNTPD.Common/Acme/AcmeTransactionJournal.cs +++ b/src/VectorNNTPD.Common/Acme/AcmeTransactionJournal.cs @@ -602,19 +602,14 @@ private static AcmeJournalWireEvent SerializeEvent(AcmeJournalEvent journalEvent /// The in-memory transaction. private static AcmeJournalTransaction ParseTransaction(JsonElement root) { - var identifiers = new List(); - if (root.TryGetProperty("identifiers", out var identifiersElement) - && identifiersElement.ValueKind == JsonValueKind.Array) - { - foreach (var item in identifiersElement.EnumerateArray()) - { - var value = item.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - identifiers.Add(value); - } - } - } + var identifiers = root.TryGetProperty("identifiers", out var identifiersElement) + && identifiersElement.ValueKind == JsonValueKind.Array + ? identifiersElement.EnumerateArray() + .Select(static item => item.GetString()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value!) + .ToList() + : []; var events = new List(); if (root.TryGetProperty("events", out var eventsElement) @@ -731,10 +726,9 @@ private static string FormatTimestamp(DateTimeOffset value) => /// The inspected fields. Serial and thumbprint come from . private static CertificateInspection InspectCertificate(byte[] pfxBytes, string password) { - X509Certificate2? cert = null; try { - cert = PfxCrypto.LoadCertificate(pfxBytes, password); + using var cert = PfxCrypto.LoadCertificate(pfxBytes, password); return new CertificateInspection( cert.SerialNumber, cert.Thumbprint, @@ -745,10 +739,6 @@ private static CertificateInspection InspectCertificate(byte[] pfxBytes, string { throw new AcmeCertificateException("journal_certificate_inspect_failed", ex.GetType().Name); } - finally - { - cert?.Dispose(); - } } /// Leaf fields copied into a successful journal transaction. diff --git a/src/VectorNNTPD.Common/Acme/AuthoritativeTxtResolver.cs b/src/VectorNNTPD.Common/Acme/AuthoritativeTxtResolver.cs index 39b9368c..664f19ba 100644 --- a/src/VectorNNTPD.Common/Acme/AuthoritativeTxtResolver.cs +++ b/src/VectorNNTPD.Common/Acme/AuthoritativeTxtResolver.cs @@ -131,7 +131,7 @@ public async Task> LookupTxtAsync(string name, Cancellatio return []; } - return (string[])[.. intersection]; + return intersection.ToArray(); } /// Resolves apex NS addresses and returns them as port-53 endpoints. An empty set throws category ns_discovery_failed. diff --git a/src/VectorNNTPD.Common/Acme/CertificateValidator.cs b/src/VectorNNTPD.Common/Acme/CertificateValidator.cs index 4cb9a1f1..aa8ce3e9 100644 --- a/src/VectorNNTPD.Common/Acme/CertificateValidator.cs +++ b/src/VectorNNTPD.Common/Acme/CertificateValidator.cs @@ -42,46 +42,45 @@ internal static CertificateStatus ValidatePfx( Reason: $"parse_failed:{ex.GetType().Name}"); } - try + using (leaf) { - if (!leaf.HasPrivateKey) + try { - return new CertificateStatus(false, true, null, "missing_private_key"); + if (!leaf.HasPrivateKey) + { + return new CertificateStatus(false, true, null, "missing_private_key"); + } + + using var privateKey = leaf.GetRSAPrivateKey() + ?? throw new AcmeCertificateException("key_mismatch", "unsupported_key_type"); + + var domains = GetDnsNames(leaf); + AssertRequiredDomains(domains, requiredDomains); + AssertValidityWindow(leaf, current); + AssertKeyMatch(leaf, privateKey); + AssertServerAuth(leaf); + + var material = new CertificateMaterial( + PfxBytes: pfxBytes.ToArray(), + Domains: domains.OrderBy(static d => d, StringComparer.Ordinal).ToArray(), + NotBefore: leaf.NotBefore.ToUniversalTime(), + NotAfter: leaf.NotAfter.ToUniversalTime()); + + var due = current >= material.NotAfter - renewalThreshold; + return new CertificateStatus( + Usable: true, + DueForRenewal: due, + Material: material, + Reason: due ? "renewal_threshold" : "ok"); + } + catch (AcmeCertificateException ex) + { + return new CertificateStatus( + Usable: false, + DueForRenewal: true, + Material: null, + Reason: ex.Category); } - - using var privateKey = leaf.GetRSAPrivateKey() - ?? throw new AcmeCertificateException("key_mismatch", "unsupported_key_type"); - - var domains = GetDnsNames(leaf); - AssertRequiredDomains(domains, requiredDomains); - AssertValidityWindow(leaf, current); - AssertKeyMatch(leaf, privateKey); - AssertServerAuth(leaf); - - var material = new CertificateMaterial( - PfxBytes: pfxBytes.ToArray(), - Domains: (string[])[.. domains.OrderBy(static d => d, StringComparer.Ordinal)], - NotBefore: leaf.NotBefore.ToUniversalTime(), - NotAfter: leaf.NotAfter.ToUniversalTime()); - - var due = current >= material.NotAfter - renewalThreshold; - return new CertificateStatus( - Usable: true, - DueForRenewal: due, - Material: material, - Reason: due ? "renewal_threshold" : "ok"); - } - catch (AcmeCertificateException ex) - { - return new CertificateStatus( - Usable: false, - DueForRenewal: true, - Material: null, - Reason: ex.Category); - } - finally - { - leaf.Dispose(); } } diff --git a/src/VectorNNTPD.Common/Acme/Dns01Solver.cs b/src/VectorNNTPD.Common/Acme/Dns01Solver.cs index 7049f743..b31d8313 100644 --- a/src/VectorNNTPD.Common/Acme/Dns01Solver.cs +++ b/src/VectorNNTPD.Common/Acme/Dns01Solver.cs @@ -420,11 +420,12 @@ private async Task> FindRecordsByContentAsync( { var records = await _client.ListAllRecordsForNameAsync(zoneId, name, cancellationToken) .ConfigureAwait(false); - return (string[])[.. records + return records .Where(r => string.Equals(r.Type, Cloudflare.CloudflareDnsRecordTypes.TXT, StringComparison.OrdinalIgnoreCase) && string.Equals(r.Content, content, StringComparison.Ordinal)) - .Select(r => r.Id)]; + .Select(r => r.Id) + .ToArray(); } /// Deletes one record. HTTP 404, a message containing not found, or Cloudflare error 81044 is treated as already deleted. diff --git a/src/VectorNNTPD.Common/Acme/DnsZoneCoverage.cs b/src/VectorNNTPD.Common/Acme/DnsZoneCoverage.cs index f43c890f..fc37a63d 100644 --- a/src/VectorNNTPD.Common/Acme/DnsZoneCoverage.cs +++ b/src/VectorNNTPD.Common/Acme/DnsZoneCoverage.cs @@ -62,9 +62,8 @@ internal static void RequireIdentitiesInDnsZone(IReadOnlyList identities throw new AcmeConfigurationException("missing_identities", "no certificate identities"); } - foreach (var raw in identities) + foreach (var host in identities.Select(static raw => NormalizeDnsHostname(raw, "certificate_identity"))) { - var host = NormalizeDnsHostname(raw, "certificate_identity"); if (!ZoneCoversHostname(apex, host)) { throw new AcmeConfigurationException( diff --git a/src/VectorNNTPD.Common/Acme/Protocol/AcmeClient.cs b/src/VectorNNTPD.Common/Acme/Protocol/AcmeClient.cs index 24221695..dc3914fa 100644 --- a/src/VectorNNTPD.Common/Acme/Protocol/AcmeClient.cs +++ b/src/VectorNNTPD.Common/Acme/Protocol/AcmeClient.cs @@ -326,14 +326,10 @@ internal async Task DownloadCertificateAsync(Uri certificateUrl _accountKey, KeyId, certificateUrl, PemChainContentType, cancellationToken) .ConfigureAwait(false); - var alternates = new List(); - foreach (AcmeLink link in response.Links) - { - if (string.Equals(link.Relation, "alternate", StringComparison.OrdinalIgnoreCase)) - { - alternates.Add(link.Url); - } - } + var alternates = response.Links + .Where(static link => string.Equals(link.Relation, "alternate", StringComparison.OrdinalIgnoreCase)) + .Select(static link => link.Url) + .ToList(); return new AcmeCertificate(response.Body, alternates); } diff --git a/src/VectorNNTPD.Common/Acme/Protocol/AcmeHttpTransport.cs b/src/VectorNNTPD.Common/Acme/Protocol/AcmeHttpTransport.cs index b18f70fe..ec96d825 100644 --- a/src/VectorNNTPD.Common/Acme/Protocol/AcmeHttpTransport.cs +++ b/src/VectorNNTPD.Common/Acme/Protocol/AcmeHttpTransport.cs @@ -627,12 +627,9 @@ private void CaptureNonce(HttpResponseMessage response) return; } - foreach (string value in values) + foreach (string value in values.Where(static v => !string.IsNullOrEmpty(v))) { - if (!string.IsNullOrEmpty(value)) - { - _nonces.Push(value); - } + _nonces.Push(value); } } diff --git a/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs b/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs index 0be0f51e..3e1b5798 100644 --- a/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs +++ b/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs @@ -496,22 +496,20 @@ private static HeaderParseOutcome TryParseHeaders( continue; } - if (currentHeaderNameOffset >= 0) + if (currentHeaderNameOffset >= 0 + && !TryAddHeader( + ref headers, + ref headerCount, + ref rentedHeaders, + options.MaxHeaderCount, + new NntpArticleHeaderEntry( + currentKnownName, + currentHeaderNameOffset, + currentHeaderNameLength, + currentHeaderValueOffset, + currentHeaderHasValue ? currentHeaderValueEndExclusive - currentHeaderValueOffset : 0))) { - if (!TryAddHeader( - ref headers, - ref headerCount, - ref rentedHeaders, - options.MaxHeaderCount, - new NntpArticleHeaderEntry( - currentKnownName, - currentHeaderNameOffset, - currentHeaderNameLength, - currentHeaderValueOffset, - currentHeaderHasValue ? currentHeaderValueEndExclusive - currentHeaderValueOffset : 0))) - { - return HeaderParseOutcome.Fail(NntpArticleParseFailureCode.TooManyHeaders, articleBytes, headerStart, lineContentEnd, headers[..headerCount]); - } + return HeaderParseOutcome.Fail(NntpArticleParseFailureCode.TooManyHeaders, articleBytes, headerStart, lineContentEnd, headers[..headerCount]); } if (headerCount >= options.MaxHeaderCount) @@ -579,6 +577,7 @@ private static HeaderParseOutcome TryParseHeaders( /// Hard cap. The rented array is this long. /// Header to store. /// when is already at . + // codeql[cs/too-many-ref-parameters] Hot-path ArrayPool growth; refs avoid allocating a mutable header-buffer holder per article. private static bool TryAddHeader( ref Span headers, ref int headerCount, diff --git a/src/VectorNNTPD.Common/Articles/Validation/NntpMessageIdValidationSimd.cs b/src/VectorNNTPD.Common/Articles/Validation/NntpMessageIdValidationSimd.cs index 66c6b766..483f3db0 100644 --- a/src/VectorNNTPD.Common/Articles/Validation/NntpMessageIdValidationSimd.cs +++ b/src/VectorNNTPD.Common/Articles/Validation/NntpMessageIdValidationSimd.cs @@ -99,8 +99,12 @@ private static bool TryConsumeDotAtom(ReadOnlySpan messageId, int end, ref private static bool TryConsumeAtom(ReadOnlySpan messageId, int end, ref int index) { var start = index; - while (TryConsumeAtext(messageId, end, ref index)) + while (true) { + if (!TryConsumeAtext(messageId, end, ref index)) + { + break; + } } return index > start; @@ -119,8 +123,12 @@ private static bool TryConsumeAtom(ReadOnlySpan messageId, int end, ref in private static bool TryConsumeNoFoldLiteral(ReadOnlySpan messageId, int end, ref int index) { index++; - while (TryConsumeMdtext(messageId, end, ref index)) + while (true) { + if (!TryConsumeMdtext(messageId, end, ref index)) + { + break; + } } if ((uint)index >= (uint)end || messageId[index] != (byte)']') diff --git a/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciler.cs b/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciler.cs index c1f4cbbc..005c7073 100644 --- a/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciler.cs +++ b/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciler.cs @@ -318,6 +318,7 @@ public async Task> ListRecordIdsAsync( { var records = await ListExactFqdnRecordsAsync(zoneId, fqdn, cancellationToken).ConfigureAwait(false); var ids = new List(records.Count); + // codeql[cs/linq/missed-where] Gate-held list projection; Where would allocate under SemaphoreSlim. foreach (var record in records) { if (!string.IsNullOrWhiteSpace(record.Id)) @@ -564,6 +565,7 @@ private static void CollectUnmanaged( string type, List<(string Type, CloudflareDnsRecord Record)> restore) { + // codeql[cs/linq/missed-where] Gate-held restore snapshot; Where would allocate under SemaphoreSlim. foreach (var record in records) { if (!CloudflareManagedDnsPolicy.MatchesManagedAttributes(record)) @@ -1143,6 +1145,7 @@ private static HashSet ToContentSet(IReadOnlyList addresses) private static HashSet ToObservedContentSet(IReadOnlyList records, string type) { var set = new HashSet(StringComparer.OrdinalIgnoreCase); + // codeql[cs/linq/missed-select] Gate-held verification path; Select/Where would allocate under SemaphoreSlim. foreach (var record in records) { var content = NormalizeContent(record.Content, type); diff --git a/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciliationService.cs b/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciliationService.cs index 9f9572b7..09aa4381 100644 --- a/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciliationService.cs +++ b/src/VectorNNTPD.Common/Cloudflare/CloudflareDnsReconciliationService.cs @@ -331,14 +331,7 @@ private async Task TryDeleteRecordsCreatedByThisAttemptAsync( .ListRecordIdsAsync(zoneId, fqdn, cleanupCts.Token) .ConfigureAwait(false); var before = new HashSet(idsBeforeAttempt, StringComparer.Ordinal); - var created = new List(); - foreach (var id in after) - { - if (!before.Contains(id)) - { - created.Add(id); - } - } + var created = after.Where(id => !before.Contains(id)).ToList(); await _reconciler .DeleteRecordIdsAsync( diff --git a/src/VectorNNTPD.Common/Core/ApplicationLifecycle.cs b/src/VectorNNTPD.Common/Core/ApplicationLifecycle.cs index a2ce53a5..23badecc 100644 --- a/src/VectorNNTPD.Common/Core/ApplicationLifecycle.cs +++ b/src/VectorNNTPD.Common/Core/ApplicationLifecycle.cs @@ -49,6 +49,7 @@ public sealed class ApplicationLifecycle : IAsyncDisposable private readonly ILogger _logger; /// Serializes and . + // codeql[cs/missed-using-statement] Owned by ApplicationLifecycle until DisposeAsync; a using at field declaration would dispose when the constructor returns. private readonly SemaphoreSlim _gate = new(1, 1); /// Guards reads and writes. diff --git a/src/VectorNNTPD.Common/Core/ApplicationServiceManager.cs b/src/VectorNNTPD.Common/Core/ApplicationServiceManager.cs index ea537058..5529c00d 100644 --- a/src/VectorNNTPD.Common/Core/ApplicationServiceManager.cs +++ b/src/VectorNNTPD.Common/Core/ApplicationServiceManager.cs @@ -65,7 +65,7 @@ public ApplicationServiceManager( ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(logger); - _services = services as IReadOnlyList ?? (List)[.. services]; + _services = services as IReadOnlyList ?? [.. services]; _options = options; _logger = logger; } @@ -85,7 +85,7 @@ internal IReadOnlyList StartedServices { lock (_startedSync) { - return (IApplicationService[])[.. _started]; + return [.. _started]; } } } @@ -246,7 +246,9 @@ internal async Task StopAsync(CancellationToken cancellationToken) ServiceManagerLogMessages.StoppingService(_logger, service.Name, i + 1); + // codeql[cs/missed-using-statement] Conditionally constructed timeout/linked sources disposed in finally after StopAsync; a using would dispose before the stop await completes. CancellationTokenSource? timeoutCts = null; + // codeql[cs/missed-using-statement] Conditionally constructed timeout/linked sources disposed in finally after StopAsync; a using would dispose before the stop await completes. CancellationTokenSource? linkedCts = null; try { @@ -569,38 +571,44 @@ private async Task WatchServiceAsync(IApplicationService service, CancellationTo /// private async Task StopExecutionMonitoringAsync() { + // codeql[cs/missed-using-statement] CTS ownership is the manager field lifetime; Exchange takes it for cancel/await then finally Dispose below. A using at exchange would dispose before the monitor finishes. var cts = Interlocked.Exchange(ref _executionCts, null); var monitor = Interlocked.Exchange(ref _executionMonitor, null); - if (cts is not null) + try { - try + if (cts is not null) { - await cts.CancelAsync().ConfigureAwait(false); + try + { + await cts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + // Already disposed. + } } - catch (ObjectDisposedException) + + if (monitor is not null) { - // Already disposed. + try + { + await monitor.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected. + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + ServiceManagerLogMessages.ExecutionMonitorExceptionDuringShutdown(_logger, ex); + } } } - - if (monitor is not null) + finally { - try - { - await monitor.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // Expected. - } - catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) - { - ServiceManagerLogMessages.ExecutionMonitorExceptionDuringShutdown(_logger, ex); - } + cts?.Dispose(); } - - cts?.Dispose(); } } diff --git a/src/VectorNNTPD.Common/Dns/ZoneApexNameserverDiscovery.cs b/src/VectorNNTPD.Common/Dns/ZoneApexNameserverDiscovery.cs index cf7f3e9e..aaa14944 100644 --- a/src/VectorNNTPD.Common/Dns/ZoneApexNameserverDiscovery.cs +++ b/src/VectorNNTPD.Common/Dns/ZoneApexNameserverDiscovery.cs @@ -345,16 +345,9 @@ private static async Task> ResolveHostAddressesAsync( return []; } - List ips = new(all.Length); - foreach (IPAddress ip in all) - { - if (ip.AddressFamily is AddressFamily.InterNetwork or AddressFamily.InterNetworkV6) - { - ips.Add(ip); - } - } - - return ips; + return all + .Where(static ip => ip.AddressFamily is AddressFamily.InterNetwork or AddressFamily.InterNetworkV6) + .ToList(); } catch (OperationCanceledException) { diff --git a/src/VectorNNTPD.Common/Logging/GzipRollingFileSink.cs b/src/VectorNNTPD.Common/Logging/GzipRollingFileSink.cs index 14d0e5e2..056f008e 100644 --- a/src/VectorNNTPD.Common/Logging/GzipRollingFileSink.cs +++ b/src/VectorNNTPD.Common/Logging/GzipRollingFileSink.cs @@ -385,6 +385,7 @@ private void ApplyRetention() for (var index = _retainedFileCountLimit.Value; index < ranked.Count; index++) { + // codeql[cs/linq/missed-where] Retention runs under _sync; Where would allocate while holding the sink lock. foreach (var path in ranked[index].Paths) { if (!IsCurrent(path)) @@ -459,10 +460,8 @@ private bool IsCurrent(string path) => internal DateTime? NextCheckpoint(DateTime instant) => _interval switch { RollingInterval.Infinite => null, - RollingInterval.Year => new DateTime(instant.Year + 1, 1, 1, 0, 0, 0, instant.Kind), - RollingInterval.Month => instant.Month == 12 - ? new DateTime(instant.Year + 1, 1, 1, 0, 0, 0, instant.Kind) - : new DateTime(instant.Year, instant.Month + 1, 1, 0, 0, 0, instant.Kind), + RollingInterval.Year => new DateTime(instant.Year, 1, 1, 0, 0, 0, instant.Kind).AddYears(1), + RollingInterval.Month => new DateTime(instant.Year, instant.Month, 1, 0, 0, 0, instant.Kind).AddMonths(1), RollingInterval.Day => instant.Date.AddDays(1), RollingInterval.Hour => new DateTime(instant.Year, instant.Month, instant.Day, instant.Hour, 0, 0, instant.Kind).AddHours(1), RollingInterval.Minute => new DateTime( diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientAsyncConfirmPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientAsyncConfirmPublishChannel.cs index de0127cc..71ce5984 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientAsyncConfirmPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientAsyncConfirmPublishChannel.cs @@ -1,6 +1,7 @@ using System.Globalization; using RabbitMQ.Client; using RabbitMQ.Client.Events; +using Serilog.Debugging; using VectorNNTPD.Common.Core; namespace VectorNNTPD.Common.Messaging.RabbitMq @@ -113,6 +114,7 @@ public async ValueTask DisposeAsync() } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { + SelfLog.WriteLine("RabbitMQ async-confirm publish channel close failed and disposal will continue. " + ex); } await _channel.DisposeAsync().ConfigureAwait(false); diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs index 72c7669b..c7f512bc 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs @@ -2,6 +2,7 @@ using RabbitMQ.Client; using RabbitMQ.Client.Events; using RabbitMQ.Client.Exceptions; +using Serilog.Debugging; using VectorNNTPD.Common.Core; namespace VectorNNTPD.Common.Messaging.RabbitMq @@ -212,6 +213,7 @@ public async ValueTask DisposeAsync() } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { + SelfLog.WriteLine("RabbitMQ publish channel close failed and disposal will continue. " + ex); } // Dispose the channel first so in-flight confirm tracking finishes; the channel only holds a diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs index 8e944633..ee2923a5 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs @@ -280,7 +280,7 @@ internal RabbitMqRuntimeOptions ToRuntimeOptions() throw new InvalidOperationException("RabbitMQ Hosts must contain at least one entry."); } - string[] hosts = [.. (Hosts ?? []) + string[] hosts = [.. Hosts .Where(static x => !string.IsNullOrWhiteSpace(x)) .Select(static x => x.Trim()) .Distinct(StringComparer.OrdinalIgnoreCase)]; diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs index 2af4aa6a..e15f068d 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs @@ -273,8 +273,10 @@ public async ValueTask DisposeAsync() { await execution.ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException ex) { + // Expected when StopAsync cancels the run loop; keep the exception observed. + _ = ex; } } @@ -302,8 +304,10 @@ private async Task WatchConnectionAsync(CancellationToken cancellationToken) await RecoverAsync(cancellationToken).ConfigureAwait(false); } } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) { + // Expected cooperative cancel of the recovery watch; keep the exception observed. + _ = ex; } } diff --git a/src/VectorNNTPD.Common/Transport/ArticleTransfer/VatpFrameEncoder.cs b/src/VectorNNTPD.Common/Transport/ArticleTransfer/VatpFrameEncoder.cs index 692091f0..b0c4299b 100644 --- a/src/VectorNNTPD.Common/Transport/ArticleTransfer/VatpFrameEncoder.cs +++ b/src/VectorNNTPD.Common/Transport/ArticleTransfer/VatpFrameEncoder.cs @@ -87,11 +87,7 @@ internal static EncodedFrame EncodeMeta(uint streamId, ReadOnlyMemory meta internal static EncodedFrame EncodeData(uint streamId, ReadOnlyMemory payload, bool fin = false) { ThrowIfConnectionStream(streamId); - if (payload.Length > int.MaxValue) - { - throw new ArgumentOutOfRangeException(nameof(payload)); - } - + // ReadOnlyMemory.Length is int; Encode checks payload length via checked uint cast. var flags = fin ? VatpProtocol.FlagFin : 0u; return Encode(VatpFrameType.Data, streamId, payload, flags); } diff --git a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj index 5029413e..e5317bb7 100644 --- a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj +++ b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj @@ -12,8 +12,10 @@ - - + + + + diff --git a/src/VectorNNTPD.ReaderService/.editorconfig b/src/VectorNNTPD.ReaderService/.editorconfig new file mode 100644 index 00000000..c774aab7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/.editorconfig @@ -0,0 +1,3 @@ +# All C# namespaces must be block-scoped; file-scoped namespaces are not permitted. +[*.cs] +csharp_style_namespace_declarations = block_scoped:warning diff --git a/src/VectorNNTPD.ReaderService/Acme/AcmeCertificateApplicationService.cs b/src/VectorNNTPD.ReaderService/Acme/AcmeCertificateApplicationService.cs new file mode 100644 index 00000000..0a1c53a1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Acme/AcmeCertificateApplicationService.cs @@ -0,0 +1,35 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Acme +{ + /// + /// NNTPD lifecycle wrapper around the shared ACME certificate implementation. + /// + public sealed class AcmeCertificateApplicationService : IApplicationService, IAsyncDisposable + { + private readonly AcmeCertificateService _inner; + + /// Initializes a new wrapper. + public AcmeCertificateApplicationService(AcmeCertificateService inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + /// + public string Name => AcmeCertificateService.Name; + + /// + public Task? Execution => _inner.Execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) => _inner.StartAsync(cancellationToken); + + /// + public Task StopAsync(CancellationToken cancellationToken) => _inner.StopAsync(cancellationToken); + + /// + public ValueTask DisposeAsync() => _inner.DisposeAsync(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/Article.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/Article.cs new file mode 100644 index 00000000..6dda9e43 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/Article.cs @@ -0,0 +1,83 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Owned destuffed article produced by downstream IHAVE interpretation. + /// + /// + /// + /// and are owned destuffed copies built by + /// for benches and classifier + /// tests. Production IHAVE queues CanonicalV1 ArticleRecord instead. + /// + /// + /// is the destuffed complete-article length (headers, the header/body + /// blank line, and body). The NNTP terminating . line is not included. This matches + /// ArticleIngestionOptions.MaxArticleBytes (“after dot-unstuffing”) and is the + /// value later Transit routing should use. + /// + /// + /// Headers are raw destuffed header field lines (CRLF-terminated), not a parsed object + /// model. The body is the destuffed received representation (wire yEnc/BASE64/uuencode + /// text when those encodings were used — not decoded). + /// + /// + public readonly struct Article + { + /// Initializes a new owned article from destuffed header and body bytes. + public Article(ReadOnlyMemory headers, ReadOnlyMemory body, ArticleType type) + { + Headers = headers; + Body = body; + Type = type == ArticleType.None ? ArticleType.Default : type; + var separator = headers.Length > 0 || body.Length > 0 ? 2 : 0; + Size = checked(headers.Length + separator + body.Length); + Payload = Combine(headers, body, separator); + } + + /// Gets destuffed header field lines, each CRLF-terminated, without the blank separator. + public ReadOnlyMemory Headers { get; } + + /// Gets destuffed body bytes (received representation; not decoded). + public ReadOnlyMemory Body { get; } + + /// Gets the destuffed complete article (headers + blank line + body). + public ReadOnlyMemory Payload { get; } + + /// + /// Gets destuffed complete-article bytes (headers + blank line + body). + /// Does not include the NNTP multiline terminator. + /// + public int Size { get; } + + /// Gets the deterministic Diablo-mapped classification. + public ArticleType Type { get; } + + private static ReadOnlyMemory Combine( + ReadOnlyMemory headers, + ReadOnlyMemory body, + int separator) + { + if (headers.IsEmpty && body.IsEmpty) + { + return ReadOnlyMemory.Empty; + } + + var buffer = new byte[headers.Length + separator + body.Length]; + headers.Span.CopyTo(buffer); + if (separator > 0) + { + buffer[headers.Length] = (byte)'\r'; + buffer[headers.Length + 1] = (byte)'\n'; + } + + if (!body.IsEmpty) + { + body.Span.CopyTo(buffer.AsSpan(headers.Length + separator)); + } + + return buffer; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleEnqueueResult.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleEnqueueResult.cs new file mode 100644 index 00000000..0b0da5e2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleEnqueueResult.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// Outcome of attempting to accept an article into the ingestion queue. + public enum ArticleEnqueueResult + { + /// Article was accepted into the bounded ingestion queue (239 path). + Accepted = 0, + + /// + /// Queue is unavailable (completed/shutting down or otherwise not accepting). + /// Maps to RFC 4644 temporary failure (400 + close) for TAKETHIS. + /// + Unavailable = 1, + + /// + /// Article cannot be admitted because its payload exceeds + /// Nntpd:TransitQueueMemoryLimit. Waiting cannot free enough budget. + /// Maps to IHAVE 437 and TAKETHIS 439. + /// + Rejected = 2, + + /// + /// Article cannot be reserved immediately without waiting for capacity. + /// IHAVE maps this to 436 and must not wait. TAKETHIS still uses + /// (waits) and does not + /// observe this result on the production path. + /// + Full = 3, + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleIngestionQueue.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleIngestionQueue.cs new file mode 100644 index 00000000..ec6479c0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleIngestionQueue.cs @@ -0,0 +1,505 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Byte-budgeted -backed Transit article ingestion queue. + /// + /// + /// + /// The admission bound is + /// (Nntpd:TransitQueueMemoryLimit, default 1 GiB). Accounting uses each + /// article's owned payload length (.Length): + /// canonical / ArtData length. Every queued + /// item is a CanonicalV1 ArticleRecord. Object overhead is not counted. + /// This is not a process-wide memory cap. + /// + /// + /// Producers reserve payload bytes atomically before the article is visible + /// to consumers. Multiple producers therefore cannot oversubscribe the + /// budget. An article larger than the configured budget is rejected immediately + /// so admission cannot deadlock waiting for capacity. The historical + /// 256-article count bound is not an admission limit; the channel is unbounded + /// and the byte reservation is the resource bound. + /// + /// + /// Every successful reservation has exactly one release: consumer dequeue, + /// failed write after reserve, or (for waiters) cancel/shutdown before admit. + /// + /// + /// The channel is multi-reader. Concurrent callers are + /// supported. Items leave the channel in FIFO order; worker processing/completion + /// order may differ. Byte-budget accounting still serializes on the existing + /// reservation gate. A successful dequeue transfers ownership and releases the + /// reservation inside ; consumers must not release again. + /// + /// + public sealed class ArticleIngestionQueue : IArticleIngestionQueue + { + private readonly Channel _channel; + private readonly object _gate = new(); + private readonly Queue _waiters = new(); + private readonly long _memoryLimit; + private readonly int _maxArticleBytes; + private long _queuedBytes; + private long _peakQueuedBytes; + private int _count; + private int _peakCount; + private int _completed; + private long _admissionFailures; + private long _admissionWaitTicks; + + /// Initializes a new instance of the class. + public ArticleIngestionQueue(IOptions options) + : this(GetIngestionOptions(options), GetMemoryLimit(options)) + { + } + + /// + /// Initializes a new instance with explicit ingestion options and the default 1 GiB budget (tests). + /// + public ArticleIngestionQueue(ArticleIngestionOptions options) + : this(options, NntpdOptions.DefaultTransitQueueMemoryLimit) + { + } + + /// Initializes a new instance with an explicit Transit queue memory budget (tests). + public ArticleIngestionQueue(ArticleIngestionOptions options, long transitQueueMemoryLimit) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentOutOfRangeException.ThrowIfLessThan(options.MaxArticleBytes, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(transitQueueMemoryLimit, 1); + + _memoryLimit = transitQueueMemoryLimit; + _maxArticleBytes = options.MaxArticleBytes; + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public long MemoryLimitBytes => _memoryLimit; + + /// + public long QueuedBytes => Volatile.Read(ref _queuedBytes); + + /// + public long PeakQueuedBytes => Volatile.Read(ref _peakQueuedBytes); + + /// + public int MaxArticleBytes => _maxArticleBytes; + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public int PeakCount => Math.Max(0, Volatile.Read(ref _peakCount)); + + /// + public bool IsAccepting => Volatile.Read(ref _completed) == 0; + + /// + public int WaitingProducerCount => DebugWaiterCount; + + /// + public long AdmissionFailureCount => Volatile.Read(ref _admissionFailures); + + /// + public long AdmissionWaitTicks => Volatile.Read(ref _admissionWaitTicks); + + /// Gets the number of producers waiting for byte-budget capacity (tests). + internal int DebugWaiterCount + { + get + { + lock (_gate) + { + var n = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var waiter in _waiters) + { + if (waiter.IsWaiting) + { + n++; + } + } + + return n; + } + } + } + + /// + public async ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + EnsureCanonical(article); + + var bytes = article.Payload.Length; + if (bytes > _memoryLimit) + { + RecordAdmission(ArticleEnqueueResult.Rejected, waitTicks: 0); + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + RecordAdmission(ArticleEnqueueResult.Unavailable, waitTicks: 0); + return ArticleEnqueueResult.Unavailable; + } + + var waitStarted = System.Diagnostics.Stopwatch.GetTimestamp(); + var reserved = await ReserveAsync(bytes, cancellationToken).ConfigureAwait(false); + var waitTicks = System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted; + IngestionPipelineMetrics.Shared.RecordEnqueueWaitTicks(waitTicks); + if (reserved.WaitedForBudget) + { + IngestionPipelineMetrics.Shared.RecordBudgetWaitTicks(waitTicks); + } + + if (reserved.Result != ArticleEnqueueResult.Accepted) + { + RecordAdmission(reserved.Result, waitTicks); + return reserved.Result; + } + + var written = TryWriteReserved(article, bytes) + ? ArticleEnqueueResult.Accepted + : ArticleEnqueueResult.Unavailable; + RecordAdmission(written, waitTicks); + return written; + } + + /// + public bool TryProbeCapacity() + { + lock (_gate) + { + return _completed == 0 && _queuedBytes < _memoryLimit; + } + } + + /// + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + EnsureCanonical(article); + + var bytes = article.Payload.Length; + if (bytes > _memoryLimit) + { + RecordAdmission(ArticleEnqueueResult.Rejected, waitTicks: 0); + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + RecordAdmission(ArticleEnqueueResult.Unavailable, waitTicks: 0); + return ArticleEnqueueResult.Unavailable; + } + + if (!TryReserveImmediate(bytes)) + { + RecordAdmission(ArticleEnqueueResult.Full, waitTicks: 0); + return ArticleEnqueueResult.Full; + } + + var written = TryWriteReserved(article, bytes) + ? ArticleEnqueueResult.Accepted + : ArticleEnqueueResult.Unavailable; + RecordAdmission(written, waitTicks: 0); + return written; + } + + /// + public bool TryEnqueue(InboundArticle article) => TryAdmit(article) == ArticleEnqueueResult.Accepted; + + /// + public void Complete() + { + List? pending = null; + lock (_gate) + { + Interlocked.Exchange(ref _completed, 1); + _channel.Writer.TryComplete(); + if (_waiters.Count > 0) + { + pending = new List(_waiters.Count); + while (_waiters.TryDequeue(out var waiter)) + { + if (waiter.TryCancel()) + { + pending.Add(waiter); + } + } + } + } + + if (pending is null) + { + return; + } + + foreach (var waiter in pending) + { + waiter.Tcs.TrySetResult(false); + } + } + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + // WaitToReadAsync + TryRead is the Channel multi-reader contract: another + // consumer may take the item between the wait and the read, so retry. + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (!_channel.Reader.TryRead(out var article) || article is null) + { + continue; + } + + Interlocked.Decrement(ref _count); + Release(article.Payload.Length); + return article; + } + + return null; + } + + /// + /// Completes the channel writer without marking the queue unavailable (tests). + /// The next reserve-then-write fails and must release the reservation. + /// + internal void CompleteChannelWithoutMarkingUnavailableForTests() => _channel.Writer.TryComplete(); + + private async ValueTask<(ArticleEnqueueResult Result, bool WaitedForBudget)> ReserveAsync( + int bytes, + CancellationToken cancellationToken) + { + ByteWaiter? waiter; + lock (_gate) + { + if (_completed != 0) + { + return (ArticleEnqueueResult.Unavailable, false); + } + + if (bytes <= _memoryLimit - _queuedBytes) + { + AddReservation(bytes); + return (ArticleEnqueueResult.Accepted, false); + } + + waiter = new ByteWaiter(bytes); + _waiters.Enqueue(waiter); + } + + using var registration = cancellationToken.Register( + static state => ((ByteWaiter)state!).CancelFromToken(), + waiter); + + try + { + var admitted = await waiter.Tcs.Task.ConfigureAwait(false); + return ( + admitted ? ArticleEnqueueResult.Accepted : ArticleEnqueueResult.Unavailable, + true); + } + catch (OperationCanceledException) + { + waiter.CancelFromToken(); + return (ArticleEnqueueResult.Unavailable, true); + } + } + + private void RecordAdmission(ArticleEnqueueResult result, long waitTicks) + { + if (result is ArticleEnqueueResult.Rejected + or ArticleEnqueueResult.Unavailable + or ArticleEnqueueResult.Full) + { + Interlocked.Increment(ref _admissionFailures); + } + + if (waitTicks > 0) + { + Interlocked.Add(ref _admissionWaitTicks, waitTicks); + } + } + + private bool TryReserveImmediate(int bytes) + { + lock (_gate) + { + if (_completed != 0 || bytes > _memoryLimit - _queuedBytes) + { + return false; + } + + AddReservation(bytes); + return true; + } + } + + private void AddReservation(int bytes) + { + _queuedBytes += bytes; + if (_queuedBytes > _peakQueuedBytes) + { + _peakQueuedBytes = _queuedBytes; + } + } + + private static void EnsureCanonical(InboundArticle article) + { + if (article.Record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException( + "The ingress queue accepts CanonicalV1 ArticleRecord items only.", + nameof(article)); + } + } + + private bool TryWriteReserved(InboundArticle article, int bytes) + { + if (_channel.Writer.TryWrite(article)) + { + var now = Interlocked.Increment(ref _count); + UpdatePeakCount(now); + return true; + } + + Release(bytes); + return false; + } + + private void Release(int bytes) + { + List? admitted; + lock (_gate) + { + if (bytes > 0) + { + var next = _queuedBytes - bytes; + _queuedBytes = next < 0 ? 0 : next; + } + + admitted = AdmitWaiters(); + } + + if (admitted is null) + { + return; + } + + foreach (var waiter in admitted) + { + waiter.Tcs.TrySetResult(true); + } + } + + private List? AdmitWaiters() + { + List? admitted = null; + while (_waiters.TryPeek(out var waiter)) + { + if (!waiter.IsWaiting) + { + _waiters.Dequeue(); + continue; + } + + if (_completed != 0) + { + break; + } + + if (waiter.Bytes > _memoryLimit - _queuedBytes) + { + break; + } + + if (!waiter.TryAdmit()) + { + _waiters.Dequeue(); + continue; + } + + _waiters.Dequeue(); + AddReservation(waiter.Bytes); + admitted ??= []; + admitted.Add(waiter); + } + + return admitted; + } + + private void UpdatePeakCount(int now) + { + var peak = Volatile.Read(ref _peakCount); + while (now > peak) + { + var previous = Interlocked.CompareExchange(ref _peakCount, now, peak); + if (previous == peak) + { + break; + } + + peak = previous; + } + } + + private static ArticleIngestionOptions GetIngestionOptions(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Value.ArticleIngestion ?? new ArticleIngestionOptions(); + } + + private static long GetMemoryLimit(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Value.TransitQueueMemoryLimit; + } + + private sealed class ByteWaiter + { + private const int Waiting = 0; + private const int Admitted = 1; + private const int Cancelled = 2; + + private int _state; + + public ByteWaiter(int bytes) + { + Bytes = bytes; + } + + public int Bytes { get; } + + public TaskCompletionSource Tcs { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool IsWaiting => Volatile.Read(ref _state) == Waiting; + + public bool TryAdmit() => Interlocked.CompareExchange(ref _state, Admitted, Waiting) == Waiting; + + public bool TryCancel() => Interlocked.CompareExchange(ref _state, Cancelled, Waiting) == Waiting; + + public void CancelFromToken() + { + if (TryCancel()) + { + Tcs.TrySetResult(false); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticlePlacementLogMessages.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticlePlacementLogMessages.cs new file mode 100644 index 00000000..44d42184 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticlePlacementLogMessages.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Source-generated placement logs. NNTPD places one copy. A + /// result + /// is logged as attempt 1 and may be followed by one same-target retry logged as attempt 2. + /// + internal static partial class ArticlePlacementLogMessages + { + [LoggerMessage( + EventId = 2620, + Level = LogLevel.Information, + Message = "Article placement accepted artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Accepted elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Accepted(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2621, + Level = LogLevel.Information, + Message = "Article placement duplicate artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Duplicate elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Duplicate(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2622, + Level = LogLevel.Warning, + Message = "Article placement conflict artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Conflict elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Conflict(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2623, + Level = LogLevel.Warning, + Message = "Article placement rejected capacity artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedCapacity elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedCapacity(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2624, + Level = LogLevel.Warning, + Message = "Article placement rejected pressure artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedPressure elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedPressure(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2625, + Level = LogLevel.Warning, + Message = "Article placement rejected invalid artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedInvalid elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedInvalid(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2626, + Level = LogLevel.Warning, + Message = "Article placement found no active server artId={ArtId} outcome=NoActiveServer elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void NoActiveServer(ILogger logger, string ArtId, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2627, + Level = LogLevel.Warning, + Message = "Article placement transport failure artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} failure={Failure} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void TransportFailed( + ILogger logger, + string ArtId, + int ServerId, + int VatpPort, + string Fqdn, + string Failure, + long ElapsedMs, + int Attempt); + + [LoggerMessage( + EventId = 2628, + Level = LogLevel.Information, + Message = "Article placement cancelled artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Cancelled(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2629, + Level = LogLevel.Warning, + Message = "Article placement acknowledgement not observed artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void AcknowledgementNotObserved( + ILogger logger, + string ArtId, + int ServerId, + int VatpPort, + string Fqdn, + long ElapsedMs, + int Attempt); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleRecordIngress.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleRecordIngress.cs new file mode 100644 index 00000000..ebd98e80 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/ArticleRecordIngress.cs @@ -0,0 +1,155 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Builds a Common at the NNTP ingress boundary. + /// + /// + /// Destuff is this type's only extra step and only for stuffed wire. Parsing, + /// classification, Date/Path materialization, ArtId, and ArtHash stay in + /// . The destuffed source is discarded after + /// materialize; the queued record references the factory ArtData buffer. + /// + internal static class ArticleRecordIngress + { + /// + /// Parses already-destuffed article bytes into a CanonicalV1 record. + /// + /// Session-scoped Common parser (local identity for Path hops). + /// Unstuffed article bytes. Not retained after success. + /// Factory result. Accepted is the canonical buffer. + public static ArticleRecordCreateResult TryCreateFromDestuffed( + NntpArticleParser parser, + ReadOnlyMemory destuffedArticle) + { + ArgumentNullException.ThrowIfNull(parser); + return ArticleRecordFactory.TryCreate(parser, destuffedArticle, ArticlePathMode.Traverse); + } + + /// + /// Parses already-destuffed article bytes with one captured size limit and Path tracker. + /// + /// Session parser. Its local identity remains the application hop. + /// Unstuffed article bytes. + /// Captured maxartsize. + /// Captured sitename bytes. + /// Factory result. + public static ArticleRecordCreateResult TryCreateFromDestuffed( + NntpArticleParser parser, + ReadOnlyMemory destuffedArticle, + int maxArticleBytes, + ReadOnlySpan siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(parser); + return ArticleRecordFactory.TryCreate( + parser, + destuffedArticle, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + } + + /// + /// Destuffs terminator-omitted wire once, then builds a CanonicalV1 record. + /// + /// Session-scoped Common parser (local identity for Path hops). + /// Stuffed article bytes without the NNTP terminator. + /// Destuff and materialize ceiling. + /// + /// Factory result, or parse-rejected + /// when destuff exceeds . + /// + public static ArticleRecordCreateResult TryCreateFromStuffedWire( + NntpArticleParser parser, + ReadOnlyMemory stuffedWire, + int maxArticleBytes) + { + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (!NntpArticleDestuffer.TryDestuffStuffedWire(stuffedWire.Span, maxArticleBytes, out var destuffed)) + { + return ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.ArticleTooLarge); + } + + return ArticleRecordFactory.TryCreate( + parser, + destuffed, + ArticlePathMode.Traverse, + maxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + } + + /// + /// Destuffs stuffed wire and materializes it with one captured size limit and Path tracker. + /// + /// Session parser. + /// Stuffed article bytes. + /// Captured maxartsize. Also the destuff ceiling. + /// Captured sitename bytes. + /// Factory result, or when destuff exceeds the limit. + public static ArticleRecordCreateResult TryCreateFromStuffedWire( + NntpArticleParser parser, + ReadOnlyMemory stuffedWire, + int maxArticleBytes, + ReadOnlySpan siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (!NntpArticleDestuffer.TryDestuffStuffedWire(stuffedWire.Span, maxArticleBytes, out var destuffed)) + { + return ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.ArticleTooLarge); + } + + return ArticleRecordFactory.TryCreate( + parser, + destuffed, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + } + + /// + /// Creates a queue item whose aliases + /// .ArtData (no article-sized copy). + /// + /// Command or POST Message-ID text. + /// Accepted CanonicalV1 record. + /// Session client identity at admission. + /// Queue-admission timestamp. + /// TAKETHIS, POST, IHAVE, or BackFiller. + /// Inbound Transit identifier bytes captured from the session. + /// Queue item carrying . + /// + /// Throws only when is empty or + /// is not CanonicalV1. POST calls this only after + /// TryCreateFromDestuffed accepted CanonicalV1 with a non-empty + /// Message-ID; those throws are unreachable on that path. + /// + public static InboundArticle CreateQueued( + string messageId, + in ArticleRecord record, + ConnectionClientIdentity clientIdentity, + DateTimeOffset receivedAtUtc, + InboundArticleProducer producer, + ReadOnlyMemory feed = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + if (record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException("Queued ArticleRecord must be CanonicalV1.", nameof(record)); + } + + return new InboundArticle( + messageId, + clientIdentity, + receivedAtUtc, + producer, + record, + feed); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/DisabledArticleIngestionQueue.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/DisabledArticleIngestionQueue.cs new file mode 100644 index 00000000..e08e455c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/DisabledArticleIngestionQueue.cs @@ -0,0 +1,72 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// No-op ingestion queue used when no real queue is supplied (tests / early wiring). + /// + /// Always reports unavailable so TAKETHIS follows the RFC 4644 400 temporary path. + public sealed class DisabledArticleIngestionQueue : IArticleIngestionQueue + { + /// Shared disabled instance. + public static DisabledArticleIngestionQueue Instance { get; } = new(); + + private DisabledArticleIngestionQueue() + { + } + + /// + public long MemoryLimitBytes => 0; + + /// + public long QueuedBytes => 0; + + /// + public long PeakQueuedBytes => 0; + + /// + public int MaxArticleBytes => Configuration.ArticleIngestionOptions.DefaultMaxArticleBytes; + + /// + public int Count => 0; + + /// + public int PeakCount => 0; + + /// + public bool IsAccepting => false; + + /// + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + return ValueTask.FromResult(ArticleEnqueueResult.Unavailable); + } + + /// + public bool TryProbeCapacity() => false; + + /// + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + return ArticleEnqueueResult.Unavailable; + } + + /// + public bool TryEnqueue(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + return false; + } + + /// + public void Complete() + { + } + + /// + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + ValueTask.FromResult(null); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IArticleIngestionQueue.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IArticleIngestionQueue.cs new file mode 100644 index 00000000..732a741b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IArticleIngestionQueue.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Byte-budgeted in-memory ingestion boundary between NNTP receive handlers and the spool writer. + /// + /// + /// Enqueue completes when payload-byte capacity is available (or fails when the queue is + /// completed, or when a single article exceeds the configured budget). Callers must not + /// perform disk I/O; persistence is owned by the background spool writer. + /// + public interface IArticleIngestionQueue + { + /// + /// Gets the configured Transit article-queue memory budget in bytes + /// (Nntpd:TransitQueueMemoryLimit). + /// + long MemoryLimitBytes { get; } + + /// + /// Gets reserved queued article payload bytes + /// (sum of owned lengths). + /// + long QueuedBytes { get; } + + /// Gets the high-water mark of since construction. + long PeakQueuedBytes { get; } + + /// Gets the maximum accepted article payload size in bytes. + int MaxArticleBytes { get; } + + /// Gets the approximate number of articles currently queued. + int Count { get; } + + /// Gets the high-water mark of since construction. + int PeakCount { get; } + + /// + /// Gets the number of producers waiting for byte-budget capacity. + /// + int WaitingProducerCount => 0; + + /// + /// Gets lifetime admission failures (rejected, full, or unavailable). + /// + long AdmissionFailureCount => 0; + + /// + /// Gets accumulated producer wait ticks while blocked on byte-budget capacity. + /// + long AdmissionWaitTicks => 0; + + /// Gets a value indicating whether the queue is still accepting articles. + bool IsAccepting { get; } + + /// + /// Enqueues an accepted article, waiting for byte-budget capacity when the queue is full. + /// + /// + /// when queued; + /// when the article is larger than the budget; + /// when the queue is completed/shutting down + /// or the wait is cancelled. + /// + ValueTask EnqueueAsync(InboundArticle article, CancellationToken cancellationToken); + + /// + /// Returns whether the queue is accepting and has at least one free payload byte. + /// + /// + /// Does not reserve bytes. IHAVE uses this before 335 because the article + /// size is unknown. A later still performs the atomic + /// reservation against the actual payload length. + /// + bool TryProbeCapacity(); + + /// + /// Attempts to reserve the article's payload bytes and enqueue it without waiting. + /// + /// + /// when queued; + /// when the article is larger than the budget; + /// when the remaining budget cannot admit it now; + /// when the queue is completed/shutting down. + /// + ArticleEnqueueResult TryAdmit(InboundArticle article); + + /// + /// Attempts to enqueue without waiting. Returns when the budget + /// cannot admit the article immediately, the article exceeds the budget, or the queue + /// is unavailable. + /// + bool TryEnqueue(InboundArticle article); + + /// + /// Completes the writer side so no further articles are accepted. + /// Already-queued articles remain readable until drained. + /// + void Complete(); + + /// + /// Reads the next queued article, or when the queue is completed and empty. + /// + /// + /// Safe for concurrent callers. A successful read transfers consumer ownership and + /// releases reserved payload bytes exactly once. Callers must not release again. + /// Items leave the queue in FIFO order; concurrent processing order may differ. + /// + ValueTask DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/INewsLogWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/INewsLogWriter.cs new file mode 100644 index 00000000..cfcc79ea --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/INewsLogWriter.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Emits an already-decided INN news event. + /// + /// + /// Production uses a dedicated Serilog pipeline + /// (). File lifecycle is Serilog's + /// responsibility. Implementations must throw on write failure rather than + /// pretending the event was recorded. + /// + public interface INewsLogWriter + { + /// + /// Appends one complete INN news line for . + /// + /// Already-decided news event. + /// The line could not be written. + void Write(in NewsLogEvent evt); + + /// Flushes buffered news lines when the implementation buffers independently of Serilog. + void Flush(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IPathSurveyWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IPathSurveyWriter.cs new file mode 100644 index 00000000..24fae1a0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IPathSurveyWriter.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Persists one canonical Path observation to the durable Path-survey stream. + /// + /// + /// Production uses a dedicated Serilog pipeline + /// (). File lifecycle is Serilog's + /// responsibility. Implementations must throw on write failure rather than + /// pretending the observation was recorded. Callers treat this as an + /// operational survey stream: a failure must not change NNTP responses or + /// ingestion. + /// + public interface IPathSurveyWriter + { + /// + /// Appends one INN-style Path-survey line for . + /// + /// + /// Canonical ArticleRecord.Path value bytes. Empty follows + /// semantics + /// (absent or zero-length field) and still writes a Path line. + /// + /// The line could not be written. + void Write(ReadOnlySpan canonicalPath); + + /// Flushes buffered Path lines when the implementation buffers independently of Serilog. + void Flush(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IhaveArticleInterpreter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IhaveArticleInterpreter.cs new file mode 100644 index 00000000..957f505d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IhaveArticleInterpreter.cs @@ -0,0 +1,77 @@ +using System.Buffers; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Destuffs IHAVE/TAKETHIS wire into for benches and + /// classifier tests. + /// + /// + /// Production IHAVE no longer uses this type. After receive, IHAVE destuffs + /// once in ArticleRecordIngress.TryCreateFromStuffedWire and + /// materializes CanonicalV1 through . + /// Remaining IHAVE-specific behaviour lives in the IHAVE command (History, + /// 335/235/435/436/437, non-blocking probe/admit). Common owns parse, + /// Date/Path materialize, ArtId, ArtHash, and ArtType. + /// + public static class IhaveArticleInterpreter + { + /// + /// Destuffs stuffed IHAVE wire (no terminator) into . + /// + public static Article DestuffToArticle(ReadOnlySpan stuffedWire, int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var headers = new ArrayBufferWriter(1024); + var body = new ArrayBufferWriter(1024); + var type = ArticleType.None; + var exceeded = false; + var inHeaders = true; + var offset = 0; + while (offset < stuffedWire.Length && !exceeded) + { + var remaining = stuffedWire[offset..]; + var crlf = remaining.IndexOf(NntpDelimiterSearch.Crlf); + if (crlf < 0) + { + break; + } + + var line = remaining[..crlf]; + var destuffed = DestuffLine(line); + if (inHeaders) + { + if (destuffed.IsEmpty) + { + inHeaders = false; + } + else + { + NntpArticleDestuffer.AppendUnstuffedLine(headers, line, maxArticleBytes, ref exceeded); + ArticleTypeClassifier.ObserveLine(destuffed, inHeader: true, ref type); + } + } + else + { + NntpArticleDestuffer.AppendUnstuffedLine(body, line, maxArticleBytes, ref exceeded); + ArticleTypeClassifier.ObserveLine(destuffed, inHeader: false, ref type); + } + + offset += crlf + 2; + } + + var headerBytes = headers.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : headers.WrittenMemory.ToArray(); + var bodyBytes = body.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : body.WrittenMemory.ToArray(); + return new Article(headerBytes, bodyBytes, type); + } + + private static ReadOnlySpan DestuffLine(ReadOnlySpan stuffedLine) => + stuffedLine.Length > 0 && stuffedLine[0] == (byte)'.' ? stuffedLine[1..] : stuffedLine; + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticle.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticle.cs new file mode 100644 index 00000000..de3c7559 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticle.cs @@ -0,0 +1,65 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// An article accepted into the ingestion queue as a CanonicalV1 . + /// + /// + /// TAKETHIS, POST, and IHAVE all construct the record before admission. + /// aliases . The queue + /// rejects items that are not CanonicalV1. Workers must not destuff or parse. + /// + public sealed class InboundArticle + { + /// Initializes a CanonicalV1 queue item. + public InboundArticle( + string messageId, + ConnectionClientIdentity clientIdentity, + DateTimeOffset receivedAtUtc, + InboundArticleProducer producer, + ArticleRecord record, + ReadOnlyMemory feed = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentNullException.ThrowIfNull(clientIdentity); + if (record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException("Queued ArticleRecord must be CanonicalV1.", nameof(record)); + } + + MessageId = messageId; + ClientIdentity = clientIdentity; + ReceivedAtUtc = receivedAtUtc; + Producer = producer; + Record = record; + Payload = record.ArtData; + Feed = feed; + } + + /// Gets the message-id supplied with the transfer command or POST. + public string MessageId { get; } + + /// Gets canonical ArtData (alias of ). + public ReadOnlyMemory Payload { get; } + + /// Gets the effective client identity at acceptance time. + public ConnectionClientIdentity ClientIdentity { get; } + + /// Gets the UTC timestamp when the article was accepted into the queue. + public DateTimeOffset ReceivedAtUtc { get; } + + /// Gets which command produced this item. + public InboundArticleProducer Producer { get; } + + /// + /// Gets the inbound Transit identifier captured at queue admission, or empty + /// when the session was not a named peer. + /// + public ReadOnlyMemory Feed { get; } + + /// Gets the CanonicalV1 ingress record. + public ArticleRecord Record { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticleProducer.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticleProducer.cs new file mode 100644 index 00000000..00068cb8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/InboundArticleProducer.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// Identifies which ingest command produced an . + public enum InboundArticleProducer + { + /// + /// TAKETHIS. Ingress builds before admission; + /// aliases canonical ArtData. + /// + TakeThis = 0, + + /// + /// IHAVE. Ingress destuffs stuffed wire and builds + /// before admission. + /// aliases canonical ArtData. + /// + IHave = 1, + + /// + /// POST. Ingress builds after existing POST + /// receive/validate/normalize and before admission. + /// aliases canonical ArtData. + /// + Post = 2, + + /// + /// BackFiller VATP retrieval. The CanonicalV1 + /// is already validated; aliases ArtData with no destuff/parse. + /// + BackFiller = 3, + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IncomingSpoolWriterService.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IncomingSpoolWriterService.cs new file mode 100644 index 00000000..d0a23b2f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IncomingSpoolWriterService.cs @@ -0,0 +1,648 @@ +using System.Buffers; +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.OverviewDb; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Storage; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Background application service that drains CanonicalV1 queued articles through + /// a dynamically sized and hands OverviewDB + /// payloads to an independent publisher stage. + /// + /// + /// + /// Article workers encode OverviewArticleV1, enqueue the owned bytes onto a bounded + /// in-process , then write the news log and Path + /// survey. The article body is then discarded. StorageServer/VATP placement remains + /// on this type and is temporarily not invoked. + /// Workers do not await RabbitMQ publication or publisher confirmation. + /// Successful enqueue is not durable RabbitMQ handoff; broker confirmation remains + /// owned by via asynchronous outstanding + /// confirms on . + /// + /// + /// OverviewDB publisher workers scale independently from article workers based on + /// OverviewDB work-queue pressure. Publisher confirms, mandatory publishing, + /// persistent delivery, and the 2-second AMQP expiration are unchanged. + /// + /// + public sealed class IncomingSpoolWriterService : IApplicationService + { + private static readonly TimeSpan RejectedPressureRetryDelay = TimeSpan.FromSeconds(5); + + /// + /// Gets whether StorageServer placement runs after Path survey. + /// + /// + /// Temporarily . Placement methods stay on this type so the + /// bypass is reversed by returning . + /// + private static bool ArticlePlacementEnabled => false; + + private readonly IArticleIngestionQueue _queue; + private readonly INewsLogWriter _newsLog; + private readonly IPathSurveyWriter _pathSurvey; + private readonly INewsgroupCatalogue? _catalogue; + private readonly IOverviewDbHandoffPublisher _overviewHandoff; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly Func? _samplePressure; + private readonly IArticlePlacementClient? _placement; + private readonly IStorageServerRegistry? _placementRegistry; + private readonly CancellationTokenSource _articleRunCts = new(); + private readonly CancellationTokenSource _overviewRunCts = new(); + private Task? _execution; + private int _started; + private IngestionWorkerPool? _pool; + private OverviewDbWorkQueue? _overviewWorkQueue; + private OverviewDbPublisherPool? _overviewPublisherPool; + private Task? _overviewPublisherExecution; + + /// Initializes a new instance of the class. + public IncomingSpoolWriterService( + IArticleIngestionQueue queue, + IOptions options, + ILogger logger, + IFeedDiagnostics? feedDiagnostics = null, + INewsLogWriter? newsLog = null, + INewsgroupCatalogue? catalogue = null, + TimeProvider? timeProvider = null, + IOverviewDbHandoffPublisher? overviewHandoff = null, + IngestionPipelineMetrics? pipelineMetrics = null, + IPathSurveyWriter? pathSurvey = null, + Func? samplePressure = null, + IArticlePlacementClient? placement = null, + IStorageServerRegistry? placementRegistry = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _options = options; + _logger = logger; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _pathSurvey = pathSurvey ?? NullPathSurveyWriter.Instance; + _catalogue = catalogue; + _time = timeProvider ?? TimeProvider.System; + _overviewHandoff = overviewHandoff ?? NullOverviewDbHandoffPublisher.Instance; + _pipeline = pipelineMetrics; + _samplePressure = samplePressure; + _placement = placement; + _placementRegistry = placementRegistry; + PressureRetryDelay = (delay, cancellationToken) => Task.Delay(delay, _time, cancellationToken); + } + + /// + public string Name => "IncomingSpoolWriter"; + + /// + public Task? Execution => _execution; + + /// Gets the active article worker pool (tests). + internal IngestionWorkerPool? Pool => _pool; + + /// + /// Wait between a result and the one retry. + /// Production waits five seconds on . + /// + internal Func PressureRetryDelay { get; set; } + + /// Gets the OverviewDB work queue (tests). + internal IOverviewDbWorkQueue? OverviewWorkQueue => _overviewWorkQueue; + + /// Gets the OverviewDB publisher pool (tests). + internal OverviewDbPublisherPool? OverviewPublisherPool => _overviewPublisherPool; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + var ingestion = _options.Value.ArticleIngestion ?? new ArticleIngestionOptions(); + SpoolLogMessages.WriterStarted( + _logger, + _queue.MemoryLimitBytes, + _queue.MaxArticleBytes); + + _overviewWorkQueue = new OverviewDbWorkQueue(ingestion.OverviewDbWorkQueueMemoryLimit); + _overviewPublisherPool = new OverviewDbPublisherPool( + _overviewWorkQueue, + _overviewHandoff, + ingestion, + _logger, + _pipeline, + _time); + _overviewPublisherExecution = Task.Run( + () => _overviewPublisherPool.RunAsync(_overviewRunCts.Token), + CancellationToken.None); + + _pool = new IngestionWorkerPool( + _queue, + ProcessArticleAsync, + ingestion, + _logger, + _pipeline, + _samplePressure, + _time); + _execution = Task.Run(() => _pool.RunAsync(_articleRunCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + // 1) Stop admitting article ingress and drain article workers (they may still + // enqueue OverviewDB work while draining). Publisher pool keeps running. + _queue.Complete(); + await _articleRunCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + SpoolLogMessages.StopCanceledWithBufferedArticles(_logger, _queue.Count); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.WriterStoppedWithError(_logger, ex); + } + } + + // 2) Complete OverviewDB work queue and drain publisher workers for a bounded period. + // Outstanding OverviewDB confirms may be abandoned after the configured timeout. + _overviewWorkQueue?.Complete(); + await _overviewRunCts.CancelAsync().ConfigureAwait(false); + var overviewExecution = _overviewPublisherExecution; + if (overviewExecution is not null) + { + var shutdownSeconds = Math.Max( + 1, + (_options.Value.ArticleIngestion ?? new ArticleIngestionOptions()) + .OverviewDbPublisherShutdownSeconds); + using var overviewDrainCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + overviewDrainCts.CancelAfter(TimeSpan.FromSeconds(shutdownSeconds)); + try + { + await overviewExecution.WaitAsync(overviewDrainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.WriterStoppedWithError(_logger, ex); + } + } + + _overviewHandoff.AbandonOutstanding(); + + try + { + _newsLog.Flush(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.NewsLogFlushFailed(_logger, ex); + } + + try + { + _pathSurvey.Flush(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.PathSurveyFlushFailed(_logger, ex); + } + + SpoolLogMessages.WriterStopped(_logger); + } + + private async Task ProcessArticleAsync( + InboundArticle article, + long itemStart, + CancellationToken cancellationToken) + { + var overviewAccepted = false; + _feedDiagnostics.BeginSpoolWork(); + try + { + await EnqueueOverviewWorkAsync(article, itemStart, cancellationToken).ConfigureAwait(false); + overviewAccepted = true; + var newsStart = System.Diagnostics.Stopwatch.GetTimestamp(); + WriteNewsLog(article); + _pipeline?.RecordNews(newsStart); + WritePathSurvey(article); + if (ArticlePlacementEnabled) + { + if (article.Producer == InboundArticleProducer.BackFiller) + { + await StoreBackFillerAsync(article, cancellationToken).ConfigureAwait(false); + } + else + { + await PlaceAfterPersistAsync(article, cancellationToken).ConfigureAwait(false); + } + } + else + { + DiscardAcceptedArticle(article); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (!overviewAccepted) + { + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + ex, + article.MessageId, + article.Payload.Length); + await RequeueAsync(article).ConfigureAwait(false); + } + else + { + SpoolLogMessages.FinishFailed( + _logger, + ex, + article.MessageId, + article.Payload.Length); + } + } + finally + { + _feedDiagnostics.EndSpoolWork(article.Payload.Length, persisted: false); + } + } + + /// + /// One VATP STORE of a BackFiller article to one active StorageServer. + /// + /// + /// The target is the active registry entry with a dialable VATP port and the lowest + /// , then ordinal FQDN. Advertised free + /// space is ignored. This method does not call + /// or . + /// Accepted and Duplicate are terminal success. + /// is logged and retried once, + /// after five seconds, against the same server and the same article. Every other + /// is logged once and not retried. NNTPD does not + /// place a second copy. + /// + private async Task StoreBackFillerAsync(InboundArticle article, CancellationToken cancellationToken) + { + if (_placement is null || _placementRegistry is null) + { + return; + } + + var started = Stopwatch.GetTimestamp(); + var artId = article.Record.ArtId.ToLowerHexString(); + try + { + if (cancellationToken.IsCancellationRequested) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + return; + } + + if (!TrySelectBackFillerTarget(_placementRegistry.GetActive(_time.GetUtcNow()), out var target) + || target.VatpPort is not int port) + { + ArticlePlacementLogMessages.NoActiveServer(_logger, artId, Elapsed(started), Attempt: 1); + return; + } + + await PlaceOnSelectedTargetAsync(article.Record, target, port, artId, started, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + 0, + 0, + string.Empty, + ex.GetType().Name, + Elapsed(started), + Attempt: 1); + } + } + + /// + /// Chooses one dialable active StorageServer for . + /// + /// + /// Lowest , then ordinal FQDN. + /// Entries without a VATP port from 1 through 65535 are skipped. Free space + /// and placement persistence are not used. + /// + private static bool TrySelectBackFillerTarget( + IReadOnlyList active, + out StorageServerFleetEntry selected) + { + ArgumentNullException.ThrowIfNull(active); + selected = default; + var found = false; + foreach (var entry in active) + { + if (entry.VatpPort is not (>= 1 and <= 65535)) + { + continue; + } + + if (!found + || entry.ServerId < selected.ServerId + || (entry.ServerId == selected.ServerId + && string.CompareOrdinal(entry.Fqdn, selected.Fqdn) < 0)) + { + selected = entry; + found = true; + } + } + + return found; + } + + /// + /// One STORE after persist. Accepted and Duplicate complete placement. + /// is retried once against the same + /// target and article. Failures stay in this method so they cannot requeue OverviewDB + /// work or undo persistence. NNTPD does not select or store a second copy. + /// + private async Task PlaceAfterPersistAsync(InboundArticle article, CancellationToken cancellationToken) + { + if (article.Producer is not ( + InboundArticleProducer.TakeThis + or InboundArticleProducer.IHave + or InboundArticleProducer.Post)) + { + return; + } + + if (_placement is null || _placementRegistry is null) + { + return; + } + + var started = Stopwatch.GetTimestamp(); + var artId = article.Record.ArtId.ToLowerHexString(); + try + { + if (cancellationToken.IsCancellationRequested) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + return; + } + + var active = _placementRegistry.GetActive(_time.GetUtcNow()); + if (!StorageServerPlacementSelector.TrySelect(active, out var target) || target.VatpPort is not int port) + { + ArticlePlacementLogMessages.NoActiveServer(_logger, artId, Elapsed(started), Attempt: 1); + return; + } + + await PlaceOnSelectedTargetAsync(article.Record, target, port, artId, started, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + 0, + 0, + string.Empty, + ex.GetType().Name, + Elapsed(started), + Attempt: 1); + } + } + + /// + /// Stores on . + /// is logged as attempt 1 and + /// retried once, after , with the same record + /// and target. The second result is settled by . + /// + private async Task PlaceOnSelectedTargetAsync( + ArticleRecord record, + StorageServerFleetEntry target, + int port, + string artId, + long started, + CancellationToken cancellationToken) + { + var result = await _placement!.PlaceAsync(record, target, cancellationToken).ConfigureAwait(false); + if (result.Kind != ArticlePlacementKind.RejectedPressure) + { + LogPlacementResult(result, artId, target, port, Elapsed(started), attempt: 1); + return; + } + + LogPlacementResult(result, artId, target, port, Elapsed(started), attempt: 1); + await PressureRetryDelay(RejectedPressureRetryDelay, cancellationToken).ConfigureAwait(false); + var retryStarted = Stopwatch.GetTimestamp(); + result = await _placement.PlaceAsync(record, target, cancellationToken).ConfigureAwait(false); + LogPlacementResult(result, artId, target, port, Elapsed(retryStarted), attempt: 2); + } + + /// Logs one placement result. The attempt number is the STORE call that produced it. + private void LogPlacementResult( + ArticlePlacementResult result, + string artId, + StorageServerFleetEntry target, + int port, + long elapsedMs, + int attempt) + { + switch (result.Kind) + { + case ArticlePlacementKind.Accepted: + ArticlePlacementLogMessages.Accepted(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Duplicate: + ArticlePlacementLogMessages.Duplicate(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Conflict: + ArticlePlacementLogMessages.Conflict(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedCapacity: + ArticlePlacementLogMessages.RejectedCapacity(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedPressure: + ArticlePlacementLogMessages.RejectedPressure(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedInvalid: + ArticlePlacementLogMessages.RejectedInvalid(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Cancelled: + ArticlePlacementLogMessages.Cancelled(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.AcknowledgementNotObserved: + ArticlePlacementLogMessages.AcknowledgementNotObserved( + _logger, + artId, + target.ServerId, + port, + target.Fqdn, + elapsedMs, + attempt); + break; + default: + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + target.ServerId, + port, + target.Fqdn, + result.Failure ?? result.Kind.ToString(), + elapsedMs, + attempt); + break; + } + } + + private static long Elapsed(long started) => + (long)Stopwatch.GetElapsedTime(started).TotalMilliseconds; + + /// + /// Writes the post-queue INN news event. Failures are logged and + /// swallowed so the already-accepted article still reaches Path survey. + /// + private void WriteNewsLog(InboundArticle article) + { + try + { + var transit = _options.Value.Transit ?? new TransitOptions(); + if (!IngressNewsDisposition.TryCreateEvent( + article, + transit, + _catalogue, + _time.GetLocalNow(), + out var evt)) + { + return; + } + + _newsLog.Write(in evt); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.NewsLogFailed(_logger, ex, article.MessageId); + } + } + + /// + /// Writes the canonical Path-survey observation. Failures are logged and + /// swallowed so the worker still discards the article and continues. + /// + private void WritePathSurvey(InboundArticle article) + { + try + { + _pathSurvey.Write(article.Record.Path); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.PathSurveyFailed(_logger, ex, article.MessageId); + } + } + + /// + /// Drops after Path survey. The body is not copied, + /// written, or passed to StorageServer selection or VATP placement. + /// + private static void DiscardAcceptedArticle(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + } + + /// + /// Encodes OverviewArticleV1 and enqueues it onto the in-process OverviewDB work + /// queue. Does not call RabbitMQ. + /// + private async Task EnqueueOverviewWorkAsync( + InboundArticle article, + long itemStart, + CancellationToken cancellationToken) + { + var workQueue = _overviewWorkQueue + ?? throw new InvalidOperationException("OverviewDB work queue is not started."); + + var max = OverviewArticleV1Codec.GetMaxEncodedSize(article.Record); + var rented = ArrayPool.Shared.Rent(Math.Max(max, 1)); + try + { + var encodeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var written = OverviewArticleV1Codec.Encode(article.Record, rented); + _pipeline?.RecordEncode(encodeStart); + _pipeline?.RecordToPublishStart(itemStart); + + var owned = new byte[written]; + rented.AsSpan(0, written).CopyTo(owned); + var item = new OverviewDbWorkItem(owned, article.MessageId); + var result = await workQueue.EnqueueAsync(item, cancellationToken).ConfigureAwait(false); + if (result != ArticleEnqueueResult.Accepted) + { + throw new InvalidOperationException( + $"OverviewDB work queue rejected handoff enqueue ({result})."); + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + private async Task RequeueAsync(InboundArticle article) + { + var result = await _queue.EnqueueAsync(article, CancellationToken.None).ConfigureAwait(false); + if (result == ArticleEnqueueResult.Accepted) + { + OverviewDbHandoffLogMessages.Requeued(_logger, article.MessageId); + return; + } + + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, article.MessageId); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionPressureSnapshot.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionPressureSnapshot.cs new file mode 100644 index 00000000..f8505c7a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionPressureSnapshot.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// One sample of Transit ingestion-queue pressure used by the worker-pool scaler. + /// + /// + /// Combined pressure in [0, 1]: the greater of byte utilisation and a + /// waiting-producer signal (1 when any producer waits for byte budget). + /// + /// Current reserved payload bytes in the ingestion queue. + /// Configured Transit queue byte budget. + /// Current queued article count. + /// Producers blocked on the byte budget. + public readonly record struct IngestionPressureSnapshot( + double Pressure, + long QueuedBytes, + long MemoryLimitBytes, + int ArticleCount, + int WaitingProducers) + { + /// Builds a snapshot from the live ingestion queue. + public static IngestionPressureSnapshot FromQueue(IArticleIngestionQueue queue) + { + ArgumentNullException.ThrowIfNull(queue); + var limit = queue.MemoryLimitBytes; + var bytes = queue.QueuedBytes; + var waiting = queue.WaitingProducerCount; + var utilisation = limit <= 0 ? 0d : Math.Clamp(bytes / (double)limit, 0d, 1d); + var waitingSignal = waiting > 0 ? 1d : 0d; + return new IngestionPressureSnapshot( + Math.Max(utilisation, waitingSignal), + bytes, + limit, + queue.Count, + waiting); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPool.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPool.cs new file mode 100644 index 00000000..09b9047a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPool.cs @@ -0,0 +1,415 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Dynamically sized pool of ingestion workers that drain + /// concurrently. + /// + /// + /// Each worker processes one article at a time. OverviewDB RabbitMQ publication + /// is performed by a separate publisher pool; article workers only enqueue onto + /// the in-process OverviewDB work queue. Scaling uses sustained queue pressure + /// with hysteresis. A single worker fault does not stop the pool. + /// + internal sealed class IngestionWorkerPool + { + private readonly IArticleIngestionQueue _queue; + private readonly Func _processArticleAsync; + private readonly Func _samplePressure; + private readonly ArticleIngestionOptions _options; + private readonly ILogger _logger; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly List _workers = []; + private int _highPressureStreak; + private int _lowPressureStreak; + private int _scaleUps; + private int _scaleDowns; + + /// Initializes a new ingestion worker pool. + public IngestionWorkerPool( + IArticleIngestionQueue queue, + Func processArticleAsync, + ArticleIngestionOptions options, + ILogger logger, + IngestionPipelineMetrics? pipelineMetrics = null, + Func? samplePressure = null, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(processArticleAsync); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _processArticleAsync = processArticleAsync; + _options = options; + _logger = logger; + _pipeline = pipelineMetrics; + _samplePressure = samplePressure ?? (() => IngestionPressureSnapshot.FromQueue(queue)); + _time = timeProvider ?? TimeProvider.System; + } + + /// Gets the current worker count. + public int WorkerCount + { + get + { + lock (_gate) + { + return _workers.Count; + } + } + } + + /// Gets how many scale-up decisions have been applied (tests). + public int ScaleUpCount => Volatile.Read(ref _scaleUps); + + /// Gets how many scale-down decisions have been applied (tests). + public int ScaleDownCount => Volatile.Read(ref _scaleDowns); + + /// Runs workers and the scaler until is cancelled. + public async Task RunAsync(CancellationToken cancellationToken) + { + IngestionWorkerPoolLogMessages.PoolStarting( + _logger, + _options.MinWorkers, + _options.MaxWorkers, + _options.MaxPublishConcurrency); + + for (var i = 0; i < _options.MinWorkers; i++) + { + AddWorker(cancellationToken); + } + + PublishPoolObservation(); + + var scaler = Task.Run(() => ScaleLoopAsync(cancellationToken), CancellationToken.None); + try + { + await scaler.ConfigureAwait(false); + } + finally + { + List snapshot; + lock (_gate) + { + snapshot = [.. _workers]; + _workers.Clear(); + } + + foreach (var worker in snapshot) + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + } + + foreach (var worker in snapshot) + { + try + { + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + } + } + + PublishPoolObservation(); + IngestionWorkerPoolLogMessages.PoolStopped(_logger, snapshot.Count); + } + } + + /// Applies one scaling decision from (tests). + internal void ApplyScaleDecisionForTests(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + ApplyScaleDecision(snapshot, workerToken); + PublishPoolObservation(); + } + + private async Task ScaleLoopAsync(CancellationToken cancellationToken) + { + var interval = TimeSpan.FromSeconds(Math.Max(1, _options.ScaleIntervalSeconds)); + using var timer = new PeriodicTimer(interval, _time); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + var snapshot = _samplePressure(); + ApplyScaleDecision(snapshot, cancellationToken); + PublishPoolObservation(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private void ApplyScaleDecision(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + if (snapshot.Pressure >= _options.ScaleUpPressureThreshold) + { + _highPressureStreak++; + _lowPressureStreak = 0; + if (_highPressureStreak >= _options.ScaleUpConsecutiveIntervals) + { + _highPressureStreak = 0; + if (TryAddWorker(workerToken)) + { + Interlocked.Increment(ref _scaleUps); + _pipeline?.RecordScaleUp(); + IngestionWorkerPoolLogMessages.ScaledUp(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + if (snapshot.Pressure <= _options.ScaleDownPressureThreshold) + { + _lowPressureStreak++; + _highPressureStreak = 0; + if (_lowPressureStreak >= _options.ScaleDownConsecutiveIntervals) + { + _lowPressureStreak = 0; + if (TryRemoveWorker()) + { + Interlocked.Increment(ref _scaleDowns); + _pipeline?.RecordScaleDown(); + IngestionWorkerPoolLogMessages.ScaledDown(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + _highPressureStreak = 0; + _lowPressureStreak = 0; + } + + private bool TryAddWorker(CancellationToken workerToken) + { + lock (_gate) + { + if (_workers.Count >= _options.MaxWorkers) + { + return false; + } + + AddWorkerAlreadyLocked(workerToken); + return true; + } + } + + private void AddWorker(CancellationToken workerToken) + { + lock (_gate) + { + AddWorkerAlreadyLocked(workerToken); + } + } + + private void AddWorkerAlreadyLocked(CancellationToken workerToken) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(workerToken); + var worker = new Worker(cts) + { + Execution = Task.CompletedTask, + }; + worker.Execution = Task.Run(() => WorkerLoopAsync(worker, cts.Token), CancellationToken.None); + _workers.Add(worker); + } + + private bool TryRemoveWorker() + { + Worker? victim; + lock (_gate) + { + if (_workers.Count <= _options.MinWorkers) + { + return false; + } + + victim = _workers[^1]; + _workers.RemoveAt(_workers.Count - 1); + } + + _ = RetireWorkerAsync(victim); + return true; + } + + private async Task RetireWorkerAsync(Worker worker) + { + try + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + PublishPoolObservation(); + } + } + + /// + /// Processes queued articles until this worker is cancelled. + /// + /// + /// Graceful shutdown completes the queue before cancelling workers. Cancellation + /// observed while blocked in , or + /// between articles, drains leftovers without the worker token. + /// is approximate and is not proof the + /// channel is empty. Scale-down cancels one worker while the queue is still + /// accepting; that worker exits without draining. + /// + private async Task WorkerLoopAsync(Worker worker, CancellationToken cancellationToken) + { + try + { + while (true) + { + InboundArticle? article; + var idleStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + article = await _queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _pipeline?.AddIdleTicks(System.Diagnostics.Stopwatch.GetTimestamp() - idleStart); + _pipeline?.RecordDequeueWait(idleStart); + await DrainIfShuttingDownAsync().ConfigureAwait(false); + break; + } + + _pipeline?.AddIdleTicks(System.Diagnostics.Stopwatch.GetTimestamp() - idleStart); + _pipeline?.RecordDequeueWait(idleStart); + + if (article is null) + { + break; + } + + await ProcessOneAsync(article).ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + continue; + } + + await DrainIfShuttingDownAsync().ConfigureAwait(false); + break; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + } + + /// + /// Processes articles still queued after graceful shutdown has stopped admission. + /// + /// + /// No-ops while the queue is accepting, so scale-down cancellation does not steal + /// work from the remaining workers. Concurrent callers are safe: each article is + /// dequeued by one caller, and a completed empty queue returns null. + /// + private async Task DrainIfShuttingDownAsync() + { + if (!_queue.IsAccepting) + { + await DrainRemainingAsync().ConfigureAwait(false); + } + } + + /// + /// Dequeues and processes until the queue is completed and empty. + /// + /// + /// Uses so worker cancellation cannot abandon + /// an article that was already accepted. Safe to run from more than one worker. + /// + private async Task DrainRemainingAsync() + { + while (true) + { + InboundArticle? article; + try + { + article = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (article is null) + { + break; + } + + await ProcessOneAsync(article).ConfigureAwait(false); + } + } + + private async Task ProcessOneAsync(InboundArticle article) + { + var itemStart = System.Diagnostics.Stopwatch.GetTimestamp(); + _pipeline?.ObserveQueueDepth(_queue.Count + 1); + try + { + await _processArticleAsync(article, itemStart, CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + _pipeline?.AddBusyTicks(System.Diagnostics.Stopwatch.GetTimestamp() - itemStart); + _pipeline?.RecordWorkerItem(itemStart); + } + } + + private void PublishPoolObservation() + { + _pipeline?.ObserveWorkerPool(WorkerCount, _options.MinWorkers, _options.MaxWorkers); + } + + private sealed class Worker(CancellationTokenSource cts) + { + public CancellationTokenSource Cts { get; } = cts; + + public required Task Execution { get; set; } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs new file mode 100644 index 00000000..65ce2ecf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// Source-generated ingestion worker-pool log messages. + internal static partial class IngestionWorkerPoolLogMessages + { + [LoggerMessage( + EventId = 2010, + Level = LogLevel.Information, + Message = "Ingestion worker pool starting (min={MinWorkers}, max={MaxWorkers}, publishConcurrency={PublishConcurrency})")] + public static partial void PoolStarting( + ILogger logger, + int MinWorkers, + int MaxWorkers, + int PublishConcurrency); + + [LoggerMessage( + EventId = 2011, + Level = LogLevel.Information, + Message = "Ingestion worker pool scaled up to {WorkerCount} (pressure={Pressure:F3})")] + public static partial void ScaledUp(ILogger logger, int WorkerCount, double Pressure); + + [LoggerMessage( + EventId = 2012, + Level = LogLevel.Information, + Message = "Ingestion worker pool scaled down to {WorkerCount} (pressure={Pressure:F3})")] + public static partial void ScaledDown(ILogger logger, int WorkerCount, double Pressure); + + [LoggerMessage( + EventId = 2013, + Level = LogLevel.Warning, + Message = "Ingestion worker faulted; remaining workers continue")] + public static partial void WorkerFaulted(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2014, + Level = LogLevel.Information, + Message = "Ingestion worker pool stopped (workers={WorkerCount})")] + public static partial void PoolStopped(ILogger logger, int WorkerCount); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsDisposition.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsDisposition.cs new file mode 100644 index 00000000..c24ae149 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsDisposition.cs @@ -0,0 +1,297 @@ +using System.Text; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Post-queue ingress disposition: decides + vs j and whether to log. + /// + /// + /// + /// Uses in-memory . A successful lookup + /// means the group exists in the published catalogue. + /// is the RFC 6048 j status (peer articles permitted, nothing locally filed). + /// Unknown means returned false. A locally + /// carried non-junk group is a catalogue hit whose status is not PeerOnly + /// (y/n/m/x). + /// + /// + /// POST is never WantTrash junk. An unavailable catalogue is treated as + /// accepted rather than invented junk. Newsgroups bytes come from the + /// CanonicalV1 field table; the header is not rewritten. + /// + /// + internal static class IngressNewsDisposition + { + /// Classifies without writing a news line. + public static NewsLogDisposition Classify( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue) => + Classify(article, transit, catalogue, out _); + + /// + /// Classifies and, for junk, the source-backed reason + /// including the Newsgroups tokens that caused that decision. + /// + public static NewsLogDisposition Classify( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + out string? reason) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + reason = null; + if (article.Producer == InboundArticleProducer.Post || catalogue is null) + { + return NewsLogDisposition.Accepted; + } + + var snapshot = catalogue.Current; + var newsgroups = article.Record.Newsgroups; + if (HasLocalNonJunkGroup(newsgroups, snapshot)) + { + return NewsLogDisposition.Accepted; + } + + if (HasPeerOnlyGroup(newsgroups, snapshot)) + { + reason = FormatResponsibleGroups( + IngressNewsReasons.PeerOnly, + newsgroups, + snapshot, + peerOnly: true); + return NewsLogDisposition.Junk; + } + + if (transit.WantTrash) + { + reason = FormatResponsibleGroups( + IngressNewsReasons.NewsgroupNotCarried, + newsgroups, + snapshot, + peerOnly: false); + return NewsLogDisposition.Junk; + } + + return NewsLogDisposition.Accepted; + } + + /// + /// TAKETHIS/IHAVE WantTrash=false rejection: every listed group is unknown. + /// PeerOnly catalogue hits remain accepted junk. POST and a missing catalogue + /// do not reject. + /// + public static bool IsUncarriedWantTrashRejection( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue) => + IsUncarriedWantTrashRejection(article, transit, catalogue, out _); + + /// + /// TAKETHIS/IHAVE WantTrash=false rejection, with the uncarried group names + /// that caused it. Classification is unchanged from the bool overload. + /// + /// Queued inbound article already constructed. + /// Transit options; WantTrash=true never rejects here. + /// Published catalogue; missing catalogue does not reject. + /// + /// When the method returns , newsgroup not carried + /// plus the unknown header tokens in article order. + /// + public static bool IsUncarriedWantTrashRejection( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + out string reason) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + reason = string.Empty; + if (transit.WantTrash + || article.Producer == InboundArticleProducer.Post + || catalogue is null) + { + return false; + } + + var snapshot = catalogue.Current; + var newsgroups = article.Record.Newsgroups; + if (HasLocalNonJunkGroup(newsgroups, snapshot) + || HasPeerOnlyGroup(newsgroups, snapshot)) + { + return false; + } + + reason = FormatResponsibleGroups( + IngressNewsReasons.NewsgroupNotCarried, + newsgroups, + snapshot, + peerOnly: false); + return true; + } + + /// + /// Builds a news event when the disposition should be recorded. + /// + public static bool TryCreateEvent( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + DateTimeOffset timestamp, + out NewsLogEvent evt) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + var disposition = Classify(article, transit, catalogue, out var reason); + if (disposition == NewsLogDisposition.Junk && !transit.LogTrash) + { + evt = default; + return false; + } + + evt = new NewsLogEvent( + disposition, + MessageIdMemory(article), + feed: article.Feed, + sites: default, + timestamp, + reason: Ascii(reason), + size: article.Payload.Length); + return true; + } + + private static ReadOnlyMemory Ascii(string? value) => + string.IsNullOrEmpty(value) ? default : Encoding.ASCII.GetBytes(value); + + private static ReadOnlyMemory MessageIdMemory(InboundArticle article) + { + var range = article.Record.Fields.MessageId; + if (!range.IsPresent || range.Length == 0) + { + return ReadOnlyMemory.Empty; + } + + return article.Record.ArtData.Slice(range.Offset, range.Length); + } + + private static bool HasLocalNonJunkGroup(ReadOnlySpan newsgroups, NewsgroupSnapshot snapshot) + { + foreach (var token in EnumerateGroups(newsgroups)) + { + if (snapshot.TryGet(token, out var group) + && group.PostingStatus != NewsgroupPostingStatus.PeerOnly) + { + return true; + } + } + + return false; + } + + /// + /// Builds plus the header tokens that match the + /// already-decided junk/rejection kind, in article Newsgroups order. + /// + /// Already-decided reason prefix. + /// Article Newsgroups field bytes. + /// Published catalogue snapshot used by the existing decision. + /// + /// collects PeerOnly catalogue hits; + /// collects unknown tokens. + /// + /// The prefix, or prefix plus responsible header tokens. + private static string FormatResponsibleGroups( + string prefix, + ReadOnlySpan newsgroups, + NewsgroupSnapshot snapshot, + bool peerOnly) + { + List? names = null; + foreach (var token in EnumerateGroups(newsgroups)) + { + var hit = snapshot.TryGet(token, out var group); + var include = peerOnly + ? hit && group.PostingStatus == NewsgroupPostingStatus.PeerOnly + : !hit; + if (!include) + { + continue; + } + + names ??= new List(2); + names.Add(Encoding.ASCII.GetString(token)); + } + + return names is null + ? prefix + : IngressNewsReasons.WithGroups(prefix, names); + } + + private static bool HasPeerOnlyGroup(ReadOnlySpan newsgroups, NewsgroupSnapshot snapshot) + { + foreach (var token in EnumerateGroups(newsgroups)) + { + if (snapshot.TryGet(token, out var group) + && group.PostingStatus == NewsgroupPostingStatus.PeerOnly) + { + return true; + } + } + + return false; + } + + private static GroupTokenEnumerable EnumerateGroups(ReadOnlySpan newsgroups) => new(newsgroups); + + private readonly ref struct GroupTokenEnumerable(ReadOnlySpan newsgroups) + { + private readonly ReadOnlySpan _newsgroups = newsgroups; + + public Enumerator GetEnumerator() => new(_newsgroups); + + public ref struct Enumerator(ReadOnlySpan newsgroups) + { + private ReadOnlySpan _remaining = newsgroups; + private ReadOnlySpan _current; + + public ReadOnlySpan Current => _current; + + public bool MoveNext() + { + while (!_remaining.IsEmpty) + { + var comma = _remaining.IndexOf((byte)','); + var token = comma < 0 ? _remaining : _remaining[..comma]; + _remaining = comma < 0 ? default : _remaining[(comma + 1)..]; + token = Trim(token); + if (!token.IsEmpty) + { + _current = token; + return true; + } + } + + return false; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + while (!value.IsEmpty && (value[0] == (byte)' ' || value[0] == (byte)'\t')) + { + value = value[1..]; + } + + while (!value.IsEmpty && (value[^1] == (byte)' ' || value[^1] == (byte)'\t')) + { + value = value[..^1]; + } + + return value; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsEvents.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsEvents.cs new file mode 100644 index 00000000..02567efb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsEvents.cs @@ -0,0 +1,73 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Builds already-decided news events at ingress decision points. + /// + internal static class IngressNewsEvents + { + /// WantTrash=false reason for unknown/non-carried Newsgroups. + internal const string NewsgroupNotCarried = IngressNewsReasons.NewsgroupNotCarried; + + /// Writes a rejection news event. Failures do not change protocol state. + public static void TryWriteRejected( + NntpSession session, + string? messageId, + int responseCode, + string reason, + int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Rejected, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + responseCode: responseCode, + reason: Ascii(reason), + size: size)); + } + + /// Writes an accepted news event for a path that never enters the queue. + public static void TryWriteAccepted(NntpSession session, string? messageId, int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Accepted, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + size: size)); + } + + /// Writes a moderated news event for a successful POST moderation submission. + public static void TryWriteModerated(NntpSession session, string? messageId, int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Moderated, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + size: size)); + } + + /// + /// Copies the session's inbound Transit identifier for a self-contained news event. + /// + internal static ReadOnlyMemory SnapshotInboundFeed(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var name = session.Authorization.TransitPeerName; + return string.IsNullOrEmpty(name) ? default : Encoding.ASCII.GetBytes(name); + } + + private static ReadOnlyMemory Ascii(string? value) => + string.IsNullOrEmpty(value) ? default : Encoding.ASCII.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsReasons.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsReasons.cs new file mode 100644 index 00000000..8eb82281 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/IngressNewsReasons.cs @@ -0,0 +1,237 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session.Commands.Posting; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Maps already-decided ingress failures to concise operator-facing news reasons. + /// + /// + /// The formatter does not inspect parser enums, response codes, or validator + /// state. Callers supply the existing decision value; this type only chooses + /// the news-journal text. It does not change NNTP responses or admission. + /// + internal static class IngressNewsReasons + { + /// Invalid or missing Message-ID. + internal const string MessageIdInvalid = "message-id invalid"; + + /// Invalid or missing Date. + internal const string DateInvalid = "date invalid"; + + /// + /// Unknown/non-carried Newsgroups: WantTrash=false rejection, or WantTrash=true junk. + /// Decision sites append : group[, group...] via . + /// + internal const string NewsgroupNotCarried = "newsgroup not carried"; + + /// Receive newsfeeds expression did not subscribe the article. + internal const string NewsgroupPatternRejected = "newsgroup pattern rejected"; + + /// RFC 6048 j / catalogue hit. + internal const string PeerOnly = "peer-only"; + + /// Receive-size or ArticleRecord size rejection. + internal const string ArticleTooLarge = "article too large"; + + /// Session article-type capability rejection. + internal const string ArticleTypeNotPermitted = "article type not permitted"; + + /// Hard queue memory-budget rejection (). + internal const string QueueCapacityExceeded = "queue capacity exceeded"; + + /// yEnc markers present and validation failed. + internal const string YEncodingInvalid = "yEncoding invalid"; + + /// + /// Formats an already-decided prefix plus the responsible group names. + /// + /// Already-decided reason prefix. + /// Responsible group names in header order. + /// The prefix, or prefix: name[, name...]. + /// + /// Does not inspect catalogue or header state. Callers supply the groups + /// that caused the existing decision, in header order. + /// + internal static string WithGroups(string prefix, IReadOnlyList groups) + { + ArgumentException.ThrowIfNullOrEmpty(prefix); + ArgumentNullException.ThrowIfNull(groups); + if (groups.Count == 0) + { + return prefix; + } + + var builder = new StringBuilder(prefix.Length + 2 + (groups.Count * 16)); + builder.Append(prefix); + builder.Append(": "); + for (var i = 0; i < groups.Count; i++) + { + if (i > 0) + { + builder.Append(", "); + } + + builder.Append(groups[i]); + } + + return builder.ToString(); + } + + /// Maps an ArticleRecord construction failure to a news reason. + public static string ForArticleRecord(in ArticleRecordCreateResult created) + { + if (created.ParseFailure != NntpArticleParseFailureCode.None) + { + return ForParseFailure(created.ParseFailure); + } + + return ForMaterializeFailure(created.MaterializeFailure); + } + + /// Maps a parser failure code already produced by ArticleRecord construction. + public static string ForParseFailure(NntpArticleParseFailureCode code) => + code switch + { + NntpArticleParseFailureCode.YEncDecodingFailed => YEncodingInvalid, + NntpArticleParseFailureCode.InvalidMessageId + or NntpArticleParseFailureCode.MissingMessageId => MessageIdInvalid, + NntpArticleParseFailureCode.MissingOrInvalidDate => DateInvalid, + NntpArticleParseFailureCode.ArticleTooLarge => ArticleTooLarge, + NntpArticleParseFailureCode.EmptyArticle => "empty article", + NntpArticleParseFailureCode.MissingHeaderBodySeparator => "missing header/body separator", + NntpArticleParseFailureCode.HeaderSectionTooLarge => "header too large", + NntpArticleParseFailureCode.TooManyHeaders => "too many headers", + NntpArticleParseFailureCode.HeaderLineTooLong => "header line too long", + NntpArticleParseFailureCode.MalformedHeader + or NntpArticleParseFailureCode.MalformedHeaderContinuation => "malformed header", + NntpArticleParseFailureCode.HeaderNameTooLong => "header name too long", + NntpArticleParseFailureCode.HeaderValueTooLong => "header value too long", + NntpArticleParseFailureCode.ContainsNul => "embedded NUL", + NntpArticleParseFailureCode.ContainsIllegalControlByte => "illegal control byte", + NntpArticleParseFailureCode.MissingNewsgroups => "newsgroups missing", + NntpArticleParseFailureCode.InvalidNewsgroups => "newsgroups invalid", + NntpArticleParseFailureCode.InvalidFrom => "from invalid", + NntpArticleParseFailureCode.InvalidPath => "path invalid", + NntpArticleParseFailureCode.DuplicateMessageId => "duplicate message-id", + NntpArticleParseFailureCode.DuplicateNewsgroups => "duplicate newsgroups", + NntpArticleParseFailureCode.DuplicatePath => "duplicate path", + NntpArticleParseFailureCode.BodyLineTooLong => "body line too long", + _ => "article rejected", + }; + + /// Maps a materializer failure code already produced by ArticleRecord construction. + public static string ForMaterializeFailure(NntpArticleCanonicalFailureCode code) => + code switch + { + NntpArticleCanonicalFailureCode.ArticleTooLarge => ArticleTooLarge, + NntpArticleCanonicalFailureCode.MissingSelectedDateHeader => DateInvalid, + NntpArticleCanonicalFailureCode.EmptyArticle => "empty article", + NntpArticleCanonicalFailureCode.DuplicatePath => "duplicate path", + NntpArticleCanonicalFailureCode.InvalidHeaderSeparator => "malformed header", + NntpArticleCanonicalFailureCode.DateLineTooLong => "date line too long", + NntpArticleCanonicalFailureCode.PathRewriteLineTooLong + or NntpArticleCanonicalFailureCode.PathInsertionLineTooLong => "path invalid", + _ => "article record rejected", + }; + + /// + /// Maps an existing IHAVE/TAKETHIS reject-detail string (kept for command + /// completion / TX logs) to the news-journal reason. + /// + public static string ForExistingRejectDetail(string? detail) + { + if (string.IsNullOrEmpty(detail)) + { + return "article rejected"; + } + + if (detail is "rejected too large") + { + return ArticleTooLarge; + } + + if (detail is "rejected article type") + { + return ArticleTypeNotPermitted; + } + + if (detail is "rejected newsgroup pattern") + { + return NewsgroupPatternRejected; + } + + if (detail is "rejected exceeds queue budget") + { + return QueueCapacityExceeded; + } + + if (detail.StartsWith("rejected article record ", StringComparison.Ordinal)) + { + var token = detail["rejected article record ".Length..]; + if (Enum.TryParse(token, out NntpArticleParseFailureCode parse) + && parse != NntpArticleParseFailureCode.None) + { + return ForParseFailure(parse); + } + + if (Enum.TryParse(token, out NntpArticleCanonicalFailureCode materialize) + && materialize != NntpArticleCanonicalFailureCode.None) + { + return ForMaterializeFailure(materialize); + } + } + + return detail; + } + + /// Maps a POST already produced by the command path. + public static string ForPostingFailure(in PostingFailure failure) + { + if (TryMapArticleRecordEnumName(failure.Detail, out var fromEnum)) + { + return fromEnum; + } + + return failure.Category switch + { + PostingFailureCategory.ArticleTooLarge => ArticleTooLarge, + PostingFailureCategory.InvalidMessageId => MessageIdInvalid, + PostingFailureCategory.InvalidDate => DateInvalid, + PostingFailureCategory.PolicyRejected when failure.Detail == "arttype-capability" + => ArticleTypeNotPermitted, + PostingFailureCategory.PolicyRejected when failure.Detail == "unknown newsgroup" + => NewsgroupNotCarried, + PostingFailureCategory.PersistenceFailure when failure.Detail is "Rejected" or "Full" + => QueueCapacityExceeded, + _ => string.IsNullOrEmpty(failure.Detail) ? "article rejected" : failure.Detail, + }; + } + + private static bool TryMapArticleRecordEnumName(string? detail, out string reason) + { + if (!string.IsNullOrEmpty(detail) + && Enum.TryParse(detail, out NntpArticleParseFailureCode parse) + && parse != NntpArticleParseFailureCode.None) + { + reason = ForParseFailure(parse); + return true; + } + + if (!string.IsNullOrEmpty(detail) + && Enum.TryParse(detail, out NntpArticleCanonicalFailureCode materialize) + && materialize != NntpArticleCanonicalFailureCode.None) + { + reason = ForMaterializeFailure(materialize); + return true; + } + + reason = string.Empty; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogDisposition.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogDisposition.cs new file mode 100644 index 00000000..547aaced --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogDisposition.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// INN news log disposition codes implemented for v1 ingress. + /// + /// + /// Values match innd(8) LOGGING / newslog(5) where VectorNNTP actually + /// produces the event: + accepted for propagation, j accepted + /// locally as junk (not propagated), - deliberately rejected. m + /// is a VectorNNTP disposition for a POST sealed for moderation (not an INN + /// innd code) and is not currently propagated by egress routing. Cancel + /// processing (c) is not produced until a cancel path exists. INN's + /// informational ? (isolated CR/LF) is not emitted. Only + /// includes the outbound-site field. + /// + public enum NewsLogDisposition : byte + { + /// Accepted for local store and propagation (+); includes the outbound-site field. + Accepted = (byte)'+', + + /// Accepted locally as junk (j); not propagated, so no outbound-site field. + Junk = (byte)'j', + + /// Deliberately rejected article (-); no outbound-site field. + Rejected = (byte)'-', + + /// POST sealed for moderation (m); not propagated, so no outbound-site field. + Moderated = (byte)'m', + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogEvent.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogEvent.cs new file mode 100644 index 00000000..3df03eb6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogEvent.cs @@ -0,0 +1,89 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Domain-neutral INN news log event already decided by ingress processing. + /// + /// + /// This type does not classify junk or invent rejection reasons. The writer + /// serializes the fields supplied here. An empty is written + /// as INN's unavailable token ? in the inbound-peer position. Outbound + /// exist only for articles accepted for propagation + /// (); empty Sites then emit ? + /// until egress routing exists. Junk, rejected, moderated, and future cancel + /// lines omit the outbound-site field. is the INN article + /// size field after the Message-ID (canonical payload bytes). Timestamp is the + /// news-log event time, not the article Date header. + /// may be stored for but is not + /// rendered. is the already-decided operator-facing text + /// for and + /// , including any responsible newsgroup + /// names supplied by the decision site. + /// + public readonly struct NewsLogEvent + { + /// Initializes a news-log event. + /// Already-decided +, j, -, or m. + /// Article Message-ID bytes, including angle brackets when present. + /// Authoritative inbound feed identity, or empty to emit ?. + /// + /// Outbound site list for ; empty emits + /// ? on that disposition only and is omitted on j/-/m. + /// + /// + /// Event time. lets the writer stamp + /// processing time. + /// + /// NNTP response code for -; retained on the event, not rendered. + /// Already-decided operator-facing reason bytes for - or j. + /// INN article size in bytes (canonical payload length). + public NewsLogEvent( + NewsLogDisposition disposition, + ReadOnlyMemory messageId, + ReadOnlyMemory feed = default, + ReadOnlyMemory sites = default, + DateTimeOffset timestamp = default, + int responseCode = 0, + ReadOnlyMemory reason = default, + int size = 0) + { + Disposition = disposition; + MessageId = messageId; + Feed = feed; + Sites = sites; + Timestamp = timestamp; + ResponseCode = responseCode; + Reason = reason; + Size = size; + } + + /// Gets the news-log event timestamp (processing time, not article Date). + public DateTimeOffset Timestamp { get; } + + /// Gets the already-decided disposition. + public NewsLogDisposition Disposition { get; } + + /// Gets the inbound peer/feed identity captured at decision time. + public ReadOnlyMemory Feed { get; } + + /// Gets the article Message-ID value bytes. + public ReadOnlyMemory MessageId { get; } + + /// Gets the INN article size in bytes (canonical payload length). + public int Size { get; } + + /// + /// Gets whether this event includes the outbound-site field. + /// Only accepted-for-propagation (+) articles have it. + /// + public bool HasOutboundSiteField => Disposition == NewsLogDisposition.Accepted; + + /// Gets optional future outbound site tokens; empty emits ? when is true. + public ReadOnlyMemory Sites { get; } + + /// Gets the NNTP response code carried for a rejection; 0 means none. Not rendered. + public int ResponseCode { get; } + + /// Gets the already-decided rejection or junk reason; empty omits it. + public ReadOnlyMemory Reason { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogLineFormatter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogLineFormatter.cs new file mode 100644 index 00000000..cfd8fb34 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/NewsLogLineFormatter.cs @@ -0,0 +1,189 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Serializes one INN news line per innd(8) LOGGING / ARTlog. + /// + /// + /// Verified INN field order (innd.pod LOGGING and innd/art.c ARTlog): + /// mon dd hh:mm:ss.mmm disposition feedsite message-id size [sites|reason]. + /// Timestamp uses English month abbreviations and local wall time at millisecond + /// resolution. Empty inbound Feed is INN's unavailable token ?. + /// Outbound Sites are written only for + /// (+); empty Sites then emit ? until routing exists. Junk, rejected, + /// and moderated lines omit Sites and append the already-decided reason after size + /// when present. The NNTP response code is not a news field. Disposition characters + /// and field order are hard-coded. + /// + internal static class NewsLogLineFormatter + { + private static ReadOnlySpan Months => "JanFebMarAprMayJunJulAugSepOctNovDec"u8; + + private static ReadOnlySpan UnknownFeed => "?"u8; + + /// Formats into as one LF-terminated line. + /// Bytes written. + public static int Write(Span destination, in NewsLogEvent evt, DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(timestamp.Offset); + var dt = local.DateTime; + var written = 0; + written += WriteMonth(destination[written..], dt.Month); + destination[written++] = (byte)' '; + written += WriteSpacePaddedDay(destination[written..], dt.Day); + destination[written++] = (byte)' '; + written += WriteTwoDigits(destination[written..], dt.Hour); + destination[written++] = (byte)':'; + written += WriteTwoDigits(destination[written..], dt.Minute); + destination[written++] = (byte)':'; + written += WriteTwoDigits(destination[written..], dt.Second); + destination[written++] = (byte)'.'; + written += WriteThreeDigits(destination[written..], dt.Millisecond); + destination[written++] = (byte)' '; + destination[written++] = (byte)evt.Disposition; + destination[written++] = (byte)' '; + var feed = evt.Feed.IsEmpty ? UnknownFeed : evt.Feed.Span; + feed.CopyTo(destination[written..]); + written += feed.Length; + destination[written++] = (byte)' '; + evt.MessageId.Span.CopyTo(destination[written..]); + written += evt.MessageId.Length; + destination[written++] = (byte)' '; + written += WriteUnsignedDecimal(destination[written..], evt.Size); + if (evt.HasOutboundSiteField) + { + destination[written++] = (byte)' '; + var sites = evt.Sites.IsEmpty ? UnknownFeed : evt.Sites.Span; + sites.CopyTo(destination[written..]); + written += sites.Length; + } + + if (WritesReason(evt.Disposition) && !evt.Reason.IsEmpty) + { + destination[written++] = (byte)' '; + evt.Reason.Span.CopyTo(destination[written..]); + written += evt.Reason.Length; + } + + destination[written++] = (byte)'\n'; + return written; + } + + /// Byte count required to format including the trailing LF. + public static int RequiredLength(in NewsLogEvent evt) + { + var feedLength = evt.Feed.IsEmpty ? 1 : evt.Feed.Length; + var sites = evt.HasOutboundSiteField + ? 1 + (evt.Sites.IsEmpty ? 1 : evt.Sites.Length) + : 0; + var reason = WritesReason(evt.Disposition) && !evt.Reason.IsEmpty + ? 1 + evt.Reason.Length + : 0; + return 24 + feedLength + evt.MessageId.Length + 1 + DecimalDigitCount(evt.Size) + + sites + reason; + } + + private static bool WritesReason(NewsLogDisposition disposition) => + disposition is NewsLogDisposition.Rejected or NewsLogDisposition.Junk; + + private static int WriteMonth(Span dest, int month) + { + var index = (month - 1) * 3; + Months.Slice(index, 3).CopyTo(dest); + return 3; + } + + private static int WriteSpacePaddedDay(Span dest, int day) + { + if (day < 10) + { + dest[0] = (byte)' '; + dest[1] = (byte)('0' + day); + } + else + { + dest[0] = (byte)('0' + (day / 10)); + dest[1] = (byte)('0' + (day % 10)); + } + + return 2; + } + + private static int WriteTwoDigits(Span dest, int value) + { + dest[0] = (byte)('0' + (value / 10)); + dest[1] = (byte)('0' + (value % 10)); + return 2; + } + + private static int WriteThreeDigits(Span dest, int value) + { + dest[0] = (byte)('0' + (value / 100)); + dest[1] = (byte)('0' + ((value / 10) % 10)); + dest[2] = (byte)('0' + (value % 10)); + return 3; + } + + private static int WriteUnsignedDecimal(Span dest, int value) + { + var digits = DecimalDigitCount(value); + var remaining = value; + for (var i = digits - 1; i >= 0; i--) + { + dest[i] = (byte)('0' + (remaining % 10)); + remaining /= 10; + } + + return digits; + } + + private static int DecimalDigitCount(int value) + { + if (value < 10) + { + return 1; + } + + if (value < 100) + { + return 2; + } + + if (value < 1_000) + { + return 3; + } + + if (value < 10_000) + { + return 4; + } + + if (value < 100_000) + { + return 5; + } + + if (value < 1_000_000) + { + return 6; + } + + if (value < 10_000_000) + { + return 7; + } + + if (value < 100_000_000) + { + return 8; + } + + if (value < 1_000_000_000) + { + return 9; + } + + return 10; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/NullNewsLogWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/NullNewsLogWriter.cs new file mode 100644 index 00000000..7efb556d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/NullNewsLogWriter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// No-op for tests that do not assert news output. + public sealed class NullNewsLogWriter : INewsLogWriter + { + /// Gets the shared no-op instance. + public static NullNewsLogWriter Instance { get; } = new(); + + /// + public void Write(in NewsLogEvent evt) + { + } + + /// + public void Flush() + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/NullPathSurveyWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/NullPathSurveyWriter.cs new file mode 100644 index 00000000..ec1fa4ec --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/NullPathSurveyWriter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// No-op for tests that do not assert Path-survey output. + public sealed class NullPathSurveyWriter : IPathSurveyWriter + { + /// Gets the shared no-op instance. + public static NullPathSurveyWriter Instance { get; } = new(); + + /// + public void Write(ReadOnlySpan canonicalPath) + { + } + + /// + public void Flush() + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..6a610088 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// One-way OverviewDB RabbitMQ handoff with asynchronous publisher confirms. + /// + /// + /// returns after the message is written into the outstanding + /// confirmation window. Broker confirmation is not awaited on the publish path. + /// Crash loss of outstanding unconfirmed work is acceptable for OverviewDB. + /// + public interface IOverviewDbHandoffPublisher + { + /// + /// Publishes a compact OverviewDB payload to overviewdb.queue without waiting + /// for the broker confirmation. + /// + /// Owned OverviewDB work item (already-encoded payload). + /// Token used to cancel waiting for outstanding capacity or the write. + Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken); + + /// Gets the number of publishes awaiting broker confirmation. + int OutstandingCount { get; } + + /// + /// Tries to dequeue one nack/return failure for in-process requeue handling. + /// + /// Must not be called from inside a RabbitMQ ack/nack/return callback. + bool TryDequeuePublishFailure(out OverviewDbWorkItem item); + + /// + /// Abandons all outstanding unconfirmed work (bounded shutdown / crash-loss path). + /// + void AbandonOutstanding(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs new file mode 100644 index 00000000..c934297b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// Bounded, byte-budgeted in-process queue between article workers and OverviewDB + /// RabbitMQ publisher workers. + /// + /// + /// This queue is not durable. Enqueue success does not imply RabbitMQ confirmation. + /// + internal interface IOverviewDbWorkQueue + { + /// Gets the configured byte budget. + long MemoryLimitBytes { get; } + + /// Gets reserved queued payload bytes. + long QueuedBytes { get; } + + /// Gets the approximate number of queued work items. + int Count { get; } + + /// Gets producers waiting for byte-budget capacity. + int WaitingProducerCount { get; } + + /// Gets whether the queue still accepts new work. + bool IsAccepting { get; } + + /// + /// Enqueues work, waiting for byte-budget capacity when full. + /// + ValueTask EnqueueAsync(OverviewDbWorkItem item, CancellationToken cancellationToken); + + /// + /// Dequeues the next item, or when completed and empty. + /// Releases the byte reservation on successful dequeue. + /// + ValueTask DequeueAsync(CancellationToken cancellationToken); + + /// Stops accepting new work; queued items remain readable until drained. + void Complete(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..87a4d11c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// Test-only OverviewDB publisher that completes without contacting a broker. + /// Production DI registers instead. + /// + internal sealed class NullOverviewDbHandoffPublisher : IOverviewDbHandoffPublisher + { + /// Shared succeeding instance used when tests construct the worker directly. + public static NullOverviewDbHandoffPublisher Instance { get; } = new(); + + private NullOverviewDbHandoffPublisher() + { + } + + /// + public Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + + /// + public int OutstandingCount => 0; + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + item = null!; + return false; + } + + /// + public void AbandonOutstanding() + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs new file mode 100644 index 00000000..09fb6f32 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// Source-generated OverviewDB handoff log messages. + internal static partial class OverviewDbHandoffLogMessages + { + [LoggerMessage( + EventId = 2100, + Level = LogLevel.Error, + Message = "OverviewDB RabbitMQ handoff failed for {MessageId} ({Bytes} payload bytes)")] + public static partial void PublishFailed(ILogger logger, Exception exception, string MessageId, int Bytes); + + [LoggerMessage( + EventId = 2101, + Level = LogLevel.Warning, + Message = "OverviewDB handoff was requeued for {MessageId}")] + public static partial void Requeued(ILogger logger, string MessageId); + + [LoggerMessage( + EventId = 2102, + Level = LogLevel.Error, + Message = "OverviewDB handoff could not be requeued for {MessageId}; queue is unavailable")] + public static partial void RequeueUnavailable(ILogger logger, string MessageId); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..0456f1e5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs @@ -0,0 +1,448 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Text; +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// Publishes OverviewDB protobuf payloads to overviewdb.queue with an asynchronous + /// outstanding-confirmation window. + /// + /// + /// + /// Uses one confirm-enabled RabbitMQ channel (library tracking disabled) so + /// BasicPublishAsync returns after the write. Confirms arrive via + /// BasicAcksAsync / BasicNacksAsync / BasicReturnAsync and are + /// correlated by publish sequence number. Channel publishes are serialized; callbacks + /// only update outstanding state and must not publish. + /// + /// + /// Crash or shutdown loss of outstanding unconfirmed OverviewDB work is acceptable. + /// + /// + internal sealed class OverviewDbHandoffPublisher : IOverviewDbHandoffPublisher, IAsyncDisposable + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly ConcurrentDictionary _outstanding = new(); + private readonly ConcurrentDictionary _returned = new(); + private readonly Channel _failures = Channel.CreateUnbounded( + new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + private SemaphoreSlim? _outstandingWindow; + private int _windowSize; + private IRabbitMqAsyncConfirmPublishChannel? _channel; + private int _disposed; + + /// Initializes a new OverviewDB handoff publisher. + public OverviewDbHandoffPublisher( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IOptions rabbitMqOptions, + IngestionPipelineMetrics? pipelineMetrics = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(rabbitMqOptions); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _pipeline = pipelineMetrics; + _ = rabbitMqOptions; + } + + /// + public int OutstandingCount => _outstanding.Count; + + /// Gets the configured outstanding window size (tests). + internal int WindowSize => Volatile.Read(ref _windowSize); + + /// + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + var fqdn = _nntpdOptions.Value.Fqdn; + if (string.IsNullOrWhiteSpace(fqdn)) + { + throw new InvalidOperationException("OverviewDB handoff requires the generated application FQDN."); + } + + var window = EnsureOutstandingWindow(); + var publishStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await window.WaitAsync(cancellationToken).ConfigureAwait(false); + _pipeline?.RecordGateWait(publishStart); + _pipeline?.ObserveInFlightPublishes(OutstandingCount + 1); + + try + { + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var channel = await EnsureChannelAlreadyLockedAsync(cancellationToken).ConfigureAwait(false); + var sequence = await channel + .GetNextPublishSequenceNumberAsync(cancellationToken) + .ConfigureAwait(false); + if (!_outstanding.TryAdd(sequence, item)) + { + throw new InvalidOperationException( + $"Duplicate OverviewDB publish sequence number {sequence}."); + } + + try + { + var writeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await channel.PublishAsync( + OverviewDbTopology.DefaultExchange, + OverviewDbTopology.RoutingKey, + Guid.NewGuid().ToString(), + fqdn, + OverviewDbTopology.ExpirationMilliseconds, + sequence, + item.Payload, + cancellationToken) + .ConfigureAwait(false); + // Write latency only; broker confirms are asynchronous. + _pipeline?.RecordPublishAndConfirm(writeStart); + _pipeline?.RecordHandoff(publishStart); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + _outstanding.TryRemove(sequence, out _); + _returned.TryRemove(sequence, out _); + throw; + } + } + finally + { + _publishGate.Release(); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + ReleaseWindowSlot(); + _pipeline?.RecordConfirmFailure(); + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + throw; + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) => + _failures.Reader.TryRead(out item!); + + /// + public void AbandonOutstanding() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _outstanding) + { + if (_outstanding.TryRemove(pair.Key, out _)) + { + _returned.TryRemove(pair.Key, out _); + ReleaseWindowSlot(); + } + } + + while (_failures.Reader.TryRead(out _)) + { + // Drain residual failure notifications before observe. + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + AbandonOutstanding(); + await DisposeChannelAsync().ConfigureAwait(false); + _publishGate.Dispose(); + Interlocked.Exchange(ref _outstandingWindow, null)?.Dispose(); + } + + private SemaphoreSlim EnsureOutstandingWindow() + { + var existing = Volatile.Read(ref _outstandingWindow); + if (existing is not null) + { + return existing; + } + + var size = _nntpdOptions.Value.ArticleIngestion?.OverviewDbPublisherBatchSize + ?? ArticleIngestionOptions.DefaultOverviewDbPublisherBatchSize; + if (size < 1) + { + size = 1; + } + + var created = new SemaphoreSlim(size, size); + // codeql[cs/missed-using-statement] Ownership transfers into _outstandingWindow on successful CompareExchange; using would dispose the live window. toDispose is cleared only after publish succeeds. + SemaphoreSlim? toDispose = created; + try + { + var prior = Interlocked.CompareExchange(ref _outstandingWindow, created, null); + if (prior is not null) + { + return prior; + } + + toDispose = null; + Volatile.Write(ref _windowSize, size); + return created; + } + finally + { + toDispose?.Dispose(); + } + } + + /// + /// Ensures a usable async-confirm channel. Caller must hold . + /// + private async Task EnsureChannelAlreadyLockedAsync( + CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + throw new InvalidOperationException("RabbitMQ connection is not ready for OverviewDB handoff."); + } + + var existing = _channel; + if (existing is not null + && existing.Generation == handle.Generation + && existing.IsOpen) + { + return existing; + } + + if (existing is not null) + { + DetachHandlers(existing); + try + { + await existing.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort channel dispose must not fail channel recovery. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored during channel recovery dispose + } + + FailOutstandingForChannelLoss(); + _channel = null; + } + + var created = await handle.Connection + .CreateAsyncConfirmPublishChannelAsync(handle.Generation, cancellationToken) + .ConfigureAwait(false); + AttachHandlers(created); + _channel = created; + return created; + } + + private void AttachHandlers(IRabbitMqAsyncConfirmPublishChannel channel) + { + channel.BasicAcksAsync += OnBasicAckAsync; + channel.BasicNacksAsync += OnBasicNackAsync; + channel.BasicReturnAsync += OnBasicReturnAsync; + } + + private void DetachHandlers(IRabbitMqAsyncConfirmPublishChannel channel) + { + channel.BasicAcksAsync -= OnBasicAckAsync; + channel.BasicNacksAsync -= OnBasicNackAsync; + channel.BasicReturnAsync -= OnBasicReturnAsync; + } + + private Task OnBasicAckAsync(object sender, BasicAckEventArgs args) + { + // Lightweight: settle outstanding only — never publish from this callback. + CompleteOutstanding(args.DeliveryTag, args.Multiple, nack: false); + return Task.CompletedTask; + } + + private Task OnBasicNackAsync(object sender, BasicNackEventArgs args) + { + CompleteOutstanding(args.DeliveryTag, args.Multiple, nack: true); + return Task.CompletedTask; + } + + private Task OnBasicReturnAsync(object sender, BasicReturnEventArgs args) + { + // Mark returned; settlement (window release + failure dequeue) waits for ack/nack. + // RabbitMQ typically acks after returning a mandatory unroutable message. + if (TryReadPublishSequence(args.BasicProperties, out var sequence)) + { + _returned[sequence] = 0; + } + + return Task.CompletedTask; + } + + private void CompleteOutstanding(ulong deliveryTag, bool multiple, bool nack) + { + if (multiple) + { + // codeql[cs/linq/missed-where] ConcurrentDictionary keys with SettleOne side effects. + foreach (var key in _outstanding.Keys) + { + if (key <= deliveryTag) + { + SettleOne(key, nack); + } + } + } + else + { + SettleOne(deliveryTag, nack); + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + private void SettleOne(ulong sequence, bool nack) + { + if (!_outstanding.TryRemove(sequence, out var item)) + { + return; + } + + var wasReturned = _returned.TryRemove(sequence, out _); + ReleaseWindowSlot(); + + if (nack || wasReturned) + { + _ = _failures.Writer.TryWrite(item); + _pipeline?.RecordConfirmFailure(); + } + } + + private void FailOutstandingForChannelLoss() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _outstanding) + { + if (_outstanding.TryRemove(pair.Key, out var item)) + { + _returned.TryRemove(pair.Key, out _); + _ = _failures.Writer.TryWrite(item); + ReleaseWindowSlot(); + _pipeline?.RecordConfirmFailure(); + } + } + } + + private void ReleaseWindowSlot() + { + try + { + EnsureOutstandingWindow().Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (SemaphoreFullException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task DisposeChannelAsync() + { + var channel = Interlocked.Exchange(ref _channel, null); + if (channel is null) + { + return; + } + + DetachHandlers(channel); + try + { + await channel.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort channel dispose must not fail publisher teardown. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored during publisher teardown dispose + } + } + + /// + /// Recovers the publish sequence from Basic.Return headers. + /// + /// + /// Matches RabbitMQ.Client 7.2.2 HandleReturn representations: + /// long, decimal string, and longstr byte[] (ASCII digits). + /// + internal static bool TryReadPublishSequence(IReadOnlyBasicProperties? properties, out ulong sequence) + { + sequence = 0; + if (properties?.Headers is null + || !properties.Headers.TryGetValue(Constants.PublishSequenceNumberHeader, out var raw) + || raw is null) + { + return false; + } + + return TryParsePublishSequenceNumberHeader(raw, out sequence); + } + + /// + /// Parses an AMQP table value written for . + /// + internal static bool TryParsePublishSequenceNumberHeader(object raw, out ulong sequence) + { + switch (raw) + { + case long l when l >= 0: + sequence = (ulong)l; + return true; + case int i when i >= 0: + sequence = (ulong)i; + return true; + case string s: + return ulong.TryParse( + s, + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence); + case byte[] bytes: + // Wire longstr ('S') for a decimal string arrives as UTF-8/ASCII bytes. + return ulong.TryParse( + Encoding.ASCII.GetString(bytes), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence); + default: + sequence = 0; + return false; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs new file mode 100644 index 00000000..bd0771a3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs @@ -0,0 +1,447 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// Dynamically sized pool that drains and + /// publishes through with asynchronous confirms. + /// + /// + /// Workers do not await broker confirmation per message. They wait only for outstanding + /// confirmation-window capacity and the network write. Nack/return failures are drained + /// from the publisher and requeued in-process (bounded; abandoned on shutdown). + /// + internal sealed class OverviewDbPublisherPool + { + private readonly IOverviewDbWorkQueue _queue; + private readonly IOverviewDbHandoffPublisher _publisher; + private readonly ArticleIngestionOptions _options; + private readonly ILogger _logger; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly List _workers = []; + private int _highPressureStreak; + private int _lowPressureStreak; + private int _scaleUps; + private int _scaleDowns; + + /// Initializes a new OverviewDB publisher pool. + public OverviewDbPublisherPool( + IOverviewDbWorkQueue queue, + IOverviewDbHandoffPublisher publisher, + ArticleIngestionOptions options, + ILogger logger, + IngestionPipelineMetrics? pipelineMetrics = null, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(publisher); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _publisher = publisher; + _options = options; + _logger = logger; + _pipeline = pipelineMetrics; + _time = timeProvider ?? TimeProvider.System; + } + + /// Gets the current publisher worker count. + public int WorkerCount + { + get + { + lock (_gate) + { + return _workers.Count; + } + } + } + + /// Gets scale-up count (tests). + public int ScaleUpCount => Volatile.Read(ref _scaleUps); + + /// Gets scale-down count (tests). + public int ScaleDownCount => Volatile.Read(ref _scaleDowns); + + /// Runs publisher workers and the scaler until cancelled. + public async Task RunAsync(CancellationToken cancellationToken) + { + OverviewDbPublisherPoolLogMessages.PoolStarting( + _logger, + _options.OverviewDbMinPublisherWorkers, + _options.OverviewDbMaxPublisherWorkers, + _options.OverviewDbPublisherBatchSize); + + for (var i = 0; i < _options.OverviewDbMinPublisherWorkers; i++) + { + AddWorker(cancellationToken); + } + + PublishPoolObservation(); + + var scaler = Task.Run(() => ScaleLoopAsync(cancellationToken), CancellationToken.None); + try + { + await scaler.ConfigureAwait(false); + } + finally + { + List snapshot; + lock (_gate) + { + snapshot = [.. _workers]; + _workers.Clear(); + } + + foreach (var worker in snapshot) + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + } + + var drainTimeout = TimeSpan.FromSeconds( + Math.Max(1, _options.OverviewDbPublisherShutdownSeconds)); + using var drainCts = new CancellationTokenSource(drainTimeout); + foreach (var worker in snapshot) + { + try + { + await worker.Execution.WaitAsync(drainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + } + } + + // Bounded shutdown: abandon any remaining outstanding confirms. + _publisher.AbandonOutstanding(); + PublishPoolObservation(); + OverviewDbPublisherPoolLogMessages.PoolStopped(_logger, snapshot.Count); + } + } + + /// Applies one scaling decision (tests). + internal void ApplyScaleDecisionForTests(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + ApplyScaleDecision(snapshot, workerToken); + PublishPoolObservation(); + } + + private async Task ScaleLoopAsync(CancellationToken cancellationToken) + { + var interval = TimeSpan.FromSeconds(Math.Max(1, _options.ScaleIntervalSeconds)); + using var timer = new PeriodicTimer(interval, _time); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + ApplyScaleDecision(SamplePressure(), cancellationToken); + PublishPoolObservation(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private IngestionPressureSnapshot SamplePressure() + { + var limit = _queue.MemoryLimitBytes; + var bytes = _queue.QueuedBytes; + var waiting = _queue.WaitingProducerCount; + var utilisation = limit <= 0 ? 0d : Math.Clamp(bytes / (double)limit, 0d, 1d); + var waitingSignal = waiting > 0 ? 1d : 0d; + return new IngestionPressureSnapshot( + Math.Max(utilisation, waitingSignal), + bytes, + limit, + _queue.Count, + waiting); + } + + private void ApplyScaleDecision(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + if (snapshot.Pressure >= _options.ScaleUpPressureThreshold) + { + _highPressureStreak++; + _lowPressureStreak = 0; + if (_highPressureStreak >= _options.ScaleUpConsecutiveIntervals) + { + _highPressureStreak = 0; + if (TryAddWorker(workerToken)) + { + Interlocked.Increment(ref _scaleUps); + _pipeline?.RecordOverviewDbPublisherScaleUp(); + OverviewDbPublisherPoolLogMessages.ScaledUp(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + if (snapshot.Pressure <= _options.ScaleDownPressureThreshold) + { + _lowPressureStreak++; + _highPressureStreak = 0; + if (_lowPressureStreak >= _options.ScaleDownConsecutiveIntervals) + { + _lowPressureStreak = 0; + if (TryRemoveWorker()) + { + Interlocked.Increment(ref _scaleDowns); + _pipeline?.RecordOverviewDbPublisherScaleDown(); + OverviewDbPublisherPoolLogMessages.ScaledDown(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + _highPressureStreak = 0; + _lowPressureStreak = 0; + } + + private bool TryAddWorker(CancellationToken workerToken) + { + lock (_gate) + { + if (_workers.Count >= _options.OverviewDbMaxPublisherWorkers) + { + return false; + } + + AddWorkerAlreadyLocked(workerToken); + return true; + } + } + + private void AddWorker(CancellationToken workerToken) + { + lock (_gate) + { + AddWorkerAlreadyLocked(workerToken); + } + } + + private void AddWorkerAlreadyLocked(CancellationToken workerToken) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(workerToken); + var worker = new Worker(cts) { Execution = Task.CompletedTask }; + worker.Execution = Task.Run(() => WorkerLoopAsync(worker, cts.Token), CancellationToken.None); + _workers.Add(worker); + } + + private bool TryRemoveWorker() + { + Worker? victim; + lock (_gate) + { + if (_workers.Count <= _options.OverviewDbMinPublisherWorkers) + { + return false; + } + + victim = _workers[^1]; + _workers.RemoveAt(_workers.Count - 1); + } + + _ = RetireWorkerAsync(victim); + return true; + } + + private async Task RetireWorkerAsync(Worker worker) + { + try + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + PublishPoolObservation(); + } + } + + private async Task WorkerLoopAsync(Worker worker, CancellationToken cancellationToken) + { + try + { + while (true) + { + DrainPublishFailures(); + + OverviewDbWorkItem? item; + try + { + item = await _queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (!_queue.IsAccepting) + { + await DrainRemainingAsync().ConfigureAwait(false); + } + + DrainPublishFailures(); + break; + } + + if (item is null) + { + DrainPublishFailures(); + break; + } + + await PublishOneAsync(item, cancellationToken).ConfigureAwait(false); + DrainPublishFailures(); + + if (cancellationToken.IsCancellationRequested && _queue.Count == 0) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + } + + private async Task DrainRemainingAsync() + { + while (true) + { + DrainPublishFailures(); + OverviewDbWorkItem? item; + try + { + item = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (item is null) + { + break; + } + + await PublishOneAsync(item, CancellationToken.None).ConfigureAwait(false); + } + + DrainPublishFailures(); + } + + private async Task PublishOneAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + try + { + // Does not await broker confirmation — only outstanding-window capacity + write. + await _publisher.PublishAsync(item, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown: abandon in-flight work. Do not claim the OverviewDB work queue + // is unavailable — cancellation is not a queue-admission failure. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // Local serialization / publish-write failures (e.g. WireFormattingException) + // never reached the broker; treat as publish failure + in-memory requeue. + OverviewDbHandoffLogMessages.PublishFailed(_logger, ex, item.MessageId, item.ByteLength); + await TryRequeueFailureAsync(item).ConfigureAwait(false); + } + } + + private void DrainPublishFailures() + { + while (_publisher.TryDequeuePublishFailure(out var failed)) + { + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + new InvalidOperationException("OverviewDB publish was nacked or returned."), + failed.MessageId, + failed.ByteLength); + // Fire-and-forget requeue onto the in-process work queue while accepting. + _ = TryRequeueFailureAsync(failed); + } + } + + private async Task TryRequeueFailureAsync(OverviewDbWorkItem item) + { + if (!_queue.IsAccepting) + { + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, item.MessageId); + return; + } + + var result = await _queue.EnqueueAsync(item, CancellationToken.None).ConfigureAwait(false); + if (result == ArticleEnqueueResult.Accepted) + { + OverviewDbHandoffLogMessages.Requeued(_logger, item.MessageId); + return; + } + + if (result == ArticleEnqueueResult.Unavailable) + { + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, item.MessageId); + return; + } + + // Rejected (e.g. oversized) — not an OverviewDB work-queue unavailable condition. + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + new InvalidOperationException($"OverviewDB work-queue rejected requeue ({result})."), + item.MessageId, + item.ByteLength); + } + + private void PublishPoolObservation() + { + _pipeline?.ObserveOverviewDbPublisherPool( + WorkerCount, + _options.OverviewDbMinPublisherWorkers, + _options.OverviewDbMaxPublisherWorkers); + _pipeline?.ObserveOverviewDbWorkQueue(_queue.Count, _queue.QueuedBytes, _queue.WaitingProducerCount); + } + + private sealed class Worker(CancellationTokenSource cts) + { + public CancellationTokenSource Cts { get; } = cts; + + public Task Execution { get; set; } = Task.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs new file mode 100644 index 00000000..cb6329c7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// Source-generated OverviewDB publisher-pool log messages. + internal static partial class OverviewDbPublisherPoolLogMessages + { + [LoggerMessage( + EventId = 2110, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool starting (min={MinWorkers}, max={MaxWorkers}, publishConcurrency={PublishConcurrency})")] + public static partial void PoolStarting( + ILogger logger, + int MinWorkers, + int MaxWorkers, + int PublishConcurrency); + + [LoggerMessage( + EventId = 2111, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool stopped (workers={Workers})")] + public static partial void PoolStopped(ILogger logger, int Workers); + + [LoggerMessage( + EventId = 2112, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool scaled up to {Workers} (pressure={Pressure:F3})")] + public static partial void ScaledUp(ILogger logger, int Workers, double Pressure); + + [LoggerMessage( + EventId = 2113, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool scaled down to {Workers} (pressure={Pressure:F3})")] + public static partial void ScaledDown(ILogger logger, int Workers, double Pressure); + + [LoggerMessage( + EventId = 2114, + Level = LogLevel.Error, + Message = "OverviewDB publisher worker faulted")] + public static partial void WorkerFaulted(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs new file mode 100644 index 00000000..08b762d8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// One in-process OverviewDB handoff unit: an already-encoded + /// OverviewArticleV1 payload awaiting RabbitMQ publication. + /// + /// + /// Ownership of transfers with the item. The publisher + /// stage publishes these bytes and does not re-encode. Successful enqueue into + /// the OverviewDB work queue is not equivalent to a RabbitMQ publisher + /// confirmation; broker durability still occurs only after the OverviewDB publisher + /// stage receives a broker confirmation for the outstanding publish. + /// + public sealed class OverviewDbWorkItem + { + /// Initializes a work item that owns . + public OverviewDbWorkItem(byte[] payload, string messageId) + { + ArgumentNullException.ThrowIfNull(payload); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + Payload = payload; + MessageId = messageId; + } + + /// Gets the owned encoded OverviewArticleV1 bytes. + public byte[] Payload { get; } + + /// Gets the article Message-ID for diagnostics. + public string MessageId { get; } + + /// Gets the reserved byte-budget length (payload length). + public int ByteLength => Payload.Length; + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs new file mode 100644 index 00000000..7424a063 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs @@ -0,0 +1,241 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb +{ + /// + /// Byte-budgeted -backed OverviewDB handoff work queue. + /// + /// + /// In-process only. The admission bound is . + /// Accounting uses each item's . + /// + internal sealed class OverviewDbWorkQueue : IOverviewDbWorkQueue + { + private readonly Channel _channel; + private readonly object _gate = new(); + private readonly Queue _waiters = new(); + private readonly long _memoryLimit; + private long _queuedBytes; + private int _count; + private int _completed; + + /// Initializes a new OverviewDB work queue with the given byte budget. + public OverviewDbWorkQueue(long memoryLimitBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(memoryLimitBytes, 1); + _memoryLimit = memoryLimitBytes; + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public long MemoryLimitBytes => _memoryLimit; + + /// + public long QueuedBytes => Volatile.Read(ref _queuedBytes); + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public bool IsAccepting => Volatile.Read(ref _completed) == 0; + + /// + public int WaitingProducerCount + { + get + { + lock (_gate) + { + var n = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var waiter in _waiters) + { + if (waiter.IsWaiting) + { + n++; + } + } + + return n; + } + } + } + + /// + public async ValueTask EnqueueAsync( + OverviewDbWorkItem item, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + var bytes = item.ByteLength; + if (bytes > _memoryLimit) + { + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + return ArticleEnqueueResult.Unavailable; + } + + var reserved = await ReserveAsync(bytes, cancellationToken).ConfigureAwait(false); + if (reserved != ArticleEnqueueResult.Accepted) + { + return reserved; + } + + if (!_channel.Writer.TryWrite(item)) + { + Release(bytes); + return ArticleEnqueueResult.Unavailable; + } + + Interlocked.Increment(ref _count); + return ArticleEnqueueResult.Accepted; + } + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var item)) + { + Release(item.ByteLength); + Interlocked.Decrement(ref _count); + return item; + } + } + + return null; + } + + /// + public void Complete() + { + if (Interlocked.Exchange(ref _completed, 1) == 1) + { + return; + } + + _channel.Writer.TryComplete(); + List waiters; + lock (_gate) + { + waiters = [.. _waiters]; + _waiters.Clear(); + } + + foreach (var waiter in waiters) + { + waiter.TrySetUnavailable(); + } + } + + private async ValueTask ReserveAsync(int bytes, CancellationToken cancellationToken) + { + while (true) + { + ByteWaiter waiter; + lock (_gate) + { + if (_completed != 0) + { + return ArticleEnqueueResult.Unavailable; + } + + if (_queuedBytes + bytes <= _memoryLimit) + { + _queuedBytes += bytes; + return ArticleEnqueueResult.Accepted; + } + + waiter = new ByteWaiter(bytes); + _waiters.Enqueue(waiter); + } + + try + { + return await waiter.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + lock (_gate) + { + waiter.Cancel(); + } + + throw; + } + } + } + + private void Release(int bytes) + { + lock (_gate) + { + _queuedBytes = Math.Max(0, _queuedBytes - bytes); + while (_waiters.Count > 0) + { + var next = _waiters.Peek(); + if (!next.IsWaiting) + { + _waiters.Dequeue(); + continue; + } + + if (_queuedBytes + next.Bytes > _memoryLimit) + { + break; + } + + _waiters.Dequeue(); + _queuedBytes += next.Bytes; + next.TrySetAccepted(); + } + } + } + + private sealed class ByteWaiter + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _state; // 0 waiting, 1 completed + + public ByteWaiter(int bytes) => Bytes = bytes; + + public int Bytes { get; } + + public bool IsWaiting => Volatile.Read(ref _state) == 0; + + public Task Task => _tcs.Task; + + public void TrySetAccepted() + { + if (Interlocked.Exchange(ref _state, 1) == 0) + { + _tcs.TrySetResult(ArticleEnqueueResult.Accepted); + } + } + + public void TrySetUnavailable() + { + if (Interlocked.Exchange(ref _state, 1) == 0) + { + _tcs.TrySetResult(ArticleEnqueueResult.Unavailable); + } + } + + public void Cancel() + { + Interlocked.Exchange(ref _state, 1); + _tcs.TrySetCanceled(); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/PathSurveyLineFormatter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/PathSurveyLineFormatter.cs new file mode 100644 index 00000000..11e6fa75 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/PathSurveyLineFormatter.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Serializes one INN-style Path-survey observation as Path: {canonical-path}\n. + /// + /// + /// The wire format is an application invariant. This type copies + /// ArticleRecord.Path bytes as-is: it does not parse Path, invent hops, + /// or add Message-ID, Newsgroups, size, timestamp, peer, or disposition. + /// + internal static class PathSurveyLineFormatter + { + /// ASCII Path: prefix including the header-style space. + internal static ReadOnlySpan Prefix => "Path: "u8; + + /// Formats into as one LF-terminated line. + /// Bytes written. + public static int Write(Span destination, ReadOnlySpan canonicalPath) + { + var prefix = Prefix; + prefix.CopyTo(destination); + var written = prefix.Length; + canonicalPath.CopyTo(destination[written..]); + written += canonicalPath.Length; + destination[written++] = (byte)'\n'; + return written; + } + + /// Byte count required to format including the trailing LF. + public static int RequiredLength(ReadOnlySpan canonicalPath) => + Prefix.Length + canonicalPath.Length + 1; + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogNewsLogWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogNewsLogWriter.cs new file mode 100644 index 00000000..feefcece --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogNewsLogWriter.cs @@ -0,0 +1,58 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Emits already-decided news events through the dedicated Serilog news pipeline. + /// + /// + /// File path, rolling, retention, compression, and buffering come from + /// Serilog:News. The INN line is produced only by + /// . This writer does not open, rotate, or + /// compress files itself. Serilog host disposal flushes the pipeline. + /// + public sealed class SerilogNewsLogWriter : INewsLogWriter, IDisposable + { + private readonly Serilog.Core.Logger _logger; + + /// Initializes a writer that owns a dedicated Serilog news logger. + public SerilogNewsLogWriter(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _logger = NntpdNewsLogging.CreateLogger(configuration); + } + + /// Initializes a writer that uses an already-constructed news logger (tests). + public SerilogNewsLogWriter(Serilog.Core.Logger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// + public void Write(in NewsLogEvent evt) + { + var messageId = Encoding.ASCII.GetString(evt.MessageId.Span); + var feed = evt.Feed.IsEmpty ? "?" : Encoding.ASCII.GetString(evt.Feed.Span); + var sites = evt.Sites.IsEmpty ? string.Empty : Encoding.ASCII.GetString(evt.Sites.Span); + var reason = evt.Reason.IsEmpty ? string.Empty : Encoding.ASCII.GetString(evt.Reason.Span); + _logger + .ForContext(NntpdNewsLogging.DispositionProperty, (char)evt.Disposition) + .ForContext(NntpdNewsLogging.FeedProperty, feed) + .ForContext(NntpdNewsLogging.MessageIdProperty, messageId) + .ForContext(NntpdNewsLogging.SitesProperty, sites) + .ForContext(NntpdNewsLogging.SizeProperty, evt.Size) + .ForContext(NntpdNewsLogging.ResponseCodeProperty, evt.ResponseCode) + .ForContext(NntpdNewsLogging.ReasonProperty, reason) + .Information("news"); + } + + /// + public void Flush() => _logger.Dispose(); + + /// + public void Dispose() => _logger.Dispose(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogPathSurveyWriter.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogPathSurveyWriter.cs new file mode 100644 index 00000000..2550feb4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/SerilogPathSurveyWriter.cs @@ -0,0 +1,61 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// + /// Emits canonical Path observations through the dedicated Serilog Path-survey pipeline. + /// + /// + /// File path, rolling, retention, compression, and buffering come from + /// Serilog:Inpaths. The Path line is produced only by + /// . This writer does not open, rotate, or + /// compress files itself. Serilog host disposal flushes the pipeline. Path + /// bytes are copied at so later ArtData release cannot + /// mutate a buffered event. Observations are not retained in an in-memory + /// collection; the Path-survey file is the persistence mechanism. + /// + public sealed class SerilogPathSurveyWriter : IPathSurveyWriter, IDisposable + { + private readonly Serilog.Core.Logger _logger; + + /// Initializes a writer that owns a dedicated Serilog Path-survey logger. + public SerilogPathSurveyWriter( + IConfiguration configuration, + ICompletedPathSurveyFileHandler? completedFileHandler = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(configuration); + _logger = NntpdPathSurveyLogging.CreateLogger( + configuration, + completedFileHandler ?? NullCompletedPathSurveyFileHandler.Instance, + logger ?? NullLogger.Instance); + } + + /// Initializes a writer that uses an already-constructed Path-survey logger (tests). + public SerilogPathSurveyWriter(Serilog.Core.Logger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// + public void Write(ReadOnlySpan canonicalPath) + { + var path = canonicalPath.IsEmpty + ? string.Empty + : Encoding.Latin1.GetString(canonicalPath); + _logger + .ForContext(NntpdPathSurveyLogging.PathProperty, path) + .Information("inpaths"); + } + + /// + public void Flush() => _logger.Dispose(); + + /// + public void Dispose() => _logger.Dispose(); + } +} diff --git a/src/VectorNNTPD.ReaderService/ArticleIngestion/SpoolLogMessages.cs b/src/VectorNNTPD.ReaderService/ArticleIngestion/SpoolLogMessages.cs new file mode 100644 index 00000000..cb439e72 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/ArticleIngestion/SpoolLogMessages.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.ReaderService.ArticleIngestion +{ + /// Source-generated incoming spool writer log messages. + internal static partial class SpoolLogMessages + { + [LoggerMessage( + EventId = 2000, + Level = LogLevel.Warning, + Message = "Incoming spool writer stop canceled with {Queued} article(s) still buffered")] + public static partial void StopCanceledWithBufferedArticles(ILogger logger, int Queued); + + [LoggerMessage( + EventId = 2001, + Level = LogLevel.Error, + Message = "Incoming spool writer stopped with an error")] + public static partial void WriterStoppedWithError(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2002, + Level = LogLevel.Information, + Message = "Incoming spool writer started (transit queue memory limit {MemoryLimit} bytes, max article {MaxBytes} bytes)")] + public static partial void WriterStarted(ILogger logger, long MemoryLimit, int MaxBytes); + + [LoggerMessage( + EventId = 2003, + Level = LogLevel.Error, + Message = "Incoming article {MessageId} ({Bytes} bytes) failed after OverviewDB handoff and was not requeued")] + public static partial void FinishFailed(ILogger logger, Exception exception, string MessageId, int Bytes); + + [LoggerMessage( + EventId = 2004, + Level = LogLevel.Information, + Message = "Incoming spool writer stopped")] + public static partial void WriterStopped(ILogger logger); + + [LoggerMessage( + EventId = 2005, + Level = LogLevel.Error, + Message = "Failed to write news log for incoming article {MessageId}; article remains accepted and is not reprocessed")] + public static partial void NewsLogFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2006, + Level = LogLevel.Error, + Message = "Failed to flush the news log during incoming spool writer stop")] + public static partial void NewsLogFlushFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2007, + Level = LogLevel.Error, + Message = "Failed to write Path-survey log for incoming article {MessageId}; article remains accepted and is not reprocessed")] + public static partial void PathSurveyFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2008, + Level = LogLevel.Error, + Message = "Failed to flush the Path-survey log during incoming spool writer stop")] + public static partial void PathSurveyFlushFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/AccountCacheKeys.cs b/src/VectorNNTPD.ReaderService/Authentication/AccountCacheKeys.cs new file mode 100644 index 00000000..b61065d3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/AccountCacheKeys.cs @@ -0,0 +1,55 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Redis key material for the short-lived AUTHINFO account-record cache. + /// + /// + /// Key: nntpd:account:{sha256hex(accountName)}. The plaintext AUTHINFO + /// username is hashed with the same UTF-8 SHA-256 convention as other NNTPD + /// Redis account keys so listings do not expose account names. The value is a + /// complete MySQL nntpusers snapshot with a Redis-native TTL. This is + /// not a session, lease, or byte-ledger key. + /// + internal static class AccountCacheKeys + { + /// ASCII namespace for account-record cache keys. + public static ReadOnlySpan NamespacePrefix => "nntpd:account:"u8; + + /// Builds the account-record cache key for . + public static byte[] Create(string accountName) + { + var hex = HashHex(accountName); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + + /// + /// Returns lowercase hex SHA-256 of the UTF-8 wire account name. + /// The caller supplies the same string used for the MySQL lookup. + /// + public static string HashHex(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + return Convert.ToHexStringLower(hash); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/AccountCacheLogMessages.cs b/src/VectorNNTPD.ReaderService/Authentication/AccountCacheLogMessages.cs new file mode 100644 index 00000000..64e2942d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/AccountCacheLogMessages.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.Authentication +{ + /// Source-generated account-cache diagnostics. Never includes secrets or payloads. + internal static partial class AccountCacheLogMessages + { + [LoggerMessage(EventId = 2340, Level = LogLevel.Debug, Message = "Account cache HIT accountHash={AccountHash}")] + public static partial void Hit(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2341, Level = LogLevel.Debug, Message = "Account cache MISS accountHash={AccountHash}")] + public static partial void Miss(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2342, Level = LogLevel.Debug, Message = "Account cache unavailable; falling back to MySQL accountHash={AccountHash}")] + public static partial void Unavailable(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2343, Level = LogLevel.Debug, Message = "Account cache populate skipped accountHash={AccountHash}")] + public static partial void PopulateSkipped(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2344, Level = LogLevel.Debug, Message = "Account cache payload rejected accountHash={AccountHash}")] + public static partial void PayloadRejected(ILogger logger, string accountHash); + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/ArticleTypeAccessPolicy.cs b/src/VectorNNTPD.ReaderService/Authentication/ArticleTypeAccessPolicy.cs new file mode 100644 index 00000000..5be93f0c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/ArticleTypeAccessPolicy.cs @@ -0,0 +1,45 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Session; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Shared account ArtType capability checks. Distinct from PostFilter policy + /// art_type ENUM rows and from flags. + /// + public static class ArticleTypeAccessPolicy + { + /// + /// Returns whether the session may inject . + /// Unauthenticated sessions and accounts without a capability snapshot are + /// unrestricted. Authenticated accounts use the flags loaded at AUTHINFO. + /// + public static bool CanPostArticleType(NntpSession session, ArticleType articleType) + { + ArgumentNullException.ThrowIfNull(session); + return Allows(session.AccountPolicy, articleType); + } + + /// + /// Returns whether the session may read . + /// Uses the same capability mask as posting. Retrieval cannot call this + /// until stored articles expose . + /// + public static bool CanReadArticleType(NntpSession session, ArticleType articleType) + { + ArgumentNullException.ThrowIfNull(session); + return Allows(session.AccountPolicy, articleType); + } + + /// Evaluates a captured account policy, or unrestricted when none applies. + public static bool Allows(NntpAccountPolicy? policy, ArticleType articleType) + { + if (policy is null) + { + return true; + } + + return ArticleTypeCapabilities.Allows(policy.AllowedArtTypes, articleType); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/AuthenticationLogMessages.cs b/src/VectorNNTPD.ReaderService/Authentication/AuthenticationLogMessages.cs new file mode 100644 index 00000000..9849332b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/AuthenticationLogMessages.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.ReaderService.Authentication +{ + /// Source-generated authentication diagnostics. Never includes secrets. + internal static partial class AuthenticationLogMessages + { + [LoggerMessage(EventId = 2302, Level = LogLevel.Debug, Message = "Authentication rejected: invalid credentials mechanism={Mechanism} user={Username} ip={ClientIp}")] + public static partial void InvalidCredentials(ILogger logger, string mechanism, string username, string clientIp); + + [LoggerMessage(EventId = 2303, Level = LogLevel.Information, Message = "Authentication succeeded mechanism={Mechanism} user={Username} ip={ClientIp} byteLimit={ByteLimit} rateLimitBps={RateLimitBps} customer={CustomerId}")] + public static partial void Succeeded( + ILogger logger, + string mechanism, + string username, + string clientIp, + long byteLimit, + int rateLimitBps, + string customerId); + + [LoggerMessage(EventId = 2304, Level = LogLevel.Error, Message = "Authentication backend failure mechanism={Mechanism} user={Username}")] + public static partial void BackendFailed(ILogger logger, Exception exception, string mechanism, string username); + + [LoggerMessage(EventId = 2305, Level = LogLevel.Error, Message = "nntpusers lookup failed user={Username}")] + public static partial void UserLookupFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2306, Level = LogLevel.Information, Message = "Authentication admission rejected user={Username} reason={Reason} ip={ClientIp}")] + public static partial void AdmissionRejected(ILogger logger, string username, string reason, string clientIp); + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/CachedNntpUserRecordStore.cs b/src/VectorNNTPD.ReaderService/Authentication/CachedNntpUserRecordStore.cs new file mode 100644 index 00000000..8be692d9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/CachedNntpUserRecordStore.cs @@ -0,0 +1,227 @@ +using System.Collections.Concurrent; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Short-lived Redis cache around MySQL SELECT. + /// + /// + /// Caches a complete account-record snapshot only. Does not cache password + /// verification or session admission. Redis GET/SET failure falls back to the + /// inner store and does not trip the shared Redis availability circuit. + /// Concurrent cold lookups for the same wire account name share one inner + /// SELECT. Missing accounts are not cached. Idle single-flight state is + /// removed when the shared lookup completes. + /// + internal sealed class CachedNntpUserRecordStore : INntpUserRecordStore + { + internal static readonly TimeSpan CacheTtl = TimeSpan.FromSeconds(10); + + private readonly INntpUserRecordStore _inner; + private readonly IRedisService _redis; + private readonly ILogger _logger; + private readonly ConcurrentDictionary> _inflight = + new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + public CachedNntpUserRecordStore( + INntpUserRecordStore inner, + IRedisService redis, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + _inner = inner; + _redis = redis; + _logger = logger; + } + + /// Gets the number of in-flight account-scoped lookups (tests). + internal int InFlightCount => _inflight.Count; + + /// + public async ValueTask TryGetUserAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + + var cached = await TryReadCacheAsync(accountName, cancellationToken).ConfigureAwait(false); + if (cached.Status == CacheReadStatus.Hit) + { + AccountCacheLogMessages.Hit(_logger, AccountCacheKeys.HashHex(accountName)); + return cached.Record; + } + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var shared = _inflight.GetOrAdd(accountName, tcs.Task); + if (!ReferenceEquals(shared, tcs.Task)) + { + return await shared.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + try + { + var record = await LoadAndPopulateAsync(accountName, cached.Status).ConfigureAwait(false); + tcs.SetResult(record); + return record; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + tcs.SetException(ex); + throw; + } + finally + { + _ = _inflight.TryRemove(new KeyValuePair>(accountName, tcs.Task)); + } + } + + private async Task LoadAndPopulateAsync(string accountName, CacheReadStatus priorStatus) + { + var again = await TryReadCacheAsync(accountName, CancellationToken.None).ConfigureAwait(false); + if (again.Status == CacheReadStatus.Hit) + { + AccountCacheLogMessages.Hit(_logger, AccountCacheKeys.HashHex(accountName)); + return again.Record; + } + + if (priorStatus == CacheReadStatus.Miss || again.Status == CacheReadStatus.Miss) + { + AccountCacheLogMessages.Miss(_logger, AccountCacheKeys.HashHex(accountName)); + } + + var record = await _inner.TryGetUserAsync(accountName, CancellationToken.None).ConfigureAwait(false); + if (record is not null) + { + await TryWriteCacheAsync(accountName, record).ConfigureAwait(false); + } + + return record; + } + + private async ValueTask TryReadCacheAsync(string accountName, CancellationToken cancellationToken) + { + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + byte[]? payload; + try + { + payload = await database.GetAsync(AccountCacheKeys.Create(accountName), cancellationToken) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + // Cache is an optimization. Do not trip the shared Redis circuit. + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + if (payload is null) + { + return CacheRead.Missed; + } + + if (!NntpAccountCacheCodec.TryDecode(payload, accountName, out var record) || record is null) + { + AccountCacheLogMessages.PayloadRejected(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Missed; + } + + return CacheRead.Found(record); + } + + private async ValueTask TryWriteCacheAsync(string accountName, NntpUserRecord record) + { + byte[] payload; + try + { + payload = NntpAccountCacheCodec.Encode(record); + } + catch (ArgumentOutOfRangeException) + { + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + try + { + await database.SetAsync( + AccountCacheKeys.Create(accountName), + payload, + CacheTtl, + CancellationToken.None) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + } + } + + private enum CacheReadStatus + { + Hit, + Miss, + Unavailable, + } + + private readonly record struct CacheRead(CacheReadStatus Status, NntpUserRecord? Record) + { + public static CacheRead Found(NntpUserRecord record) => new(CacheReadStatus.Hit, record); + + public static CacheRead Missed { get; } = new(CacheReadStatus.Miss, null); + + public static CacheRead Unavailable { get; } = new(CacheReadStatus.Unavailable, null); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/CompositeNntpAuthenticationProvider.cs b/src/VectorNNTPD.ReaderService/Authentication/CompositeNntpAuthenticationProvider.cs new file mode 100644 index 00000000..287b9c62 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/CompositeNntpAuthenticationProvider.cs @@ -0,0 +1,71 @@ +using System.Net; +using VectorNNTPD.ReaderService.Session.Authentication; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// AUTHINFO USER/PASS provider: newsmaster first (when configured), then MySQL reader accounts. + /// + /// + /// A newsmaster username match never falls through to MySQL. Transit peer AUTHINFO is + /// selected only when the session AUTHINFO authority is Transit (MODE STREAM) and is + /// handled by , + /// not this type. This provider is Reader-authority only. + /// + public sealed class CompositeNntpAuthenticationProvider : INntpAuthenticationProvider + { + private readonly INntpAuthenticationProvider _newsmaster; + private readonly string? _newsmasterUsername; + private readonly MySqlNntpCredentialValidator _mysql; + + /// Initializes a composite provider. + public CompositeNntpAuthenticationProvider( + INntpAuthenticationProvider newsmaster, + string? newsmasterUsername, + MySqlNntpCredentialValidator mysql) + { + ArgumentNullException.ThrowIfNull(newsmaster); + ArgumentNullException.ThrowIfNull(mysql); + _newsmaster = newsmaster; + _newsmasterUsername = string.IsNullOrWhiteSpace(newsmasterUsername) ? null : newsmasterUsername.Trim(); + _mysql = mysql; + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + AuthenticateAsync(username, password, IPAddress.None, cancellationToken); + + /// + public async ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(clientIp); + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + if (_newsmasterUsername is not null + && string.Equals(username, _newsmasterUsername, StringComparison.Ordinal)) + { + return await _newsmaster.AuthenticateAsync(username, password, cancellationToken).ConfigureAwait(false); + } + + return await _mysql + .ValidatePasswordAsync( + NntpAuthMechanisms.AuthInfoUserPass, + username, + password, + clientIp, + cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/INntpUserRecordStore.cs b/src/VectorNNTPD.ReaderService/Authentication/INntpUserRecordStore.cs new file mode 100644 index 00000000..03634935 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/INntpUserRecordStore.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.ReaderService.Authentication +{ + /// Looks up an NNTP account row by plaintext wire username. + public interface INntpUserRecordStore + { + /// + /// Returns the account snapshot, or when no row exists. + /// Disabled accounts are still returned; enablement is enforced by the validator. + /// + /// Backend failure. Must not be mapped to invalid credentials. + ValueTask TryGetUserAsync(string accountName, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/MySqlNntpCredentialValidator.cs b/src/VectorNNTPD.ReaderService/Authentication/MySqlNntpCredentialValidator.cs new file mode 100644 index 00000000..e4635485 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/MySqlNntpCredentialValidator.cs @@ -0,0 +1,217 @@ +using System.Net; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.SessionState; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Validates AUTHINFO PASS / SASL password mechanisms and finalizes SCRAM/CRAM against MySQL. + /// + /// + /// Account rows come from (Redis 10s cache, then + /// MySQL). This type does not keep a process-local full-record cache. Password + /// and SASL proofs are verified on every attempt. + /// + public sealed class MySqlNntpCredentialValidator + { + /// Reader + posting privileges granted to every successful MySQL account. + public static NntpAuthorization ReaderPrivileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private readonly INntpUserRecordStore _store; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlNntpCredentialValidator(INntpUserRecordStore store, ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _logger = logger; + } + + /// Validates a cleartext password for AUTHINFO PASS, PLAIN, or LOGIN. + public async ValueTask ValidatePasswordAsync( + string mechanism, + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(mechanism); + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(clientIp); + var ip = FormatClientIp(clientIp); + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !record.AllowAuthPlain + || !NntpPasswordComparer.EqualsAscii(record.AccountPassword, password)) + { + AuthenticationLogMessages.InvalidCredentials(_logger, mechanism, username, ip); + return NntpAuthenticationResult.Failed; + } + + return Succeed(mechanism, record, ip); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.BackendFailed(_logger, ex, mechanism, username); + return NntpAuthenticationResult.TransientFailure; + } + } + + /// + /// Loads the nntpusers row for a SCRAM username without applying + /// enabled/SCRAM policy. Callers decide real vs dummy verifier. + /// + public async ValueTask TryLoadUserAsync( + string username, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + try + { + return await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + } + + /// Loads SCRAM material when the account may use SCRAM-SHA-256. + public async ValueTask TryGetScramRecordAsync( + string username, + CancellationToken cancellationToken = default) + { + var record = await TryLoadUserAsync(username, cancellationToken).ConfigureAwait(false); + if (!ScramStoredCredential.TryGetAuthorized(record, out _)) + { + return null; + } + + return record; + } + + /// Loads a CRAM-MD5 HMAC key from the decrypted password. + public async ValueTask TryGetCramRecordAsync( + string username, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !record.AllowAuthPlain + || !NntpPasswordComparer.IsAscii(record.AccountPassword) + || record.AccountPassword.Length == 0) + { + return null; + } + + return record; + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + } + + /// Finalizes SCRAM or CRAM after wire-level proof verification. + public async ValueTask CompleteSaslAccountAsync( + string mechanism, + string username, + IPAddress clientIp, + bool requireScram, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + ArgumentNullException.ThrowIfNull(clientIp); + var ip = FormatClientIp(clientIp); + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !(requireScram ? record.AllowAuthScram256 : record.AllowAuthPlain)) + { + AuthenticationLogMessages.InvalidCredentials(_logger, mechanism, username, ip); + return NntpAuthenticationResult.Failed; + } + + return Succeed(mechanism, record, ip); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.BackendFailed(_logger, ex, mechanism, username); + return NntpAuthenticationResult.TransientFailure; + } + } + + private NntpAuthenticationResult Succeed(string mechanism, NntpUserRecord record, string clientIp) + { + AuthenticationLogMessages.Succeeded( + _logger, + mechanism, + record.AccountName, + clientIp, + record.ByteLimit, + record.RateLimitBps, + record.CustomerId); + return NntpAuthenticationResult.Success( + record.AccountName, + ReaderPrivileges, + NntpAccountPolicy.FromRecord(record)); + } + + internal static string FormatClientIp(IPAddress clientIp) => + SourceAddressIdentity.Format(clientIp); + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/MySqlUserRecordStore.cs b/src/VectorNNTPD.ReaderService/Authentication/MySqlUserRecordStore.cs new file mode 100644 index 00000000..0c4e732f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/MySqlUserRecordStore.cs @@ -0,0 +1,50 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Loads nntpusers through the existing NntpDB connection pool. + /// + public sealed class MySqlUserRecordStore : INntpUserRecordStore + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlUserRecordStore(NntpDbService nntpDb, ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask TryGetUserAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryUserAccountAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.UserLookupFailed(_logger, ex, accountName); + throw new NntpDbUnavailableException("MySQL nntpusers lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpAccountCacheCodec.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpAccountCacheCodec.cs new file mode 100644 index 00000000..634e3a8f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpAccountCacheCodec.cs @@ -0,0 +1,237 @@ +using System.Buffers.Binary; +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Explicit binary snapshot of one MySQL nntpusers row for Redis. + /// + /// + /// Versioned little-endian layout. Preserves nullability as empty strings or + /// empty byte spans, booleans as 0/1, and integer widths without truncation. + /// Does not include account_type. The payload contains credential + /// material and must not be logged. + /// + internal static class NntpAccountCacheCodec + { + internal const byte Version = 2; + + internal const byte Version1 = 1; + + private const int MaxAccountNameBytes = 512; + private const int MaxPasswordBytes = 4096; + private const int MaxScramBytes = 1024; + private const int MaxCustomerIdBytes = 128; + + /// Serializes to the version-1 cache payload. + public static byte[] Encode(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + var accountName = Encoding.UTF8.GetBytes(record.AccountName); + var password = Encoding.UTF8.GetBytes(record.AccountPassword); + var customerId = Encoding.UTF8.GetBytes(record.CustomerId); + var salt = record.ScramSalt; + var storedKey = record.ScramStoredKey; + var serverKey = record.ScramServerKey; + if (accountName.Length > MaxAccountNameBytes + || password.Length > MaxPasswordBytes + || salt.Length > MaxScramBytes + || storedKey.Length > MaxScramBytes + || serverKey.Length > MaxScramBytes + || customerId.Length > MaxCustomerIdBytes) + { + throw new ArgumentOutOfRangeException(nameof(record), "Account cache field exceeds encoding limits."); + } + + var length = 1 + + 4 + accountName.Length + + 4 + password.Length + + 1 + + 1 + + 4 + salt.Length + + 4 + + 4 + storedKey.Length + + 4 + serverKey.Length + + 4 + + 8 + + 4 + + 4 + + 1 + + 4 + customerId.Length + + 4; + var buffer = new byte[length]; + var span = buffer.AsSpan(); + span[0] = Version; + var offset = 1; + WriteBytes(span, ref offset, accountName); + WriteBytes(span, ref offset, password); + span[offset++] = record.AllowAuthPlain ? (byte)1 : (byte)0; + span[offset++] = record.AllowAuthScram256 ? (byte)1 : (byte)0; + WriteBytes(span, ref offset, salt.Span); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.ScramIterations); + offset += 4; + WriteBytes(span, ref offset, storedKey.Span); + WriteBytes(span, ref offset, serverKey.Span); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.RateLimitBps); + offset += 4; + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(offset), record.ByteLimit); + offset += 8; + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.SessionLimit); + offset += 4; + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.SrcIpLimit); + offset += 4; + span[offset++] = record.IsEnabled ? (byte)1 : (byte)0; + WriteBytes(span, ref offset, customerId); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), (int)record.AllowedArtTypes); + return buffer; + } + + /// + /// Decodes when it is a well-formed snapshot for + /// . Returns for any + /// corrupt, truncated, or account-mismatched payload. + /// + public static bool TryDecode(ReadOnlySpan payload, string accountName, out NntpUserRecord? record) + { + record = null; + if (payload.Length < 2 || (payload[0] != Version && payload[0] != Version1)) + { + return false; + } + + var version = payload[0]; + var offset = 1; + if (!TryReadBytes(payload, ref offset, MaxAccountNameBytes, out var cachedName) + || !TryReadBytes(payload, ref offset, MaxPasswordBytes, out var password) + || !TryReadByte(payload, ref offset, out var allowPlain) + || allowPlain > 1 + || !TryReadByte(payload, ref offset, out var allowScram) + || allowScram > 1 + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var salt) + || !TryReadInt32(payload, ref offset, out var iterations) + || iterations < 0 + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var storedKey) + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var serverKey) + || !TryReadInt32(payload, ref offset, out var rateLimitBps) + || !TryReadInt64(payload, ref offset, out var byteLimit) + || !TryReadInt32(payload, ref offset, out var sessionLimit) + || !TryReadInt32(payload, ref offset, out var srcIpLimit) + || !TryReadByte(payload, ref offset, out var enabled) + || enabled > 1 + || !TryReadBytes(payload, ref offset, MaxCustomerIdBytes, out var customerId)) + { + return false; + } + + var allowedArtTypes = ArticleTypeCapabilities.All; + if (version == Version) + { + if (!TryReadInt32(payload, ref offset, out var artTypeBits) || offset != payload.Length) + { + return false; + } + + allowedArtTypes = (ArticleType)artTypeBits; + } + else if (offset != payload.Length) + { + return false; + } + + var decodedName = Encoding.UTF8.GetString(cachedName); + if (!string.Equals(decodedName, accountName, StringComparison.Ordinal)) + { + return false; + } + + record = new NntpUserRecord( + decodedName, + Encoding.UTF8.GetString(password), + allowPlain == 1, + allowScram == 1, + salt.ToArray(), + iterations, + storedKey.ToArray(), + serverKey.ToArray(), + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + enabled == 1, + Encoding.UTF8.GetString(customerId), + allowedArtTypes); + return true; + } + + private static void WriteBytes(Span destination, ref int offset, ReadOnlySpan value) + { + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset), value.Length); + offset += 4; + value.CopyTo(destination.Slice(offset)); + offset += value.Length; + } + + private static bool TryReadBytes( + ReadOnlySpan source, + ref int offset, + int maxLength, + out ReadOnlySpan value) + { + value = default; + if (offset > source.Length - 4) + { + return false; + } + + var length = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(offset)); + offset += 4; + if (length < 0 || length > maxLength || offset > source.Length - length) + { + return false; + } + + value = source.Slice(offset, length); + offset += length; + return true; + } + + private static bool TryReadByte(ReadOnlySpan source, ref int offset, out byte value) + { + if (offset >= source.Length) + { + value = 0; + return false; + } + + value = source[offset++]; + return true; + } + + private static bool TryReadInt32(ReadOnlySpan source, ref int offset, out int value) + { + if (offset > source.Length - 4) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(offset)); + offset += 4; + return true; + } + + private static bool TryReadInt64(ReadOnlySpan source, ref int offset, out long value) + { + if (offset > source.Length - 8) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt64LittleEndian(source.Slice(offset)); + offset += 8; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpAccountPolicy.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpAccountPolicy.cs new file mode 100644 index 00000000..aed878b8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpAccountPolicy.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// Authenticated account policy copied from nntpusers after a successful credential check. + /// + /// + /// Every authenticated account has both remaining-byte and rate policies. + /// Session and source-IP limits are enforced by + /// . + /// is enforced by + /// + /// after SessionState admission. is the AUTHINFO-time + /// remaining-byte snapshot and is not enforced here; live remaining is observed by + /// . + /// + public sealed class NntpAccountPolicy + { + /// Initializes a new account policy snapshot. + public NntpAccountPolicy( + string username, + int rateLimitBps, + long byteLimit, + int sessionLimit, + int srcIpLimit, + string customerId, + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + Username = username; + RateLimitBps = rateLimitBps; + ByteLimit = byteLimit; + SessionLimit = sessionLimit; + SrcIpLimit = srcIpLimit; + CustomerId = customerId ?? string.Empty; + AllowedArtTypes = allowedArtTypes; + } + + /// Gets the plaintext wire username. + public string Username { get; } + + /// + /// Gets account_rate_limit in bits per second. + /// Examples: 240 = 240 bps, 2_400 = 2,400 bps = 300 B/s, + /// 1_000_000 = 1 Mbps, 10_000_000 = 10 Mbps. + /// 0 is unlimited. Negative values are treated as unlimited. + /// + public int RateLimitBps { get; } + + /// + /// Gets the AUTHINFO-time snapshot of account_byte_limit. + /// This is remaining bytes at last user-record load: 0 is exhausted, + /// not unlimited. Live remaining is observed by + /// and must not + /// be taken from this cached snapshot. + /// + public long ByteLimit { get; } + + /// Gets the cluster-wide authenticated session cap. 0 is unlimited. + public int SessionLimit { get; } + + /// Gets the cluster-wide distinct source-IP cap. 0 is unlimited. + public int SrcIpLimit { get; } + + /// Gets the customer/tenant identifier. + public string CustomerId { get; } + + /// + /// Gets the account ArtType capability flags loaded at AUTHINFO. + /// is unrestricted and is the + /// default for existing users. + /// + public ArticleType AllowedArtTypes { get; } + + /// + /// Gets whether cluster admission must run: session limit, source-IP limit, + /// or a positive rate that needs a cluster session count. + /// + public bool RequiresAdmission => SessionLimit > 0 || SrcIpLimit > 0 || RequiresRateTracking; + + /// + /// Gets whether this account has a positive account_rate_limit. + /// 0 remains unlimited and does not participate in rate allocation. + /// + public bool RequiresRateTracking => RateLimitBps > 0; + + /// Builds policy from a validated user record. + public static NntpAccountPolicy FromRecord(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return new( + record.AccountName, + record.RateLimitBps, + record.ByteLimit, + record.SessionLimit, + record.SrcIpLimit, + record.CustomerId, + record.AllowedArtTypes); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpAuthMechanisms.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpAuthMechanisms.cs new file mode 100644 index 00000000..4974b92c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpAuthMechanisms.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.ReaderService.Authentication +{ + /// Canonical authentication mechanism labels used in logs and SASL dispatch. + internal static class NntpAuthMechanisms + { + public const string AuthInfoUserPass = "AUTHINFO USER/PASS"; + public const string SaslPlain = "SASL PLAIN"; + public const string SaslLogin = "SASL LOGIN"; + public const string SaslCramMd5 = "SASL CRAM-MD5"; + public const string SaslScramSha256 = "SASL SCRAM-SHA-256"; + + public const string WirePlain = "PLAIN"; + public const string WireLogin = "LOGIN"; + public const string WireCramMd5 = "CRAM-MD5"; + public const string WireScramSha256 = "SCRAM-SHA-256"; + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpPasswordComparer.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpPasswordComparer.cs new file mode 100644 index 00000000..5b205497 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpPasswordComparer.cs @@ -0,0 +1,76 @@ +using System.Buffers; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// Constant-time ASCII password comparison used by AUTHINFO PASS, PLAIN, and LOGIN. + internal static class NntpPasswordComparer + { + private const int StackallocThresholdBytes = 4096; + + /// + /// Returns only when both strings are non-null ASCII of equal length + /// and identical content. + /// + public static bool EqualsAscii(string? storedPassword, string? suppliedPassword) + { + if (storedPassword is null || suppliedPassword is null) + { + return false; + } + + if (!IsAscii(storedPassword) || !IsAscii(suppliedPassword)) + { + return false; + } + + var storedLength = storedPassword.Length; + var suppliedLength = suppliedPassword.Length; + var maxLength = Math.Max(storedLength, suppliedLength); + if (maxLength <= StackallocThresholdBytes) + { + Span left = stackalloc byte[maxLength]; + Span right = stackalloc byte[maxLength]; + left.Clear(); + right.Clear(); + Encoding.ASCII.GetBytes(storedPassword, left); + Encoding.ASCII.GetBytes(suppliedPassword, right); + return CryptographicOperations.FixedTimeEquals(left, right) && storedLength == suppliedLength; + } + + var leftArray = ArrayPool.Shared.Rent(maxLength); + var rightArray = ArrayPool.Shared.Rent(maxLength); + try + { + var left = leftArray.AsSpan(0, maxLength); + var right = rightArray.AsSpan(0, maxLength); + left.Clear(); + right.Clear(); + Encoding.ASCII.GetBytes(storedPassword, left); + Encoding.ASCII.GetBytes(suppliedPassword, right); + return CryptographicOperations.FixedTimeEquals(left, right) && storedLength == suppliedLength; + } + finally + { + ArrayPool.Shared.Return(leftArray, clearArray: true); + ArrayPool.Shared.Return(rightArray, clearArray: true); + } + } + + /// Returns whether contains only ASCII code points. + public static bool IsAscii(string value) + { + ArgumentNullException.ThrowIfNull(value); + foreach (var c in value) + { + if (c > 0x7F) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpSaslExchange.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpSaslExchange.cs new file mode 100644 index 00000000..9626a2fd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpSaslExchange.cs @@ -0,0 +1,54 @@ +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Session.Authentication; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// In-progress AUTHINFO SASL exchange isolated to one NNTP session. + internal sealed class NntpSaslExchange + { + public NntpSaslExchange(string mechanism) + { + ArgumentException.ThrowIfNullOrWhiteSpace(mechanism); + Mechanism = mechanism; + } + + public string Mechanism { get; } + public string? LoginUsername { get; set; } + /// Gets or sets the native RFC 2195 CRAM-MD5 challenge (not Base64). + public string? CramChallenge { get; set; } + public ScramMechanism? Scram { get; set; } + public string? ScramUsername { get; set; } + + /// + /// Gets or sets whether this exchange used the server-local dummy verifier. + /// Dummy success must never authenticate. + /// + public bool ScramUsedDummy { get; set; } + + public bool WaitingPlainCredentials { get; set; } + } + + /// SASL start/continuation outcome for the AUTHINFO handler. + internal sealed class NntpSaslReply + { + public NntpSaslReply( + ReadOnlyMemory wire, + string statusLine, + bool completed, + NntpSaslExchange? exchange = null, + NntpAuthenticationResult? authentication = null) + { + Wire = wire; + StatusLine = statusLine; + Completed = completed; + Exchange = exchange; + Authentication = authentication; + } + + public ReadOnlyMemory Wire { get; } + public string StatusLine { get; } + public bool Completed { get; } + public NntpSaslExchange? Exchange { get; } + public NntpAuthenticationResult? Authentication { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpSaslService.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpSaslService.cs new file mode 100644 index 00000000..6609391b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpSaslService.cs @@ -0,0 +1,427 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// AUTHINFO SASL exchange: PLAIN, LOGIN, CRAM-MD5, and SCRAM-SHA-256. + public sealed class NntpSaslService + { + private readonly INntpAuthenticationProvider _passwordProvider; + private readonly MySqlNntpCredentialValidator? _mysql; + private readonly string _fqdn; + private readonly ScramDummyVerifier _dummyScram; + + /// Initializes a SASL service. SCRAM/CRAM require . + public NntpSaslService( + INntpAuthenticationProvider passwordProvider, + MySqlNntpCredentialValidator? mysql = null, + IOptions? nntpd = null) + : this(passwordProvider, mysql, nntpd, dummyScram: null) + { + } + + /// Initializes a SASL service with an injected dummy SCRAM verifier (tests). + internal NntpSaslService( + INntpAuthenticationProvider passwordProvider, + MySqlNntpCredentialValidator? mysql, + IOptions? nntpd, + ScramDummyVerifier? dummyScram) + { + ArgumentNullException.ThrowIfNull(passwordProvider); + _passwordProvider = passwordProvider; + _mysql = mysql; + _fqdn = ResolveFqdn(nntpd?.Value); + _dummyScram = dummyScram ?? ScramDummyVerifier.Create(); + } + + /// Starts AUTHINFO SASL. is the optional raw token. + internal async ValueTask StartAsync( + string mechanism, + string initialResponse, + IPAddress clientIp, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(mechanism); + ArgumentNullException.ThrowIfNull(initialResponse); + ArgumentNullException.ThrowIfNull(clientIp); + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WirePlain)) + { + if (IsEmptyInitialResponse(initialResponse)) + { + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WirePlain) { WaitingPlainCredentials = true }; + return ContinueReply(NntpResponses.SaslEmptyChallenge, NntpResponseStatus.SaslEmptyChallenge, completed: false, exchange); + } + + if (!TryDecodeBase64(initialResponse, out var decoded)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + return await CompletePlainAsync(decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireLogin)) + { + if (!IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslProtocolError, NntpResponseStatus.SaslProtocolError); + } + + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireLogin); + return ContinueReply( + NntpResponses.SaslLoginUsernameChallenge, + NntpResponseStatus.SaslLoginUsernameChallenge, + completed: false, + exchange); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireCramMd5)) + { + if (_mysql is null) + { + return FailReply(NntpResponses.SaslCramUnavailable, NntpResponseStatus.SaslCramUnavailable); + } + + if (!IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslProtocolError, NntpResponseStatus.SaslProtocolError); + } + + var nativeChallenge = CramMd5Mechanism.CreateChallenge(_fqdn); + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireCramMd5) { CramChallenge = nativeChallenge }; + var wire = EncodeLine(383, Convert.ToBase64String(Encoding.ASCII.GetBytes(nativeChallenge))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslChallenge, completed: false, exchange); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireScramSha256)) + { + if (_mysql is null) + { + return FailReply(NntpResponses.SaslScramUnavailable, NntpResponseStatus.SaslScramUnavailable); + } + + if (IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslScramUnavailable, NntpResponseStatus.SaslScramUnavailable); + } + + if (!TryDecodeBase64(initialResponse, out var clientFirst)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + return await BeginScramAsync(clientFirst, clientIp, cancellationToken).ConfigureAwait(false); + } + + return FailReply(NntpResponses.SaslMechanismNotSupported, NntpResponseStatus.SaslMechanismNotSupported); + } + + /// Continues an in-progress SASL exchange. Payload * cancels. + internal async ValueTask ContinueAsync( + NntpSaslExchange exchange, + string line, + IPAddress clientIp, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(exchange); + ArgumentNullException.ThrowIfNull(line); + ArgumentNullException.ThrowIfNull(clientIp); + var payload = line.Trim(); + if (payload == "*") + { + return FailReply(NntpResponses.AuthenticationCancelled, NntpResponseStatus.AuthenticationCancelled); + } + + if (!TryDecodeBase64(payload, out var decoded)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + if (exchange.WaitingPlainCredentials) + { + return await CompletePlainAsync(decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireLogin)) + { + return await ContinueLoginAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireCramMd5)) + { + return await ContinueCramAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireScramSha256)) + { + return await FinishScramAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + return FailReply(NntpResponses.SaslContinuationNotSupported, NntpResponseStatus.SaslContinuationNotSupported); + } + + private async ValueTask ContinueLoginAsync( + NntpSaslExchange exchange, + string decoded, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (exchange.LoginUsername is null) + { + exchange.LoginUsername = decoded; + return ContinueReply( + NntpResponses.SaslLoginPasswordChallenge, + NntpResponseStatus.SaslLoginPasswordChallenge, + completed: false, + exchange); + } + + var result = await _passwordProvider + .AuthenticateAsync(exchange.LoginUsername, decoded, clientIp, cancellationToken) + .ConfigureAwait(false); + return FromPasswordResult(result); + } + + private async ValueTask CompletePlainAsync( + string decoded, + IPAddress clientIp, + CancellationToken cancellationToken) + { + var parts = decoded.Split('\0'); + if (parts.Length < 3 || string.IsNullOrEmpty(parts[1])) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _passwordProvider + .AuthenticateAsync(parts[1], parts[2], clientIp, cancellationToken) + .ConfigureAwait(false); + return FromPasswordResult(result); + } + + private async ValueTask ContinueCramAsync( + NntpSaslExchange exchange, + string response, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (_mysql is null || exchange.CramChallenge is null) + { + return FailReply(NntpResponses.SaslCramUnavailable, NntpResponseStatus.SaslCramUnavailable); + } + + var space = response.IndexOf(' '); + if (space <= 0) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var username = response[..space]; + NntpUserRecord? record; + try + { + record = await _mysql.TryGetCramRecordAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (NntpDbUnavailableException) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + if (record is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var secret = CramMd5Mechanism.SecretFromPassword(record.AccountPassword); + if (!CramMd5Mechanism.Verify(username, response, exchange.CramChallenge, secret)) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _mysql + .CompleteSaslAccountAsync(NntpAuthMechanisms.SaslCramMd5, username, clientIp, requireScram: false, cancellationToken) + .ConfigureAwait(false); + return FromSaslResult(result, NntpResponses.AuthenticationAccepted, NntpResponseStatus.AuthenticationAccepted); + } + + private async ValueTask BeginScramAsync( + string clientFirst, + IPAddress clientIp, + CancellationToken cancellationToken) + { + _ = clientIp; + if (!ScramMechanism.TryGetUsername(clientFirst, out var username) || _mysql is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + NntpUserRecord? record; + try + { + record = await _mysql.TryLoadUserAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (NntpDbUnavailableException) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + var useReal = ScramStoredCredential.TryGetAuthorized(record, out var credential); + if (!useReal) + { + credential = _dummyScram.Credential; + } + + try + { + var (state, serverFirst) = ScramMechanism.Begin(clientFirst, credential); + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireScramSha256) + { + Scram = state, + ScramUsername = username, + ScramUsedDummy = !useReal, + }; + var wire = EncodeLine(383, Convert.ToBase64String(Encoding.UTF8.GetBytes(serverFirst))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslChallenge, completed: false, exchange); + } + catch (ArgumentException) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + } + + private async ValueTask FinishScramAsync( + NntpSaslExchange exchange, + string clientFinal, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (exchange.Scram is null || exchange.ScramUsername is null || _mysql is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var serverFinal = exchange.Scram.TryFinish(clientFinal); + if (serverFinal is null || exchange.ScramUsedDummy) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _mysql + .CompleteSaslAccountAsync( + NntpAuthMechanisms.SaslScramSha256, + exchange.ScramUsername, + clientIp, + requireScram: true, + cancellationToken) + .ConfigureAwait(false); + if (!result.Succeeded) + { + return FromPasswordResult(result); + } + + var wire = EncodeLine(283, Convert.ToBase64String(Encoding.UTF8.GetBytes(serverFinal))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslSuccessWithData, completed: true, authentication: result); + } + + private static NntpSaslReply FromPasswordResult(NntpAuthenticationResult result) + { + if (result.Succeeded) + { + return new NntpSaslReply( + NntpResponses.AuthenticationAccepted, + NntpResponseStatus.AuthenticationAccepted, + completed: true, + authentication: result); + } + + if (result.Failure == NntpAuthenticationFailureKind.TransientFailure) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + private static NntpSaslReply FromSaslResult( + NntpAuthenticationResult result, + ReadOnlyMemory successWire, + string successStatus) + { + if (result.Succeeded) + { + return new NntpSaslReply(successWire, successStatus, completed: true, authentication: result); + } + + return FromPasswordResult(result); + } + + private static NntpSaslReply ContinueReply( + ReadOnlyMemory wire, + string status, + bool completed, + NntpSaslExchange exchange) => + new(wire, status, completed, exchange); + + private static NntpSaslReply FailReply(ReadOnlyMemory wire, string status) => + new(wire, status, completed: true); + + internal static bool EqualsMechanism(string actual, string expected) => + actual.Equals(expected, StringComparison.OrdinalIgnoreCase); + + internal static bool IsEmptyInitialResponse(string token) + { + ArgumentNullException.ThrowIfNull(token); + var text = token.Trim(); + return text.Length == 0 || text == "="; + } + + /// + /// Decodes a SASL token. RFC 4643 = is an empty payload. Invalid BASE64 is rejected + /// (504) rather than treated as plaintext. + /// + internal static bool TryDecodeBase64(string token, out string decoded) + { + ArgumentNullException.ThrowIfNull(token); + var text = token.Trim(); + if (text.Length == 0 || text == "=") + { + decoded = string.Empty; + return true; + } + + try + { + decoded = Encoding.UTF8.GetString(Convert.FromBase64String(text)); + return true; + } + catch (FormatException) + { + decoded = string.Empty; + return false; + } + } + + private static string ResolveFqdn(NntpdOptions? options) + { + if (options is not null && !string.IsNullOrWhiteSpace(options.Fqdn)) + { + return options.Fqdn; + } + + return NntpdOptions.FormatFqdn(1, "usenet.ninja"); + } + + private static ReadOnlyMemory EncodeLine(int code, string text) + { + var line = string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{code} {text}\r\n"); + return Encoding.ASCII.GetBytes(line); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/NntpUserRecord.cs b/src/VectorNNTPD.ReaderService/Authentication/NntpUserRecord.cs new file mode 100644 index 00000000..adf3dd0c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/NntpUserRecord.cs @@ -0,0 +1,117 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Authentication +{ + /// + /// In-process snapshot of one nntpusers row after SQL decrypt. + /// + /// + /// , , , + /// and are sensitive. They must not be logged or copied onto the + /// session identity snapshot. account_type is not selected for NNTP policy. + /// Every account has both remaining-byte and rate fields. + /// + public sealed class NntpUserRecord + { + /// Creates a validated user-record snapshot. + public NntpUserRecord( + string accountName, + string accountPassword, + bool allowAuthPlain, + bool allowAuthScram256, + ReadOnlyMemory scramSalt, + int scramIterations, + ReadOnlyMemory scramStoredKey, + ReadOnlyMemory scramServerKey, + int rateLimitBps, + long byteLimit, + int sessionLimit, + int srcIpLimit, + bool isEnabled, + string customerId, + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) + { + ArgumentException.ThrowIfNullOrEmpty(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(scramIterations); + AccountName = accountName; + AccountPassword = accountPassword ?? string.Empty; + AllowAuthPlain = allowAuthPlain; + AllowAuthScram256 = allowAuthScram256; + ScramSalt = scramSalt; + ScramIterations = scramIterations; + ScramStoredKey = scramStoredKey; + ScramServerKey = scramServerKey; + RateLimitBps = rateLimitBps; + ByteLimit = byteLimit; + SessionLimit = sessionLimit; + SrcIpLimit = srcIpLimit; + IsEnabled = isEnabled; + CustomerId = customerId ?? string.Empty; + AllowedArtTypes = allowedArtTypes; + } + + /// Gets the plaintext wire username (not the MD5 column value). + public string AccountName { get; } + + /// Gets the decrypted password. Sensitive. Empty when SQL decrypt is NULL. + public string AccountPassword { get; } + + /// Gets whether AUTHINFO PASS, PLAIN, LOGIN, and CRAM-MD5 are permitted. + public bool AllowAuthPlain { get; } + + /// Gets whether SCRAM-SHA-256 is permitted. + public bool AllowAuthScram256 { get; } + + /// Gets SCRAM salt bytes, or empty when not provisioned. + public ReadOnlyMemory ScramSalt { get; } + + /// Gets SCRAM iteration count. 0 means SCRAM is not provisioned. + public int ScramIterations { get; } + + /// Gets SCRAM StoredKey. Sensitive. + public ReadOnlyMemory ScramStoredKey { get; } + + /// Gets SCRAM ServerKey. Sensitive. + public ReadOnlyMemory ScramServerKey { get; } + + /// + /// Gets account_rate_limit in bits per second. + /// Examples: 240 = 240 bps, 1_000_000 = 1 Mbps, 10_000_000 = 10 Mbps. + /// 0 is unlimited. NULL at map time becomes 0. This is not megabits. + /// + public int RateLimitBps { get; } + + /// + /// Gets account_byte_limit remaining bytes. + /// 0 is exhausted. NULL at map time becomes 0. This snapshot is not + /// live cluster remaining. + /// + public long ByteLimit { get; } + + /// Gets account_session_limit. 0 is unlimited. + public int SessionLimit { get; } + + /// Gets account_srcip_limit. 0 is unlimited. + public int SrcIpLimit { get; } + + /// Gets whether is_enabled is Y. + public bool IsEnabled { get; } + + /// Gets customer_id. + public string CustomerId { get; } + + /// + /// Gets account_art_type flags loaded at AUTHINFO. + /// is unrestricted. + /// + public ArticleType AllowedArtTypes { get; } + + /// Gets whether stored SCRAM material is complete enough to begin an exchange. + public bool HasScramMaterial => + AllowAuthScram256 + && !ScramSalt.IsEmpty + && ScramIterations > 0 + && !ScramStoredKey.IsEmpty + && !ScramServerKey.IsEmpty; + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/Sasl/CramMd5Mechanism.cs b/src/VectorNNTPD.ReaderService/Authentication/Sasl/CramMd5Mechanism.cs new file mode 100644 index 00000000..a3b3e75b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/Sasl/CramMd5Mechanism.cs @@ -0,0 +1,96 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace VectorNNTPD.ReaderService.Authentication.Sasl +{ + /// + /// RFC 2195 CRAM-MD5 native challenge and verification. + /// + /// + /// The mechanism challenge is an RFC 822 msg-id + /// <random-digits.timestamp@fqdn>. HMAC-MD5 is computed over that + /// native string. NNTP AUTHINFO SASL Base64-encodes the same octets in a + /// 383 reply (RFC 4643); that encoding is not part of CRAM-MD5. + /// + internal static partial class CramMd5Mechanism + { + [GeneratedRegex(@"^<[0-9]+\.[0-9]+@[^@<>\s]+>$", RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture)] + private static partial Regex ChallengeForm(); + + /// + /// Creates a native RFC 2195 challenge using random digits, a UTC Unix + /// timestamp, and the application's Nntpd FQDN. + /// + public static string CreateChallenge(string fqdn, TimeProvider? timeProvider = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + var host = NormalizeFqdn(fqdn); + var unique = BinaryPrimitives.ReadUInt64LittleEndian(RandomNumberGenerator.GetBytes(8)); + var timestamp = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeSeconds(); + return string.Create( + CultureInfo.InvariantCulture, + $"<{unique}.{timestamp}@{host}>"); + } + + /// Returns when matches RFC 2195 msg-id form. + public static bool IsRfc2195Challenge(ReadOnlySpan challenge) => + ChallengeForm().IsMatch(challenge); + + /// Verifies username hexhmac against HMAC-MD5 of the native challenge octets. + public static bool Verify(string username, string response, string challenge, ReadOnlySpan secret) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(response); + ArgumentException.ThrowIfNullOrEmpty(challenge); + var space = response.IndexOf(' '); + if (space <= 0) + { + return false; + } + + var respUser = response[..space]; + if (!string.Equals(respUser, username, StringComparison.Ordinal)) + { + return false; + } + + var expectedHex = ComputeHexDigest(secret, challenge); + return string.Equals(response[(space + 1)..], expectedHex, StringComparison.OrdinalIgnoreCase); + } + + /// Computes the lowercase hex HMAC-MD5 of the native CRAM challenge. + public static string ComputeHexDigest(ReadOnlySpan secret, string nativeChallenge) + { + ArgumentException.ThrowIfNullOrEmpty(nativeChallenge); +#pragma warning disable CA5351 + using var hmac = new HMACMD5(secret.ToArray()); +#pragma warning restore CA5351 + return Convert.ToHexString(hmac.ComputeHash(Encoding.ASCII.GetBytes(nativeChallenge))).ToLowerInvariant(); + } + + /// Encodes an ASCII password as HMAC key bytes. + public static byte[] SecretFromPassword(string password) + { + ArgumentNullException.ThrowIfNull(password); + return Encoding.ASCII.GetBytes(password); + } + + private static string NormalizeFqdn(string fqdn) + { + var host = fqdn.Trim().TrimEnd('.'); + if (host.Length == 0 + || host.Contains('@', StringComparison.Ordinal) + || host.Contains('<', StringComparison.Ordinal) + || host.Contains('>', StringComparison.Ordinal) + || host.Contains(' ', StringComparison.Ordinal)) + { + throw new ArgumentException("CRAM-MD5 host identity must be the Nntpd FQDN.", nameof(fqdn)); + } + + return host; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramDummyVerifier.cs b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramDummyVerifier.cs new file mode 100644 index 00000000..51b80b94 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramDummyVerifier.cs @@ -0,0 +1,56 @@ +using System.Security.Cryptography; + +namespace VectorNNTPD.ReaderService.Authentication.Sasl +{ + /// + /// Server-local dummy SCRAM-SHA-256 verifier used so the 383 server-first + /// exchange does not reveal whether a username exists or has SCRAM material. + /// + /// + /// Material is generated once and is not derived from any client-controlled + /// input. A client that somehow obtained these keys still cannot authenticate: + /// the SASL exchange records that the dummy verifier was used and forces 481. + /// Never log this material. + /// + internal sealed class ScramDummyVerifier + { + /// RFC 7677 minimum iteration count. Same shape as a real server-first. + public const int IterationCount = 4096; + + /// Salt length used for dummy server-first s=. + public const int SaltLength = 16; + + /// SHA-256 key length for dummy StoredKey and ServerKey. + public const int KeyLength = 32; + + /// Creates a dummy verifier with cryptographically random keys. + public static ScramDummyVerifier Create() => + new(new ScramStoredCredential( + RandomNumberGenerator.GetBytes(SaltLength), + IterationCount, + RandomNumberGenerator.GetBytes(KeyLength), + RandomNumberGenerator.GetBytes(KeyLength))); + + /// Creates a dummy verifier from injected material (tests). + public static ScramDummyVerifier Create(ScramStoredCredential credential) + { + if (credential.Salt.Length != SaltLength + || credential.IterationCount != IterationCount + || credential.StoredKey.Length != KeyLength + || credential.ServerKey.Length != KeyLength) + { + throw new ArgumentException("Dummy SCRAM material must match the production dummy shape.", nameof(credential)); + } + + return new ScramDummyVerifier(credential); + } + + private ScramDummyVerifier(ScramStoredCredential credential) + { + Credential = credential; + } + + /// Gets the immutable dummy stored credential. Do not log. + public ScramStoredCredential Credential { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramMechanism.cs b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramMechanism.cs new file mode 100644 index 00000000..57f82d79 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramMechanism.cs @@ -0,0 +1,157 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.Authentication.Sasl +{ + /// RFC 5802 SCRAM-SHA-256 server-side exchange using stored keys (no PBKDF2 at auth time). + internal sealed class ScramMechanism + { + private readonly ScramStoredCredential _credential; + private readonly string _gs2Header; + private readonly string _clientFirstBare; + private readonly string _serverFirst; + private readonly string _combinedNonce; + + private ScramMechanism( + ScramStoredCredential credential, + string gs2Header, + string clientFirstBare, + string serverFirst, + string combinedNonce) + { + _credential = credential; + _gs2Header = gs2Header; + _clientFirstBare = clientFirstBare; + _serverFirst = serverFirst; + _combinedNonce = combinedNonce; + } + + /// Begins SCRAM with a client-first message and returns server-first. + public static (ScramMechanism State, string ServerFirst) Begin(string clientFirst, ScramStoredCredential credential) + { + if (!TrySplitClientFirst(clientFirst, out var gs2Header, out var clientFirstBare) + || !TryParseAttribute(clientFirstBare, 'r', out var clientNonce) + || string.IsNullOrEmpty(clientNonce)) + { + throw new ArgumentException("Invalid SCRAM client-first message.", nameof(clientFirst)); + } + + var serverNonce = Convert.ToBase64String(RandomNumberGenerator.GetBytes(18)); + var combinedNonce = clientNonce + serverNonce; + var saltB64 = Convert.ToBase64String(credential.Salt.Span); + var serverFirst = $"r={combinedNonce},s={saltB64},i={credential.IterationCount}"; + return (new ScramMechanism(credential, gs2Header, clientFirstBare, serverFirst, combinedNonce), serverFirst); + } + + /// Verifies client-final and returns v= server-final, or on failure. + public string? TryFinish(string clientFinal) + { + if (!TryParseAttribute(clientFinal, 'p', out var proofB64) + || !TryParseAttribute(clientFinal, 'r', out var combinedNonce) + || !TryParseAttribute(clientFinal, 'c', out var channelBinding)) + { + return null; + } + + if (!string.Equals(combinedNonce, _combinedNonce, StringComparison.Ordinal)) + { + return null; + } + + if (!string.Equals(channelBinding, Convert.ToBase64String(Encoding.ASCII.GetBytes(_gs2Header)), StringComparison.Ordinal)) + { + return null; + } + + byte[] clientProof; + try + { + clientProof = Convert.FromBase64String(proofB64); + } + catch (FormatException) + { + return null; + } + + var clientFinalWithoutProof = clientFinal.Replace($",p={proofB64}", string.Empty, StringComparison.Ordinal); + var authMessage = $"{_clientFirstBare},{_serverFirst},{clientFinalWithoutProof}"; + var storedKey = _credential.StoredKey.ToArray(); + var clientSignature = HmacSha256(storedKey, Encoding.UTF8.GetBytes(authMessage)); + var clientKey = Xor(clientProof, clientSignature); + var computedStoredKey = SHA256.HashData(clientKey); + if (computedStoredKey.Length != storedKey.Length + || !CryptographicOperations.FixedTimeEquals(computedStoredKey, storedKey)) + { + return null; + } + + var serverSignature = HmacSha256(_credential.ServerKey.ToArray(), Encoding.UTF8.GetBytes(authMessage)); + return $"v={Convert.ToBase64String(serverSignature)}"; + } + + /// Extracts the n= username from a client-first message. + public static bool TryGetUsername(string clientFirst, [NotNullWhen(true)] out string? username) + { + username = null; + if (!TrySplitClientFirst(clientFirst, out _, out var bare)) + { + return false; + } + + return TryParseAttribute(bare, 'n', out username) && !string.IsNullOrEmpty(username); + } + + private static byte[] HmacSha256(byte[] key, byte[] data) + { + using var hmac = IncrementalHash.CreateHMAC(HashAlgorithmName.SHA256, key); + hmac.AppendData(data); + return hmac.GetHashAndReset(); + } + + private static byte[] Xor(byte[] a, byte[] b) + { + var len = Math.Min(a.Length, b.Length); + var result = new byte[len]; + for (var i = 0; i < len; i++) + { + result[i] = (byte)(a[i] ^ b[i]); + } + + return result; + } + + private static bool TrySplitClientFirst( + string clientFirst, + [NotNullWhen(true)] out string? gs2Header, + [NotNullWhen(true)] out string? clientFirstBare) + { + var idx = clientFirst.IndexOf(",,", StringComparison.Ordinal); + if (idx < 0) + { + gs2Header = null; + clientFirstBare = null; + return false; + } + + gs2Header = clientFirst[..(idx + 2)]; + clientFirstBare = clientFirst[(idx + 2)..]; + return !string.IsNullOrEmpty(clientFirstBare); + } + + private static bool TryParseAttribute(string message, char key, [NotNullWhen(true)] out string? value) + { + foreach (var part in message.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (part.Length >= 2 && part[0] == key && part[1] == '=') + { + value = part[2..]; + return true; + } + } + + value = null; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramStoredCredential.cs b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramStoredCredential.cs new file mode 100644 index 00000000..c57fb5c9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Authentication/Sasl/ScramStoredCredential.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.ReaderService.Authentication.Sasl +{ + /// Stored SCRAM-SHA-256 material from nntpusers. + internal readonly struct ScramStoredCredential + { + /// Initializes stored SCRAM fields. + public ScramStoredCredential( + ReadOnlyMemory salt, + int iterationCount, + ReadOnlyMemory storedKey, + ReadOnlyMemory serverKey) + { + Salt = salt; + IterationCount = iterationCount; + StoredKey = storedKey; + ServerKey = serverKey; + } + + public ReadOnlyMemory Salt { get; } + public int IterationCount { get; } + public ReadOnlyMemory StoredKey { get; } + public ReadOnlyMemory ServerKey { get; } + + public static ScramStoredCredential FromRecord(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return new(record.ScramSalt, record.ScramIterations, record.ScramStoredKey, record.ScramServerKey); + } + + /// + /// Returns the real stored verifier only for an enabled account with usable + /// SCRAM-SHA-256 material. Disabled, denied, missing, or malformed rows + /// must use the dummy verifier so server-first does not enumerate accounts. + /// + public static bool TryGetAuthorized(NntpUserRecord? record, out ScramStoredCredential credential) + { + credential = default; + if (record is null + || !record.IsEnabled + || !record.AllowAuthScram256 + || record.ScramSalt.IsEmpty + || record.ScramIterations <= 0 + || record.ScramStoredKey.Length != 32 + || record.ScramServerKey.Length != 32) + { + return false; + } + + credential = FromRecord(record); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ArticleIngestionOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/ArticleIngestionOptions.cs new file mode 100644 index 00000000..705b0931 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ArticleIngestionOptions.cs @@ -0,0 +1,222 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Article ingestion worker and per-article size settings for the Transit ingestion path. + /// + /// + /// Used by TAKETHIS and IHAVE as the + /// network → queue → background ingestion worker boundary. + /// The worker does not write the article body to local durable storage. + /// Queue admission is bounded by , + /// not by . + /// + public sealed class ArticleIngestionOptions + { + /// + /// Legacy default article-count setting. Not an admission bound. + /// + public const int DefaultQueueCapacity = 256; + + /// Default maximum article payload size in bytes (4 MiB). + public const int DefaultMaxArticleBytes = 4 * 1024 * 1024; + + /// + /// Gets or sets a leftover article-count setting retained for binding compatibility. + /// + /// + /// Not an admission bound. The historical 256-entry cap was a memory-safety + /// choke and has been replaced by . + /// Valid range remains 1–100000 so existing configuration files still validate. + /// + [Range(1, 100_000)] + public int QueueCapacity { get; set; } = DefaultQueueCapacity; + + /// + /// Gets or sets the maximum accepted IHAVE/TAKETHIS article size in bytes + /// (after dot-unstuffing). + /// + /// + /// Valid range is 1–104857600 (100 MiB). Default is 4 MiB. + /// This is not the POST destuffed limit; POST uses + /// . + /// + [Range(1, 100 * 1024 * 1024)] + public int MaxArticleBytes { get; set; } = DefaultMaxArticleBytes; + + /// Default outstanding STORE streams on one placement connection. + public const int DefaultMaxOutstandingStoresPerConnection = 2; + + /// Default minimum concurrent ingestion workers. + public const int DefaultMinWorkers = 2; + + /// Default maximum concurrent ingestion workers. + public const int DefaultMaxWorkers = 32; + + /// Default maximum outstanding OverviewDB publisher confirms. + public const int DefaultMaxPublishConcurrency = 32; + + /// Default worker-pool scaling sample interval in seconds. + public const int DefaultScaleIntervalSeconds = 2; + + /// Default sustained high-pressure threshold (byte utilisation / wait signal). + public const double DefaultScaleUpPressureThreshold = 0.40; + + /// Default sustained low-pressure threshold. + public const double DefaultScaleDownPressureThreshold = 0.10; + + /// Default consecutive high-pressure intervals required before +1 worker. + public const int DefaultScaleUpConsecutiveIntervals = 2; + + /// Default consecutive low-pressure intervals required before −1 worker. + public const int DefaultScaleDownConsecutiveIntervals = 3; + + /// Default OverviewDB work-queue byte budget (1 GiB). + public const long DefaultOverviewDbWorkQueueMemoryLimit = 1L * 1024 * 1024 * 1024; + + /// Default minimum OverviewDB RabbitMQ publisher workers. + public const int DefaultOverviewDbMinPublisherWorkers = 2; + + /// Default maximum OverviewDB RabbitMQ publisher workers. + public const int DefaultOverviewDbMaxPublisherWorkers = 32; + + /// Default outstanding OverviewDB publisher confirmation window. + public const int DefaultOverviewDbPublisherBatchSize = 100; + + /// Default bounded OverviewDB publisher shutdown drain seconds. + public const int DefaultOverviewDbPublisherShutdownSeconds = 2; + + /// + /// Gets or sets how many STORE streams may be outstanding on one placement connection. + /// + /// + /// Valid values are 1 and 2. One keeps a single STORE in flight. Two is the measured safe overlap. + /// This does not enable placement. The spool writer still skips placement while that path is disabled. + /// + [Range(1, 2)] + public int MaxOutstandingStoresPerConnection { get; set; } = DefaultMaxOutstandingStoresPerConnection; + + /// + /// Gets or sets the minimum number of concurrent ingestion workers. + /// + /// Valid range is 1–512. Default is . + [Range(1, 512)] + public int MinWorkers { get; set; } = DefaultMinWorkers; + + /// + /// Gets or sets the maximum number of concurrent ingestion workers. + /// + /// + /// Must be greater than or equal to . Valid range is 1–512. + /// Default is . + /// + [Range(1, 512)] + public int MaxWorkers { get; set; } = DefaultMaxWorkers; + + /// + /// Gets or sets the maximum number of outstanding OverviewDB publisher confirms + /// (legacy channel-pool bound retained for binding compatibility). + /// + /// + /// Prefer for the asynchronous + /// confirmation window. Valid range is 1–512. + /// Default is . + /// + [Range(1, 512)] + public int MaxPublishConcurrency { get; set; } = DefaultMaxPublishConcurrency; + + /// + /// Gets or sets the outstanding OverviewDB RabbitMQ confirmation window. + /// + /// + /// Maximum unconfirmed publishes in flight. Publishers stop dequeuing/publishing + /// when the window is full and resume as confirms free capacity. + /// Valid range is 1–10000. Default is . + /// + [Range(1, 10_000)] + public int OverviewDbPublisherBatchSize { get; set; } = DefaultOverviewDbPublisherBatchSize; + + /// + /// Gets or sets the bounded OverviewDB publisher shutdown drain period in seconds. + /// + /// + /// After this period, outstanding OverviewDB confirms are abandoned. + /// Valid range is 1–60. Default is . + /// + [Range(1, 60)] + public int OverviewDbPublisherShutdownSeconds { get; set; } = DefaultOverviewDbPublisherShutdownSeconds; + + /// + /// Gets or sets the in-process OverviewDB work-queue byte budget. + /// + /// + /// Bounded handoff between article workers and OverviewDB publisher workers. + /// Not durable; not a RabbitMQ queue. Valid range is 1–9223372036854775807. + /// Default is . + /// + [Range(1, long.MaxValue)] + public long OverviewDbWorkQueueMemoryLimit { get; set; } = DefaultOverviewDbWorkQueueMemoryLimit; + + /// + /// Gets or sets the minimum OverviewDB RabbitMQ publisher workers. + /// + /// Valid range is 1–512. Default is . + [Range(1, 512)] + public int OverviewDbMinPublisherWorkers { get; set; } = DefaultOverviewDbMinPublisherWorkers; + + /// + /// Gets or sets the maximum OverviewDB RabbitMQ publisher workers. + /// + /// + /// Must be greater than or equal to . + /// Scales independently of /. + /// Valid range is 1–512. Default is . + /// + [Range(1, 512)] + public int OverviewDbMaxPublisherWorkers { get; set; } = DefaultOverviewDbMaxPublisherWorkers; + + /// + /// Gets or sets the worker-pool scaling sample interval in seconds. + /// + /// + /// Shared by article and OverviewDB publisher pool scalers. + /// Valid range is 1–3600. Default is . + /// + [Range(1, 3600)] + public int ScaleIntervalSeconds { get; set; } = DefaultScaleIntervalSeconds; + + /// + /// Gets or sets the sustained pressure threshold that triggers scale-up. + /// + /// + /// Pressure is the greater of queue byte utilisation and a waiting-producer + /// signal. Must be strictly greater than . + /// Valid range is 0–1. + /// + [Range(0, 1)] + public double ScaleUpPressureThreshold { get; set; } = DefaultScaleUpPressureThreshold; + + /// + /// Gets or sets the sustained pressure threshold that triggers scale-down. + /// + /// + /// Must be strictly less than . + /// Valid range is 0–1. + /// + [Range(0, 1)] + public double ScaleDownPressureThreshold { get; set; } = DefaultScaleDownPressureThreshold; + + /// + /// Gets or sets how many consecutive high-pressure samples are required before adding one worker. + /// + [Range(1, 100)] + public int ScaleUpConsecutiveIntervals { get; set; } = DefaultScaleUpConsecutiveIntervals; + + /// + /// Gets or sets how many consecutive low-pressure samples are required before removing one worker. + /// + [Range(1, 100)] + public int ScaleDownConsecutiveIntervals { get; set; } = DefaultScaleDownConsecutiveIntervals; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ControlOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/ControlOptions.cs new file mode 100644 index 00000000..bb816ee4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ControlOptions.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Top-level control-message infrastructure options. + /// + /// + /// + /// The configuration section name is (Control; + /// case-insensitive). This section is a passive catalogue. It does not enable + /// control-message processing, dispatch, authorization, or PGP verification. + /// + /// + /// An omitted or empty section is valid. NNTPD startup does not require a + /// populated authority catalogue. + /// + /// + public sealed class ControlOptions + { + /// Top-level configuration section name. + public const string SectionName = "Control"; + + /// + /// Gets or sets the authoritative Usenet PGP control-authority catalogue. + /// + public PgpAuthoritiesOptions PgpAuthorities { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ControlOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/ControlOptionsValidator.cs new file mode 100644 index 00000000..35fa4ab2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ControlOptionsValidator.cs @@ -0,0 +1,108 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Validates when the section is bound. + /// + /// + /// An omitted or empty catalogue is valid. This validator does not authorize + /// control messages and does not load PGP keys. + /// + public sealed class ControlOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, ControlOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + var authorities = options.PgpAuthorities?.Authorities; + if (authorities is null || authorities.Length == 0) + { + return ValidateOptionsResult.Success; + } + + for (var i = 0; i < authorities.Length; i++) + { + var authority = authorities[i]; + if (authority is null) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}] must not be null."); + continue; + } + + if (string.IsNullOrWhiteSpace(authority.Name)) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.Name)} must not be empty."); + } + + if (authority.KeyFingerprint is not null + && !string.IsNullOrWhiteSpace(authority.KeyFingerprint) + && !PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out _)) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.KeyFingerprint)} must be hexadecimal."); + } + + var authorizations = authority.Authorizations; + if (authorizations is null) + { + continue; + } + + for (var j = 0; j < authorizations.Length; j++) + { + var rule = authorizations[j]; + if (rule is null) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.Authorizations)}[{j}] must not be null."); + continue; + } + + RequireNonEmpty( + failures, + rule.Message, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.Message)); + RequireNonEmpty( + failures, + rule.From, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.From)); + RequireNonEmpty( + failures, + rule.Newsgroups, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.Newsgroups)); + RequireNonEmpty( + failures, + rule.VerificationIdentity, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.VerificationIdentity)); + } + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void RequireNonEmpty( + List failures, + string? value, + int authorityIndex, + int ruleIndex, + string propertyName) + { + if (string.IsNullOrWhiteSpace(value)) + { + failures.Add( + $"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{authorityIndex}].{nameof(PgpAuthorityOptions.Authorizations)}[{ruleIndex}].{propertyName} must not be empty."); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/EmailOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/EmailOptions.cs new file mode 100644 index 00000000..76e69a87 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/EmailOptions.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Top-level outbound email options for the generic Email subsystem. + /// + /// + /// + /// The configuration section name is (Email; + /// case-insensitive). This is application email infrastructure, not an NNTP or + /// moderation setting. Moderation is one producer. + /// + /// + /// When is (the production default), + /// IEmailService.SendAsync rejects immediately. No spool file is written + /// and no SMTP connection is opened. SMTP host and credentials are not required. + /// + /// + /// Credentials must be supplied via environment variables + /// (Email__Smtp__Username, Email__Smtp__Password) or a secret store. + /// Do not place real credentials in VectorNNTPD.ReaderService.json. + /// + /// + public sealed class EmailOptions + { + /// Top-level configuration section name. + public const string SectionName = "Email"; + + /// + /// Gets or sets whether the outbound email subsystem accepts work. + /// Default . + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the default RFC 5322 From mailbox used when a producer + /// does not supply one. Required when is . + /// + public string DefaultFrom { get; set; } = string.Empty; + + /// + /// Gets or sets the SMTP envelope sender (MAIL FROM). + /// When empty, is used. Never derived from recipients. + /// + public string EnvelopeSender { get; set; } = string.Empty; + + /// Gets or sets SMTP transport options. + public SmtpOptions Smtp { get; set; } = new(); + + /// Gets or sets the durable filesystem spool options. + public EmailSpoolOptions Spool { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/EmailOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/EmailOptionsValidator.cs new file mode 100644 index 00000000..7ae54de8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/EmailOptionsValidator.cs @@ -0,0 +1,186 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Validates at bind / startup time. + /// Never includes credentials in failure messages. + public sealed class EmailOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, EmailOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + + var failures = new List(); + ValidateSpool(options.Spool, failures); + + if (!options.Enabled) + { + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + ValidateMailbox( + options.DefaultFrom, + $"{EmailOptions.SectionName}:{nameof(EmailOptions.DefaultFrom)}", + failures); + + if (!string.IsNullOrWhiteSpace(options.EnvelopeSender)) + { + ValidateMailbox( + options.EnvelopeSender, + $"{EmailOptions.SectionName}:{nameof(EmailOptions.EnvelopeSender)}", + failures); + } + + ValidateSmtp(options.Smtp, failures); + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateSpool(EmailSpoolOptions spool, List failures) + { + if (string.IsNullOrWhiteSpace(spool.Directory) || spool.Directory.Trim().Length > 512) + { + failures.Add($"{EmailOptions.SectionName}:Spool:Directory must be a non-empty filesystem path of 512 characters or fewer."); + } + + if (spool.ShutdownTimeout < TimeSpan.FromSeconds(1) + || spool.ShutdownTimeout > TimeSpan.FromMinutes(5)) + { + failures.Add( + $"{EmailOptions.SectionName}:Spool:ShutdownTimeout must be between 1 second and 5 minutes."); + } + + if (spool.ScanInterval < TimeSpan.FromMilliseconds(20) + || spool.ScanInterval > TimeSpan.FromMinutes(5)) + { + failures.Add( + $"{EmailOptions.SectionName}:Spool:ScanInterval must be between 20ms and 5 minutes."); + } + } + + private static void ValidateSmtp(SmtpOptions smtp, List failures) + { + if (string.IsNullOrWhiteSpace(smtp.Host)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Host is required when Email is enabled."); + } + else if (ContainsControl(smtp.Host) || smtp.Host.Trim().Length > 253) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Host must be a hostname or IP without control characters."); + } + + if (smtp.Port is < 1 or > 65535) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Port must be between 1 and 65535."); + } + + if (!Enum.IsDefined(smtp.Security)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Security must be None, StartTls, or ImplicitTls."); + } + + if (smtp.ConnectTimeout < TimeSpan.FromMilliseconds(100) + || smtp.ConnectTimeout > TimeSpan.FromMinutes(5)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:ConnectTimeout must be between 100ms and 5 minutes."); + } + + if (smtp.CommandTimeout < TimeSpan.FromMilliseconds(100) + || smtp.CommandTimeout > TimeSpan.FromMinutes(10)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:CommandTimeout must be between 100ms and 10 minutes."); + } + + if (smtp.MaxAttempts is < 1 or > 20) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:MaxAttempts must be between 1 and 20."); + } + + if (smtp.InitialRetryDelay < TimeSpan.Zero || smtp.InitialRetryDelay > TimeSpan.FromHours(1)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:InitialRetryDelay must be between 0 and 1 hour."); + } + + if (smtp.MaximumRetryDelay < smtp.InitialRetryDelay || smtp.MaximumRetryDelay > TimeSpan.FromHours(1)) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:MaximumRetryDelay must be at least InitialRetryDelay and at most 1 hour."); + } + + var userSet = !string.IsNullOrWhiteSpace(smtp.Username); + var passwordSet = !string.IsNullOrEmpty(smtp.Password); + if (userSet != passwordSet) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:Username and Password must both be set or both omitted " + + "(use Email__Smtp__Password or secrets; never commit the value)."); + } + else if (userSet) + { + if (ContainsControl(smtp.Username) || smtp.Username.Trim().Length > 256) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Username must be 1–256 characters without control characters."); + } + + if (smtp.RequireTlsForAuthentication && smtp.Security == SmtpSecurityMode.None) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:RequireTlsForAuthentication is true, so Security cannot be None. " + + "Set RequireTlsForAuthentication to false only when a controlled plaintext relay is intended."); + } + } + } + + internal static bool TryValidateMailbox(string? value, out string mailbox) + { + mailbox = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (trimmed.Length is 0 or > 254 || ContainsControl(trimmed)) + { + return false; + } + + var at = trimmed.IndexOf('@'); + if (at <= 0 || at != trimmed.LastIndexOf('@') || at == trimmed.Length - 1) + { + return false; + } + + mailbox = trimmed; + return true; + } + + private static void ValidateMailbox(string value, string key, List failures) + { + if (!TryValidateMailbox(value, out _)) + { + failures.Add($"{key} must be a mailbox address."); + } + } + + private static bool ContainsControl(string value) + { + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/EmailSpoolOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/EmailSpoolOptions.cs new file mode 100644 index 00000000..3da75e0a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/EmailSpoolOptions.cs @@ -0,0 +1,31 @@ +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Filesystem outbound email spool settings nested under . + public sealed class EmailSpoolOptions + { + /// Default application-relative spool directory. + public const string DefaultDirectory = "spool/smtp"; + + /// + /// Gets or sets the spool directory. Relative paths resolve through Common + /// against + /// . Default . + /// Created automatically when email is enabled. + /// + public string Directory { get; set; } = DefaultDirectory; + + /// + /// Gets or sets how long shutdown waits for an in-flight SMTP operation. + /// Default 15 seconds. Pending .eml files are left on disk. + /// + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(15); + + /// + /// Gets or sets how often the worker rescans the spool when idle. + /// Default 1 second. A write also wakes the worker. + /// + public TimeSpan ScanInterval { get; set; } = TimeSpan.FromSeconds(1); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/FeedDiagnosticsOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/FeedDiagnosticsOptions.cs new file mode 100644 index 00000000..06ff99a8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/FeedDiagnosticsOptions.cs @@ -0,0 +1,69 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Temporary opt-in real-feed observability under Nntpd:FeedDiagnostics. + /// + /// + /// Off by default. When disabled, the reporter does not run and session probes are not + /// attached. Enable with Nntpd:FeedDiagnostics:Enabled or environment variable + /// . This is diagnostic instrumentation, not a + /// permanent per-article log path. + /// + public sealed class FeedDiagnosticsOptions + { + /// Environment variable that enables feed diagnostics (VECTORNNTP_FEED_DIAGNOSTICS). + public const string EnvironmentVariableName = "VECTORNNTP_FEED_DIAGNOSTICS"; + + /// Default snapshot interval in seconds. + public const int DefaultIntervalSeconds = 5; + + /// Minimum accepted interval in seconds. + public const int MinIntervalSeconds = 1; + + /// Maximum accepted interval in seconds. + public const int MaxIntervalSeconds = 60; + + /// Gets or sets whether the periodic feed snapshot reporter is enabled. + public bool Enabled { get; set; } + + /// + /// Gets or sets the snapshot interval in seconds. + /// + /// Default is 10. Valid range is 1–60. + [Range(MinIntervalSeconds, MaxIntervalSeconds)] + public int IntervalSeconds { get; set; } = DefaultIntervalSeconds; + + /// Gets or sets whether each snapshot includes compact per-session lines. + public bool IncludeSessions { get; set; } = true; + + /// + /// Returns whether feed diagnostics should run for . + /// The environment variable wins when set to a truthy value. + /// + public static bool ResolveEnabled(NntpdOptions? options) + { + if (IsEnvironmentEnabled()) + { + return true; + } + + return options?.FeedDiagnostics?.Enabled == true; + } + + /// Returns whether is a truthy value. + public static bool IsEnvironmentEnabled() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + return value.Equals("1", StringComparison.OrdinalIgnoreCase) + || value.Equals("true", StringComparison.OrdinalIgnoreCase) + || value.Equals("yes", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/IpPrefix.cs b/src/VectorNNTPD.ReaderService/Configuration/IpPrefix.cs new file mode 100644 index 00000000..f4de9b63 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/IpPrefix.cs @@ -0,0 +1,203 @@ +using System.Net; +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// An IPv4 or IPv6 network prefix used by Transit AllowFrom matching. + /// + /// + /// Does not apply routability tests such as IPAddress.IsGlobal. Private, ULA, + /// documentation, and lab addresses are valid. + /// + public readonly struct IpPrefix : IEquatable + { + /// Initializes a prefix from a network address and length. + public IpPrefix(IPAddress network, int prefixLength) + { + ArgumentNullException.ThrowIfNull(network); + network = TrustedProxyHosts.Canonicalize(network); + var max = network.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + if (prefixLength < 0 || prefixLength > max) + { + throw new ArgumentOutOfRangeException(nameof(prefixLength)); + } + + Network = Mask(network, prefixLength); + PrefixLength = prefixLength; + AddressFamily = Network.AddressFamily; + } + + /// Gets the masked network address. + public IPAddress Network { get; } + + /// Gets the prefix length in bits. + public int PrefixLength { get; } + + /// Gets the address family of this prefix. + public AddressFamily AddressFamily { get; } + + /// Creates a host prefix (/32 or /128) for . + public static IpPrefix Host(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + address = TrustedProxyHosts.Canonicalize(address); + var length = address.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + return new IpPrefix(address, length); + } + + /// + /// Parses a literal IPv4/IPv6 address or CIDR prefix. + /// + public static bool TryParse(string text, out IpPrefix prefix) + { + prefix = default; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var trimmed = text.Trim(); + var slash = trimmed.LastIndexOf('/'); + if (slash < 0) + { + if (!IPAddress.TryParse(trimmed, out var address)) + { + return false; + } + + prefix = Host(address); + return true; + } + + if (slash == 0 || slash == trimmed.Length - 1) + { + return false; + } + + if (!IPAddress.TryParse(trimmed[..slash], out var network)) + { + return false; + } + + if (!int.TryParse(trimmed[(slash + 1)..], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out var length)) + { + return false; + } + + var max = network.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + if (network.AddressFamily == AddressFamily.InterNetworkV6 && network.IsIPv4MappedToIPv6) + { + max = 32; + } + + if (length < 0 || length > max) + { + return false; + } + + prefix = new IpPrefix(network, length); + return true; + } + + /// Returns whether is contained by this prefix. + public bool Contains(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + address = TrustedProxyHosts.Canonicalize(address); + if (address.AddressFamily != AddressFamily) + { + return false; + } + + return PrefixBitsEqual(Network, address, PrefixLength); + } + + /// + /// Returns whether this prefix and share any address. + /// + public bool Overlaps(IpPrefix other) + { + if (AddressFamily != other.AddressFamily) + { + return false; + } + + var length = Math.Min(PrefixLength, other.PrefixLength); + return PrefixBitsEqual(Network, other.Network, length); + } + + /// + public bool Equals(IpPrefix other) => + AddressFamily == other.AddressFamily + && PrefixLength == other.PrefixLength + && Network.Equals(other.Network); + + /// + public override bool Equals(object? obj) => obj is IpPrefix other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(AddressFamily, PrefixLength, Network); + + /// + public override string ToString() => + $"{Network}/{PrefixLength}"; + + private static IPAddress Mask(IPAddress address, int prefixLength) + { + var bytes = address.GetAddressBytes(); + ApplyPrefixMask(bytes, prefixLength); + return new IPAddress(bytes); + } + + private static bool PrefixBitsEqual(IPAddress left, IPAddress right, int prefixLength) + { + if (prefixLength == 0) + { + return true; + } + + var leftBytes = left.GetAddressBytes(); + var rightBytes = right.GetAddressBytes(); + if (leftBytes.Length != rightBytes.Length) + { + return false; + } + + var fullBytes = prefixLength / 8; + for (var i = 0; i < fullBytes; i++) + { + if (leftBytes[i] != rightBytes[i]) + { + return false; + } + } + + var remaining = prefixLength % 8; + if (remaining == 0) + { + return true; + } + + var mask = (byte)(0xFF << (8 - remaining)); + return (leftBytes[fullBytes] & mask) == (rightBytes[fullBytes] & mask); + } + + private static void ApplyPrefixMask(byte[] bytes, int prefixLength) + { + var fullBytes = prefixLength / 8; + var remaining = prefixLength % 8; + if (remaining != 0) + { + bytes[fullBytes] &= (byte)(0xFF << (8 - remaining)); + fullBytes++; + } + + for (var i = fullBytes; i < bytes.Length; i++) + { + bytes[i] = 0; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ModerationOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/ModerationOptions.cs new file mode 100644 index 00000000..9b67e30a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ModerationOptions.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Top-level ordinary-newsgroup moderation options. + /// + /// + /// + /// The configuration section name is (Moderation; + /// case-insensitive). This section is not Control:PgpAuthorities. PGP control + /// authorities do not authorize moderated-newsgroup approval. + /// + /// + /// An omitted or empty section is valid. Runtime authorization is loaded from + /// nntpmoderators. A leftover list fails startup. + /// Missing database routes reject moderated POST; they do not bypass moderation. + /// + /// + public sealed class ModerationOptions + { + /// Top-level configuration section name. + public const string SectionName = "Moderation"; + + /// + /// Gets or sets provenance for the imported INN samples/moderators snapshot. + /// + public ModerationSourceOptions Source { get; set; } = new(); + + /// + /// Gets or sets a leftover static mapping list. Must be empty. + /// Runtime authorization is loaded from nntpmoderators. + /// + public ModeratorMappingOptions[] Moderators { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ModerationOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/ModerationOptionsValidator.cs new file mode 100644 index 00000000..50eb90ec --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ModerationOptionsValidator.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Validates . Runtime authorization is loaded from + /// nntpmoderators; a leftover list is rejected. + /// + public sealed class ModerationOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, ModerationOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Moderators is { Length: > 0 }) + { + return ValidateOptionsResult.Fail( + "Moderation:Moderators is no longer a runtime authorization source. " + + "Load moderator routes from nntpmoderators."); + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ModerationSourceOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/ModerationSourceOptions.cs new file mode 100644 index 00000000..5a56b851 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ModerationSourceOptions.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Provenance for the historical INN samples/moderators import. + /// Runtime authorization is nntpmoderators, not this metadata. + /// + public sealed class ModerationSourceOptions + { + /// + /// Gets or sets the human-facing INN samples/moderators URL. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the INN GitHub raw samples/moderators URL. + /// + public string? InnUrl { get; set; } + + /// + /// Gets or sets which configured URL was actually retrieved. + /// + public string? RetrievedFrom { get; set; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/ModeratorMappingOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/ModeratorMappingOptions.cs new file mode 100644 index 00000000..bb469230 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/ModeratorMappingOptions.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Leftover static moderator-route shape retained only so a non-empty + /// Moderation:Moderators list can fail startup validation. + /// + /// + /// + /// is the unapproved-article submission destination. + /// A static mailbox is used as-is. An INN template containing %s is + /// expanded to the matched newsgroup name with dots changed to dashes. + /// The expanded mailbox does not authenticate an NNTP client. + /// + /// + /// is optional. It is required only for local + /// authenticated reinjection. Routing-only INN destinations omit it. + /// Credentials are not stored here. + /// + /// + public sealed class ModeratorMappingOptions + { + /// + /// Gets or sets the RFC 3977 wildmat matched against the newsgroup name. + /// + public string Pattern { get; set; } = string.Empty; + + /// + /// Gets or sets the routing mailbox or INN %s template for groups + /// matching . + /// + public string Address { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO username authorized to approve groups matching + /// , or empty when this entry is routing-only. + /// + public string Username { get; set; } = string.Empty; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/NntpServerJsonFiles.cs b/src/VectorNNTPD.ReaderService/Configuration/NntpServerJsonFiles.cs new file mode 100644 index 00000000..02973b95 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/NntpServerJsonFiles.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Shared JSON filenames used by Reader and Transit hosts. + public static class NntpServerJsonFiles + { + /// Shared RabbitMQ connectivity JSON registered by each NNTP server host. + public const string SharedRabbitMqFileName = "VectorNNTPD.RabbitMQ.json"; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs b/src/VectorNNTPD.ReaderService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs new file mode 100644 index 00000000..ba1533eb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Applies root-level ACME directory and renewal values without replacing + /// NNTPD-owned bind addresses, ports, or ACME state directory. Account email, + /// certificate password, and the Cloudflare API key are published from + /// nntpsharedconfig and are not read from VECTOR__ variables. + /// + internal static class NntpdAcmeCloudflareOptionsOverlay + { + /// + /// Overlays shared root directory and renewal settings, then restores + /// bound from Nntpd. + /// Bind addresses and ports remain the values bound from Nntpd. + /// + /// The NNTPD options already bound from the Nntpd section. + /// The full configuration root. + public static void OverlaySharedFromRootPreservingApplicationAcme( + NntpdOptions options, + IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(configuration); + + var stateDir = options.AcmeStateDir; + + AcmeCloudflareOptions.OverlaySharedFromRoot(options, configuration, includeSharedSecrets: false); + + options.AcmeStateDir = stateDir; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/NntpdOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/NntpdOptions.cs new file mode 100644 index 00000000..b482efc5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/NntpdOptions.cs @@ -0,0 +1,436 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Strongly typed lifecycle, hosting, and NNTPD listener options for VectorNNTPD.ReaderService. + /// + /// + /// + /// NNTPD-specific settings bind from section and + /// NNTPD__* environment variables. Bind addresses and ports and the + /// ACME state directory bind from Nntpd only. ACME directory URL, + /// renewal threshold, account email, certificate password, Cloudflare API key, + /// zone id, and DNS suffix are published from nntpsharedconfig. + /// RabbitMQ hosts and Management settings bind from the top-level + /// RabbitMQ section. RabbitMQ username and password are published from + /// nntpsharedconfig and are not read from VECTOR__RABBITMQ__USERNAME + /// or VECTOR__RABBITMQ__PASSWORD. + /// + /// + /// is generated from and and cannot + /// be bound or overridden from configuration or environment variables. + /// + /// + /// Never log complete instances: + /// , + /// , , + /// / , and + /// are secrets. + /// + /// + public sealed class NntpdOptions : AcmeCloudflareOptions, IApplicationLifecycleOptions + { + /// NNTPD-specific configuration section name. + public new const string SectionName = "Nntpd"; + + /// + /// Legacy combined-host FQDN prefix retained for documentation and older fixtures. + /// Live Reader/Transit hosts use . + /// + public const string ApplicationPrefix = "nntpd"; + + /// FQDN host-label prefix for the Reader service. + public const string ReaderApplicationPrefix = "reader"; + + /// FQDN host-label prefix for the Transit service. + public const string TransitApplicationPrefix = "transit"; + + /// + /// Returns the FQDN host-label prefix for . + /// + /// Reader or Transit host role. + /// The role-specific prefix. + public static string ApplicationPrefixFor(Hosting.NntpServerRole role) => + role switch + { + Hosting.NntpServerRole.Transit => TransitApplicationPrefix, + _ => ReaderApplicationPrefix, + }; + + /// + /// Gets or sets the FQDN host-label prefix for this process. + /// Set by hosting registration; not bound from configuration. + /// + public string HostApplicationPrefix { get; set; } = ApplicationPrefix; + + /// + /// Environment variable that supplies + /// (NNTPD__SERVERID). + /// + public const string ServerIdEnvironmentVariable = "NNTPD__SERVERID"; + + /// Configuration key for the POST X-Trace AES-256 key secret. + public const string XTraceKeyConfigurationKey = "XTraceKey"; + + /// Configuration key for the optional previous POST X-Trace AES-256 key. + public const string XTracePreviousKeyConfigurationKey = "XTracePreviousKey"; + + /// + /// Environment variable that supplies + /// (NNTPD__XTRACEKEY). + /// + public const string XTraceKeyEnvironmentVariable = "NNTPD__XTRACEKEY"; + + /// + /// Environment variable that supplies + /// (NNTPD__XTRACEPREVIOUSKEY). + /// + public const string XTracePreviousKeyEnvironmentVariable = "NNTPD__XTRACEPREVIOUSKEY"; + + /// Configuration key for the optional newsmaster AUTHINFO username. + public const string NewsmasterUserConfigurationKey = "NewsmasterUser"; + + /// Configuration key for the optional newsmaster AUTHINFO password secret. + public const string NewsmasterPasswordConfigurationKey = "NewsmasterPassword"; + + /// + /// Environment variable that supplies + /// (NNTPD__NEWSMASTERUSER). + /// + public const string NewsmasterUserEnvironmentVariable = "NNTPD__NEWSMASTERUSER"; + + /// + /// Environment variable that supplies + /// (NNTPD__NEWSMASTERPASSWORD). + /// + public const string NewsmasterPasswordEnvironmentVariable = "NNTPD__NEWSMASTERPASSWORD"; + + /// + /// Gets or sets the application display name used by lifecycle messages. + /// + /// + /// Defaults to . + /// Log file names, the Serilog application property, the Windows service name, + /// and the RabbitMQ connection prefix use the entry assembly name directly. + /// + [Required(AllowEmptyStrings = false)] + [MaxLength(128)] + public string ApplicationName { get; set; } = ApplicationJsonConfiguration.EntryAssemblyName; + + /// + /// Gets or sets the maximum time allowed for the overall graceful shutdown of application services. + /// + /// + /// This is a single wall-clock budget for the entire reverse-order stop sequence, not a fresh + /// timeout granted independently to each service. + /// awaits each service stop (no abandon). Services that ignore cancellation can block that await + /// until they return; the host ShutdownTimeout (configured from this value) and external + /// supervisors remain the process-level backstop. + /// + public TimeSpan GracefulShutdownTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the maximum time allowed for application startup before cancellation is considered a failure. + /// + /// + /// When , startup is bounded only by the host cancellation token. + /// + public TimeSpan? StartupTimeout { get; set; } + + /// + /// Gets or sets a value indicating whether the process should exit if an application service + /// terminates unexpectedly while running. + /// + public bool StopHostOnUnexpectedServiceTermination { get; set; } = true; + + /// + /// Gets or sets systemd-specific notification and watchdog options. + /// + [Required] + public SystemdOptions Systemd { get; set; } = new(); + + /// + /// Gets or sets trusted HAProxy PROXY-protocol peer addresses. + /// + /// + /// + /// Optional. Each entry must be a literal IPv4 or IPv6 address (not a DNS name, not a CIDR). + /// When empty or omitted, PROXY protocol processing is disabled and every connection's + /// effective client endpoint is the TCP peer. + /// + /// + /// When non-empty, a TCP peer whose address matches an entry is treated as a trusted proxy + /// and must present a valid PROXY v1/v2 header before TLS/NNTP. Untrusted peers are not + /// permitted to supply PROXY metadata that replaces their TCP identity. + /// + /// + public string[] ProxyHosts { get; set; } = []; + + /// + /// Gets or sets whether AUTHINFO USER/PASS is permitted when the connection is not TLS-protected. + /// + /// + /// + /// Default is so cleartext NNTP can authenticate. AUTHINFO PASS + /// transmits the password in clear text at the NNTP protocol layer; prefer TLS in production. + /// + /// + /// When and the connection is not TLS-protected, AUTHINFO USER/PASS + /// return 483 and AUTHINFO USER is not advertised in CAPABILITIES. + /// TLS connections always permit AUTHINFO USER/PASS regardless of this setting. + /// + /// + public bool AllowCleartextAuth { get; set; } = true; + + /// Default relative Serilog file-log and INN news directory. + public const string DefaultLogDir = "logs/"; + + /// + /// Gets or sets the filesystem directory for Serilog daily rolling application logs + /// and the INN-compatible news file. + /// + public string LogDir { get; set; } = DefaultLogDir; + + /// + /// Gets or sets the numeric server identity used when generating . + /// + /// + /// + /// Required. No default. Must be explicitly configured as an integer in 1–255. + /// + /// + /// Typed as so a missing configuration value remains distinguishable + /// from an explicitly configured 0 (both fail validation). + /// + /// + [Required] + [Range(ServerIdRules.MinimumInclusive, ServerIdRules.MaximumInclusive)] + public int? ServerId { get; set; } + + /// + /// Gets the generated server FQDN nntpd{ServerId:00}.{DnsSuffix}. + /// + /// + /// Not independently configurable. There is no dot between nntpd and the two-digit id. + /// Example: ServerId 1 → nntpd01.usenet.ninja. + /// Returns an empty string when is unset so data-annotation validation + /// can inspect the object; dependent services must use this only after validation succeeds. + /// + public override string Fqdn => + ServerId is { } serverId + && ServerIdRules.IsInRange(serverId) + && !string.IsNullOrWhiteSpace(DnsSuffix) + ? FormatFqdn( + string.IsNullOrWhiteSpace(HostApplicationPrefix) ? ApplicationPrefix : HostApplicationPrefix, + serverId, + DnsSuffix) + : string.Empty; + + /// + /// Formats the generated FQDN for a validated server id and DNS suffix. + /// + /// Server id in 1–255. + /// DNS suffix without a trailing dot. + /// The FQDN string. + public static string FormatFqdn(int serverId, string dnsSuffix) => + ApplicationFqdn.Build(ApplicationPrefix, serverId, dnsSuffix); + + /// + /// Formats the generated FQDN for a role-specific host-label prefix. + /// + /// Host-label prefix (reader or transit). + /// Server id in 1–255. + /// DNS suffix without a trailing dot. + /// The FQDN string. + public static string FormatFqdn(string applicationPrefix, int serverId, string dnsSuffix) => + ApplicationFqdn.Build(applicationPrefix, serverId, dnsSuffix); + + /// + /// Gets or sets NNTPD-local transit runtime options (STREAM TX depth and + /// WantTrash/LogTrash). + /// + /// + /// Named Transit peers bind from the top-level Transit section + /// (), not from this property. + /// + [Required] + public TransitOptions Transit { get; set; } = new(); + + /// + /// Gets or sets SPEEDTEST diagnostic limits. + /// + /// + /// Optional. Defaults cap duration, bytes, and concurrency for the VectorNNTP + /// SPEEDTEST extension. Not part of the article data plane. + /// + [Required] + public SpeedTestOptions SpeedTest { get; set; } = new(); + + /// + /// Gets or sets temporary real-feed observability. + /// + /// + /// Off by default. Periodic snapshots only; not per-article logging. + /// Also enabled by environment variable VECTORNNTP_FEED_DIAGNOSTICS. + /// + [Required] + public FeedDiagnosticsOptions FeedDiagnostics { get; set; } = new(); + + /// + /// Gets or sets how long a HistoryDB marker is retained in local memory and Redis. + /// + /// + /// Default is two hours. Redis uses native key TTL. The in-memory store expires the same + /// digest after this period. HistoryDB policy lives here, not on . + /// + public TimeSpan HistoryTime { get; set; } = TimeSpan.FromHours(2); + + /// Default NNTP command idle timeout in seconds. + public const int DefaultIdleTime = 300; + + /// Minimum accepted in seconds. + public const int MinIdleTime = 1; + + /// Maximum accepted in seconds (24 hours). + public const int MaxIdleTime = 86_400; + + /// + /// Gets or sets how many seconds an established NNTP session may remain without + /// executing an NNTP command before the session is disconnected. + /// + /// + /// Unit is seconds. Default is . Valid range is + /// –. 0 is invalid + /// (not “disabled”); it follows the same minimum-1-second rule as other + /// required timeouts such as . + /// The timer starts after session setup (greeting) and resets when a command + /// is accepted for processing. In-flight commands (including pipelined CHECK + /// and TAKETHIS) keep the session non-idle. + /// + public int IdleTime { get; set; } = DefaultIdleTime; + + /// Default POST article size limit: 5 MiB. + public const int DefaultMaxArticleSize = 5 * 1024 * 1024; + + /// Minimum accepted in bytes. + public const int MinMaxArticleSize = 1; + + /// Maximum accepted in bytes (100 MiB). + public const int MaxMaxArticleSize = 100 * 1024 * 1024; + + /// + /// Gets or sets the maximum destuffed POST article size in bytes. + /// + /// + /// Default is (5,242,880). Valid range is + /// –. The limit is + /// enforced while the multiline article is streamed (destuffed headers + blank + /// separator + body; NNTP terminator excluded; stuffing dots are not counted). + /// Distinct from ArticleIngestion:MaxArticleBytes, which bounds + /// IHAVE/TAKETHIS ingest and IHAVE worker destuff only. + /// + public int MaxArticleSize { get; set; } = DefaultMaxArticleSize; + + /// Default POST Injection-Info complaint mailbox. + public const string DefaultMailComplaintsTo = "abuse@usenet.ninja"; + + /// + /// Gets or sets the mailbox emitted as mail-complaints-to on server-owned + /// POST Injection-Info. + /// + /// + /// Default is . Required; must be a plausible + /// mailbox. Client-supplied Injection-Info is never used. + /// + public string MailComplaintsTo { get; set; } = DefaultMailComplaintsTo; + + /// + /// Gets or sets Path-survey (ninpaths) report recipient mailboxes. + /// + /// + /// Missing, null, or empty disables internal ninpaths reporting. Whitespace-only + /// entries are ignored. There is no separate enable flag and no environment-variable + /// overlay. Reports are emailed through the existing Email subsystem. + /// + public string[] Top1000 { get; set; } = []; + + /// + /// Gets or sets the persisted AES-256 key used to protect POST X-Trace values. + /// + /// + /// Required secret. Supply 32 bytes as 64 hex characters or Base64 via + /// or deployment secrets — never commit + /// this value. The process does not generate a key at startup; restarting with the + /// same key keeps previously issued tokens decryptable. + /// + public string XTraceKey { get; set; } = string.Empty; + + /// + /// Gets or sets the previous AES-256 key retained for one-generation X-Trace rotation. + /// + /// + /// Optional. When set, trusted decrypt accepts tokens produced with this key. + /// New tokens always use . After this value is removed, + /// tokens produced only with the retired key cannot be decrypted. + /// + public string XTracePreviousKey { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO username that may POST a well-formed cancel control article. + /// + /// + /// Optional. When omitted with , AUTHINFO remains deny-all + /// (unless another provider is registered). When set, is + /// required. Ordinary POST clients never receive this identity. + /// + public string NewsmasterUser { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO password for . + /// + /// + /// Optional secret. Required when is set. Supply via + /// or deployment secrets — never commit + /// this value. Do not log it or include it in exception messages. + /// + public string NewsmasterPassword { get; set; } = string.Empty; + + /// + /// Default Transit article-queue memory budget: 1 GiB (1,073,741,824 bytes). + /// + public const long DefaultTransitQueueMemoryLimit = 1_073_741_824L; + + /// + /// Gets or sets the Transit article-queue memory budget in bytes. + /// + /// + /// + /// Bounds the sum of owned queued article payload lengths + /// (InboundArticle.Payload.Length): complete NNTP article bytes as + /// queued (IHAVE: stuffed wire, terminating CRLF . CRLF excluded). + /// It does not include object overhead and is not a process-wide memory cap. + /// Memory is released when queued articles are consumed. + /// + /// + /// Default is 1 GiB. Valid range is 1 through + /// (signed 64-bit accounting). An individual article larger than this budget + /// is rejected rather than waited for, so admission cannot deadlock. + /// + /// + public long TransitQueueMemoryLimit { get; set; } = DefaultTransitQueueMemoryLimit; + + /// + /// Gets or sets article ingestion worker options. + /// + /// + /// Defaults: max article 4 MiB. The ingestion worker does not write article bodies to disk. + /// Queue admission is bounded by , not + /// an article-count cap. Used by TAKETHIS and IHAVE. + /// + [Required] + public ArticleIngestionOptions ArticleIngestion { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/NntpdOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/NntpdOptionsValidator.cs new file mode 100644 index 00000000..cebb227c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/NntpdOptionsValidator.cs @@ -0,0 +1,609 @@ +using System.Globalization; +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Validates at options bind / startup time. + /// + /// + /// Does not bind sockets or call Cloudflare APIs. Never includes secret values in failure messages. + /// + public sealed class NntpdOptionsValidator : IValidateOptions + { + private readonly ILocalIpAddressAssignee _localIpAddressAssignee; + + /// + /// Initializes a new instance of the class. + /// + /// Probe used to verify explicit bind addresses. + public NntpdOptionsValidator(ILocalIpAddressAssignee localIpAddressAssignee) + { + ArgumentNullException.ThrowIfNull(localIpAddressAssignee); + _localIpAddressAssignee = localIpAddressAssignee; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + + ValidateApplicationName(options, failures); + ValidateTimeouts(options, failures); + ValidateTop1000(options, failures); + ValidateSystemd(options, failures); + AcmeCloudflareOptionsValidator.CollectBindAddressFailures(options, _localIpAddressAssignee, failures); + ValidateProxyHosts(options, failures); + ValidatePorts(options, failures); + ValidateDnsSuffixAndServerId(options, failures); + ValidateAcme(options, failures); + ValidateLogDir(options, failures); + ValidateArticleIngestion(options, failures); + ValidateTransitQueueMemoryLimit(options, failures); + ValidateTransit(options, failures); + ValidateSpeedTest(options, failures); + ValidateFeedDiagnostics(options, failures); + ValidateXTraceKeys(options, failures); + ValidateNewsmaster(options, failures); + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateSpeedTest(NntpdOptions options, List failures) + { + var speed = options.SpeedTest ?? new SpeedTestOptions(); + if (speed.MaxDurationSeconds is < SpeedTestOptions.MinDurationSeconds + or > SpeedTestOptions.MaxDurationSecondsLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxDurationSeconds)} must be between {SpeedTestOptions.MinDurationSeconds} and {SpeedTestOptions.MaxDurationSecondsLimit}."); + } + + if (speed.MaxBytes is < SpeedTestOptions.MinBytes or > SpeedTestOptions.MaxBytesLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxBytes)} must be between {SpeedTestOptions.MinBytes} and {SpeedTestOptions.MaxBytesLimit}."); + } + + if (speed.MaxConcurrent is < SpeedTestOptions.MinConcurrent + or > SpeedTestOptions.MaxConcurrentLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxConcurrent)} must be between {SpeedTestOptions.MinConcurrent} and {SpeedTestOptions.MaxConcurrentLimit}."); + } + + if (speed.MaxConcurrentPerPeer is < SpeedTestOptions.MinConcurrentPerPeer + or > SpeedTestOptions.MaxConcurrentPerPeerLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxConcurrentPerPeer)} must be between {SpeedTestOptions.MinConcurrentPerPeer} and {SpeedTestOptions.MaxConcurrentPerPeerLimit}."); + } + } + + private static void ValidateFeedDiagnostics(NntpdOptions options, List failures) + { + var feed = options.FeedDiagnostics ?? new FeedDiagnosticsOptions(); + if (feed.IntervalSeconds is < FeedDiagnosticsOptions.MinIntervalSeconds + or > FeedDiagnosticsOptions.MaxIntervalSeconds) + { + failures.Add( + $"{nameof(NntpdOptions.FeedDiagnostics)}.{nameof(FeedDiagnosticsOptions.IntervalSeconds)} must be between {FeedDiagnosticsOptions.MinIntervalSeconds} and {FeedDiagnosticsOptions.MaxIntervalSeconds}."); + } + } + + private static void ValidateTransit(NntpdOptions options, List failures) + { + var transit = options.Transit ?? new TransitOptions(); + var depth = transit.StreamOutstandingArticleDepth; + if (!NntpStreamArticleTxScheduler.IsValidDepth(depth)) + { + failures.Add( + $"{nameof(NntpdOptions.Transit)}.{nameof(TransitOptions.StreamOutstandingArticleDepth)} must be between {NntpStreamArticleTxScheduler.MinDepth} and {NntpStreamArticleTxScheduler.MaxDepth}."); + } + } + + private static void ValidateArticleIngestion(NntpdOptions options, List failures) + { + var ingestion = options.ArticleIngestion ?? new ArticleIngestionOptions(); + if (ingestion.QueueCapacity is < 1 or > 100_000) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.QueueCapacity)} must be between 1 and 100000."); + } + + if (ingestion.MaxArticleBytes is < 1 or > 100 * 1024 * 1024) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxArticleBytes)} must be between 1 and 104857600."); + } + + if (ingestion.MaxOutstandingStoresPerConnection is not (1 or 2)) + { + failures.Add( + $"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxOutstandingStoresPerConnection)} must be 1 or 2."); + } + + if (ingestion.MinWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MinWorkers)} must be between 1 and 512."); + } + + if (ingestion.MaxWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxWorkers)} must be between 1 and 512."); + } + + if (ingestion.MaxWorkers < ingestion.MinWorkers) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxWorkers)} must be greater than or equal to {nameof(ArticleIngestionOptions.MinWorkers)}."); + } + + if (ingestion.MaxPublishConcurrency is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxPublishConcurrency)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbPublisherBatchSize is < 1 or > 10_000) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbPublisherBatchSize)} must be between 1 and 10000."); + } + + if (ingestion.OverviewDbPublisherShutdownSeconds is < 1 or > 60) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbPublisherShutdownSeconds)} must be between 1 and 60."); + } + + if (ingestion.OverviewDbWorkQueueMemoryLimit < 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbWorkQueueMemoryLimit)} must be at least 1."); + } + + if (ingestion.OverviewDbMinPublisherWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMinPublisherWorkers)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbMaxPublisherWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMaxPublisherWorkers)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbMaxPublisherWorkers < ingestion.OverviewDbMinPublisherWorkers) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMaxPublisherWorkers)} must be greater than or equal to {nameof(ArticleIngestionOptions.OverviewDbMinPublisherWorkers)}."); + } + + if (ingestion.ScaleIntervalSeconds is < 1 or > 3600) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleIntervalSeconds)} must be between 1 and 3600."); + } + + if (ingestion.ScaleUpPressureThreshold is < 0 or > 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpPressureThreshold)} must be between 0 and 1."); + } + + if (ingestion.ScaleDownPressureThreshold is < 0 or > 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleDownPressureThreshold)} must be between 0 and 1."); + } + + if (ingestion.ScaleUpPressureThreshold <= ingestion.ScaleDownPressureThreshold) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpPressureThreshold)} must be greater than {nameof(ArticleIngestionOptions.ScaleDownPressureThreshold)}."); + } + + if (ingestion.ScaleUpConsecutiveIntervals is < 1 or > 100) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpConsecutiveIntervals)} must be between 1 and 100."); + } + + if (ingestion.ScaleDownConsecutiveIntervals is < 1 or > 100) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleDownConsecutiveIntervals)} must be between 1 and 100."); + } + } + + private static void ValidateTransitQueueMemoryLimit(NntpdOptions options, List failures) + { + if (options.TransitQueueMemoryLimit < 1) + { + failures.Add( + $"{nameof(NntpdOptions.TransitQueueMemoryLimit)} must be a positive byte count (1 through {long.MaxValue})."); + } + } + + private static void ValidateApplicationName(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.ApplicationName)) + { + failures.Add($"{nameof(NntpdOptions.ApplicationName)} must be a non-empty string."); + } + else if (options.ApplicationName.Length > 128) + { + failures.Add($"{nameof(NntpdOptions.ApplicationName)} must be 128 characters or fewer."); + } + } + + private static void ValidateTimeouts(NntpdOptions options, List failures) + { + if (options.GracefulShutdownTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.GracefulShutdownTimeout)} must be at least 1 second."); + } + + if (options.GracefulShutdownTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.GracefulShutdownTimeout)} must not exceed 1 hour."); + } + + if (options.StartupTimeout is { } startupTimeout) + { + if (startupTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.StartupTimeout)} must be at least 1 second when specified."); + } + + if (startupTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.StartupTimeout)} must not exceed 1 hour."); + } + } + + if (options.CloudFlareOperationTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.CloudFlareOperationTimeout)} must be at least 1 second."); + } + + if (options.CloudFlareOperationTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.CloudFlareOperationTimeout)} must not exceed 1 hour."); + } + + if (options.HistoryTime < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.HistoryTime)} must be at least 1 second."); + } + + if (options.HistoryTime > TimeSpan.FromDays(7)) + { + failures.Add($"{nameof(NntpdOptions.HistoryTime)} must not exceed 7 days."); + } + + if (options.IdleTime < NntpdOptions.MinIdleTime) + { + failures.Add( + $"{nameof(NntpdOptions.IdleTime)} must be at least {NntpdOptions.MinIdleTime} second."); + } + + if (options.IdleTime > NntpdOptions.MaxIdleTime) + { + failures.Add( + $"{nameof(NntpdOptions.IdleTime)} must not exceed {NntpdOptions.MaxIdleTime} seconds."); + } + + if (options.MaxArticleSize < NntpdOptions.MinMaxArticleSize + || options.MaxArticleSize > NntpdOptions.MaxMaxArticleSize) + { + failures.Add( + $"{nameof(NntpdOptions.MaxArticleSize)} must be between {NntpdOptions.MinMaxArticleSize} and {NntpdOptions.MaxMaxArticleSize}."); + } + + if (string.IsNullOrWhiteSpace(options.MailComplaintsTo) + || !IsPlausibleEmail(options.MailComplaintsTo)) + { + failures.Add( + $"{nameof(NntpdOptions.MailComplaintsTo)} must be a valid mailbox address."); + } + } + + private static void ValidateTop1000(NntpdOptions options, List failures) + { + if (options.Top1000 is null || options.Top1000.Length == 0) + { + return; + } + + for (var i = 0; i < options.Top1000.Length; i++) + { + var entry = options.Top1000[i]; + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + if (!EmailOptionsValidator.TryValidateMailbox(entry, out _)) + { + failures.Add( + $"{nameof(NntpdOptions.Top1000)}[{i}] must be a mailbox address or whitespace."); + } + } + } + + private static void ValidateXTraceKeys(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.XTraceKey) + || !XTraceKeyParser.TryDecode(options.XTraceKey, out _)) + { + failures.Add( + $"{NntpdOptions.XTraceKeyConfigurationKey} must be a 32-byte AES-256 key encoded as 64 hex characters or Base64 " + + $"(use environment variable {NntpdOptions.XTraceKeyEnvironmentVariable} or secrets; never commit the value)."); + } + + if (!string.IsNullOrWhiteSpace(options.XTracePreviousKey) + && !XTraceKeyParser.TryDecode(options.XTracePreviousKey, out _)) + { + failures.Add( + $"{NntpdOptions.XTracePreviousKeyConfigurationKey} must be a 32-byte AES-256 key encoded as 64 hex characters or Base64 " + + $"when set (use environment variable {NntpdOptions.XTracePreviousKeyEnvironmentVariable} or secrets; never commit the value)."); + } + } + + private static void ValidateNewsmaster(NntpdOptions options, List failures) + { + var userSet = !string.IsNullOrWhiteSpace(options.NewsmasterUser); + var passwordSet = !string.IsNullOrEmpty(options.NewsmasterPassword); + if (userSet != passwordSet) + { + failures.Add( + $"{NntpdOptions.NewsmasterUserConfigurationKey} and {NntpdOptions.NewsmasterPasswordConfigurationKey} must both be set or both omitted " + + $"(use {NntpdOptions.NewsmasterPasswordEnvironmentVariable} or secrets for the password; never commit the value)."); + return; + } + + if (!userSet) + { + return; + } + + var user = options.NewsmasterUser.Trim(); + if (user.Length is < 1 or > 64 || ContainsControl(user)) + { + failures.Add($"{NntpdOptions.NewsmasterUserConfigurationKey} must be 1–64 characters without control characters."); + } + + if (options.NewsmasterPassword.Length > 256) + { + failures.Add($"{NntpdOptions.NewsmasterPasswordConfigurationKey} must not exceed 256 characters."); + } + } + + private static bool ContainsControl(string value) + { + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + return true; + } + } + + return false; + } + + private static void ValidateSystemd(NntpdOptions options, List failures) + { + if (options.Systemd is null) + { + failures.Add($"{nameof(NntpdOptions.Systemd)} must be provided."); + return; + } + + var fraction = options.Systemd.WatchdogIntervalFraction; + if (double.IsNaN(fraction) || double.IsInfinity(fraction) || fraction is <= 0 or >= 1) + { + failures.Add( + $"{nameof(NntpdOptions.Systemd)}.{nameof(SystemdOptions.WatchdogIntervalFraction)} must be in the open interval (0, 1)."); + } + else if (fraction is < 0.05 or > 0.9) + { + failures.Add( + $"{nameof(NntpdOptions.Systemd)}.{nameof(SystemdOptions.WatchdogIntervalFraction)} must be between 0.05 and 0.9 inclusive."); + } + } + + private static void ValidateProxyHosts(NntpdOptions options, List failures) + { + if (options.ProxyHosts is null) + { + return; + } + + for (var i = 0; i < options.ProxyHosts.Length; i++) + { + var entry = options.ProxyHosts[i]; + if (string.IsNullOrWhiteSpace(entry)) + { + failures.Add($"{nameof(NntpdOptions.ProxyHosts)}[{i}] must not be empty."); + continue; + } + + var trimmed = entry.Trim(); + if (trimmed.Contains('/') || trimmed.Contains('*')) + { + failures.Add( + $"{nameof(NntpdOptions.ProxyHosts)}[{i}] must be a literal IPv4 or IPv6 address (CIDR and wildcards are not supported)."); + continue; + } + + if (!IPAddress.TryParse(trimmed, out var address)) + { + failures.Add($"{nameof(NntpdOptions.ProxyHosts)}[{i}] is not a valid IPv4 or IPv6 address."); + continue; + } + + if (address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any)) + { + failures.Add( + $"{nameof(NntpdOptions.ProxyHosts)}[{i}] must not be an any-address wildcard; use an explicit proxy peer address."); + } + } + } + + private static void ValidatePorts(NntpdOptions options, List failures) + { + if (options.BindPort is < 1 or > 65535) + { + failures.Add($"{nameof(NntpdOptions.BindPort)} must be an integer in the range 1–65535."); + } + + if (options.BindPortTls is < 0 or > 65535) + { + failures.Add( + $"{nameof(NntpdOptions.BindPortTls)} must be 0 (TLS disabled) or an integer in the range 1–65535."); + } + } + + private static void ValidateAcme(NntpdOptions options, List failures) + { + // Directory URL, renewal threshold, zone id, DNS suffix, account email, + // certificate password, and Cloudflare API key come from nntpsharedconfig. + // State directory stays on this application. + if (string.IsNullOrWhiteSpace(options.AcmeStateDir)) + { + failures.Add($"{nameof(NntpdOptions.AcmeStateDir)} must be a non-empty filesystem path."); + } + } + + private static void ValidateLogDir(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.LogDir)) + { + failures.Add($"{nameof(NntpdOptions.LogDir)} must be a non-empty filesystem path."); + } + } + + private static bool IsPlausibleEmail(string email) + { + var trimmed = email.Trim(); + if (trimmed.Length is 0 or > 254) + { + return false; + } + + var at = trimmed.IndexOf('@'); + if (at <= 0 || at != trimmed.LastIndexOf('@') || at == trimmed.Length - 1) + { + return false; + } + + var domain = trimmed[(at + 1)..]; + return domain.Contains('.', StringComparison.Ordinal) && IsValidDnsSuffix(domain); + } + + private static void ValidateDnsSuffixAndServerId(NntpdOptions options, List failures) + { + switch (ServerIdRules.Classify(options.ServerId)) + { + case ServerIdValidationStatus.Missing: + failures.Add( + $"{nameof(NntpdOptions.ServerId)} is required and must be an integer in the range {ServerIdRules.MinimumInclusive}–{ServerIdRules.MaximumInclusive} (no default; set {NntpdOptions.SectionName}:{nameof(NntpdOptions.ServerId)} or {NntpdOptions.ServerIdEnvironmentVariable})."); + break; + case ServerIdValidationStatus.OutOfRange: + failures.Add( + $"{nameof(NntpdOptions.ServerId)} must be an integer in the range {ServerIdRules.MinimumInclusive}–{ServerIdRules.MaximumInclusive}."); + break; + } + } + + /// + /// Validates DNS suffix / name syntax (labels, length, allowed characters). + /// + internal static bool IsValidDnsSuffix(string suffix) + { + if (string.IsNullOrWhiteSpace(suffix)) + { + return false; + } + + var value = suffix.Trim().TrimEnd('.'); + if (value.Length is 0 or > 253) + { + return false; + } + + if (value.Contains("..", StringComparison.Ordinal)) + { + return false; + } + + var labels = value.Split('.'); + if (labels.Length == 0) + { + return false; + } + + foreach (var label in labels) + { + if (label.Length is 0 or > 63) + { + return false; + } + + if (label[0] == '-' || label[^1] == '-') + { + return false; + } + + foreach (var ch in label) + { + if (!IsDnsLabelChar(ch)) + { + return false; + } + } + } + + return true; + } + + private static bool IsDnsLabelChar(char ch) => + char.IsAsciiLetterOrDigit(ch) || ch == '-'; + + /// + /// Returns whether contains a configured secret (for tests / diagnostics hygiene). + /// + public static bool ContainsSecret(string text, string? secret) + { + if (string.IsNullOrEmpty(secret) || string.IsNullOrEmpty(text)) + { + return false; + } + + return text.Contains(secret, StringComparison.Ordinal); + } + + /// + /// Joins validation failures for assertions without exposing caller-supplied secrets. + /// + public static string JoinFailures(ValidateOptionsResult result) + { + if (result.Failures is null) + { + return result.FailureMessage ?? string.Empty; + } + + var sb = new StringBuilder(); + foreach (var failure in result.Failures) + { + if (sb.Length > 0) + { + sb.Append(CultureInfo.InvariantCulture, $"; {failure}"); + } + else + { + sb.Append(failure); + } + } + + return sb.ToString(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/NntpdRabbitMqOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/NntpdRabbitMqOptionsValidator.cs new file mode 100644 index 00000000..3ccddee9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/NntpdRabbitMqOptionsValidator.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// NNTPD RabbitMQ options validator: Common connection rules plus required Management HTTP settings. + /// + /// + /// NNTPD ArticleWork availability discovery requires Management BaseUrl. + /// Broker username and password are published from nntpsharedconfig after this validator runs. + /// Connectivity-only hosts should register alone. + /// + public sealed class NntpdRabbitMqOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, RabbitMqOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + RabbitMqOptionsValidator.ValidateConnection(options, failures); + ValidateRequiredManagement(options, failures); + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateRequiredManagement(RabbitMqOptions rabbitMq, List failures) + { + rabbitMq.Management ??= new RabbitMqManagementOptions(); + var management = rabbitMq.Management; + + if (string.IsNullOrWhiteSpace(management.BaseUrl)) + { + failures.Add("RabbitMQ:Management:BaseUrl is required."); + return; + } + + RabbitMqOptionsValidator.ValidateManagementShape(management, failures); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritiesOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritiesOptions.cs new file mode 100644 index 00000000..88f643fe --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritiesOptions.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Authoritative Usenet PGP control-authority catalogue derived from ISC/INN + /// control.ctl. + /// + /// + /// This is trusted reference data for a later control-message implementation. + /// It is not an authorization engine and does not verify signatures. + /// + public sealed class PgpAuthoritiesOptions + { + /// + /// Gets or sets provenance for the control.ctl snapshot used to populate + /// . + /// + public PgpAuthoritySourceOptions Source { get; set; } = new(); + + /// + /// Gets or sets the catalogue entries. An empty array is valid. + /// + public PgpAuthorityOptions[] Authorities { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityAuthorizationOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityAuthorizationOptions.cs new file mode 100644 index 00000000..00c4abd0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityAuthorizationOptions.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// One explicit PGP control-rule association from ISC/INN control.ctl. + /// + /// + /// Corresponds to a <message>:<from>:<newsgroups>:verify-<identity> + /// line. Stored for a later authorization implementation; not evaluated now. + /// + public sealed class PgpAuthorityAuthorizationOptions + { + /// + /// Gets or sets the control-message type (newgroup, rmgroup, + /// checkgroups, or all). + /// + public string Message { get; set; } = string.Empty; + + /// + /// Gets or sets the source From-address pattern. + /// + public string From { get; set; } = string.Empty; + + /// + /// Gets or sets the source newsgroup/hierarchy pattern. + /// + public string Newsgroups { get; set; } = string.Empty; + + /// + /// Gets or sets the PGP user-id from the source verify- action. + /// + public string VerificationIdentity { get; set; } = string.Empty; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityFingerprint.cs b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityFingerprint.cs new file mode 100644 index 00000000..8aafa1e0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityFingerprint.cs @@ -0,0 +1,91 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Normalizes PGP key fingerprints from ISC/INN control.ctl grouping + /// into a machine-readable uppercase hexadecimal form. + /// + public static class PgpAuthorityFingerprint + { + /// + /// Attempts to normalize a fingerprint to uppercase hexadecimal without + /// whitespace. + /// + /// Source fingerprint text, possibly grouped with spaces. + /// Normalized fingerprint when the method returns . + /// + /// when contains only hexadecimal + /// digits and whitespace, is non-empty after stripping whitespace, and has an + /// even number of hex digits. + /// + /// + /// Normalization removes whitespace and uppercases hex digits. It does not + /// reinterpret or invent bytes. + /// + public static bool TryNormalize(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = null; + if (value is null) + { + return false; + } + + var hexLength = 0; + foreach (var c in value) + { + if (char.IsWhiteSpace(c)) + { + continue; + } + + if (!IsHex(c)) + { + return false; + } + + hexLength++; + } + + if (hexLength == 0 || (hexLength & 1) != 0) + { + return false; + } + + var chars = new char[hexLength]; + var i = 0; + foreach (var c in value) + { + if (char.IsWhiteSpace(c)) + { + continue; + } + + chars[i++] = char.ToUpperInvariant(c); + } + + normalized = new string(chars); + return true; + } + + /// + /// Normalizes a fingerprint or throws when the value is not hexadecimal. + /// + /// Source fingerprint text. + /// Uppercase hexadecimal without whitespace. + /// The value is not a fingerprint. + public static string Normalize(string value) + { + ArgumentNullException.ThrowIfNull(value); + if (!TryNormalize(value, out var normalized)) + { + throw new ArgumentException("Value is not a hexadecimal PGP key fingerprint.", nameof(value)); + } + + return normalized; + } + + private static bool IsHex(char c) => + c is (>= '0' and <= '9') or (>= 'A' and <= 'F') or (>= 'a' and <= 'f'); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityOptions.cs new file mode 100644 index 00000000..32a0ce35 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthorityOptions.cs @@ -0,0 +1,67 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// One Usenet PGP control authority from ISC/INN control.ctl. + /// + /// + /// Optional metadata properties are omitted from JSON when the source does not + /// provide them. This type stores public catalogue metadata only: no private + /// keys, passphrases, or armored key blocks. + /// + public sealed class PgpAuthorityOptions + { + /// + /// Gets or sets the hierarchy heading name from control.ctl. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the source Contact value, when present. + /// + public string? Contact { get; set; } + + /// + /// Gets or sets the source Admin group value, when present. + /// + public string? AdminGroup { get; set; } + + /// + /// Gets or sets the source URL value, when present. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the source Key URL value, when present. + /// + public string? KeyUrl { get; set; } + + /// + /// Gets or sets the source PGP key fingerprint, when present. + /// + /// + /// Stored as uppercase hexadecimal without spaces. Source grouping whitespace + /// is discarded only; the hex digits are unchanged. + /// + public string? KeyFingerprint { get; set; } + + /// + /// Gets or sets the source Key mail value, when present. + /// + public string? KeyMail { get; set; } + + /// + /// Gets or sets the source Syncable server value, when present. + /// + public string? SyncableServer { get; set; } + + /// + /// Gets or sets explicit PGP verify-* control rules from the source. + /// + /// + /// Populated only when control.ctl states a control-message type, + /// From pattern, newsgroup pattern, and verification identity. Rules are + /// not collapsed. This array is not evaluated at runtime. + /// + public PgpAuthorityAuthorizationOptions[] Authorizations { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritySourceOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritySourceOptions.cs new file mode 100644 index 00000000..d3374da4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PgpAuthoritySourceOptions.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Provenance for the ISC/INN control.ctl snapshot that populated + /// . + /// + public sealed class PgpAuthoritySourceOptions + { + /// + /// Gets or sets the ISC canonical control.ctl URL. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the INN GitHub samples/control.ctl URL. + /// + public string? InnUrl { get; set; } + + /// + /// Gets or sets the source Last modified date from control.ctl + /// (yyyy-MM-dd). + /// + public string? LastModified { get; set; } + + /// + /// Gets or sets which configured URL was actually retrieved. + /// + /// + /// InnUrl when the ISC URL was unavailable and the INN copy was used. + /// + public string? RetrievedFrom { get; set; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PostFilterLeftoverConfigurationValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/PostFilterLeftoverConfigurationValidator.cs new file mode 100644 index 00000000..e21ef9d7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PostFilterLeftoverConfigurationValidator.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Rejects leftover Nntpd:PostFilter so cluster policy cannot split from NntpDB. + /// + public sealed class PostFilterLeftoverConfigurationValidator : IValidateOptions + { + private readonly IConfiguration _configuration; + + /// Initializes the leftover-section check. + public PostFilterLeftoverConfigurationValidator(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _configuration = configuration; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var section = _configuration.GetSection("Nntpd:PostFilter"); + if (!section.Exists() || !section.GetChildren().Any()) + { + return ValidateOptionsResult.Success; + } + + return ValidateOptionsResult.Fail( + "Nntpd:PostFilter is not node-local configuration. Cluster PostFilter policy is loaded from NntpDB table nntppostfiltercurrent."); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/PostFilterOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/PostFilterOptions.cs new file mode 100644 index 00000000..0e4cb92d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/PostFilterOptions.cs @@ -0,0 +1,201 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Operational gate for NNTPD PostFilter. + public enum PostFilterGateState + { + /// Skip remaining PostFilter stages, including quota and SpamAssassin. + Disabled = 0, + + /// Evaluate deny, ArtType, quotas, and SpamAssassin. + Active = 1, + + /// Reject every POST that reaches PostFilter. + Closed = 2, + } + + /// + /// SpamAssassin scanner-fault action. Required when + /// is . + /// + public enum PostFilterSpamOnFailure + { + /// Scanner fault rejects the article (441). + Reject = 0, + + /// Scanner fault continues with the existing reservation. + Accept = 1, + } + + /// Deterministic SPAMD host selection compiled into the PostFilter snapshot. + public enum PostFilterSpamAssassinHostSelection + { + /// Rotate across ; failover on connect failure only. + RoundRobin = 0, + + /// Always try hosts in list order; failover on connect failure only. + Failover = 1, + } + + /// In-memory PostFilter compile model produced from NntpDB. + /// + /// Not bound from Nntpd:PostFilter. Cluster policy is nntppostfiltercurrent. + /// POST does not parse this object. The compiler produces an immutable snapshot. + /// + public sealed class PostFilterOptions + { + /// Gets or sets the operational gate. + public PostFilterGateState Gate { get; set; } = PostFilterGateState.Disabled; + + /// + /// Gets or sets deny-list entries. Plaintext usernames are hashed to + /// lowercase MD5 hex at compile time. Stored NntpDB values are already hashed. + /// + public string[] DeniedAccounts { get; set; } = []; + + /// Gets or sets client CIDRs that are denied. + public string[] DeniedCidrs { get; set; } = []; + + /// + /// Gets or sets allow-list entries (same MD5 identity rules as + /// ) that skip SpamAssassin only. + /// Does not skip the gate, deny, ArtType, or quotas. + /// + public string[] AllowlistedAccounts { get; set; } = []; + + /// Gets or sets client CIDRs that skip SpamAssassin only. + public string[] AllowlistedCidrs { get; set; } = []; + + /// + /// Gets or sets names rejected by Stage 3. + /// Empty means no type policy. + /// + public string[] RejectArtTypes { get; set; } = []; + + /// Gets or sets accept-quota windows and ceilings. + [Required] + public PostFilterQuotaOptions Quota { get; set; } = new(); + + /// Gets or sets SpamAssassin eligibility and failure policy. + [Required] + public PostFilterSpamAssassinOptions SpamAssassin { get; set; } = new(); + } + + /// Quota windows and ceilings compiled into the PostFilter snapshot. + public sealed class PostFilterQuotaOptions + { + /// Gets or sets the sustained window. Must be positive. + public TimeSpan LongWindow { get; set; } = TimeSpan.FromDays(1); + + /// Gets or sets the burst window. Must be positive. + public TimeSpan ShortWindow { get; set; } = TimeSpan.FromMinutes(10); + + /// Gets or sets the L message ceiling. 0 disables the dimension. + public long MaxMessagesLong { get; set; } + + /// Gets or sets the L byte ceiling. 0 disables the dimension. + public long MaxBytesLong { get; set; } + + /// Gets or sets the L identical-body ceiling. 0 disables the dimension. + public long MaxIdenticalLong { get; set; } + + /// Gets or sets the S message ceiling. 0 disables the dimension. + public long MaxMessagesShort { get; set; } + + /// Gets or sets the S byte ceiling. 0 disables the dimension. + public long MaxBytesShort { get; set; } + + /// Gets or sets the S identical-body ceiling. 0 disables the dimension. + public long MaxIdenticalShort { get; set; } + } + + /// SpamAssassin stage options. Disabled until explicitly enabled. + /// + /// SpamAssassin is a targeted small-article scanner. The default eligibility + /// matches the historical production boundary: + /// (131072) exclusive maximum and excluded. + /// MaxArticleSize = 0 disables the size gate. An empty + /// array disables type exclusion. + /// Ineligible articles skip CHECK; that is not an event. + /// Spam findings always reject in v1 (OnSpam is not independently configurable). + /// is required when is . + /// + public sealed class PostFilterSpamAssassinOptions + { + /// Historical exclusive ArtSize maximum used as the safe default (131072). + public const int DefaultMaxArticleSize = 131_072; + + /// Default pooled concurrent SPAMD connections. + public const int DefaultMaxConnections = 4; + + /// Minimum when SpamAssassin is enabled. + public const int MinMaxConnections = 1; + + /// Maximum when SpamAssassin is enabled. + public const int MaxMaxConnections = 32; + + /// Default SPAMC protocol version sent on CHECK. + public const string DefaultProtocolVersion = "1.5"; + + /// Gets or sets whether CHECK is invoked for eligible articles. + public bool Enabled { get; set; } + + /// + /// Gets or sets scanner-fault behaviour. Required when is true. + /// + public PostFilterSpamOnFailure? OnFailure { get; set; } + + /// + /// Gets or sets the exclusive ArtSize eligibility maximum. + /// Articles with ArtSize >= MaxArticleSize skip CHECK. + /// 0 disables the size gate. Default is . + /// + public int MaxArticleSize { get; set; } = DefaultMaxArticleSize; + + /// + /// Gets or sets names excluded from CHECK. + /// Default is . Empty means no type exclusion. + /// + public string[] ExcludeArtTypes { get; set; } = ["YEncoded"]; + + /// Gets or sets SPAMD hosts. Required when is true. + public string[] Hosts { get; set; } = []; + + /// Gets or sets the shared SPAMD port for every host. + public int Port { get; set; } = 783; + + /// Gets or sets the SPAMC protocol version used in the CHECK request line. + public string ProtocolVersion { get; set; } = DefaultProtocolVersion; + + /// + /// Gets or sets the maximum number of concurrent persistent SPAMD connections. + /// Each connection serializes CHECK requests; concurrency is the pool size. + /// + public int MaxConnections { get; set; } = DefaultMaxConnections; + + /// Gets or sets deterministic host selection. + public PostFilterSpamAssassinHostSelection HostSelection { get; set; } = + PostFilterSpamAssassinHostSelection.RoundRobin; + + /// Gets or sets the connect timeout. + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Gets or sets the CHECK operation timeout. + /// + /// This is the wall-clock budget for one CHECK, including connect. + /// When SpamAssassin is enabled, the PostFilter reservation hold is + /// max(10s, OperationTimeout + 1s) so the reservation cannot expire + /// during a legitimate CHECK. + /// + public TimeSpan OperationTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Minimum when SpamAssassin is enabled. + public static readonly TimeSpan MinOperationTimeout = TimeSpan.FromSeconds(1); + + /// Maximum when SpamAssassin is enabled. + /// Caps derived reservation hold so crash-recovery TTL cannot become unbounded. + public static readonly TimeSpan MaxOperationTimeout = TimeSpan.FromMinutes(2); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/RedisOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/RedisOptions.cs new file mode 100644 index 00000000..ba2186dc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/RedisOptions.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Top-level Redis connection options. Hosts are StackExchange.Redis endpoints/seeds + /// for one shared topology, not independently round-robined servers. + /// + public sealed class RedisOptions + { + /// Configuration section name. + public const string SectionName = "Redis"; + + /// Default Redis TCP port. + public const int DefaultPort = 6379; + + /// Gets or sets Redis hostnames or IP addresses used as multiplexer endpoints. + public string[] Host { get; set; } = []; + + /// Gets or sets the Redis TCP port applied to every configured host. + public int Port { get; set; } = DefaultPort; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/RedisOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/RedisOptionsValidator.cs new file mode 100644 index 00000000..5d611aea --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/RedisOptionsValidator.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Validates at bind / startup time. + public sealed class RedisOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, RedisOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + if (options.Host is null || options.Host.Length == 0) + { + failures.Add($"{nameof(RedisOptions.Host)} must contain at least one Redis hostname or IP address."); + } + else + { + for (var i = 0; i < options.Host.Length; i++) + { + if (string.IsNullOrWhiteSpace(options.Host[i])) + { + failures.Add($"{nameof(RedisOptions.Host)}[{i}] must not be empty."); + } + } + } + + if (options.Port is < 1 or > 65535) + { + failures.Add($"{nameof(RedisOptions.Port)} must be an integer in the range 1–65535."); + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/SmtpOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/SmtpOptions.cs new file mode 100644 index 00000000..fd27238c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/SmtpOptions.cs @@ -0,0 +1,57 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// SMTP transport settings nested under . + /// + /// Distinguishes SMTP relay (often port 25, possibly plaintext on a private network) + /// from message submission (RFC 6409, typically 587 STARTTLS or 465 implicit TLS). + /// This client speaks SMTP; it does not assume MSA-only rules when the destination + /// is a local relay. + /// + public sealed class SmtpOptions + { + /// Default submission port used when configuration omits . + public const int DefaultPort = 587; + + /// Gets or sets the SMTP server hostname or IP. Required when email is enabled. + public string Host { get; set; } = string.Empty; + + /// Gets or sets the SMTP TCP port (1–65535). Default . + public int Port { get; set; } = DefaultPort; + + /// Gets or sets the explicit TLS mode. Never inferred from . + public SmtpSecurityMode Security { get; set; } = SmtpSecurityMode.StartTls; + + /// + /// Gets or sets the SMTP AUTH username. Empty means no authentication. + /// Override with Email__Smtp__Username; never commit real values. + /// + public string Username { get; set; } = string.Empty; + + /// + /// Gets or sets the SMTP AUTH password. Override with Email__Smtp__Password + /// or a secret store. Never logged. + /// + public string Password { get; set; } = string.Empty; + + /// + /// Gets or sets whether AUTH is refused unless the session is already TLS-protected. + /// Default . Plaintext AUTH requires an explicit . + /// + public bool RequireTlsForAuthentication { get; set; } = true; + + /// Gets or sets the TCP connect timeout. Default 15 seconds. + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(15); + + /// Gets or sets the SMTP command/read/write timeout. Default 30 seconds. + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Gets or sets the maximum delivery attempts including the first. Default 3. + public int MaxAttempts { get; set; } = 3; + + /// Gets or sets the first retry delay. Default 2 seconds. + public TimeSpan InitialRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// Gets or sets the retry delay ceiling. Default 60 seconds. + public TimeSpan MaximumRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/SmtpSecurityMode.cs b/src/VectorNNTPD.ReaderService/Configuration/SmtpSecurityMode.cs new file mode 100644 index 00000000..503939d1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/SmtpSecurityMode.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// SMTP transport security mode. Explicit configuration wins over port defaults. + /// + /// + /// + /// These modes are mutually exclusive. The client never infers a mode from port + /// number alone and never silently downgrades from a required TLS mode. + /// + /// + /// — plaintext TCP. Legitimate for a controlled local relay. + /// — RFC 3207 STARTTLS after the SMTP greeting and first EHLO. + /// — TLS handshake immediately after TCP connect (typically port 465). + /// + /// Port defaults (25 / 587 / 465) are documentation only; is authoritative. + /// + public enum SmtpSecurityMode + { + /// Plaintext SMTP (no TLS). + None = 0, + + /// STARTTLS after EHLO (RFC 3207). Typical message-submission port 587. + StartTls = 1, + + /// Implicit TLS immediately after TCP connect (RFC 8314). Typical port 465. + ImplicitTls = 2, + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/SpeedTestOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/SpeedTestOptions.cs new file mode 100644 index 00000000..c678d4a6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/SpeedTestOptions.cs @@ -0,0 +1,125 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Diagnostic limits for the VectorNNTP SPEEDTEST extension under Nntpd:SpeedTest. + /// + /// + /// + /// These bounds cap synthetic diagnostic traffic. They are not article-ingestion, + /// HistoryDB, Redis, or Transit feed settings. SPEEDTEST does not open outbound + /// peer sockets from ConnectTo. + /// + /// + /// Defaults are sized for a short operator diagnostic, not a general-purpose traffic generator. + /// + /// + public sealed class SpeedTestOptions + { + /// Default maximum test duration in seconds. + public const int DefaultMaxDurationSeconds = 10; + + /// Minimum accepted duration in seconds. + public const int MinDurationSeconds = 1; + + /// Maximum accepted duration in seconds. + public const int MaxDurationSecondsLimit = 60; + + /// Default maximum payload bytes transferred in one test (64 MiB). + public const long DefaultMaxBytes = 64L * 1024 * 1024; + + /// Minimum accepted byte limit (one payload line). + public const long MinBytes = 1024; + + /// Maximum accepted byte limit (1 GiB). + public const long MaxBytesLimit = 1024L * 1024 * 1024; + + /// Default maximum concurrent SPEEDTEST operations on this host. + public const int DefaultMaxConcurrent = 2; + + /// Minimum accepted global concurrency. + public const int MinConcurrent = 1; + + /// Maximum accepted global concurrency. + public const int MaxConcurrentLimit = 8; + + /// Default maximum concurrent SPEEDTEST operations per Transit identifier. + public const int DefaultMaxConcurrentPerPeer = 1; + + /// Minimum accepted per-peer concurrency. + public const int MinConcurrentPerPeer = 1; + + /// Maximum accepted per-peer concurrency. + public const int MaxConcurrentPerPeerLimit = 4; + + /// + /// Gets or sets the maximum wall-clock duration of one SPEEDTEST payload transfer. + /// + /// Default is 10. Valid range is 1–60. + [Range(MinDurationSeconds, MaxDurationSecondsLimit)] + public int MaxDurationSeconds { get; set; } = DefaultMaxDurationSeconds; + + /// + /// Gets or sets the maximum synthetic payload bytes written for one SPEEDTEST. + /// + /// + /// Default is 64 MiB. Valid range is 1024–1073741824. + /// Counts payload body octets (pattern lines including CRLF), not status lines. + /// + [Range(MinBytes, MaxBytesLimit)] + public long MaxBytes { get; set; } = DefaultMaxBytes; + + /// + /// Gets or sets the maximum number of SPEEDTEST operations that may run at once on this host. + /// + /// Default is 2. Valid range is 1–8. + [Range(MinConcurrent, MaxConcurrentLimit)] + public int MaxConcurrent { get; set; } = DefaultMaxConcurrent; + + /// + /// Gets or sets the maximum number of simultaneous SPEEDTEST operations for one Transit identifier. + /// + /// Default is 1. Valid range is 1–4. + [Range(MinConcurrentPerPeer, MaxConcurrentPerPeerLimit)] + public int MaxConcurrentPerPeer { get; set; } = DefaultMaxConcurrentPerPeer; + + /// Returns an immutable copy of the current limits. + public SpeedTestLimits Snapshot() => new(this); + } + + /// Immutable SPEEDTEST limits captured at acquire time. + public readonly struct SpeedTestLimits + { + /// Initializes limits from . + public SpeedTestLimits(SpeedTestOptions options) + { + ArgumentNullException.ThrowIfNull(options); + MaxDuration = TimeSpan.FromSeconds(options.MaxDurationSeconds); + MaxBytes = options.MaxBytes; + MaxConcurrent = options.MaxConcurrent; + MaxConcurrentPerPeer = options.MaxConcurrentPerPeer; + } + + /// Creates limits directly (tests). + public SpeedTestLimits(TimeSpan maxDuration, long maxBytes, int maxConcurrent, int maxConcurrentPerPeer) + { + MaxDuration = maxDuration; + MaxBytes = maxBytes; + MaxConcurrent = maxConcurrent; + MaxConcurrentPerPeer = maxConcurrentPerPeer; + } + + /// Gets the maximum payload duration. + public TimeSpan MaxDuration { get; } + + /// Gets the maximum payload bytes. + public long MaxBytes { get; } + + /// Gets the host-wide concurrency cap. + public int MaxConcurrent { get; } + + /// Gets the per-peer concurrency cap. + public int MaxConcurrentPerPeer { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/SystemdOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/SystemdOptions.cs new file mode 100644 index 00000000..49fbf55d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/SystemdOptions.cs @@ -0,0 +1,56 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Optional systemd-specific hosting behaviour for Linux deployments. + /// + /// + /// These settings never override systemd-supplied watchdog deadlines + /// (WATCHDOG_USEC / unit WatchdogSec=). Watchdog heartbeats activate only when + /// systemd has configured a watchdog, the process is the intended watchdog target, and + /// is . + /// + public sealed class SystemdOptions + { + /// + /// Gets or sets a value indicating whether watchdog keep-alives may be sent when systemd + /// has configured WatchdogSec= for this process. + /// + public bool EnableWatchdog { get; set; } = true; + + /// + /// Gets or sets a value indicating whether lifecycle STATUS= notifications are sent + /// when systemd notify is enabled. + /// + public bool ReportLifecycleStatus { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the application sends an explicit READY=1 + /// when the application lifecycle reaches . + /// + /// + /// The built-in SystemdLifetime also notifies ready after all hosted services start. + /// Sending ready at the application lifecycle boundary ensures readiness is tied to + /// successful application initialization rather than host construction alone. + /// Duplicate READY=1 notifications are harmless. + /// + public bool NotifyReadyOnApplicationRunning { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the application sends an explicit STOPPING=1 + /// when graceful shutdown begins at the application lifecycle boundary. + /// + public bool NotifyStoppingOnApplicationShutdown { get; set; } = true; + + /// + /// Gets or sets the fraction of the systemd watchdog deadline used as the heartbeat interval. + /// + /// + /// systemd recommends notifying at about half the watchdog timeout. Valid range is (0, 1). + /// + [Range(0.05, 0.9)] + public double WatchdogIntervalFraction { get; set; } = 0.5; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitAllowFromEntry.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitAllowFromEntry.cs new file mode 100644 index 00000000..fe3d3377 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitAllowFromEntry.cs @@ -0,0 +1,93 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// A parsed Transit AllowFrom entry: literal prefix or DNS hostname. + /// + public abstract class TransitAllowFromEntry + { + private TransitAllowFromEntry(string original) + { + Original = original; + } + + /// Gets the original configuration text. + public string Original { get; } + + /// Literal IPv4/IPv6 address or CIDR prefix. + public sealed class Literal : TransitAllowFromEntry + { + /// Initializes a literal prefix entry. + public Literal(string original, IpPrefix prefix) + : base(original) + { + Prefix = prefix; + } + + /// Gets the parsed prefix. + public IpPrefix Prefix { get; } + } + + /// DNS hostname that must be resolved to A/AAAA addresses. + public sealed class Hostname : TransitAllowFromEntry + { + /// Initializes a hostname entry. + public Hostname(string original, string hostname) + : base(original) + { + DnsName = hostname; + } + + /// Gets the normalized hostname (no trailing dot). + public string DnsName { get; } + } + + /// Parses one AllowFrom entry. + public static bool TryParse(string text, out TransitAllowFromEntry? entry, out string? error) + { + entry = null; + error = null; + if (string.IsNullOrWhiteSpace(text)) + { + error = "AllowFrom entry must not be empty."; + return false; + } + + var trimmed = text.Trim(); + if (trimmed.Contains('*', StringComparison.Ordinal) || trimmed.Contains('+', StringComparison.Ordinal)) + { + error = "AllowFrom does not support wildcards; use a hostname, IP address, or CIDR prefix."; + return false; + } + + if (IpPrefix.TryParse(trimmed, out var prefix)) + { + entry = new Literal(trimmed, prefix); + return true; + } + + if (trimmed.Contains('/', StringComparison.Ordinal)) + { + error = "AllowFrom CIDR prefix is not a valid IPv4 or IPv6 network."; + return false; + } + + if (IPAddress.TryParse(trimmed, out _)) + { + error = "AllowFrom IP address is not valid."; + return false; + } + + var hostname = trimmed.TrimEnd('.'); + if (!NntpdOptionsValidator.IsValidDnsSuffix(hostname)) + { + error = "AllowFrom entry is not a valid DNS hostname, IP address, or CIDR prefix."; + return false; + } + + entry = new Hostname(trimmed, hostname); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitConnectEndpoint.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitConnectEndpoint.cs new file mode 100644 index 00000000..b4a37197 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitConnectEndpoint.cs @@ -0,0 +1,124 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// A parsed Transit ConnectTo endpoint (host plus required explicit port). + /// + /// + /// Supports DNS names, IPv4, and bracketed IPv6 with a port. This type does not + /// establish outbound connections. + /// + public sealed class TransitConnectEndpoint + { + private TransitConnectEndpoint(string host, int port, bool isIpAddress) + { + Host = host; + Port = port; + IsIpAddress = isIpAddress; + } + + /// Gets the host name or IP address (unbracketed). + public string Host { get; } + + /// Gets the explicit TCP port. + public int Port { get; } + + /// Gets a value indicating whether is a literal IP address. + public bool IsIpAddress { get; } + + /// + /// Parses host:port or [IPv6]:port. An explicit port is required. + /// + public static bool TryParse(string text, out TransitConnectEndpoint? endpoint, out string? error) + { + endpoint = null; + error = null; + if (string.IsNullOrWhiteSpace(text)) + { + error = "ConnectTo entry must not be empty."; + return false; + } + + var trimmed = text.Trim(); + string host; + string portText; + if (trimmed.StartsWith('[')) + { + var close = trimmed.IndexOf(']'); + if (close <= 1) + { + error = "ConnectTo IPv6 endpoint must be written as [address]:port."; + return false; + } + + if (close == trimmed.Length - 1 || trimmed[close + 1] != ':') + { + error = "ConnectTo IPv6 endpoint must include an explicit port after the closing bracket."; + return false; + } + + host = trimmed[1..close]; + portText = trimmed[(close + 2)..]; + } + else + { + var colon = trimmed.LastIndexOf(':'); + if (colon <= 0 || colon == trimmed.Length - 1) + { + error = "ConnectTo endpoint must include an explicit port (host:port or [IPv6]:port)."; + return false; + } + + host = trimmed[..colon]; + portText = trimmed[(colon + 1)..]; + + // Unbracketed IPv6 with a port is ambiguous (multiple colons). Require brackets. + if (host.Contains(':', StringComparison.Ordinal)) + { + error = "ConnectTo IPv6 endpoint must be written as [address]:port."; + return false; + } + } + + if (!int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out var port) + || port is < 1 or > 65535) + { + error = "ConnectTo port must be an integer in the range 1–65535."; + return false; + } + + if (IPAddress.TryParse(host, out var address)) + { + if (address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any)) + { + error = "ConnectTo must not use an any-address wildcard."; + return false; + } + + var canonical = address.AddressFamily == AddressFamily.InterNetworkV6 + ? address.ToString() + : address.ToString(); + endpoint = new TransitConnectEndpoint(canonical, port, isIpAddress: true); + return true; + } + + if (!NntpdOptionsValidator.IsValidDnsSuffix(host)) + { + error = "ConnectTo host is not a valid DNS hostname or IP address."; + return false; + } + + endpoint = new TransitConnectEndpoint(host.Trim().TrimEnd('.'), port, isIpAddress: false); + return true; + } + + /// + public override string ToString() => + Host.Contains(':', StringComparison.Ordinal) + ? string.Create(CultureInfo.InvariantCulture, $"[{Host}]:{Port}") + : string.Create(CultureInfo.InvariantCulture, $"{Host}:{Port}"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitLeftoverConfigurationValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitLeftoverConfigurationValidator.cs new file mode 100644 index 00000000..2d65cb37 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitLeftoverConfigurationValidator.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Rejects JSON that still owns the Transit catalogue or the site-wide junk flags. + /// + public sealed class TransitLeftoverConfigurationValidator : IValidateOptions + { + private readonly IConfiguration _configuration; + + /// Initializes the leftover-section check. + public TransitLeftoverConfigurationValidator(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _configuration = configuration; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (_configuration.GetSection(TransitPeersOptions.SectionName).GetChildren().Any()) + { + return ValidateOptionsResult.Fail( + "Transit peers are not JSON configuration. The catalogue is loaded from NntpDB table nntptransitcurrent."); + } + + if (_configuration.GetSection("Nntpd:Transit:WantTrash").Exists() + || _configuration.GetSection("Nntpd:Transit:LogTrash").Exists()) + { + return ValidateOptionsResult.Fail( + "Nntpd:Transit:WantTrash and Nntpd:Transit:LogTrash are not node-local settings. They are loaded from nntptransitglobalrevision."); + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypes.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypes.cs new file mode 100644 index 00000000..b5c51b67 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypes.cs @@ -0,0 +1,70 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Diablo-style article message-type flags (configuration/policy only). + /// + /// + /// Names follow Diablo ArtTypeConv(). is the value for both + /// binary and binaries. is the union of every concrete flag. + /// Bit values match Common ArticleType. Receive matching uses + /// ArticleTypeCapabilities against ArticleRecord.ArtType. Send matching is not wired. + /// + [Flags] + public enum TransitMessageTypes + { + /// Diablo none — no selected types. + None = 0, + + /// Diablo default. Distinct from . + Default = 1 << 0, + + /// Diablo control. + Control = 1 << 1, + + /// Diablo cancel. + Cancel = 1 << 2, + + /// Diablo mime. + Mime = 1 << 3, + + /// Diablo binary / binaries. + Binary = 1 << 4, + + /// Diablo uuencode. + UuEncode = 1 << 5, + + /// Diablo base64. + Base64 = 1 << 6, + + /// Diablo yenc. + Yenc = 1 << 7, + + /// Diablo bommanews. + BommaNews = 1 << 8, + + /// Diablo unidata. + UniData = 1 << 9, + + /// Diablo multipart. + Multipart = 1 << 10, + + /// Diablo html. + Html = 1 << 11, + + /// Diablo ps. + PostScript = 1 << 12, + + /// Diablo binhex. + BinHex = 1 << 13, + + /// Diablo partial. + Partial = 1 << 14, + + /// Diablo pgp. + PgpMessage = 1 << 15, + + /// Union of every concrete message-type flag (Diablo all). + All = Default | Control | Cancel | Mime | Binary | UuEncode | Base64 | Yenc | BommaNews + | UniData | Multipart | Html | PostScript | BinHex | Partial | PgpMessage, + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypesParser.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypesParser.cs new file mode 100644 index 00000000..9bbd6598 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitMessageTypesParser.cs @@ -0,0 +1,134 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Parses and formats from Diablo ArtTypeConv names. + /// + public static class TransitMessageTypesParser + { + /// Canonical default when MessageTypes is omitted. + public static readonly string[] DefaultConfiguredValues = ["default"]; + + private static readonly (string Name, TransitMessageTypes Value)[] Tokens = + [ + ("none", TransitMessageTypes.None), + ("default", TransitMessageTypes.Default), + ("control", TransitMessageTypes.Control), + ("cancel", TransitMessageTypes.Cancel), + ("mime", TransitMessageTypes.Mime), + ("binary", TransitMessageTypes.Binary), + ("binaries", TransitMessageTypes.Binary), + ("uuencode", TransitMessageTypes.UuEncode), + ("base64", TransitMessageTypes.Base64), + ("yenc", TransitMessageTypes.Yenc), + ("bommanews", TransitMessageTypes.BommaNews), + ("unidata", TransitMessageTypes.UniData), + ("multipart", TransitMessageTypes.Multipart), + ("html", TransitMessageTypes.Html), + ("ps", TransitMessageTypes.PostScript), + ("binhex", TransitMessageTypes.BinHex), + ("partial", TransitMessageTypes.Partial), + ("pgp", TransitMessageTypes.PgpMessage), + ("all", TransitMessageTypes.All), + ]; + + /// + /// Parses a JSON string-array of Diablo names into flags. + /// + /// + /// Comparison is case-insensitive. Surrounding whitespace on each value is ignored. + /// Duplicate names are OR-ed (harmless). Unknown values fail. + /// + public static bool TryParse( + IReadOnlyList? values, + out TransitMessageTypes types, + out string? error) + { + types = TransitMessageTypes.None; + error = null; + if (values is null || values.Count == 0) + { + types = TransitMessageTypes.Default; + return true; + } + + for (var i = 0; i < values.Count; i++) + { + var raw = values[i]; + if (raw is null || !TryParseToken(raw, out var flag)) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"[{i}] is not a recognised MessageTypes value."); + types = TransitMessageTypes.None; + return false; + } + + types |= flag; + } + + return true; + } + + /// Parses one Diablo token (case-insensitive, trimmed). + public static bool TryParseToken(string value, out TransitMessageTypes type) + { + ArgumentNullException.ThrowIfNull(value); + type = TransitMessageTypes.None; + var token = value.AsSpan().Trim(); + if (token.IsEmpty) + { + return false; + } + + foreach (var (name, flag) in Tokens) + { + if (token.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + type = flag; + return true; + } + } + + return false; + } + + /// + /// Returns canonical lowercase Diablo names for diagnostics (never includes passwords). + /// + public static IReadOnlyList ToCanonicalNames(TransitMessageTypes types) + { + if (types == TransitMessageTypes.None) + { + return ["none"]; + } + + if (types == TransitMessageTypes.All) + { + return ["all"]; + } + + var names = new List(); + foreach (var (name, flag) in Tokens) + { + if (flag is TransitMessageTypes.None or TransitMessageTypes.All) + { + continue; + } + + if (name.Equals("binaries", StringComparison.Ordinal)) + { + continue; + } + + if (types.HasFlag(flag)) + { + names.Add(name); + } + } + + return names; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitOptions.cs new file mode 100644 index 00000000..4a6c033e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitOptions.cs @@ -0,0 +1,53 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// NNTPD-local transit/streaming runtime options under Nntpd:Transit. + /// + /// + /// Peer authorization is the published MySQL Transit catalogue. This type holds + /// the process-local STREAM TX depth. and + /// are copied from nntptransitglobalrevision when + /// that catalogue is published. JSON must not set those two flags. + /// + public sealed class TransitOptions + { + /// + /// Gets or sets the maximum number of concurrent outstanding STREAM article TX operations + /// admitted by NntpStreamArticleTxScheduler. + /// + /// + /// Default is 8. Valid range is 4–16 (rejected outside that range; not clamped). + /// Independent of TX Channel capacity, Pipe thresholds, and TAKETHIS ingestion queue capacity. + /// + [Range(4, 16)] + public int StreamOutstandingArticleDepth { get; set; } = 8; + + /// + /// Gets or sets whether TAKETHIS/IHAVE articles posted only to unknown or + /// RFC 6048 j groups are accepted and treated as junk internally. + /// + /// + /// INN wanttrash. Default is true. When , + /// TAKETHIS/IHAVE articles posted only to unknown groups are accepted and + /// classified as junk after dequeue. When , those + /// articles are rejected before enqueue with a - news event. RFC 6048 + /// j (PeerOnly) groups remain accepted junk. The original + /// Newsgroups: header is not rewritten. POST is not subject to this + /// policy. The in-memory catalogue snapshot is consulted only after + /// ArticleRecord exists. + /// + public bool WantTrash { get; set; } = true; + + /// + /// Gets or sets whether accepted-junk events are written to the INN news log. + /// + /// + /// INN logtrash. Default is true. When , + /// junk articles are still accepted; only the j news line is omitted. + /// Accepted (+) and rejected (-) events are still written. + /// + public bool LogTrash { get; set; } = true; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitPeerOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitPeerOptions.cs new file mode 100644 index 00000000..889c959e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitPeerOptions.cs @@ -0,0 +1,138 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Bindable configuration for one named Transit peer (dictionary value under top-level Transit). + /// + /// + /// The dictionary key is the protocol-safe Identifier, not the display name. + /// is the required human-readable administrative name. + /// Do not log . + /// + public sealed class TransitPeerOptions + { + /// + /// Gets or sets the human-readable administrative display name for this peer. + /// + /// + /// Required. Preserved exactly as configured (no trimming, case-folding, or hyphenation). + /// Not a protocol argument and not derived from the dictionary-key identifier. + /// May contain spaces and punctuation. Must be non-empty, not whitespace-only, + /// at most characters, + /// and must not contain control characters. + /// + public string PeerName { get; set; } = string.Empty; + + /// + /// Gets or sets the maximum simultaneous inbound connections associated with this peer. + /// + /// + /// Required. Valid range is 0–4096. This is the cluster-wide inbound + /// cap for the peer Identifier. 0 means closed: admit no new + /// inbound connections. It is not unlimited. Existing connections are not + /// dropped when the limit is lowered. + /// + public int? MaxIncomingConnections { get; set; } + + /// + /// Gets or sets the maximum simultaneous outbound connections that may later be + /// established to this peer. + /// + /// + /// Required. Valid range is 0–4096. Stored and validated only; this host does + /// not open outbound peer sockets from . + /// + public int? MaxOutgoingConnections { get; set; } + + /// + /// Gets or sets inbound source authorization entries (DNS hostnames, IPv4/IPv6 addresses, CIDR prefixes). + /// + /// + /// Empty means this peer cannot match inbound sources (outbound-only policy). + /// + public string[] AllowFrom { get; set; } = []; + + /// + /// Gets or sets outbound endpoint strings (host:port or [IPv6]:port). + /// + /// + /// Optional. Parsed and validated only; no outbound connection is established. + /// An explicit port is required. + /// + public string[] ConnectTo { get; set; } = []; + + /// + /// Gets or sets the optional peer-specific AUTHINFO username. + /// + /// + /// Configuration requires and + /// both set or both blank. Peer AUTHINFO succeeds only when both are + /// non-empty and both supplied values match. + /// + public string Username { get; set; } = string.Empty; + + /// + /// Gets or sets the optional peer-specific AUTHINFO password. + /// + /// + /// Never log this value. Authentication requires a matching non-empty + /// as well. + /// + public string Password { get; set; } = string.Empty; + + /// + /// Gets or sets the TLS mode string: blank (none), TLS, or STARTTLS. + /// + public string Ssl { get; set; } = string.Empty; + + /// + /// Gets or sets the newsfeeds(5)-style subscription expression. + /// + /// Default is exactly *. + public string Patterns { get; set; } = "*"; + + /// + /// Gets or sets whether duplicate/in-flight article offers should later be deferred + /// (431/436) rather than rejected (438/435). + /// + /// + /// Default is . CHECK and IHAVE apply it when History + /// already contains the Message-ID. TAKETHIS does not. + /// + public bool DeferOnDuplicate { get; set; } = true; + + /// + /// Gets or sets the Path-header token reserved for outbound feed loop prevention. + /// + /// + /// Preserved exactly (no case-folding, not a DNS name or IP). Empty is allowed + /// because article Path-header matching is not implemented yet. Not consumed by + /// routing code in this change. + /// + public string PathToken { get; set; } = string.Empty; + + /// + /// Gets or sets the maximum allowed incoming article size for this peer, in bytes. + /// + /// + /// When this options object is compiled into a snapshot, this is the Receive ceiling. + /// Inbound acceptance uses the smaller of this value and the published global maxartsize. + /// Default is . Valid range is 1–2147483647. + /// + public long MaxSize { get; set; } = DefaultMaxSize; + + /// Default (10 MiB). + public const long DefaultMaxSize = 10_485_760; + + /// Maximum accepted (2 GiB − 1 byte). + public const long MaxMaxSize = int.MaxValue; + + /// + /// Gets or sets Diablo message-type names accepted for this peer. + /// + /// + /// Default is ["default"]. Not regex, MIME types, or newsgroup Patterns. + /// Article classification is not implemented. + /// + public string[] MessageTypes { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptions.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptions.cs new file mode 100644 index 00000000..8574b080 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptions.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Top-level Transit configuration: peer identifiers as dictionary keys. + /// + /// + /// + /// Binds the exact JSON shape Transit:{identifier}:{fields}. There is no nested + /// Transit:Peers layer. Distinct from under + /// Nntpd:Transit (STREAM TX depth only). + /// + /// + /// The dictionary key is the stable protocol/machine identifier (single NNTP token, + /// exact ordinal match, not normalized). is + /// the human-readable display name. An empty dictionary is valid (no configured peers). + /// + /// + public sealed class TransitPeersOptions : Dictionary + { + /// Top-level configuration section name. + public const string SectionName = "Transit"; + + /// Initializes an empty peer dictionary. + public TransitPeersOptions() + : base(StringComparer.Ordinal) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptionsValidator.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptionsValidator.cs new file mode 100644 index 00000000..4f0b1c56 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitPeersOptionsValidator.cs @@ -0,0 +1,359 @@ +using System.Globalization; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Validates top-level at bind / startup / reload. + /// + /// + /// Failures name the peer and property. A malformed peer is rejected entirely; it never + /// enters the active snapshot. Passwords are never included in failure messages. + /// + public sealed class TransitPeersOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, TransitPeersOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!TryValidate(options, out var failures)) + { + return ValidateOptionsResult.Fail(failures); + } + + return ValidateOptionsResult.Success; + } + + /// Validates options and returns property-qualified failure messages. + public static bool TryValidate(TransitPeersOptions options, out IReadOnlyList failures) + { + ArgumentNullException.ThrowIfNull(options); + var list = new List(); + var parsed = new List<(string Name, TransitPeerOptions Options, List AllowFrom)>(); + + foreach (var (identifier, peer) in options) + { + if (!TryValidateIdentifier(identifier, out var identifierError)) + { + list.Add(identifierError); + continue; + } + + if (peer is null) + { + list.Add(Format(identifier, null, "peer configuration must not be null.")); + continue; + } + + ValidatePeer(identifier, peer, list, parsed); + } + + if (list.Count == 0) + { + ValidateOverlaps(parsed, list); + } + + failures = list; + return list.Count == 0; + } + + private static void ValidatePeer( + string peerName, + TransitPeerOptions peer, + List failures, + List<(string Name, TransitPeerOptions Options, List AllowFrom)> parsed) + { + if (!TryValidatePeerName(peer.PeerName, out var peerNameError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.PeerName), peerNameError)); + } + + ValidateConnectionLimit(peerName, nameof(TransitPeerOptions.MaxIncomingConnections), peer.MaxIncomingConnections, failures); + ValidateConnectionLimit(peerName, nameof(TransitPeerOptions.MaxOutgoingConnections), peer.MaxOutgoingConnections, failures); + ValidateMaxSize(peerName, peer.MaxSize, failures); + ValidatePathToken(peerName, peer.PathToken, failures); + + ValidatePairedCredentials(peerName, peer, failures); + + if (!TransitMessageTypesParser.TryParse(peer.MessageTypes, out _, out var messageTypesError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.MessageTypes), messageTypesError ?? "invalid MessageTypes.")); + } + + if (!TransitSslParser.TryParse(peer.Ssl, out _, out var sslError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.Ssl), sslError!)); + } + + if (!NewsfeedsPattern.TryParse(peer.Patterns, out _, out var patternError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.Patterns), patternError!)); + } + + var allowFrom = new List(); + var entries = peer.AllowFrom ?? []; + for (var i = 0; i < entries.Length; i++) + { + if (!TransitAllowFromEntry.TryParse(entries[i], out var entry, out var error) || entry is null) + { + failures.Add(Format(peerName, $"{nameof(TransitPeerOptions.AllowFrom)}[{i}]", error ?? "invalid AllowFrom entry.")); + continue; + } + + allowFrom.Add(entry); + } + + var connectTo = peer.ConnectTo ?? []; + for (var i = 0; i < connectTo.Length; i++) + { + if (!TransitConnectEndpoint.TryParse(connectTo[i], out _, out var error)) + { + failures.Add(Format(peerName, $"{nameof(TransitPeerOptions.ConnectTo)}[{i}]", error ?? "invalid ConnectTo endpoint.")); + } + } + + parsed.Add((peerName, peer, allowFrom)); + } + + /// + /// Maximum Transit identifier length in characters (and SPEEDTEST token octets). + /// + public const int MaxPeerIdentifierLength = 256; + + /// Maximum PeerName length in characters (human-readable labels). + public const int MaxPeerNameLength = 256; + + /// Maximum PathToken length in characters. + public const int MaxPathTokenLength = 255; + + /// + /// Transit identifier grammar: 1–256 visible ASCII characters (0x21–0x7E). + /// + /// + /// No space, TAB, or other whitespace; no control characters; no Unicode. + /// Not normalized. The exact configured string is the identity (ordinal). + /// Safe as a single NNTP command token: SPEEDTEST identifier. + /// + public static bool IsPeerIdentifier(ReadOnlySpan identifier) + { + if (identifier.Length is < 1 or > MaxPeerIdentifierLength) + { + return false; + } + + for (var i = 0; i < identifier.Length; i++) + { + if (identifier[i] is < 0x21 or > 0x7E) + { + return false; + } + } + + return true; + } + + /// + public static bool IsPeerIdentifier(ReadOnlySpan identifier) + { + if (identifier.Length is < 1 or > MaxPeerIdentifierLength) + { + return false; + } + + for (var i = 0; i < identifier.Length; i++) + { + if (identifier[i] is < (char)0x21 or > (char)0x7E) + { + return false; + } + } + + return true; + } + + private static bool TryValidateIdentifier(string? identifier, out string error) + { + error = string.Empty; + if (string.IsNullOrEmpty(identifier)) + { + error = "Transit peer identifier must be non-empty."; + return false; + } + + if (!IsPeerIdentifier(identifier.AsSpan())) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"Transit peer identifier '{identifier}' must be 1-{MaxPeerIdentifierLength} visible ASCII characters (0x21-0x7E) with no whitespace or control characters."); + return false; + } + + return true; + } + + private static bool TryValidatePeerName(string? peerName, out string error) + { + error = string.Empty; + if (string.IsNullOrWhiteSpace(peerName)) + { + error = "must be non-empty and not whitespace-only."; + return false; + } + + if (peerName.Length > MaxPeerNameLength) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"exceeds the maximum length of {MaxPeerNameLength} characters."); + return false; + } + + foreach (var ch in peerName) + { + if (char.IsControl(ch)) + { + error = "must not contain a control character."; + return false; + } + } + + return true; + } + + private static void ValidatePathToken(string peerName, string? pathToken, List failures) + { + var value = pathToken ?? string.Empty; + if (value.Length > MaxPathTokenLength) + { + failures.Add( + Format( + peerName, + nameof(TransitPeerOptions.PathToken), + string.Create( + CultureInfo.InvariantCulture, + $"must be at most {MaxPathTokenLength} characters."))); + return; + } + + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.PathToken), "must not contain control characters.")); + return; + } + } + } + + private static void ValidatePairedCredentials(string peerName, TransitPeerOptions peer, List failures) + { + var username = (peer.Username ?? string.Empty).Trim(); + var password = peer.Password ?? string.Empty; + var hasUser = username.Length > 0; + var hasPassword = password.Length > 0; + if (hasUser == hasPassword) + { + return; + } + + failures.Add( + Format( + peerName, + hasUser ? nameof(TransitPeerOptions.Password) : nameof(TransitPeerOptions.Username), + "Username and Password must both be set or both be blank.")); + } + + private static void ValidateMaxSize(string peerName, long maxSize, List failures) + { + if (maxSize is < 1 or > TransitPeerOptions.MaxMaxSize) + { + failures.Add( + Format( + peerName, + nameof(TransitPeerOptions.MaxSize), + string.Create( + CultureInfo.InvariantCulture, + $"must be an integer in the range 1–{TransitPeerOptions.MaxMaxSize}."))); + } + } + + private static void ValidateConnectionLimit(string peerName, string property, int? value, List failures) + { + if (value is null) + { + failures.Add(Format(peerName, property, "is required.")); + return; + } + + if (value < 0 || value > TransitPeerPolicy.MaxConnectionLimit) + { + failures.Add( + Format( + peerName, + property, + string.Create( + CultureInfo.InvariantCulture, + $"must be an integer in the range 0–{TransitPeerPolicy.MaxConnectionLimit}."))); + } + } + + private static void ValidateOverlaps( + List<(string Name, TransitPeerOptions Options, List AllowFrom)> parsed, + List failures) + { + for (var i = 0; i < parsed.Count; i++) + { + for (var j = i + 1; j < parsed.Count; j++) + { + if (TryDescribeOverlap(parsed[i], parsed[j], out var message)) + { + failures.Add(message); + } + } + } + } + + private static bool TryDescribeOverlap( + (string Name, TransitPeerOptions Options, List AllowFrom) left, + (string Name, TransitPeerOptions Options, List AllowFrom) right, + out string message) + { + message = string.Empty; + foreach (var leftEntry in left.AllowFrom) + { + foreach (var rightEntry in right.AllowFrom) + { + if (leftEntry is TransitAllowFromEntry.Hostname leftHost + && rightEntry is TransitAllowFromEntry.Hostname rightHost + && leftHost.DnsName.Equals(rightHost.DnsName, StringComparison.OrdinalIgnoreCase)) + { + message = string.Create( + CultureInfo.InvariantCulture, + $"Transit peers '{left.Name}' and '{right.Name}' have overlapping AllowFrom hostname '{leftHost.DnsName}'."); + return true; + } + + if (leftEntry is TransitAllowFromEntry.Literal leftLiteral + && rightEntry is TransitAllowFromEntry.Literal rightLiteral + && leftLiteral.Prefix.Overlaps(rightLiteral.Prefix)) + { + message = string.Create( + CultureInfo.InvariantCulture, + $"Transit peers '{left.Name}' and '{right.Name}' have overlapping AllowFrom prefixes '{leftLiteral.Original}' and '{rightLiteral.Original}'."); + return true; + } + } + } + + return false; + } + + private static string Format(string peerName, string? property, string detail) + { + return property is null + ? string.Create(CultureInfo.InvariantCulture, $"Transit:{peerName}: {detail}") + : string.Create(CultureInfo.InvariantCulture, $"Transit:{peerName}:{property} {detail}"); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitSslMode.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitSslMode.cs new file mode 100644 index 00000000..af484eab --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitSslMode.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Canonical TLS mode for a configured Transit peer. + /// + /// + /// Configuration accepts these values case-insensitively. Invalid values fail validation; + /// they are never silently treated as plaintext. + /// + public enum TransitSslMode + { + /// No TLS (blank configuration value). + None = 0, + + /// Native TLS on connect (TLS). + Tls = 1, + + /// Cleartext TCP upgraded with STARTTLS (STARTTLS). + StartTls = 2, + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/TransitSslParser.cs b/src/VectorNNTPD.ReaderService/Configuration/TransitSslParser.cs new file mode 100644 index 00000000..d1d086f1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/TransitSslParser.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.ReaderService.Configuration +{ + /// Parses Transit Ssl configuration values into . + public static class TransitSslParser + { + /// + /// Parses a configuration value case-insensitively and returns the canonical mode. + /// + public static bool TryParse(string? value, out TransitSslMode mode, out string? error) + { + mode = TransitSslMode.None; + error = null; + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + var trimmed = value.Trim(); + if (trimmed.Equals("TLS", StringComparison.OrdinalIgnoreCase)) + { + mode = TransitSslMode.Tls; + return true; + } + + if (trimmed.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + mode = TransitSslMode.StartTls; + return true; + } + + error = "Ssl must be blank, 'TLS', or 'STARTTLS'."; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Configuration/XTraceKeyParser.cs b/src/VectorNNTPD.ReaderService/Configuration/XTraceKeyParser.cs new file mode 100644 index 00000000..1343b4e3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Configuration/XTraceKeyParser.cs @@ -0,0 +1,56 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.ReaderService.Configuration +{ + /// + /// Decodes the persisted POST X-Trace AES-256 key from configuration text. + /// + /// + /// Accepts 64 hex characters or Base64 of exactly 32 bytes. Does not generate keys. + /// Failure paths must not include the supplied secret in messages. + /// + public static class XTraceKeyParser + { + /// Required AES-256 key length in bytes. + public const int KeyLength = 32; + + /// + /// Attempts to decode into a 32-byte AES-256 key. + /// + public static bool TryDecode(string? text, [NotNullWhen(true)] out byte[]? key) + { + key = null; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var trimmed = text.Trim(); + if (trimmed.Length == KeyLength * 2) + { + try + { + var hex = Convert.FromHexString(trimmed); + if (hex.Length == KeyLength) + { + key = hex; + return true; + } + } + catch (FormatException) + { + return false; + } + } + + Span buffer = stackalloc byte[KeyLength]; + if (Convert.TryFromBase64String(trimmed, buffer, out var written) && written == KeyLength) + { + key = buffer.ToArray(); + return true; + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsFormatter.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsFormatter.cs new file mode 100644 index 00000000..d821e60a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsFormatter.cs @@ -0,0 +1,155 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// Formats a feed-diagnostics snapshot for a single Information log. + /// + /// Output contains peer names, remote IP/port, counters, and wait totals. + /// It does not include Message-IDs, credentials, or article payloads. + /// + public static class FeedDiagnosticsFormatter + { + /// Builds the periodic snapshot text. + public static string Format(FeedDiagnosticsSnapshot snapshot, bool includeSessions) + { + ArgumentNullException.ThrowIfNull(snapshot); + var sb = new StringBuilder(1024); + sb.Append(CultureInfo.InvariantCulture, $"FEED interval={FormatSeconds(snapshot.Interval)}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={snapshot.IntervalArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" rate={snapshot.ArticleMegabitsPerSecond:F1} Mbps"); + sb.Append(CultureInfo.InvariantCulture, $" articles/sec={snapshot.ArticlesPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage rx_app bytes={snapshot.IntervalTcpBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.TcpMegabitsPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" articles_rx={snapshot.IntervalArticlesReceived}"); + sb.Append(CultureInfo.InvariantCulture, $" articles/sec={snapshot.ArticlesReceivedPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" commands={snapshot.IntervalCommands}"); + sb.Append(CultureInfo.InvariantCulture, $" commands/sec={snapshot.CommandsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage ingest admitted={snapshot.IntervalQueuedArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" bytes={snapshot.IntervalQueuedBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.IngestMegabitsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage process completed={snapshot.IntervalArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.ProcessMegabitsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(" stage routing status=not_implemented"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage spool written={snapshot.IntervalSpoolArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" bytes={snapshot.IntervalSpoolBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.SpoolMegabitsPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" workers={snapshot.SpoolActiveWorkers}"); + sb.AppendLine(); + + var queueWaitMs = 0d; + var historyWaitMs = 0d; + foreach (var session in snapshot.Sessions) + { + queueWaitMs += session.QueueWaitTime.TotalMilliseconds; + historyWaitMs += session.HistoryWaitTime.TotalMilliseconds; + } + + sb.Append(CultureInfo.InvariantCulture, $" queue articles={snapshot.QueueCount} bytes={snapshot.QueueBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" peak_articles={snapshot.QueuePeakCount} peak_bytes={snapshot.QueuePeakBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting={snapshot.QueueWaitingProducers}"); + sb.Append(CultureInfo.InvariantCulture, $" limit={snapshot.QueueMemoryLimit}"); + sb.Append(CultureInfo.InvariantCulture, $" admit_fail={snapshot.QueueAdmissionFailures}"); + sb.Append(CultureInfo.InvariantCulture, $" admission_wait_ms={queueWaitMs:F0}"); + sb.AppendLine(); + + var lookups = snapshot.HistoryHits + snapshot.HistoryMisses + snapshot.HistoryErrors; + sb.Append(CultureInfo.InvariantCulture, $" history lookups={lookups} hits={snapshot.HistoryHits} misses={snapshot.HistoryMisses} errors={snapshot.HistoryErrors}"); + sb.Append(CultureInfo.InvariantCulture, $" wait_ms={historyWaitMs:F0}"); + sb.AppendLine(); + + var idle = 0; + var receiving = 0; + var waitingHistory = 0; + var waitingQueue = 0; + var waitingWindow = 0; + var completing = 0; + foreach (var session in snapshot.Sessions) + { + switch (session.State) + { + case FeedSessionState.Receiving: + receiving++; + break; + case FeedSessionState.WaitingHistory: + waitingHistory++; + break; + case FeedSessionState.WaitingQueue: + waitingQueue++; + break; + case FeedSessionState.WaitingWindow: + waitingWindow++; + break; + case FeedSessionState.Completing: + completing++; + break; + default: + idle++; + break; + } + } + + sb.Append(CultureInfo.InvariantCulture, $" sessions active={snapshot.ActiveSessions}"); + sb.Append(CultureInfo.InvariantCulture, $" established={snapshot.ActiveSessions}"); + sb.Append(CultureInfo.InvariantCulture, $" idle={idle} receiving={receiving}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting_history={waitingHistory} waiting_queue={waitingQueue}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting_window={waitingWindow} completing={completing}"); + sb.AppendLine(); + + foreach (var peer in snapshot.Peers) + { + var avg = peer.ArticlesReceived > 0 ? peer.ArticleBytes / (double)peer.ArticlesReceived : 0; + sb.Append(CultureInfo.InvariantCulture, $" {peer.PeerName}"); + sb.Append(CultureInfo.InvariantCulture, $" active={peer.ActiveConnections}/{peer.ConfiguredMaxConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" peak={peer.PeakConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" accepted={peer.AcceptedConnections} rejected={peer.RejectedConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={peer.ArticlesCompleted} bytes={peer.ArticleBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" avg_article={avg:F0} checks={peer.Checks}"); + sb.AppendLine(); + } + + if (!includeSessions || snapshot.Sessions.Count == 0) + { + return sb.ToString(); + } + + var shown = 0; + foreach (var session in snapshot.Sessions) + { + if (shown >= 50) + { + sb.AppendLine(" ..."); + break; + } + + sb.Append(CultureInfo.InvariantCulture, $" {session.Remote} peer={session.PeerName}"); + sb.Append(CultureInfo.InvariantCulture, $" age={FormatSeconds(session.Age)} state={session.State}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={session.ArticlesCompleted} bytes={session.ArticleBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" iv_bytes={session.IntervalBytes}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_mbps={FeedRateCalculator.MegabitsPerSecond(session.IntervalBytes, snapshot.Interval):F1}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_art={session.IntervalArticlesReceived}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_art/s={FeedRateCalculator.PerSecond(session.IntervalArticlesReceived, snapshot.Interval):F1}"); + sb.Append(CultureInfo.InvariantCulture, $" last={FormatSeconds(session.SinceLastArticle)} idle={FormatSeconds(session.Idle)}"); + sb.Append(CultureInfo.InvariantCulture, $" rx={FormatSeconds(session.ReceiveTime)} hist={FormatSeconds(session.HistoryWaitTime)} q={FormatSeconds(session.QueueWaitTime)}"); + sb.AppendLine(); + shown++; + } + + return sb.ToString(); + } + + private static string FormatSeconds(TimeSpan value) => + value.TotalSeconds.ToString("F1", CultureInfo.InvariantCulture) + "s"; + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsHub.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsHub.cs new file mode 100644 index 00000000..8ba2a93e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsHub.cs @@ -0,0 +1,400 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Process-wide feed-diagnostics counters. Safe for concurrent session attach/release. + /// + public sealed class FeedDiagnosticsHub : IFeedDiagnostics + { + private readonly ConcurrentDictionary _sessions = new(); + private readonly ConcurrentDictionary _sessionPriors = new(); + private readonly ConcurrentDictionary _peers = new(StringComparer.Ordinal); + private long _priorArticles; + private long _priorArticlesReceived; + private long _priorBytes; + private long _priorCommands; + private long _priorQueuedArticles; + private long _priorQueuedBytes; + private long _priorTcpBytes; + private long _priorSpoolArticles; + private long _priorSpoolBytes; + private long _priorTimestamp; + private long _historyHits; + private long _historyMisses; + private long _historyErrors; + private long _queueFailures; + private long _tcpBytes; + private long _spoolArticles; + private long _spoolBytes; + private int _spoolBusy; + + /// + public bool IsEnabled => true; + + /// + public void OnRejected(string peerName, string remote) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + _ = remote; + var peer = GetPeer(peerName); + Interlocked.Increment(ref peer.Rejected); + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var peerName = session.Authorization.TransitPeerName ?? "none"; + var remote = string.Create( + CultureInfo.InvariantCulture, + $"{session.ClientAddress}:{session.ClientPort}"); + var probe = new FeedSessionProbe(remote, peerName); + var peer = GetPeer(peerName); + Interlocked.Increment(ref peer.Accepted); + while (true) + { + var current = Volatile.Read(ref peer.Active); + var next = current + 1; + if (Interlocked.CompareExchange(ref peer.Active, next, current) == current) + { + UpdatePeak(ref peer.Peak, next); + break; + } + } + + _sessions[probe] = 0; + return probe; + } + + /// + public void OnReleased(FeedSessionProbe? probe) + { + if (probe is null) + { + return; + } + + if (!_sessions.TryRemove(probe, out _)) + { + return; + } + + _sessionPriors.TryRemove(probe, out _); + AddLifetime(probe); + var peer = GetPeer(probe.PeerName); + while (true) + { + var current = Volatile.Read(ref peer.Active); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref peer.Active, current - 1, current) == current) + { + return; + } + } + } + + /// + public void RecordTcpBytes(int bytes) + { + if (bytes > 0) + { + Interlocked.Add(ref _tcpBytes, bytes); + } + } + + /// + public void BeginSpoolWork() => Interlocked.Increment(ref _spoolBusy); + + /// + public void EndSpoolWork(int bytes, bool persisted) + { + if (persisted) + { + Interlocked.Increment(ref _spoolArticles); + if (bytes > 0) + { + Interlocked.Add(ref _spoolBytes, bytes); + } + } + + Interlocked.Decrement(ref _spoolBusy); + } + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(store); + + var now = Stopwatch.GetTimestamp(); + var priorTs = Interlocked.Exchange(ref _priorTimestamp, now); + var sessions = _sessions.Keys.ToArray(); + long articles = 0; + long articlesReceived = 0; + long bytes = 0; + long commands = 0; + long queuedArticles = 0; + long queuedBytes = 0; + long liveHits = 0; + long liveMisses = 0; + long liveErrors = 0; + long liveQueueFail = 0; + var receiving = 0; + var idle = 0; + foreach (var session in sessions) + { + articles += session.ArticlesCompleted; + articlesReceived += session.ArticlesReceived; + bytes += session.ArticleBytes; + commands += session.Commands; + queuedArticles += session.QueueAccepted; + queuedBytes += session.QueueAdmittedBytes; + liveHits += session.HistoryHits; + liveMisses += session.HistoryMisses; + liveErrors += session.HistoryErrors; + liveQueueFail += session.QueueRejected + session.QueueUnavailable; + if (session.State == FeedSessionState.Receiving) + { + receiving++; + } + else if (session.State == FeedSessionState.Idle) + { + idle++; + } + } + + articles += SumPeer(static p => Volatile.Read(ref p.ReleasedArticles)); + articlesReceived += SumPeer(static p => Volatile.Read(ref p.ReleasedArticlesReceived)); + bytes += SumPeer(static p => Volatile.Read(ref p.ReleasedBytes)); + commands += SumPeer(static p => Volatile.Read(ref p.ReleasedCommands)); + queuedArticles += SumPeer(static p => Volatile.Read(ref p.ReleasedQueuedArticles)); + queuedBytes += SumPeer(static p => Volatile.Read(ref p.ReleasedQueuedBytes)); + + var intervalArticles = FeedRateCalculator.NonNegativeDelta( + articles, + Interlocked.Exchange(ref _priorArticles, articles)); + var intervalArticlesReceived = FeedRateCalculator.NonNegativeDelta( + articlesReceived, + Interlocked.Exchange(ref _priorArticlesReceived, articlesReceived)); + var intervalBytes = FeedRateCalculator.NonNegativeDelta( + bytes, + Interlocked.Exchange(ref _priorBytes, bytes)); + var intervalCommands = FeedRateCalculator.NonNegativeDelta( + commands, + Interlocked.Exchange(ref _priorCommands, commands)); + var intervalQueuedArticles = FeedRateCalculator.NonNegativeDelta( + queuedArticles, + Interlocked.Exchange(ref _priorQueuedArticles, queuedArticles)); + var intervalQueuedBytes = FeedRateCalculator.NonNegativeDelta( + queuedBytes, + Interlocked.Exchange(ref _priorQueuedBytes, queuedBytes)); + var tcpBytes = Volatile.Read(ref _tcpBytes); + var intervalTcpBytes = FeedRateCalculator.NonNegativeDelta( + tcpBytes, + Interlocked.Exchange(ref _priorTcpBytes, tcpBytes)); + var spoolArticles = Volatile.Read(ref _spoolArticles); + var intervalSpoolArticles = FeedRateCalculator.NonNegativeDelta( + spoolArticles, + Interlocked.Exchange(ref _priorSpoolArticles, spoolArticles)); + var spoolBytes = Volatile.Read(ref _spoolBytes); + var intervalSpoolBytes = FeedRateCalculator.NonNegativeDelta( + spoolBytes, + Interlocked.Exchange(ref _priorSpoolBytes, spoolBytes)); + + var interval = priorTs == 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(priorTs, now); + var peers = new List(); + foreach (var (name, counters) in _peers) + { + var max = 0; + if (store.Current.Peers.TryGetValue(name, out var policy)) + { + max = policy.MaxIncomingConnections; + } + + var liveArticles = 0L; + var liveCompleted = 0L; + var liveBytes = 0L; + var liveChecks = 0L; + foreach (var session in sessions) + { + if (!string.Equals(session.PeerName, name, StringComparison.Ordinal)) + { + continue; + } + + liveArticles += session.ArticlesReceived; + liveCompleted += session.ArticlesCompleted; + liveBytes += session.ArticleBytes; + liveChecks += session.Checks; + } + + peers.Add(new FeedPeerSnapshot + { + PeerName = name, + ConfiguredMaxConnections = max, + ActiveConnections = Math.Max(0, Volatile.Read(ref counters.Active)), + PeakConnections = Volatile.Read(ref counters.Peak), + AcceptedConnections = Volatile.Read(ref counters.Accepted), + RejectedConnections = Volatile.Read(ref counters.Rejected), + ArticlesReceived = liveArticles + Volatile.Read(ref counters.ReleasedArticlesReceived), + ArticlesCompleted = liveCompleted + Volatile.Read(ref counters.ReleasedArticles), + ArticleBytes = liveBytes + Volatile.Read(ref counters.ReleasedBytes), + Checks = liveChecks + Volatile.Read(ref counters.ReleasedChecks), + }); + } + + peers.Sort(static (a, b) => string.CompareOrdinal(a.PeerName, b.PeerName)); + + var sessionRows = new List(sessions.Length); + foreach (var session in sessions) + { + _sessionPriors.TryGetValue(session, out var prior); + var sessionCommands = session.Commands; + var sessionReceived = session.ArticlesReceived; + var sessionBytes = session.ArticleBytes; + sessionRows.Add(new FeedSessionSnapshot + { + Remote = session.Remote, + PeerName = session.PeerName, + Age = session.Age, + State = session.State, + Commands = sessionCommands, + Checks = session.Checks, + ArticlesReceived = sessionReceived, + ArticlesCompleted = session.ArticlesCompleted, + ArticleBytes = sessionBytes, + IntervalCommands = FeedRateCalculator.NonNegativeDelta(sessionCommands, prior.Commands), + IntervalArticlesReceived = FeedRateCalculator.NonNegativeDelta(sessionReceived, prior.ArticlesReceived), + IntervalBytes = FeedRateCalculator.NonNegativeDelta(sessionBytes, prior.Bytes), + Idle = session.Idle, + SinceLastArticle = session.SinceLastArticle, + ReceiveTime = session.ReceiveTime, + HistoryWaitTime = session.HistoryWaitTime, + QueueWaitTime = session.QueueWaitTime, + MaxReceiveTime = session.MaxReceiveTime, + MaxHistoryWaitTime = session.MaxHistoryWaitTime, + MaxQueueWaitTime = session.MaxQueueWaitTime, + }); + _sessionPriors[session] = new SessionPrior(sessionCommands, sessionReceived, sessionBytes); + } + + sessionRows.Sort(static (a, b) => string.CompareOrdinal(a.Remote, b.Remote)); + + return new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = interval, + Peers = peers, + Sessions = sessionRows, + IntervalArticles = intervalArticles, + IntervalArticlesReceived = intervalArticlesReceived, + IntervalBytes = intervalBytes, + IntervalCommands = intervalCommands, + IntervalTcpBytes = intervalTcpBytes, + IntervalQueuedBytes = intervalQueuedBytes, + IntervalQueuedArticles = intervalQueuedArticles, + IntervalSpoolArticles = intervalSpoolArticles, + IntervalSpoolBytes = intervalSpoolBytes, + TcpBytes = tcpBytes, + SpoolArticles = spoolArticles, + SpoolBytes = spoolBytes, + SpoolActiveWorkers = Math.Max(0, Volatile.Read(ref _spoolBusy)), + ActiveSessions = sessions.Length, + ReceivingSessions = receiving, + IdleSessions = idle, + QueueCount = queue.Count, + QueueBytes = queue.QueuedBytes, + QueuePeakCount = queue.PeakCount, + QueuePeakBytes = queue.PeakQueuedBytes, + QueueWaitingProducers = queue.WaitingProducerCount, + QueueMemoryLimit = queue.MemoryLimitBytes, + HistoryHits = Volatile.Read(ref _historyHits) + liveHits, + HistoryMisses = Volatile.Read(ref _historyMisses) + liveMisses, + HistoryErrors = Volatile.Read(ref _historyErrors) + liveErrors, + QueueAdmissionFailures = Volatile.Read(ref _queueFailures) + liveQueueFail, + }; + } + + private PeerCounters GetPeer(string peerName) => _peers.GetOrAdd(peerName, static _ => new PeerCounters()); + + private long SumPeer(Func selector) + { + var total = 0L; + foreach (var pair in _peers) + { + total += selector(pair.Value); + } + + return total; + } + + private void AddLifetime(FeedSessionProbe probe) + { + var peer = GetPeer(probe.PeerName); + Interlocked.Add(ref peer.ReleasedArticles, probe.ArticlesCompleted); + Interlocked.Add(ref peer.ReleasedArticlesReceived, probe.ArticlesReceived); + Interlocked.Add(ref peer.ReleasedBytes, probe.ArticleBytes); + Interlocked.Add(ref peer.ReleasedChecks, probe.Checks); + Interlocked.Add(ref peer.ReleasedCommands, probe.Commands); + Interlocked.Add(ref peer.ReleasedQueuedArticles, probe.QueueAccepted); + Interlocked.Add(ref peer.ReleasedQueuedBytes, probe.QueueAdmittedBytes); + Interlocked.Add(ref _historyHits, probe.HistoryHits); + Interlocked.Add(ref _historyMisses, probe.HistoryMisses); + Interlocked.Add(ref _historyErrors, probe.HistoryErrors); + Interlocked.Add(ref _queueFailures, probe.QueueRejected + probe.QueueUnavailable); + } + + private static void UpdatePeak(ref int location, int candidate) + { + while (true) + { + var current = Volatile.Read(ref location); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref location, candidate, current) == current) + { + return; + } + } + } + + private sealed class PeerCounters + { + public int Active; + public int Peak; + public long Accepted; + public long Rejected; + public long ReleasedArticles; + public long ReleasedArticlesReceived; + public long ReleasedBytes; + public long ReleasedChecks; + public long ReleasedCommands; + public long ReleasedQueuedArticles; + public long ReleasedQueuedBytes; + } + + private readonly struct SessionPrior(long commands, long articlesReceived, long bytes) + { + public long Commands { get; } = commands; + + public long ArticlesReceived { get; } = articlesReceived; + + public long Bytes { get; } = bytes; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsLogMessages.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsLogMessages.cs new file mode 100644 index 00000000..a0d59955 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsLogMessages.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// Source-generated feed-diagnostics reporter messages. + internal static partial class FeedDiagnosticsLogMessages + { + [LoggerMessage( + EventId = 2300, + Level = LogLevel.Information, + Message = "Feed diagnostics enabled (interval {IntervalSeconds}s, sessions={IncludeSessions})")] + public static partial void Enabled(ILogger logger, int IntervalSeconds, bool IncludeSessions); + + [LoggerMessage( + EventId = 2301, + Level = LogLevel.Information, + Message = "{Snapshot}")] + public static partial void Snapshot(ILogger logger, string Snapshot); + + [LoggerMessage( + EventId = 2302, + Level = LogLevel.Information, + Message = "Feed diagnostics stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsService.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsService.cs new file mode 100644 index 00000000..ec5a5968 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsService.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Periodic feed-diagnostics reporter. Does not run when is false. + /// + public sealed class FeedDiagnosticsService : IApplicationService, IAsyncDisposable + { + private readonly IFeedDiagnostics _feed; + private readonly IArticleIngestionQueue _queue; + private readonly TransitConfigurationStore _store; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _disposed; + + /// Initializes a new instance of the class. + public FeedDiagnosticsService( + IFeedDiagnostics feed, + IArticleIngestionQueue queue, + TransitConfigurationStore store, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(feed); + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _feed = feed; + _queue = queue; + _store = store; + _options = options; + _logger = logger; + } + + /// + public string Name => "FeedDiagnostics"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!_feed.IsEnabled) + { + return Task.CompletedTask; + } + + var interval = _options.Value.FeedDiagnostics?.IntervalSeconds + ?? FeedDiagnosticsOptions.DefaultIntervalSeconds; + var includeSessions = _options.Value.FeedDiagnostics?.IncludeSessions ?? true; + FeedDiagnosticsLogMessages.Enabled(_logger, interval, includeSessions); + _execution = RunAsync(TimeSpan.FromSeconds(interval), includeSessions, _runCts.Token); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is null) + { + return; + } + + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + if (_feed.IsEnabled) + { + FeedDiagnosticsLogMessages.Stopped(_logger); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + _runCts.Dispose(); + } + + private async Task RunAsync(TimeSpan interval, bool includeSessions, CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(interval, cancellationToken).ConfigureAwait(false); + var snapshot = _feed.CaptureSnapshot(_queue, _store); + FeedDiagnosticsLogMessages.Snapshot(_logger, FeedDiagnosticsFormatter.Format(snapshot, includeSessions)); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsSnapshot.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsSnapshot.cs new file mode 100644 index 00000000..51e6f89f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedDiagnosticsSnapshot.cs @@ -0,0 +1,236 @@ +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// Point-in-time feed-diagnostics view (off the hot path). + public sealed class FeedDiagnosticsSnapshot + { + /// Empty snapshot used when diagnostics are disabled. + public static FeedDiagnosticsSnapshot Empty { get; } = new() + { + CapturedAt = DateTimeOffset.UnixEpoch, + Interval = TimeSpan.Zero, + Peers = [], + Sessions = [], + }; + + /// Gets the capture time. + public required DateTimeOffset CapturedAt { get; init; } + + /// Gets the interval since the previous snapshot (zero on the first). + public required TimeSpan Interval { get; init; } + + /// Gets per-peer rows. + public required IReadOnlyList Peers { get; init; } + + /// Gets per-session rows. + public required IReadOnlyList Sessions { get; init; } + + /// Gets articles completed in this interval. + public long IntervalArticles { get; init; } + + /// Gets framed articles received in this interval. + public long IntervalArticlesReceived { get; init; } + + /// Gets article bytes framed in this interval. + public long IntervalBytes { get; init; } + + /// Gets commands observed in this interval. + public long IntervalCommands { get; init; } + + /// Gets application-layer upstream bytes read in this interval. + public long IntervalTcpBytes { get; init; } + + /// Gets ingestion-queue admitted payload bytes in this interval. + public long IntervalQueuedBytes { get; init; } + + /// Gets articles successfully admitted to the ingestion queue in this interval. + public long IntervalQueuedArticles { get; init; } + + /// Gets spool articles persisted in this interval. + public long IntervalSpoolArticles { get; init; } + + /// Gets spool payload bytes persisted in this interval. + public long IntervalSpoolBytes { get; init; } + + /// Gets lifetime application-layer upstream bytes read. + public long TcpBytes { get; init; } + + /// Gets lifetime spool articles persisted. + public long SpoolArticles { get; init; } + + /// Gets lifetime spool payload bytes persisted. + public long SpoolBytes { get; init; } + + /// Gets whether the spool drain loop is currently persisting an article. + public int SpoolActiveWorkers { get; init; } + + /// Gets current admitted sessions (established and running). + public int ActiveSessions { get; init; } + + /// Gets sessions in . + public int ReceivingSessions { get; init; } + + /// Gets sessions in . + public int IdleSessions { get; init; } + + /// Gets current queue depth. + public int QueueCount { get; init; } + + /// Gets current queued payload bytes. + public long QueueBytes { get; init; } + + /// Gets queue peak article count. + public int QueuePeakCount { get; init; } + + /// Gets queue peak payload bytes. + public long QueuePeakBytes { get; init; } + + /// Gets producers waiting for queue byte budget. + public int QueueWaitingProducers { get; init; } + + /// Gets queue memory limit. + public long QueueMemoryLimit { get; init; } + + /// Gets lifetime HistoryDB Seen count (all sessions, including released). + public long HistoryHits { get; init; } + + /// Gets lifetime HistoryDB Unseen count. + public long HistoryMisses { get; init; } + + /// Gets lifetime HistoryDB error count. + public long HistoryErrors { get; init; } + + /// Gets lifetime queue admission failures (rejected + unavailable + full). + public long QueueAdmissionFailures { get; init; } + + /// Gets completed articles/sec over . + public double ArticlesPerSecond => FeedRateCalculator.PerSecond(IntervalArticles, Interval); + + /// Gets framed articles received/sec over . + public double ArticlesReceivedPerSecond => + FeedRateCalculator.PerSecond(IntervalArticlesReceived, Interval); + + /// Gets commands/sec over . + public double CommandsPerSecond => FeedRateCalculator.PerSecond(IntervalCommands, Interval); + + /// Gets bits/sec of framed article payload over . + public double BitsPerSecond => FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval) * 1_000_000d; + + /// Gets framed-article Mbps over . + public double ArticleMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval); + + /// Gets application-layer receive Mbps over . + public double TcpMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalTcpBytes, Interval); + + /// Gets ingestion-queue admit Mbps over . + public double IngestMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalQueuedBytes, Interval); + + /// Gets completed-article Mbps over . + public double ProcessMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval); + + /// Gets spool persist Mbps over . + public double SpoolMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalSpoolBytes, Interval); + } + + /// Per-peer connection and article totals. + public sealed class FeedPeerSnapshot + { + /// Gets the Transit identifier. + public required string PeerName { get; init; } + + /// Gets configured MaxIncomingConnections, or 0 when unknown. + public int ConfiguredMaxConnections { get; init; } + + /// Gets current admitted sessions. + public int ActiveConnections { get; init; } + + /// Gets peak admitted sessions. + public int PeakConnections { get; init; } + + /// Gets lifetime admits. + public long AcceptedConnections { get; init; } + + /// Gets lifetime admission rejects. + public long RejectedConnections { get; init; } + + /// Gets lifetime framed articles. + public long ArticlesReceived { get; init; } + + /// Gets lifetime completed articles. + public long ArticlesCompleted { get; init; } + + /// Gets lifetime framed bytes. + public long ArticleBytes { get; init; } + + /// Gets lifetime CHECK commands. + public long Checks { get; init; } + } + + /// Compact per-session row. + public sealed class FeedSessionSnapshot + { + /// Gets ip:port. + public required string Remote { get; init; } + + /// Gets the Transit identifier. + public required string PeerName { get; init; } + + /// Gets session age. + public TimeSpan Age { get; init; } + + /// Gets coarse activity. + public FeedSessionState State { get; init; } + + /// Gets commands observed. + public long Commands { get; init; } + + /// Gets CHECK commands. + public long Checks { get; init; } + + /// Gets framed articles. + public long ArticlesReceived { get; init; } + + /// Gets completed articles. + public long ArticlesCompleted { get; init; } + + /// Gets framed bytes. + public long ArticleBytes { get; init; } + + /// Gets commands observed in this interval. + public long IntervalCommands { get; init; } + + /// Gets framed articles received in this interval. + public long IntervalArticlesReceived { get; init; } + + /// Gets framed article bytes in this interval. + public long IntervalBytes { get; init; } + + /// Gets idle time. + public TimeSpan Idle { get; init; } + + /// Gets time since last completed article. + public TimeSpan SinceLastArticle { get; init; } + + /// Gets total receive time. + public TimeSpan ReceiveTime { get; init; } + + /// Gets total HistoryDB wait. + public TimeSpan HistoryWaitTime { get; init; } + + /// Gets total queue wait. + public TimeSpan QueueWaitTime { get; init; } + + /// Gets max receive time. + public TimeSpan MaxReceiveTime { get; init; } + + /// Gets max HistoryDB wait. + public TimeSpan MaxHistoryWaitTime { get; init; } + + /// Gets max queue wait. + public TimeSpan MaxQueueWaitTime { get; init; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedRateCalculator.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedRateCalculator.cs new file mode 100644 index 00000000..98c64e98 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedRateCalculator.cs @@ -0,0 +1,45 @@ +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Interval rate helpers for feed-diagnostics snapshots. + /// + /// + /// All methods are allocation-free and safe for zero elapsed time and + /// non-monotonic counters (reset or observed reordering). + /// + public static class FeedRateCalculator + { + /// + /// Returns current - previous when the counter advanced; otherwise 0. + /// + public static long NonNegativeDelta(long current, long previous) + { + if (current < previous) + { + return 0; + } + + return current - previous; + } + + /// + /// Returns per second, or 0 when elapsed time is not positive. + /// + public static double PerSecond(long delta, TimeSpan elapsed) + { + var seconds = elapsed.TotalSeconds; + if (seconds <= 0 || delta <= 0) + { + return 0; + } + + return delta / seconds; + } + + /// + /// Returns megabits/sec from a byte delta (decimal Mbps: bits / 1e6). + /// + public static double MegabitsPerSecond(long byteDelta, TimeSpan elapsed) => + PerSecond(byteDelta, elapsed) * 8d / 1_000_000d; + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionProbe.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionProbe.cs new file mode 100644 index 00000000..31003346 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionProbe.cs @@ -0,0 +1,291 @@ +using System.Diagnostics; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.History; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Per-session counters and wait totals for opt-in feed diagnostics. + /// + /// + /// Hot-path updates are Interlocked / Volatile only. No logging. No Message-IDs or payloads. + /// + public sealed class FeedSessionProbe + { + private readonly long _startedTimestamp; + private int _state; + private long _commands; + private long _checks; + private long _articlesReceived; + private long _articlesCompleted; + private long _articleBytes; + private long _receiveTicks; + private long _historyTicks; + private long _queueWaitTicks; + private long _responseTicks; + private long _maxReceiveTicks; + private long _maxHistoryTicks; + private long _maxQueueWaitTicks; + private long _maxArticleIntervalTicks; + private long _lastActivityTimestamp; + private long _lastArticleCompleteTimestamp; + private long _historyHits; + private long _historyMisses; + private long _historyErrors; + private long _queueAccepted; + private long _queueRejected; + private long _queueUnavailable; + private long _queueAdmittedBytes; + private long _duplicates; + + /// Initializes a probe for one accepted session. + public FeedSessionProbe(string remote, string peerName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(remote); + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + Remote = remote; + PeerName = peerName; + _startedTimestamp = Stopwatch.GetTimestamp(); + _lastActivityTimestamp = _startedTimestamp; + } + + /// Gets the effective client endpoint (ip:port). + public string Remote { get; } + + /// Gets the Transit identifier, or none. + public string PeerName { get; } + + /// Gets the session age. + public TimeSpan Age => Stopwatch.GetElapsedTime(_startedTimestamp); + + /// Gets the coarse activity state. + public FeedSessionState State => (FeedSessionState)Volatile.Read(ref _state); + + /// Gets commands observed on this session. + public long Commands => Volatile.Read(ref _commands); + + /// Gets CHECK commands observed on this session. + public long Checks => Volatile.Read(ref _checks); + + /// Gets framed articles (receive completed). + public long ArticlesReceived => Volatile.Read(ref _articlesReceived); + + /// Gets articles that reached a 239/439/400 completion. + public long ArticlesCompleted => Volatile.Read(ref _articlesCompleted); + + /// Gets framed article payload bytes (terminator omitted). + public long ArticleBytes => Volatile.Read(ref _articleBytes); + + /// Gets HistoryDB Seen outcomes. + public long HistoryHits => Volatile.Read(ref _historyHits); + + /// Gets HistoryDB Unseen outcomes. + public long HistoryMisses => Volatile.Read(ref _historyMisses); + + /// Gets HistoryDB Unavailable / exception outcomes. + public long HistoryErrors => Volatile.Read(ref _historyErrors); + + /// Gets successful ingestion-queue admits. + public long QueueAccepted => Volatile.Read(ref _queueAccepted); + + /// Gets payload bytes successfully admitted to the ingestion queue. + public long QueueAdmittedBytes => Volatile.Read(ref _queueAdmittedBytes); + + /// Gets queue rejects (article larger than budget). + public long QueueRejected => Volatile.Read(ref _queueRejected); + + /// Gets queue unavailable / shutdown outcomes. + public long QueueUnavailable => Volatile.Read(ref _queueUnavailable); + + /// Gets TAKETHIS completions treated as already-seen. + public long Duplicates => Volatile.Read(ref _duplicates); + + /// Gets total time spent in article receive. + public TimeSpan ReceiveTime => TicksToTime(Volatile.Read(ref _receiveTicks)); + + /// Gets total time spent awaiting HistoryDB. + public TimeSpan HistoryWaitTime => TicksToTime(Volatile.Read(ref _historyTicks)); + + /// Gets total time spent awaiting queue admission. + public TimeSpan QueueWaitTime => TicksToTime(Volatile.Read(ref _queueWaitTicks)); + + /// Gets total time spent enqueueing the protocol response. + public TimeSpan ResponseTime => TicksToTime(Volatile.Read(ref _responseTicks)); + + /// Gets the longest single article receive. + public TimeSpan MaxReceiveTime => TicksToTime(Volatile.Read(ref _maxReceiveTicks)); + + /// Gets the longest single HistoryDB wait. + public TimeSpan MaxHistoryWaitTime => TicksToTime(Volatile.Read(ref _maxHistoryTicks)); + + /// Gets the longest single queue-admission wait. + public TimeSpan MaxQueueWaitTime => TicksToTime(Volatile.Read(ref _maxQueueWaitTicks)); + + /// Gets the longest interval between completed articles. + public TimeSpan MaxArticleInterval => TicksToTime(Volatile.Read(ref _maxArticleIntervalTicks)); + + /// Gets time since the last recorded activity. + public TimeSpan Idle => Stopwatch.GetElapsedTime(Volatile.Read(ref _lastActivityTimestamp)); + + /// Gets time since the previous completed article, or . + public TimeSpan SinceLastArticle + { + get + { + var last = Volatile.Read(ref _lastArticleCompleteTimestamp); + return last == 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(last); + } + } + + /// Sets the coarse activity state. + public void SetState(FeedSessionState state) + { + Volatile.Write(ref _state, (int)state); + Touch(); + } + + /// Increments the command counter. + public void RecordCommand() + { + Interlocked.Increment(ref _commands); + Touch(); + } + + /// Increments the CHECK counter. + public void RecordCheck() + { + Interlocked.Increment(ref _checks); + Interlocked.Increment(ref _commands); + Touch(); + } + + /// Records a completed article frame. + public void RecordArticleReceived(int bytes, long receiveTicks) + { + Interlocked.Increment(ref _articlesReceived); + Interlocked.Add(ref _articleBytes, bytes); + AddTicks(ref _receiveTicks, ref _maxReceiveTicks, receiveTicks); + Touch(); + } + + /// Records a HistoryDB Peek/Lookup outcome and wait. + public void RecordHistory(HistoryLookupResult result, long waitTicks) + { + switch (result) + { + case HistoryLookupResult.Seen: + Interlocked.Increment(ref _historyHits); + break; + case HistoryLookupResult.Unseen: + Interlocked.Increment(ref _historyMisses); + break; + default: + Interlocked.Increment(ref _historyErrors); + break; + } + + AddTicks(ref _historyTicks, ref _maxHistoryTicks, waitTicks); + Touch(); + } + + /// Records ingestion-queue admission outcome and wait. + public void RecordQueue(ArticleEnqueueResult result, long waitTicks, int articleBytes = 0) + { + switch (result) + { + case ArticleEnqueueResult.Accepted: + Interlocked.Increment(ref _queueAccepted); + if (articleBytes > 0) + { + Interlocked.Add(ref _queueAdmittedBytes, articleBytes); + } + + break; + case ArticleEnqueueResult.Rejected: + Interlocked.Increment(ref _queueRejected); + break; + default: + Interlocked.Increment(ref _queueUnavailable); + break; + } + + AddTicks(ref _queueWaitTicks, ref _maxQueueWaitTicks, waitTicks); + Touch(); + } + + /// Records a completed TAKETHIS (or serial fallback) outcome. + public void RecordArticleCompleted(bool duplicate, long responseTicks) + { + Interlocked.Increment(ref _articlesCompleted); + if (duplicate) + { + Interlocked.Increment(ref _duplicates); + } + + if (responseTicks > 0) + { + Interlocked.Add(ref _responseTicks, responseTicks); + } + + var now = Stopwatch.GetTimestamp(); + var previous = Interlocked.Exchange(ref _lastArticleCompleteTimestamp, now); + if (previous != 0) + { + var interval = now - previous; + if (interval > 0) + { + UpdateMax(ref _maxArticleIntervalTicks, interval); + } + } + + Volatile.Write(ref _state, (int)FeedSessionState.Idle); + Touch(now); + } + + /// Adds response-enqueue wait without completing an article (CHECK). + public void RecordResponseWait(long responseTicks) + { + if (responseTicks > 0) + { + Interlocked.Add(ref _responseTicks, responseTicks); + } + + Touch(); + } + + private void Touch() => Touch(Stopwatch.GetTimestamp()); + + private void Touch(long timestamp) => Volatile.Write(ref _lastActivityTimestamp, timestamp); + + private static TimeSpan TicksToTime(long ticks) => + ticks <= 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(0, ticks); + + private static void AddTicks(ref long total, ref long max, long ticks) + { + if (ticks <= 0) + { + return; + } + + Interlocked.Add(ref total, ticks); + UpdateMax(ref max, ticks); + } + + private static void UpdateMax(ref long location, long candidate) + { + while (true) + { + var current = Volatile.Read(ref location); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref location, candidate, current) == current) + { + return; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionState.cs b/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionState.cs new file mode 100644 index 00000000..c651a503 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/FeedSessionState.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// Coarse activity for one NNTP session in a feed-diagnostics snapshot. + public enum FeedSessionState : byte + { + /// Between commands, or no in-flight article work. + Idle = 0, + + /// Framing or reading an article from the input pipe. + Receiving = 1, + + /// Waiting for HistoryDB Peek/Lookup to complete. + WaitingHistory = 2, + + /// Waiting for ingestion-queue byte-budget admission. + WaitingQueue = 3, + + /// Blocked on CHECK/TAKETHIS pipeline depth. + WaitingWindow = 4, + + /// Emitting the transfer/CHECK response. + Completing = 5, + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/IFeedDiagnostics.cs b/src/VectorNNTPD.ReaderService/Diagnostics/IFeedDiagnostics.cs new file mode 100644 index 00000000..37436822 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/IFeedDiagnostics.cs @@ -0,0 +1,41 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Opt-in real-feed observability. Disabled implementations are no-ops. + /// + public interface IFeedDiagnostics + { + /// Gets whether probes should be attached and snapshots emitted. + bool IsEnabled { get; } + + /// Records a Transit admission rejection (no session runs). + void OnRejected(string peerName, string remote); + + /// Attaches and registers a probe after a session is admitted. + FeedSessionProbe? OnAccepted(NntpSession session); + + /// Unregisters a probe when the session ends. + void OnReleased(FeedSessionProbe? probe); + + /// + /// Adds application-layer bytes read from an upstream connection (plain TCP payload, + /// or TLS-decrypted octets). No-op when disabled. + /// + void RecordTcpBytes(int bytes); + + /// Marks the single spool drain loop as busy around one persist. + void BeginSpoolWork(); + + /// Records one successfully persisted spool article and clears the busy mark. + void EndSpoolWork(int bytes, bool persisted); + + /// Builds a point-in-time snapshot for the reporter. + FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store); + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/IngestionPipelineMetrics.cs b/src/VectorNNTPD.ReaderService/Diagnostics/IngestionPipelineMetrics.cs new file mode 100644 index 00000000..0bd467ed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/IngestionPipelineMetrics.cs @@ -0,0 +1,692 @@ +using System.Diagnostics; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// + /// Always-on ingestion-pipeline histograms. Samples are lock-free counters; + /// ApplicationTelemetry emits one Information snapshot per minute (no per-article logs). + /// + /// + /// OverviewDB publishes use an asynchronous outstanding-confirmation window. + /// records the publish write duration + /// (not fused with broker ack). tracks the + /// outstanding confirmation count. Confirms remain enabled; crash loss of + /// outstanding OverviewDB work is acceptable. + /// + public sealed class IngestionPipelineMetrics + { + /// Process-wide instance used by TAKETHIS and the ingestion queue. + public static IngestionPipelineMetrics Shared { get; } = new(); + + private readonly DurationHistogram _dequeueWait = new(); + private readonly DurationHistogram _toPublishStart = new(); + private readonly DurationHistogram _encode = new(); + private readonly DurationHistogram _gateWait = new(); + private readonly DurationHistogram _publish = new(); + private readonly DurationHistogram _confirm = new(); + private readonly DurationHistogram _handoff = new(); + private readonly DurationHistogram _news = new(); + private readonly DurationHistogram _workerItem = new(); + private readonly DurationHistogram _emitGateWait = new(); + private readonly DurationHistogram _takeThisEnqueue = new(); + private readonly DurationHistogram _enqueueWait = new(); + private readonly DurationHistogram _budgetWait = new(); + private readonly DurationHistogram _occupiedFullWait = new(); + + private long _workerItems; + private long _confirmFailures; + private long _confirmTimeouts; + private long _busyTicks; + private long _idleTicks; + private long _occupiedFullWaits; + private long _budgetWaits; + private int _maxQueueDepth; + private int _maxTakeThisOccupied; + private int _maxTakeThisProcessing; + private int _currentWorkers; + private int _minWorkers; + private int _maxWorkers; + private int _inFlightPublishes; + private int _maxInFlightPublishes; + private long _scaleUps; + private long _scaleDowns; + private int _overviewPublisherWorkers; + private int _overviewPublisherMinWorkers; + private int _overviewPublisherMaxWorkers; + private int _overviewWorkCount; + private long _overviewWorkBytes; + private int _overviewWorkWaiting; + private long _overviewPublisherScaleUps; + private long _overviewPublisherScaleDowns; + + /// Records time spent blocked in DequeueAsync. + public void RecordDequeueWait(long startTimestamp) => _dequeueWait.Record(startTimestamp); + + /// Records dequeue completion until OverviewDB work-queue enqueue begins. + public void RecordToPublishStart(long startTimestamp) => _toPublishStart.Record(startTimestamp); + + /// Records OverviewArticleV1 encode duration. + public void RecordEncode(long startTimestamp) => _encode.Record(startTimestamp); + + /// Records OverviewDB publisher SemaphoreSlim wait. + public void RecordGateWait(long startTimestamp) => _gateWait.Record(startTimestamp); + + /// + /// Records OverviewDB publish write duration (async confirms; not fused with broker ack). + /// + public void RecordPublishAndConfirm(long startTimestamp) + { + _publish.Record(startTimestamp); + _confirm.Record(startTimestamp); + } + + /// Records the full OverviewDB publisher call including the publish gate. + public void RecordHandoff(long startTimestamp) => _handoff.Record(startTimestamp); + + /// Records news-log Write duration. + public void RecordNews(long startTimestamp) => _news.Record(startTimestamp); + + /// Records an exact news-log duration. Tests use this so the sample is not the wall clock. + internal void RecordNews(TimeSpan elapsed) => _news.RecordExact(elapsed); + + /// Records one worker item from dequeue return through finish or requeue. + public void RecordWorkerItem(long startTimestamp) + { + _workerItem.Record(startTimestamp); + Interlocked.Increment(ref _workerItems); + } + + /// Adds Stopwatch ticks spent processing a worker item. + public void AddBusyTicks(long stopwatchTicks) + { + if (stopwatchTicks > 0) + { + Interlocked.Add(ref _busyTicks, stopwatchTicks); + } + } + + /// Adds Stopwatch ticks spent waiting for the next dequeue. + public void AddIdleTicks(long stopwatchTicks) + { + if (stopwatchTicks > 0) + { + Interlocked.Add(ref _idleTicks, stopwatchTicks); + } + } + + /// Records a broker nack, return, or other publish failure (not a timeout). + public void RecordConfirmFailure() => Interlocked.Increment(ref _confirmFailures); + + /// Records a publisher-confirm timeout. + public void RecordConfirmTimeout() => Interlocked.Increment(ref _confirmTimeouts); + + /// Records TAKETHIS _emitGate.WaitAsync duration. + public void RecordEmitGateWait(long startTimestamp) => _emitGateWait.Record(startTimestamp); + + /// Records TAKETHIS time spent inside EnqueueAsync while holding _emitGate. + public void RecordTakeThisEnqueue(long startTimestamp) => _takeThisEnqueue.Record(startTimestamp); + + /// Records every ingestion-queue EnqueueAsync wait, including immediate admits. + public void RecordEnqueueWaitTicks(long stopwatchTicks) => _enqueueWait.RecordStopwatchTicks(stopwatchTicks); + + /// Records a producer wait caused by the global byte budget. + public void RecordBudgetWaitTicks(long stopwatchTicks) + { + Interlocked.Increment(ref _budgetWaits); + _budgetWait.RecordStopwatchTicks(stopwatchTicks); + } + + /// Records time spent in WaitForCapacityAsync because Occupied is at Depth. + public void RecordOccupiedFullWait(long startTimestamp) + { + Interlocked.Increment(ref _occupiedFullWaits); + _occupiedFullWait.Record(startTimestamp); + } + + /// Tracks the high-water ingestion-queue depth observed by the worker. + public void ObserveQueueDepth(int depth) + { + if (depth <= 0) + { + return; + } + + var current = Volatile.Read(ref _maxQueueDepth); + while (depth > current) + { + var previous = Interlocked.CompareExchange(ref _maxQueueDepth, depth, current); + if (previous == current) + { + break; + } + + current = previous; + } + } + + /// Tracks peak TAKETHIS Occupied and concurrent ProcessArticleAsync counts. + public void ObserveTakeThis(int occupied, int processing) + { + UpdateMax(ref _maxTakeThisOccupied, occupied); + UpdateMax(ref _maxTakeThisProcessing, processing); + } + + /// Records the configured worker-pool bounds and current worker count. + public void ObserveWorkerPool(int currentWorkers, int minWorkers, int maxWorkers) + { + Volatile.Write(ref _currentWorkers, Math.Max(0, currentWorkers)); + Volatile.Write(ref _minWorkers, Math.Max(0, minWorkers)); + Volatile.Write(ref _maxWorkers, Math.Max(0, maxWorkers)); + } + + /// Records the current number of leased OverviewDB publish slots. + public void ObserveInFlightPublishes(int inFlight) + { + Volatile.Write(ref _inFlightPublishes, Math.Max(0, inFlight)); + UpdateMax(ref _maxInFlightPublishes, inFlight); + } + + /// Records one worker-pool scale-up decision. + public void RecordScaleUp() => Interlocked.Increment(ref _scaleUps); + + /// Records one worker-pool scale-down decision. + public void RecordScaleDown() => Interlocked.Increment(ref _scaleDowns); + + /// Records OverviewDB publisher worker-pool bounds and current count. + public void ObserveOverviewDbPublisherPool(int currentWorkers, int minWorkers, int maxWorkers) + { + Volatile.Write(ref _overviewPublisherWorkers, Math.Max(0, currentWorkers)); + Volatile.Write(ref _overviewPublisherMinWorkers, Math.Max(0, minWorkers)); + Volatile.Write(ref _overviewPublisherMaxWorkers, Math.Max(0, maxWorkers)); + } + + /// Records OverviewDB in-process work-queue depth gauges. + public void ObserveOverviewDbWorkQueue(int count, long queuedBytes, int waitingProducers) + { + Volatile.Write(ref _overviewWorkCount, Math.Max(0, count)); + Volatile.Write(ref _overviewWorkBytes, Math.Max(0, queuedBytes)); + Volatile.Write(ref _overviewWorkWaiting, Math.Max(0, waitingProducers)); + } + + /// Records one OverviewDB publisher scale-up. + public void RecordOverviewDbPublisherScaleUp() => + Interlocked.Increment(ref _overviewPublisherScaleUps); + + /// Records one OverviewDB publisher scale-down. + public void RecordOverviewDbPublisherScaleDown() => + Interlocked.Increment(ref _overviewPublisherScaleDowns); + + /// + /// Captures OverviewDB work-queue / publisher-pool gauges and interval scale counters. + /// + public OverviewDbStageSnapshot CaptureOverviewDbStage() + { + return new OverviewDbStageSnapshot( + Volatile.Read(ref _overviewPublisherWorkers), + Volatile.Read(ref _overviewPublisherMinWorkers), + Volatile.Read(ref _overviewPublisherMaxWorkers), + Volatile.Read(ref _overviewWorkCount), + Volatile.Read(ref _overviewWorkBytes), + Volatile.Read(ref _overviewWorkWaiting), + Interlocked.Exchange(ref _overviewPublisherScaleUps, 0), + Interlocked.Exchange(ref _overviewPublisherScaleDowns, 0)); + } + + /// Captures interval totals and resets histograms and counters. + public IngestionPipelineSnapshot CaptureInterval() + { + return new IngestionPipelineSnapshot( + Interlocked.Exchange(ref _workerItems, 0), + Interlocked.Exchange(ref _confirmFailures, 0), + Interlocked.Exchange(ref _confirmTimeouts, 0), + Interlocked.Exchange(ref _busyTicks, 0), + Interlocked.Exchange(ref _idleTicks, 0), + Interlocked.Exchange(ref _occupiedFullWaits, 0), + Interlocked.Exchange(ref _budgetWaits, 0), + Interlocked.Exchange(ref _maxQueueDepth, 0), + Interlocked.Exchange(ref _maxTakeThisOccupied, 0), + Interlocked.Exchange(ref _maxTakeThisProcessing, 0), + Volatile.Read(ref _currentWorkers), + Volatile.Read(ref _minWorkers), + Volatile.Read(ref _maxWorkers), + Volatile.Read(ref _inFlightPublishes), + Interlocked.Exchange(ref _maxInFlightPublishes, 0), + Interlocked.Exchange(ref _scaleUps, 0), + Interlocked.Exchange(ref _scaleDowns, 0), + _dequeueWait.CaptureAndReset(), + _toPublishStart.CaptureAndReset(), + _encode.CaptureAndReset(), + _gateWait.CaptureAndReset(), + _publish.CaptureAndReset(), + _confirm.CaptureAndReset(), + _handoff.CaptureAndReset(), + _news.CaptureAndReset(), + _workerItem.CaptureAndReset(), + _emitGateWait.CaptureAndReset(), + _takeThisEnqueue.CaptureAndReset(), + _enqueueWait.CaptureAndReset(), + _budgetWait.CaptureAndReset(), + _occupiedFullWait.CaptureAndReset()); + } + + private static void UpdateMax(ref int location, int candidate) + { + if (candidate <= 0) + { + return; + } + + var current = Volatile.Read(ref location); + while (candidate > current) + { + var previous = Interlocked.CompareExchange(ref location, candidate, current); + if (previous == current) + { + return; + } + + current = previous; + } + } + } + + /// One telemetry interval of ingestion-pipeline histograms. + public readonly struct IngestionPipelineSnapshot + { + /// Initializes a captured interval snapshot. + public IngestionPipelineSnapshot( + long workerItems, + long confirmFailures, + long confirmTimeouts, + long busyTicks, + long idleTicks, + long occupiedFullWaits, + long budgetWaits, + int maxQueueDepth, + int maxTakeThisOccupied, + int maxTakeThisProcessing, + int currentWorkers, + int minWorkers, + int maxWorkers, + int inFlightPublishes, + int maxInFlightPublishes, + long scaleUps, + long scaleDowns, + DurationSnapshot dequeueWait, + DurationSnapshot toPublishStart, + DurationSnapshot encode, + DurationSnapshot gateWait, + DurationSnapshot publish, + DurationSnapshot confirm, + DurationSnapshot handoff, + DurationSnapshot news, + DurationSnapshot workerItem, + DurationSnapshot emitGateWait, + DurationSnapshot takeThisEnqueue, + DurationSnapshot enqueueWait, + DurationSnapshot budgetWait, + DurationSnapshot occupiedFullWait) + { + WorkerItems = workerItems; + ConfirmFailures = confirmFailures; + ConfirmTimeouts = confirmTimeouts; + BusyTicks = busyTicks; + IdleTicks = idleTicks; + OccupiedFullWaits = occupiedFullWaits; + BudgetWaits = budgetWaits; + MaxQueueDepth = maxQueueDepth; + MaxTakeThisOccupied = maxTakeThisOccupied; + MaxTakeThisProcessing = maxTakeThisProcessing; + CurrentWorkers = currentWorkers; + MinWorkers = minWorkers; + MaxWorkers = maxWorkers; + InFlightPublishes = inFlightPublishes; + MaxInFlightPublishes = maxInFlightPublishes; + ScaleUps = scaleUps; + ScaleDowns = scaleDowns; + DequeueWait = dequeueWait; + ToPublishStart = toPublishStart; + Encode = encode; + GateWait = gateWait; + Publish = publish; + Confirm = confirm; + Handoff = handoff; + News = news; + WorkerItem = workerItem; + EmitGateWait = emitGateWait; + TakeThisEnqueue = takeThisEnqueue; + EnqueueWait = enqueueWait; + BudgetWait = budgetWait; + OccupiedFullWait = occupiedFullWait; + } + + /// Gets worker items completed in the interval. + public long WorkerItems { get; } + + /// Gets OverviewDB publish/confirm failures in the interval. + public long ConfirmFailures { get; } + + /// Gets OverviewDB confirm timeouts in the interval. + public long ConfirmTimeouts { get; } + + /// Gets Stopwatch ticks the worker spent processing items. + public long BusyTicks { get; } + + /// Gets Stopwatch ticks the worker spent waiting to dequeue. + public long IdleTicks { get; } + + /// Gets TAKETHIS WaitForCapacityAsync waits while Occupied was at Depth. + public long OccupiedFullWaits { get; } + + /// Gets producers that waited on the global byte budget. + public long BudgetWaits { get; } + + /// Gets the interval high-water ingestion-queue depth observed by the worker. + public int MaxQueueDepth { get; } + + /// Gets the interval high-water TAKETHIS Occupied count. + public int MaxTakeThisOccupied { get; } + + /// Gets the interval high-water concurrent ProcessArticleAsync count. + public int MaxTakeThisProcessing { get; } + + /// Gets the worker count observed at capture. + public int CurrentWorkers { get; } + + /// Gets the configured minimum worker count. + public int MinWorkers { get; } + + /// Gets the configured maximum worker count. + public int MaxWorkers { get; } + + /// Gets in-flight OverviewDB publishes at capture. + public int InFlightPublishes { get; } + + /// Gets the interval high-water in-flight OverviewDB publishes. + public int MaxInFlightPublishes { get; } + + /// Gets scale-up events in the interval. + public long ScaleUps { get; } + + /// Gets scale-down events in the interval. + public long ScaleDowns { get; } + + /// Gets dequeue-wait samples. + public DurationSnapshot DequeueWait { get; } + + /// Gets dequeue-to-OverviewDB-publish-start samples. + public DurationSnapshot ToPublishStart { get; } + + /// Gets protobuf encode samples. + public DurationSnapshot Encode { get; } + + /// Gets OverviewDB publish-gate wait samples. + public DurationSnapshot GateWait { get; } + + /// Gets fused publish-call samples (same as with tracking). + public DurationSnapshot Publish { get; } + + /// Gets fused confirm-wait samples (same as with tracking). + public DurationSnapshot Confirm { get; } + + /// Gets total OverviewDB handoff samples including the gate. + public DurationSnapshot Handoff { get; } + + /// Gets news-log write samples. + public DurationSnapshot News { get; } + + /// Gets total worker-item samples. + public DurationSnapshot WorkerItem { get; } + + /// Gets TAKETHIS emit-gate wait samples. + public DurationSnapshot EmitGateWait { get; } + + /// Gets TAKETHIS EnqueueAsync samples taken while holding the emit gate. + public DurationSnapshot TakeThisEnqueue { get; } + + /// Gets all ingestion-queue EnqueueAsync wait samples. + public DurationSnapshot EnqueueWait { get; } + + /// Gets byte-budget wait samples. + public DurationSnapshot BudgetWait { get; } + + /// Gets Occupied==Depth wait samples. + public DurationSnapshot OccupiedFullWait { get; } + + /// Busy time as a percentage of busy+idle, or 0 when the worker was unused. + public double BusyPercent => Percent(BusyTicks, BusyTicks + IdleTicks); + + /// Confirm-wait time as a percentage of worker busy time. + public double ConfirmBusyPercent => Percent(Confirm.SumMs, TicksToMilliseconds(BusyTicks)); + + /// Converts Stopwatch ticks to whole milliseconds. + public static long TicksToMilliseconds(long stopwatchTicks) + { + if (stopwatchTicks <= 0) + { + return 0; + } + + return (long)Stopwatch.GetElapsedTime(0, stopwatchTicks).TotalMilliseconds; + } + + /// Articles per second over . + public double ArticlesPerSecond(TimeSpan period) + { + var seconds = period.TotalSeconds; + if (seconds <= 0 || WorkerItems <= 0) + { + return 0; + } + + return WorkerItems / seconds; + } + + private static double Percent(long numerator, long denominator) + { + if (numerator <= 0 || denominator <= 0) + { + return 0; + } + + return 100d * numerator / denominator; + } + } + + /// Gauge/counter snapshot for the OverviewDB work-queue publisher stage. + public readonly struct OverviewDbStageSnapshot + { + /// Initializes a stage snapshot. + public OverviewDbStageSnapshot( + int publisherWorkers, + int minPublisherWorkers, + int maxPublisherWorkers, + int workQueueCount, + long workQueueBytes, + int workQueueWaiting, + long publisherScaleUps, + long publisherScaleDowns) + { + PublisherWorkers = publisherWorkers; + MinPublisherWorkers = minPublisherWorkers; + MaxPublisherWorkers = maxPublisherWorkers; + WorkQueueCount = workQueueCount; + WorkQueueBytes = workQueueBytes; + WorkQueueWaiting = workQueueWaiting; + PublisherScaleUps = publisherScaleUps; + PublisherScaleDowns = publisherScaleDowns; + } + + /// Gets current OverviewDB publisher workers. + public int PublisherWorkers { get; } + + /// Gets configured minimum publisher workers. + public int MinPublisherWorkers { get; } + + /// Gets configured maximum publisher workers. + public int MaxPublisherWorkers { get; } + + /// Gets OverviewDB work-queue item count. + public int WorkQueueCount { get; } + + /// Gets OverviewDB work-queue reserved bytes. + public long WorkQueueBytes { get; } + + /// Gets producers waiting on the OverviewDB work queue. + public int WorkQueueWaiting { get; } + + /// Gets publisher scale-ups in the interval. + public long PublisherScaleUps { get; } + + /// Gets publisher scale-downs in the interval. + public long PublisherScaleDowns { get; } + } + + /// Bucketed duration summary for one telemetry interval. + public readonly struct DurationSnapshot + { + /// Initializes a duration snapshot. + public DurationSnapshot(long count, double avgMs, long p50Ms, long p95Ms, long maxMs, long sumMs) + { + Count = count; + AvgMs = avgMs; + P50Ms = p50Ms; + P95Ms = p95Ms; + MaxMs = maxMs; + SumMs = sumMs; + } + + /// Gets the number of samples. + public long Count { get; } + + /// Gets the arithmetic mean in milliseconds. + public double AvgMs { get; } + + /// Gets an estimated 50th percentile in milliseconds. + public long P50Ms { get; } + + /// Gets an estimated 95th percentile in milliseconds. + public long P95Ms { get; } + + /// Gets the maximum sample in milliseconds. + public long MaxMs { get; } + + /// Gets the summed sample duration in milliseconds. + public long SumMs { get; } + } + + /// Lock-free millisecond histogram used by . + internal sealed class DurationHistogram + { + private static readonly double[] UpperMs = + [ + 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, + ]; + + private readonly long[] _buckets = new long[UpperMs.Length + 1]; + private long _count; + private long _sumTicks; + private long _maxTicks; + + public void Record(long startTimestamp) => + RecordElapsed(Stopwatch.GetElapsedTime(startTimestamp)); + + /// Records without reading the clock again. + internal void RecordExact(TimeSpan elapsed) => RecordElapsed(elapsed); + + public void RecordStopwatchTicks(long stopwatchTicks) + { + if (stopwatchTicks <= 0) + { + RecordElapsed(TimeSpan.Zero); + return; + } + + RecordElapsed(Stopwatch.GetElapsedTime(0, stopwatchTicks)); + } + + public DurationSnapshot CaptureAndReset() + { + var count = Interlocked.Exchange(ref _count, 0); + var sumTicks = Interlocked.Exchange(ref _sumTicks, 0); + var maxTicks = Interlocked.Exchange(ref _maxTicks, 0); + var buckets = new long[_buckets.Length]; + for (var i = 0; i < _buckets.Length; i++) + { + buckets[i] = Interlocked.Exchange(ref _buckets[i], 0); + } + + if (count <= 0) + { + return default; + } + + var sumMs = (long)Stopwatch.GetElapsedTime(0, sumTicks).TotalMilliseconds; + var maxMs = (long)Math.Ceiling(Stopwatch.GetElapsedTime(0, maxTicks).TotalMilliseconds); + var avgMs = sumMs / (double)count; + return new DurationSnapshot( + count, + avgMs, + PercentileMs(buckets, count, 0.50), + PercentileMs(buckets, count, 0.95), + maxMs, + sumMs); + } + + private void RecordElapsed(TimeSpan elapsed) + { + if (elapsed < TimeSpan.Zero) + { + elapsed = TimeSpan.Zero; + } + + var stopwatchTicks = (long)(elapsed.TotalSeconds * Stopwatch.Frequency); + Interlocked.Increment(ref _count); + Interlocked.Add(ref _sumTicks, stopwatchTicks); + UpdateMax(stopwatchTicks); + var ms = elapsed.TotalMilliseconds; + var index = UpperMs.Length; + for (var i = 0; i < UpperMs.Length; i++) + { + if (ms <= UpperMs[i]) + { + index = i; + break; + } + } + + Interlocked.Increment(ref _buckets[index]); + } + + private void UpdateMax(long stopwatchTicks) + { + var current = Volatile.Read(ref _maxTicks); + while (stopwatchTicks > current) + { + var previous = Interlocked.CompareExchange(ref _maxTicks, stopwatchTicks, current); + if (previous == current) + { + return; + } + + current = previous; + } + } + + private static long PercentileMs(long[] buckets, long count, double percentile) + { + var target = Math.Max(1, (long)Math.Ceiling(count * percentile)); + long cumulative = 0; + for (var i = 0; i < buckets.Length; i++) + { + cumulative += buckets[i]; + if (cumulative >= target) + { + return i < UpperMs.Length ? (long)UpperMs[i] : (long)UpperMs[^1]; + } + } + + return (long)UpperMs[^1]; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Diagnostics/NullFeedDiagnostics.cs b/src/VectorNNTPD.ReaderService/Diagnostics/NullFeedDiagnostics.cs new file mode 100644 index 00000000..caa7148d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Diagnostics/NullFeedDiagnostics.cs @@ -0,0 +1,54 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Diagnostics +{ + /// No-op feed diagnostics used when the reporter is disabled. + public sealed class NullFeedDiagnostics : IFeedDiagnostics + { + /// Shared disabled instance. + public static NullFeedDiagnostics Instance { get; } = new(); + + private NullFeedDiagnostics() + { + } + + /// + public bool IsEnabled => false; + + /// + public void OnRejected(string peerName, string remote) + { + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) => null; + + /// + public void OnReleased(FeedSessionProbe? probe) + { + } + + /// + public void RecordTcpBytes(int bytes) + { + } + + /// + public void BeginSpoolWork() + { + } + + /// + public void EndSpoolWork(int bytes, bool persisted) + { + } + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) => + FeedDiagnosticsSnapshot.Empty; + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailAddress.cs b/src/VectorNNTPD.ReaderService/Email/EmailAddress.cs new file mode 100644 index 00000000..5b3ec59d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailAddress.cs @@ -0,0 +1,53 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Email +{ + /// A protocol-independent mailbox with an optional display name. + /// + /// The address is an RFC 5321/5322 mailbox. Display names are sanitized by the + /// message encoder. Neither field may contain CR or LF. + /// + public sealed class EmailAddress + { + /// Initializes a new mailbox. + /// Mailbox (local@domain). + /// Optional display name. + public EmailAddress(string address, string? displayName = null) + { + if (!EmailOptionsValidator.TryValidateMailbox(address, out var mailbox)) + { + throw new ArgumentException("Mailbox address is missing or invalid.", nameof(address)); + } + + if (displayName is not null && ContainsLineBreak(displayName)) + { + throw new ArgumentException("Display name must not contain CR or LF.", nameof(displayName)); + } + + Address = mailbox; + DisplayName = string.IsNullOrWhiteSpace(displayName) ? null : displayName.Trim(); + } + + /// Gets the mailbox. + public string Address { get; } + + /// Gets the optional display name. + public string? DisplayName { get; } + + /// + public override string ToString() => Address; + + internal static bool ContainsLineBreak(ReadOnlySpan value) + { + foreach (var ch in value) + { + if (ch is '\r' or '\n') + { + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailAttachment.cs b/src/VectorNNTPD.ReaderService/Email/EmailAttachment.cs new file mode 100644 index 00000000..bba863fa --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailAttachment.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// An optional MIME attachment. + /// Sanitized file name (no path separators, CR, or LF). + /// MIME type (default application/octet-stream). + /// Raw attachment bytes. + public sealed record EmailAttachment( + string FileName, + ReadOnlyMemory Content, + string ContentType = "application/octet-stream") + { + /// Validates that the file name cannot inject headers or paths. + public void Validate() + { + if (string.IsNullOrWhiteSpace(FileName) + || EmailAddress.ContainsLineBreak(FileName) + || FileName.IndexOfAny(['/', '\\', ':', '*', '?', '"', '<', '>', '|']) >= 0) + { + throw new ArgumentException("Attachment file name is invalid.", nameof(FileName)); + } + + if (string.IsNullOrWhiteSpace(ContentType) + || EmailAddress.ContainsLineBreak(ContentType) + || ContentType.Contains('\0')) + { + throw new ArgumentException("Attachment content type is invalid.", nameof(ContentType)); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailDeliveryService.cs b/src/VectorNNTPD.ReaderService/Email/EmailDeliveryService.cs new file mode 100644 index 00000000..828fd5a7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailDeliveryService.cs @@ -0,0 +1,228 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Email.Smtp; + +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Background email worker. Scans the filesystem spool and talks SMTP. + /// + /// + /// Startup creates the spool directory and recovers leftover .wrk claims + /// back to .eml. It does not open an SMTP connection and does not + /// requeue .delivered or failed/ files. Shutdown stops new + /// submissions, allows the current SMTP operation to finish within + /// , then leaves remaining + /// pending .eml files on disk. + /// + public sealed class EmailDeliveryService : IApplicationService + { + private readonly IEmailSpool _spool; + private readonly ISmtpTransport _transport; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private readonly Random _random = new(); + private Task? _execution; + private int _started; + + /// Initializes a new delivery service. + public EmailDeliveryService( + IEmailSpool spool, + ISmtpTransport transport, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(spool); + ArgumentNullException.ThrowIfNull(transport); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _spool = spool; + _transport = transport; + _options = options; + _logger = logger; + } + + /// + public string Name => "EmailDelivery"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + var options = _options.Value; + EmailLogMessages.DeliveryStarting( + _logger, + options.Enabled, + options.Smtp.Security.ToString()); + if (!options.Enabled) + { + return Task.CompletedTask; + } + + _spool.EnsureDirectory(); + _spool.RecoverClaims(); + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _spool.StopAccepting(); + var execution = _execution; + if (execution is null) + { + await _runCts.CancelAsync().ConfigureAwait(false); + return; + } + + using var drainCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var drain = _options.Value.Spool?.ShutdownTimeout ?? TimeSpan.FromSeconds(15); + drainCts.CancelAfter(drain); + try + { + await execution.WaitAsync(drainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await _runCts.CancelAsync().ConfigureAwait(false); + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + finally + { + EmailLogMessages.DeliveryStopped(_logger); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var smtp = _options.Value.Smtp ?? new SmtpOptions(); + var scan = _options.Value.Spool?.ScanInterval ?? TimeSpan.FromSeconds(1); + while (!cancellationToken.IsCancellationRequested) + { + EmailSpoolClaim? claim; + try + { + claim = _spool.TryClaimNext(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + EmailLogMessages.DeliveryUnexpected(_logger, ex, 0, null); + claim = null; + } + + if (claim is null) + { + try + { + await _spool.WaitAsync(scan, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + continue; + } + + try + { + await DeliverAsync(claim, smtp, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _spool.Release(claim); + break; + } + } + } + + private async Task DeliverAsync(EmailSpoolClaim claim, SmtpOptions smtp, CancellationToken cancellationToken) + { + var item = claim.Item; + while (true) + { + try + { + _ = await _transport.SendAsync(item, cancellationToken).ConfigureAwait(false); + _spool.Complete(claim); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _spool.Release(claim); + throw; + } + catch (SmtpException ex) when (ex.Kind == SmtpFailureKind.Cancelled) + { + _spool.Release(claim); + throw; + } + catch (SmtpException ex) + { + var retry = ex.IsTransient && item.Attempt < smtp.MaxAttempts; + if (!retry) + { + EmailLogMessages.DeliveryFailed( + _logger, + ex.Kind.ToString(), + ex.StatusCode, + item.Recipients.Count, + item.CorrelationId); + if (ex.IsTransient) + { + _spool.Defer(claim); + } + else + { + _spool.Quarantine(claim); + } + + return; + } + + EmailLogMessages.DeliveryRetry( + _logger, + item.Attempt, + smtp.MaxAttempts, + ex.Kind.ToString(), + ex.StatusCode); + var delay = EmailRetryDelay.Compute(smtp, item.Attempt, _random); + item.Attempt++; + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + EmailLogMessages.DeliveryUnexpected( + _logger, + ex, + item.Recipients.Count, + item.CorrelationId); + _spool.Quarantine(claim); + return; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailEnqueueStatus.cs b/src/VectorNNTPD.ReaderService/Email/EmailEnqueueStatus.cs new file mode 100644 index 00000000..fc50b4c0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailEnqueueStatus.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Result of . This is durable local spool + /// acceptance, not remote SMTP delivery. + /// + public enum EmailEnqueueStatus + { + /// + /// The complete message was atomically written to the local filesystem spool. + /// Not proof of remote SMTP receipt. + /// + Accepted = 0, + + /// The outbound subsystem is disabled. No spool file was written. + Disabled = 1, + + /// The message failed validation or encoding. No spool file was written. + Rejected = 2, + + /// The service is stopping and is not accepting new submissions. + Unavailable = 3, + + /// The spool file could not be created (I/O, permission, disk, or cancellation). + Failed = 4, + } + + /// Outcome of one call. + /// Local spool acceptance status. + /// Internal detail (never a credential). + public readonly record struct EmailEnqueueResult(EmailEnqueueStatus Status, string Detail); +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailHeader.cs b/src/VectorNNTPD.ReaderService/Email/EmailHeader.cs new file mode 100644 index 00000000..667c4a0a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailHeader.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// A custom RFC 5322 header field. + /// Header field name (token; no colon, CR, or LF). + /// Unfolded field body (no CR or LF). + public sealed record EmailHeader(string Name, string Value) + { + /// Validates that and cannot inject headers. + public void Validate() + { + if (string.IsNullOrWhiteSpace(Name) + || Name.Contains(':') + || EmailAddress.ContainsLineBreak(Name) + || !IsHeaderName(Name)) + { + throw new ArgumentException("Header name is invalid or contains injection characters.", nameof(Name)); + } + + if (Value is null || EmailAddress.ContainsLineBreak(Value)) + { + throw new ArgumentException("Header value must not contain CR or LF.", nameof(Value)); + } + } + + private static bool IsHeaderName(string name) + { + foreach (var ch in name) + { + if (ch is < (char)33 or > (char)126 || ch == ':') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailLogMessages.cs b/src/VectorNNTPD.ReaderService/Email/EmailLogMessages.cs new file mode 100644 index 00000000..8bc7a6e0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailLogMessages.cs @@ -0,0 +1,104 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// Source-generated email subsystem log messages. Never includes credentials. + internal static partial class EmailLogMessages + { + [LoggerMessage( + EventId = 2600, + Level = LogLevel.Information, + Message = "Email delivery service starting (enabled {Enabled}, security {SecurityMode})")] + public static partial void DeliveryStarting(ILogger logger, bool Enabled, string SecurityMode); + + [LoggerMessage( + EventId = 2601, + Level = LogLevel.Information, + Message = "Email delivery service stopped")] + public static partial void DeliveryStopped(ILogger logger); + + [LoggerMessage( + EventId = 2602, + Level = LogLevel.Debug, + Message = "Outbound email accepted by local spool (recipients {RecipientCount}, bytes {EncodedBytes}, correlation {CorrelationId})")] + public static partial void Enqueued( + ILogger logger, + int RecipientCount, + int EncodedBytes, + string? CorrelationId); + + [LoggerMessage( + EventId = 2604, + Level = LogLevel.Information, + Message = "SMTP session starting ({Host}:{Port}, {SecurityMode})")] + public static partial void SmtpSessionStarting(ILogger logger, string Host, int Port, string SecurityMode); + + [LoggerMessage( + EventId = 2605, + Level = LogLevel.Information, + Message = "SMTP session completed ({Host}:{Port}, accepted recipients {AcceptedRecipients})")] + public static partial void SmtpSessionCompleted(ILogger logger, string Host, int Port, int AcceptedRecipients); + + [LoggerMessage( + EventId = 2606, + Level = LogLevel.Warning, + Message = "SMTP delivery will retry (attempt {Attempt}/{MaxAttempts}, kind {FailureKind}, status {StatusCode})")] + public static partial void DeliveryRetry( + ILogger logger, + int Attempt, + int MaxAttempts, + string FailureKind, + int StatusCode); + + [LoggerMessage( + EventId = 2607, + Level = LogLevel.Error, + Message = "Email delivery failed (kind {FailureKind}, status {StatusCode}, recipients {RecipientCount}, correlation {CorrelationId})")] + public static partial void DeliveryFailed( + ILogger logger, + string FailureKind, + int StatusCode, + int RecipientCount, + string? CorrelationId); + + [LoggerMessage( + EventId = 2609, + Level = LogLevel.Debug, + Message = "Email message rejected before enqueue ({Reason})")] + public static partial void MessageRejected(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 2610, + Level = LogLevel.Error, + Message = "Unexpected email delivery failure (recipients {RecipientCount}, correlation {CorrelationId})")] + public static partial void DeliveryUnexpected(ILogger logger, Exception exception, int RecipientCount, string? CorrelationId); + + [LoggerMessage( + EventId = 2611, + Level = LogLevel.Error, + Message = "Email spool write failed")] + public static partial void SpoolWriteFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2612, + Level = LogLevel.Warning, + Message = "Email spool claim recovery failed for {FileName}")] + public static partial void SpoolRecoverFailed(ILogger logger, Exception exception, string FileName); + + [LoggerMessage( + EventId = 2613, + Level = LogLevel.Error, + Message = "Email spool file {SpoolId} is malformed and was moved to failed")] + public static partial void SpoolMalformed(ILogger logger, string SpoolId); + + [LoggerMessage( + EventId = 2614, + Level = LogLevel.Error, + Message = "Email spool quarantine failed for {SpoolId}")] + public static partial void SpoolQuarantineFailed(ILogger logger, Exception exception, string SpoolId); + + [LoggerMessage( + EventId = 2615, + Level = LogLevel.Critical, + Message = "SMTP accepted the message but the spool file {SpoolId} could not be deleted; it may be resent after restart")] + public static partial void SpoolDeleteAfterAcceptFailed(ILogger logger, Exception exception, string SpoolId); + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailMessage.cs b/src/VectorNNTPD.ReaderService/Email/EmailMessage.cs new file mode 100644 index 00000000..5c750034 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailMessage.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Protocol-independent outbound email. SMTP envelope fields are distinct from headers. + /// + /// + /// Producers construct this model. The email subsystem owns MIME encoding and SMTP. + /// Credentials must never be placed in headers or the body by this type. + /// + public sealed class EmailMessage + { + /// Gets the RFC 5322 From mailbox. + public required EmailAddress From { get; init; } + + /// Gets the visible To recipients. + public required IReadOnlyList To { get; init; } + + /// Gets the Cc recipients. + public IReadOnlyList Cc { get; init; } = []; + + /// Gets the Bcc recipients (envelope only; omitted from headers). + public IReadOnlyList Bcc { get; init; } = []; + + /// Gets the Reply-To mailboxes. + public IReadOnlyList ReplyTo { get; init; } = []; + + /// + /// Gets the SMTP envelope sender. When , configuration + /// Email:EnvelopeSender / Email:DefaultFrom is used. + /// + public EmailAddress? EnvelopeSender { get; init; } + + /// Gets the subject (no CR/LF; encoded by the message encoder when needed). + public required string Subject { get; init; } + + /// Gets the body octets (interpreted using ). + public required ReadOnlyMemory Body { get; init; } + + /// Gets the body media type without parameters. Default text/plain. + public string ContentType { get; init; } = "text/plain"; + + /// Gets the body charset. Default utf-8. + public string Charset { get; init; } = "utf-8"; + + /// Gets extra header fields (not From/To/Cc/Subject/MIME). + public IReadOnlyList Headers { get; init; } = []; + + /// Gets attachments. Non-empty messages become multipart/mixed. + public IReadOnlyList Attachments { get; init; } = []; + + /// Gets an optional correlation id for structured logs (not a secret). + public string? CorrelationId { get; init; } + + /// Enumerates distinct envelope recipients (To, Cc, Bcc). + public IReadOnlyList EnvelopeRecipients() + { + var list = new List(To.Count + Cc.Count + Bcc.Count); + AddDistinct(list, To); + AddDistinct(list, Cc); + AddDistinct(list, Bcc); + return list; + } + + private static void AddDistinct(List target, IReadOnlyList source) + { + foreach (var address in source) + { + var found = false; + for (var i = 0; i < target.Count; i++) + { + if (string.Equals(target[i].Address, address.Address, StringComparison.OrdinalIgnoreCase)) + { + found = true; + break; + } + } + + if (!found) + { + target.Add(address); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailRetryDelay.cs b/src/VectorNNTPD.ReaderService/Email/EmailRetryDelay.cs new file mode 100644 index 00000000..cce41091 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailRetryDelay.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Email +{ + /// Exponential backoff with full jitter for SMTP retries. + internal static class EmailRetryDelay + { + /// Computes the delay before the next attempt ( is 1-based completed attempt). + public static TimeSpan Compute(SmtpOptions smtp, int attempt, Random random) + { + ArgumentNullException.ThrowIfNull(smtp); + ArgumentNullException.ThrowIfNull(random); + if (attempt < 1) + { + attempt = 1; + } + + var initial = smtp.InitialRetryDelay; + var maximum = smtp.MaximumRetryDelay < initial ? initial : smtp.MaximumRetryDelay; + var multiplier = 1L << Math.Min(attempt - 1, 16); + var rawTicks = initial.Ticks * multiplier; + var capped = rawTicks > maximum.Ticks ? maximum : TimeSpan.FromTicks(rawTicks); + if (capped <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + var jitter = random.NextDouble(); + return TimeSpan.FromTicks((long)(capped.Ticks * jitter)); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailService.cs b/src/VectorNNTPD.ReaderService/Email/EmailService.cs new file mode 100644 index 00000000..ff76a0e9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailService.cs @@ -0,0 +1,120 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Public email API: validate, encode, and durably spool. Does not perform SMTP I/O. + /// + public sealed class EmailService : IEmailService + { + private readonly IEmailSpool _spool; + private readonly IEmailMessageEncoder _encoder; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// Initializes a new email service. + public EmailService( + IEmailSpool spool, + IEmailMessageEncoder encoder, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(spool); + ArgumentNullException.ThrowIfNull(encoder); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _spool = spool; + _encoder = encoder; + _options = options; + _logger = logger; + } + + /// + public async ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + + var options = _options.Value; + if (!options.Enabled) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Disabled, "email subsystem is disabled"); + } + + if (!_spool.IsAccepting) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Unavailable, "email spool is not accepting work"); + } + + EmailWorkItem item; + try + { + Rfc5322MessageEncoder.Validate(message); + var encoded = _encoder.Encode(message); + item = new EmailWorkItem + { + Message = message, + EncodedMessage = encoded, + EnvelopeSender = ResolveEnvelopeSender(message, options), + Recipients = message.EnvelopeRecipients(), + CorrelationId = message.CorrelationId, + }; + } + catch (ArgumentException ex) + { + EmailLogMessages.MessageRejected(_logger, ex.Message); + return new EmailEnqueueResult(EmailEnqueueStatus.Rejected, ex.Message); + } + + try + { + _ = await _spool.WriteAsync(item, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Failed, "spool write was canceled"); + } + catch (InvalidOperationException) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Unavailable, "email spool is not accepting work"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolWriteFailed(_logger, ex); + return new EmailEnqueueResult(EmailEnqueueStatus.Failed, "spool write failed"); + } + + EmailLogMessages.Enqueued( + _logger, + item.Recipients.Count, + item.EncodedMessage.Length, + item.CorrelationId); + return new EmailEnqueueResult( + EmailEnqueueStatus.Accepted, + "durably accepted into the local email spool"); + } + + internal static EmailAddress ResolveEnvelopeSender(EmailMessage message, EmailOptions options) + { + if (message.EnvelopeSender is not null) + { + return message.EnvelopeSender; + } + + if (EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out var envelope)) + { + return new EmailAddress(envelope); + } + + if (EmailOptionsValidator.TryValidateMailbox(options.DefaultFrom, out var from)) + { + return new EmailAddress(from); + } + + return message.From; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailSpoolFileName.cs b/src/VectorNNTPD.ReaderService/Email/EmailSpoolFileName.cs new file mode 100644 index 00000000..f2463ca7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailSpoolFileName.cs @@ -0,0 +1,35 @@ +using System.Security.Cryptography; + +namespace VectorNNTPD.ReaderService.Email +{ + /// Generates opaque spool filenames: lowercase hex MD5 of a new UUID. + internal static class EmailSpoolFileName + { + /// Creates a new 32-character lowercase hex identifier. + public static string NewId() + { + var hash = MD5.HashData(Guid.NewGuid().ToByteArray()); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// Returns whether is a 32-character lowercase hex id. + public static bool IsId(ReadOnlySpan name) + { + if (name.Length != 32) + { + return false; + } + + foreach (var ch in name) + { + var hex = ch is >= '0' and <= '9' or >= 'a' and <= 'f'; + if (!hex) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailSpoolRecord.cs b/src/VectorNNTPD.ReaderService/Email/EmailSpoolRecord.cs new file mode 100644 index 00000000..7983cec0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailSpoolRecord.cs @@ -0,0 +1,112 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Deterministic spool file: SMTP envelope preamble plus the RFC 5322/MIME message. + /// + /// + /// Envelope sender/recipients are stored separately because they are not always + /// reconstructible from headers (Bcc, explicit MAIL FROM). + /// Credentials and transport state are never stored. + /// + internal static class EmailSpoolRecord + { + internal const string Magic = "VNNTP-SMTP-SPOOL/1"; + + public static byte[] Serialize(EmailWorkItem item) + { + ArgumentNullException.ThrowIfNull(item); + var builder = new StringBuilder(64 + item.EncodedMessage.Length); + builder.Append(Magic).Append("\r\n"); + builder.Append("MAIL FROM:<").Append(item.EnvelopeSender.Address).Append(">\r\n"); + foreach (var recipient in item.Recipients) + { + builder.Append("RCPT TO:<").Append(recipient.Address).Append(">\r\n"); + } + + builder.Append("\r\n"); + var header = Encoding.ASCII.GetBytes(builder.ToString()); + var output = new byte[header.Length + item.EncodedMessage.Length]; + header.CopyTo(output, 0); + item.EncodedMessage.Span.CopyTo(output.AsSpan(header.Length)); + return output; + } + + public static bool TryParse(ReadOnlySpan data, out EmailWorkItem? item) + { + item = null; + var text = Encoding.ASCII.GetString(data); + var separator = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + if (separator < 0) + { + return false; + } + + var preamble = text[..separator].Split("\r\n", StringSplitOptions.None); + if (preamble.Length < 3 || !string.Equals(preamble[0], Magic, StringComparison.Ordinal)) + { + return false; + } + + if (!TryMailbox(preamble[1], "MAIL FROM:", out var sender)) + { + return false; + } + + var recipients = new List(); + for (var i = 2; i < preamble.Length; i++) + { + if (string.IsNullOrEmpty(preamble[i])) + { + continue; + } + + if (!TryMailbox(preamble[i], "RCPT TO:", out var recipient)) + { + return false; + } + + recipients.Add(recipient); + } + + if (recipients.Count == 0) + { + return false; + } + + var message = data[(separator + 4)..].ToArray(); + item = new EmailWorkItem + { + EncodedMessage = message, + EnvelopeSender = sender, + Recipients = recipients, + }; + return true; + } + + private static bool TryMailbox(string line, string prefix, out EmailAddress address) + { + address = null!; + if (!line.StartsWith(prefix, StringComparison.Ordinal) + || line.Length < prefix.Length + 3 + || line[prefix.Length] != '<' + || !line.EndsWith('>')) + { + return false; + } + + var mailbox = line[(prefix.Length + 1)..^1]; + if (!EmailOptionsValidatorMailbox(mailbox)) + { + return false; + } + + address = new EmailAddress(mailbox); + return true; + } + + private static bool EmailOptionsValidatorMailbox(string mailbox) => + Configuration.EmailOptionsValidator.TryValidateMailbox(mailbox, out _); + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/EmailWorkItem.cs b/src/VectorNNTPD.ReaderService/Email/EmailWorkItem.cs new file mode 100644 index 00000000..b9fdfcba --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/EmailWorkItem.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// One outbound message after MIME encoding (in memory or loaded from spool). + public sealed class EmailWorkItem + { + /// Gets the original message when still in the producer process; otherwise . + public EmailMessage? Message { get; init; } + + /// Gets the RFC 5322/MIME wire representation (CRLF, not SMTP-dot-stuffed). + public required ReadOnlyMemory EncodedMessage { get; init; } + + /// Gets the SMTP envelope sender. + public required EmailAddress EnvelopeSender { get; init; } + + /// Gets the SMTP envelope recipients. + public required IReadOnlyList Recipients { get; init; } + + /// Gets an optional correlation id for logs (not a secret). + public string? CorrelationId { get; init; } + + /// Gets the 1-based in-process delivery attempt count. Not persisted on disk. + public int Attempt { get; set; } = 1; + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/FilesystemEmailSpool.cs b/src/VectorNNTPD.ReaderService/Email/FilesystemEmailSpool.cs new file mode 100644 index 00000000..8223502f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/FilesystemEmailSpool.cs @@ -0,0 +1,418 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Email +{ + /// Filesystem spool under spool/smtp (configurable). + /// + /// + /// The delivery scanner enumerates only the spool directory itself + /// (), never failed/: + /// + /// + /// Suffix / pathMeaning + /// .tmpIncomplete atomic write. Never delivered. + /// .emlPending outbound message. The only automatically deliverable state. + /// .wrkIn-flight claim. Crash recovery may rename this back to .eml. + /// failed/*.emlPermanent failure. Retained for operators. Never automatically delivered or moved back. + /// .deliveredSMTP already accepted the message but local delete failed. Retained for operators. Never automatically delivered, claimed, or renamed to .eml (retry would duplicate). + /// + /// + public sealed class FilesystemEmailSpool : IEmailSpool + { + private readonly EmailSpoolOptions _options; + private readonly ILogger _logger; + private readonly SemaphoreSlim _signal = new(0); + private readonly object _claimGate = new(); + private readonly HashSet _claimed = new(StringComparer.Ordinal); + private readonly HashSet _deferred = new(StringComparer.Ordinal); + private readonly string _directory; + private readonly string _failedDirectory; + private int _accepting = 1; + + /// + /// Optional replacement for the work-file delete inside . + /// Production leaves this unset, so runs. + /// Tests assign a delegate that throws or + /// to prove the delivered-marker path. + /// + internal Action? CompleteWorkFileDelete { get; set; } + + /// Initializes a new filesystem spool from . + public FilesystemEmailSpool(IOptions options, ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _options = options.Value.Spool ?? new EmailSpoolOptions(); + _logger = logger; + _directory = ApplicationLocalPath.ResolveApplicationLocalPath( + string.IsNullOrWhiteSpace(_options.Directory) + ? EmailSpoolOptions.DefaultDirectory + : _options.Directory.Trim(), + AppContext.BaseDirectory); + _failedDirectory = Path.Join(_directory, "failed"); + } + + /// + public string Directory => _directory; + + /// + public bool IsAccepting => Volatile.Read(ref _accepting) != 0; + + /// + public void EnsureDirectory() + { + System.IO.Directory.CreateDirectory(_directory); + System.IO.Directory.CreateDirectory(_failedDirectory); + } + + /// + public void StopAccepting() => Volatile.Write(ref _accepting, 0); + + /// + public void RecoverClaims() + { + lock (_claimGate) + { + RecoverClaimsCore(); + } + } + + private void RecoverClaimsCore() + { + EnsureDirectory(); + foreach (var work in System.IO.Directory.EnumerateFiles( + _directory, + "*.wrk", + SearchOption.TopDirectoryOnly)) + { + var id = Path.GetFileNameWithoutExtension(work); + lock (_claimed) + { + if (_claimed.Contains(id)) + { + continue; + } + } + + var eml = Path.ChangeExtension(work, ".eml"); + try + { + if (File.Exists(eml)) + { + File.Delete(work); + continue; + } + + File.Move(work, eml); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolRecoverFailed(_logger, ex, Path.GetFileName(work)); + } + } + } + + /// + public async Task WriteAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + if (!IsAccepting) + { + throw new InvalidOperationException("The email spool is not accepting submissions."); + } + + EnsureDirectory(); + var payload = EmailSpoolRecord.Serialize(item); + for (var attempt = 0; attempt < 8; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + var id = EmailSpoolFileName.NewId(); + var tmp = Path.Join(_directory, id + ".tmp"); + var eml = Path.Join(_directory, id + ".eml"); + if (File.Exists(eml) || File.Exists(tmp) || File.Exists(Path.Join(_directory, id + ".wrk"))) + { + continue; + } + + try + { + await using (var stream = new FileStream( + tmp, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + FileOptions.Asynchronous)) + { + await stream.WriteAsync(payload, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + stream.Flush(flushToDisk: true); + } + + File.Move(tmp, eml); + Signal(); + return eml; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TryDelete(tmp); + if (ex is IOException && attempt < 7) + { + continue; + } + + throw; + } + } + + throw new IOException("Could not allocate a unique email spool filename."); + } + + /// + public EmailSpoolClaim? TryClaimNext() + { + lock (_claimGate) + { + return TryClaimNextCore(); + } + } + + private EmailSpoolClaim? TryClaimNextCore() + { + EnsureDirectory(); + string[] pending; + try + { + pending = System.IO.Directory.GetFiles( + _directory, + "*.eml", + SearchOption.TopDirectoryOnly); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return null; + } + + Array.Sort(pending, StringComparer.Ordinal); + foreach (var eml in pending) + { + var id = Path.GetFileNameWithoutExtension(eml); + if (!EmailSpoolFileName.IsId(id)) + { + continue; + } + + lock (_deferred) + { + if (_deferred.Contains(id)) + { + continue; + } + } + + var wrk = Path.Join(_directory, id + ".wrk"); + try + { + File.Move(eml, wrk); + lock (_claimed) + { + _claimed.Add(id); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + continue; + } + + byte[] bytes; + try + { + bytes = File.ReadAllBytes(wrk); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + ForgetClaim(id); + TryReleasePath(wrk, eml); + continue; + } + + if (!EmailSpoolRecord.TryParse(bytes, out var item) || item is null) + { + var claim = new EmailSpoolClaim(id, wrk, new EmailWorkItem + { + EncodedMessage = bytes, + EnvelopeSender = new EmailAddress("invalid@invalid.invalid"), + Recipients = [new EmailAddress("invalid@invalid.invalid")], + }); + Quarantine(claim); + EmailLogMessages.SpoolMalformed(_logger, id); + continue; + } + + return new EmailSpoolClaim(id, wrk, item); + } + + return null; + } + + /// + public void Release(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + var eml = Path.Join(_directory, claim.Id + ".eml"); + TryReleasePath(claim.WorkPath, eml); + } + + /// + public void Defer(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + lock (_deferred) + { + _deferred.Add(claim.Id); + } + + Release(claim); + } + + /// + public void Quarantine(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + EnsureDirectory(); + var dest = Path.Join(_failedDirectory, claim.Id + ".eml"); + try + { + if (File.Exists(dest)) + { + File.Delete(dest); + } + + File.Move(claim.WorkPath, dest); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolQuarantineFailed(_logger, ex, claim.Id); + } + } + + /// + public void Complete(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + try + { + var delete = CompleteWorkFileDelete ?? File.Delete; + delete(claim.WorkPath); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + var delivered = Path.Join(_directory, claim.Id + ".delivered"); + try + { + if (File.Exists(delivered)) + { + File.Delete(claim.WorkPath); + return; + } + + File.Move(claim.WorkPath, delivered); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception moveEx) when (moveEx is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolDeleteAfterAcceptFailed(_logger, moveEx, claim.Id); + return; + } + + EmailLogMessages.SpoolDeleteAfterAcceptFailed(_logger, ex, claim.Id); + } + } + + /// + public void Signal() + { + try + { + _signal.Release(); + } + catch (SemaphoreFullException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async Task WaitAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + try + { + _ = await _signal.WaitAsync(timeout, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + } + + private void ForgetClaim(string id) + { + lock (_claimed) + { + _claimed.Remove(id); + } + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private static void TryReleasePath(string work, string eml) + { + try + { + if (!File.Exists(work)) + { + return; + } + + if (File.Exists(eml)) + { + File.Delete(work); + return; + } + + File.Move(work, eml); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/IEmailMessageEncoder.cs b/src/VectorNNTPD.ReaderService/Email/IEmailMessageEncoder.cs new file mode 100644 index 00000000..d3b28aed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/IEmailMessageEncoder.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// Encodes an to RFC 5322 / MIME wire bytes. + /// + /// Separate from SMTP transport. Output uses CRLF. SMTP DATA dot-stuffing is + /// applied later by the SMTP client, not here. + /// + public interface IEmailMessageEncoder + { + /// Validates and encodes . + ReadOnlyMemory Encode(EmailMessage message); + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/IEmailService.cs b/src/VectorNNTPD.ReaderService/Email/IEmailService.cs new file mode 100644 index 00000000..5534e4a3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/IEmailService.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Application-facing outbound email API. Producers spool work; they do not + /// perform SMTP. + /// + /// + /// + /// means the complete RFC 5322/MIME message (plus SMTP + /// envelope) has been durably accepted into the local filesystem spool + /// (spool/smtp by default). It does not mean a remote SMTP server accepted + /// the message. + /// + /// + /// SMTP delivery is asynchronous. Successful remote acceptance deletes the spool + /// file. Undelivered files survive process restart. SMTP acceptance followed by + /// filesystem deletion is at-least-once and cannot be exactly-once. + /// + /// + /// Callers must not depend on TcpClient, SslStream, SMTP commands, + /// STARTTLS, authentication, retries, spool paths, or connection reuse. + /// + /// + public interface IEmailService + { + /// + /// Validates, encodes, and durably writes to the outbound spool. + /// + /// Protocol-independent message. + /// Cancels the spool write. + /// Durable local-spool acceptance, not remote SMTP delivery. + ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/IEmailSpool.cs b/src/VectorNNTPD.ReaderService/Email/IEmailSpool.cs new file mode 100644 index 00000000..c6a6f922 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/IEmailSpool.cs @@ -0,0 +1,92 @@ +namespace VectorNNTPD.ReaderService.Email +{ + /// + /// Durable filesystem outbound email spool. This is the queue. + /// + public interface IEmailSpool + { + /// Gets the resolved absolute spool directory. + string Directory { get; } + + /// Gets whether new submissions are accepted. + bool IsAccepting { get; } + + /// Creates the spool directory (and failed) if missing. + void EnsureDirectory(); + + /// Stops accepting new submissions. Existing .eml files remain. + void StopAccepting(); + + /// + /// Renames leftover .wrk claims back to .eml so a crash cannot + /// leave messages stuck in-flight. Does not touch .delivered or + /// failed/. + /// + void RecoverClaims(); + + /// + /// Atomically publishes a complete spool file. Returns the final .eml path. + /// + Task WriteAsync(EmailWorkItem item, CancellationToken cancellationToken); + + /// + /// Claims the next pending top-level .eml by renaming it to .wrk. + /// Does not scan failed/, .delivered, .wrk, or .tmp. + /// In-process callers are serialized. Returns when none are available. + /// + EmailSpoolClaim? TryClaimNext(); + + /// Returns a claimed file to pending .eml after a transient failure or cancel. + void Release(EmailSpoolClaim claim); + + /// + /// Returns the file to pending .eml but skips it for the rest of this + /// process lifetime so in-process retries do not hammer SMTP. After restart + /// the file is eligible again. + /// + void Defer(EmailSpoolClaim claim); + + /// + /// Moves a claimed file to failed/ after a permanent failure or parse error. + /// Failed files are never automatically requeued. + /// + void Quarantine(EmailSpoolClaim claim); + + /// + /// Deletes the claimed file after SMTP accepted the message. + /// If delete fails, the file is renamed to .delivered and must never be + /// automatically retransmitted (SMTP already accepted it). + /// + void Complete(EmailSpoolClaim claim); + + /// Wakes a waiting delivery worker after a successful write. + void Signal(); + + /// Waits for a write signal or , then returns. + Task WaitAsync(TimeSpan timeout, CancellationToken cancellationToken); + } + + /// One claimed spool file being delivered. + public sealed class EmailSpoolClaim + { + /// Initializes a claimed spool item. + public EmailSpoolClaim(string id, string workPath, EmailWorkItem item) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + ArgumentException.ThrowIfNullOrWhiteSpace(workPath); + ArgumentNullException.ThrowIfNull(item); + Id = id; + WorkPath = workPath; + Item = item; + } + + /// Gets the opaque filename id (no extension). + public string Id { get; } + + /// Gets the claimed .wrk path. + public string WorkPath { get; } + + /// Gets the loaded message and envelope. + public EmailWorkItem Item { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Rfc5322MessageEncoder.cs b/src/VectorNNTPD.ReaderService/Email/Rfc5322MessageEncoder.cs new file mode 100644 index 00000000..78d97ec1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Rfc5322MessageEncoder.cs @@ -0,0 +1,360 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Email +{ + /// RFC 5322 / MIME encoder with header-injection rejection. + public sealed class Rfc5322MessageEncoder : IEmailMessageEncoder + { + private static readonly Encoding Utf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + + /// + public ReadOnlyMemory Encode(EmailMessage message) + { + ArgumentNullException.ThrowIfNull(message); + Validate(message); + + var builder = new StringBuilder(512 + message.Body.Length); + WriteHeader(builder, "From", FormatMailbox(message.From)); + WriteHeader(builder, "To", FormatMailboxList(message.To)); + if (message.Cc.Count > 0) + { + WriteHeader(builder, "Cc", FormatMailboxList(message.Cc)); + } + + if (message.ReplyTo.Count > 0) + { + WriteHeader(builder, "Reply-To", FormatMailboxList(message.ReplyTo)); + } + + WriteHeader(builder, "Subject", EncodeHeaderValue(message.Subject)); + WriteHeader(builder, "Date", DateTimeOffset.UtcNow.ToString("r", CultureInfo.InvariantCulture)); + WriteHeader(builder, "MIME-Version", "1.0"); + WriteHeader(builder, "Message-ID", CreateMessageId(message.From.Address)); + + foreach (var header in message.Headers) + { + header.Validate(); + RejectReservedHeader(header.Name); + WriteHeader(builder, header.Name, EncodeHeaderValue(header.Value)); + } + + if (message.Attachments.Count == 0) + { + WriteSinglePart(builder, message); + } + else + { + WriteMultipart(builder, message); + } + + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + internal static void Validate(EmailMessage message) + { + ArgumentNullException.ThrowIfNull(message.From); + if (message.To is null || message.To.Count == 0) + { + throw new ArgumentException("At least one To recipient is required.", nameof(message)); + } + + if (message.Subject is null || EmailAddress.ContainsLineBreak(message.Subject)) + { + throw new ArgumentException("Subject must not contain CR or LF.", nameof(message)); + } + + if (string.IsNullOrWhiteSpace(message.ContentType) + || EmailAddress.ContainsLineBreak(message.ContentType)) + { + throw new ArgumentException("ContentType must be a media type without CR or LF.", nameof(message)); + } + + if (string.IsNullOrWhiteSpace(message.Charset) || EmailAddress.ContainsLineBreak(message.Charset)) + { + throw new ArgumentException("Charset is invalid.", nameof(message)); + } + + foreach (var header in message.Headers) + { + header.Validate(); + } + + foreach (var attachment in message.Attachments) + { + attachment.Validate(); + } + + if (message.EnvelopeRecipients().Count == 0) + { + throw new ArgumentException("The message has no envelope recipients.", nameof(message)); + } + } + + private static void WriteSinglePart(StringBuilder builder, EmailMessage message) + { + var body = NormalizeBody(message.Body); + var needsQuotedPrintable = RequiresQuotedPrintable(body); + var contentType = message.ContentType; + if ((contentType.Equals("text/plain", StringComparison.OrdinalIgnoreCase) + || contentType.Equals("text/html", StringComparison.OrdinalIgnoreCase)) + && contentType.IndexOf("charset=", StringComparison.OrdinalIgnoreCase) < 0) + { + contentType = $"{contentType}; charset={message.Charset}"; + } + + WriteHeader(builder, "Content-Type", contentType); + WriteHeader(builder, "Content-Transfer-Encoding", needsQuotedPrintable ? "quoted-printable" : "7bit"); + builder.Append("\r\n"); + builder.Append(needsQuotedPrintable ? EncodeQuotedPrintable(body) : Encoding.ASCII.GetString(body)); + if (body.Length == 0 || body[^1] != (byte)'\n') + { + builder.Append("\r\n"); + } + } + + private static void WriteMultipart(StringBuilder builder, EmailMessage message) + { + var boundary = "vnntp-" + Guid.NewGuid().ToString("N"); + WriteHeader(builder, "Content-Type", $"multipart/mixed; boundary=\"{boundary}\""); + builder.Append("\r\n"); + builder.Append("This is a multi-part message in MIME format.\r\n\r\n"); + builder.Append("--").Append(boundary).Append("\r\n"); + + var bodyMessage = new EmailMessage + { + From = message.From, + To = message.To, + Subject = message.Subject, + Body = message.Body, + ContentType = message.ContentType, + Charset = message.Charset, + }; + WriteSinglePart(builder, bodyMessage); + + foreach (var attachment in message.Attachments) + { + builder.Append("--").Append(boundary).Append("\r\n"); + WriteHeader(builder, "Content-Type", attachment.ContentType); + WriteHeader(builder, "Content-Transfer-Encoding", "base64"); + WriteHeader( + builder, + "Content-Disposition", + $"attachment; filename=\"{attachment.FileName}\""); + builder.Append("\r\n"); + builder.Append( + Convert.ToBase64String(attachment.Content.Span, Base64FormattingOptions.InsertLineBreaks) + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace("\n", "\r\n", StringComparison.Ordinal)); + if (!builder.ToString().EndsWith("\r\n", StringComparison.Ordinal)) + { + builder.Append("\r\n"); + } + } + + builder.Append("--").Append(boundary).Append("--\r\n"); + } + + private static void WriteHeader(StringBuilder builder, string name, string value) + { + builder.Append(name).Append(": ").Append(Fold(name, value)).Append("\r\n"); + } + + private static string Fold(string name, string value) + { + if (name.Length + 2 + value.Length <= 78) + { + return value; + } + + var folded = new StringBuilder(); + var budget = 78 - name.Length - 2; + var offset = 0; + while (offset < value.Length) + { + var take = Math.Min(budget, value.Length - offset); + if (offset > 0) + { + folded.Append("\r\n "); + } + + folded.Append(value, offset, take); + offset += take; + budget = 77; + } + + return folded.ToString(); + } + + internal static string FormatMailbox(EmailAddress address) + { + if (string.IsNullOrEmpty(address.DisplayName)) + { + return address.Address; + } + + return $"{QuoteDisplayName(address.DisplayName)} <{address.Address}>"; + } + + private static string FormatMailboxList(IReadOnlyList addresses) + { + var builder = new StringBuilder(); + for (var i = 0; i < addresses.Count; i++) + { + if (i > 0) + { + builder.Append(", "); + } + + builder.Append(FormatMailbox(addresses[i])); + } + + return builder.ToString(); + } + + private static string QuoteDisplayName(string displayName) + { + var escaped = displayName.Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + return "\"" + escaped + "\""; + } + + internal static string EncodeHeaderValue(string value) + { + var ascii = true; + // codeql[cs/linq/missed-all] Foreach over string avoids CharEnumerator allocation from All(). + foreach (var ch in value) + { + if (ch > 127) + { + ascii = false; + break; + } + } + + if (ascii) + { + return value; + } + + var bytes = Utf8.GetBytes(value); + return "=?UTF-8?B?" + Convert.ToBase64String(bytes) + "?="; + } + + private static string CreateMessageId(string from) + { + var at = from.LastIndexOf('@'); + var domain = at >= 0 ? from[(at + 1)..] : "localhost"; + return "<" + Guid.NewGuid().ToString("N") + "@" + domain + ">"; + } + + private static void RejectReservedHeader(string name) + { + if (name.Equals("From", StringComparison.OrdinalIgnoreCase) + || name.Equals("To", StringComparison.OrdinalIgnoreCase) + || name.Equals("Cc", StringComparison.OrdinalIgnoreCase) + || name.Equals("Bcc", StringComparison.OrdinalIgnoreCase) + || name.Equals("Subject", StringComparison.OrdinalIgnoreCase) + || name.Equals("MIME-Version", StringComparison.OrdinalIgnoreCase) + || name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase) + || name.Equals("Content-Transfer-Encoding", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException($"Header '{name}' is reserved and must be set on EmailMessage."); + } + } + + internal static byte[] NormalizeBody(ReadOnlyMemory body) + { + if (body.Length == 0) + { + return []; + } + + var span = body.Span; + var output = new List(span.Length + 8); + for (var i = 0; i < span.Length; i++) + { + var b = span[i]; + if (b == (byte)'\r') + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + if (i + 1 < span.Length && span[i + 1] == (byte)'\n') + { + i++; + } + } + else if (b == (byte)'\n') + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + else + { + output.Add(b); + } + } + + return [.. output]; + } + + internal static bool RequiresQuotedPrintable(ReadOnlySpan body) + { + var lineLength = 0; + foreach (var b in body) + { + if (b > 127 || b == 0) + { + return true; + } + + if (b == (byte)'\n') + { + lineLength = 0; + } + else if (b != (byte)'\r') + { + lineLength++; + if (lineLength > 998) + { + return true; + } + } + } + + return false; + } + + internal static string EncodeQuotedPrintable(ReadOnlySpan body) + { + var builder = new StringBuilder(body.Length + (body.Length / 8)); + var line = 0; + for (var i = 0; i < body.Length; i++) + { + var b = body[i]; + if (b == (byte)'\r' && i + 1 < body.Length && body[i + 1] == (byte)'\n') + { + builder.Append("\r\n"); + line = 0; + i++; + continue; + } + + var encode = b > 127 || b < 32 || b == (byte)'='; + var token = encode + ? "=" + b.ToString("X2", CultureInfo.InvariantCulture) + : ((char)b).ToString(); + if (line + token.Length > 75) + { + builder.Append("=\r\n"); + line = 0; + } + + builder.Append(token); + line += token.Length; + } + + return builder.ToString(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/ISmtpTransport.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/ISmtpTransport.cs new file mode 100644 index 00000000..8c2e559c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/ISmtpTransport.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// SMTP delivery transport. Implementations own TCP/TLS/AUTH. + public interface ISmtpTransport + { + /// Delivers one encoded message on a new or reset SMTP session. + Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpCapabilities.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpCapabilities.cs new file mode 100644 index 00000000..aa349ea8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpCapabilities.cs @@ -0,0 +1,101 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Server capabilities advertised by EHLO (RFC 5321). + public sealed class SmtpCapabilities + { + /// Gets whether STARTTLS (RFC 3207) is advertised. + public bool StartTls { get; init; } + + /// Gets whether 8BITMIME is advertised. + public bool EightBitMime { get; init; } + + /// Gets whether SMTPUTF8 (RFC 6531) is advertised. + public bool SmtpUtf8 { get; init; } + + /// Gets the advertised SIZE limit in octets, when present. + public long? MaxSize { get; init; } + + /// Gets AUTH mechanism names in server order (uppercased). + public IReadOnlyList AuthMechanisms { get; init; } = []; + + /// Parses EHLO reply lines after the greeting line. + public static SmtpCapabilities Parse(SmtpResponse ehlo) + { + ArgumentNullException.ThrowIfNull(ehlo); + var startTls = false; + var eightBit = false; + var utf8 = false; + long? size = null; + var auth = new List(); + + foreach (var line in ehlo.Lines) + { + var keyword = Keyword(line); + if (keyword.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + startTls = true; + } + else if (keyword.Equals("8BITMIME", StringComparison.OrdinalIgnoreCase)) + { + eightBit = true; + } + else if (keyword.Equals("SMTPUTF8", StringComparison.OrdinalIgnoreCase)) + { + utf8 = true; + } + else if (keyword.Equals("SIZE", StringComparison.OrdinalIgnoreCase)) + { + var rest = Rest(line); + if (long.TryParse(rest, out var parsed) && parsed > 0) + { + size = parsed; + } + } + else if (keyword.Equals("AUTH", StringComparison.OrdinalIgnoreCase)) + { + foreach (var mechanism in Rest(line).Split(' ', StringSplitOptions.RemoveEmptyEntries)) + { + auth.Add(mechanism.ToUpperInvariant()); + } + } + } + + return new SmtpCapabilities + { + StartTls = startTls, + EightBitMime = eightBit, + SmtpUtf8 = utf8, + MaxSize = size, + AuthMechanisms = auth, + }; + } + + /// Returns whether was advertised. + public bool SupportsAuth(string mechanism) + { + foreach (var advertised in AuthMechanisms) + { + if (advertised.Equals(mechanism, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static string Keyword(string line) + { + var span = line.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space < 0 ? span.ToString() : span[..space].ToString(); + } + + private static string Rest(string line) + { + var span = line.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space < 0 ? string.Empty : span[(space + 1)..].Trim().ToString(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpConnection.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpConnection.cs new file mode 100644 index 00000000..3a35b47d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpConnection.cs @@ -0,0 +1,451 @@ +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// + /// One SMTP TCP/TLS session. Supports sequential messages, QUIT, and reconnect. + /// Not a connection pool. + /// + internal sealed class SmtpConnection : IAsyncDisposable + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + + private readonly SmtpOptions _options; + private readonly string _ehloHostname; + private readonly IReadOnlyCollection? _extraTrustedRoots; + private TcpClient? _tcp; + private Stream? _stream; + private SmtpResponseReader? _reader; + private bool _tls; + private bool _quit; + + public SmtpConnection( + SmtpOptions options, + string ehloHostname, + IReadOnlyCollection? extraTrustedRoots = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(ehloHostname); + _options = options; + _ehloHostname = ehloHostname; + _extraTrustedRoots = extraTrustedRoots; + } + + public SmtpCapabilities Capabilities { get; private set; } = new(); + + public bool IsAuthenticated { get; private set; } + + public bool IsTls => _tls; + + public async Task ConnectAsync(CancellationToken cancellationToken) + { + await DisposeSocketAsync().ConfigureAwait(false); + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(_options.ConnectTimeout); + + _tcp = new TcpClient(); + try + { + await _tcp.ConnectAsync(_options.Host.Trim(), _options.Port, connectCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connect timed out."); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is SocketException or IOException) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connect failed.", ex); + } + + _tcp.NoDelay = true; + _stream = _tcp.GetStream(); + _reader = new SmtpResponseReader(_stream); + + if (_options.Security == SmtpSecurityMode.ImplicitTls) + { + await UpgradeTlsAsync(cancellationToken).ConfigureAwait(false); + } + + var greeting = await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + if (!greeting.IsPositiveCompletion) + { + throw Classify(greeting, "SMTP greeting rejected."); + } + } + + public async Task EhloAsync(CancellationToken cancellationToken) + { + var response = await CommandAsync("EHLO", SanitizeEhlo(_ehloHostname), cancellationToken) + .ConfigureAwait(false); + if (!response.IsPositiveCompletion) + { + var helo = await CommandAsync("HELO", SanitizeEhlo(_ehloHostname), cancellationToken) + .ConfigureAwait(false); + if (!helo.IsPositiveCompletion) + { + throw Classify(helo, "EHLO/HELO failed."); + } + + Capabilities = new SmtpCapabilities(); + return; + } + + Capabilities = SmtpCapabilities.Parse(response); + } + + public async Task StartTlsAsync(CancellationToken cancellationToken) + { + if (!Capabilities.StartTls) + { + throw new SmtpException( + SmtpFailureKind.Tls, + "STARTTLS is required but the server did not advertise STARTTLS."); + } + + var response = await CommandAsync("STARTTLS", argument: null, cancellationToken).ConfigureAwait(false); + if (response.Code != 220) + { + throw Classify(response, "STARTTLS was rejected."); + } + + await UpgradeTlsAsync(cancellationToken).ConfigureAwait(false); + await EhloAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task AuthenticateAsync(CancellationToken cancellationToken) + { + var username = _options.Username.Trim(); + if (string.IsNullOrEmpty(username)) + { + return; + } + + if (_options.RequireTlsForAuthentication && !_tls) + { + throw new SmtpException( + SmtpFailureKind.Authentication, + "SMTP AUTH requires TLS and the session is not TLS-protected."); + } + + if (Capabilities.SupportsAuth("PLAIN")) + { + await AuthPlainAsync(username, _options.Password, cancellationToken).ConfigureAwait(false); + IsAuthenticated = true; + return; + } + + if (Capabilities.SupportsAuth("LOGIN")) + { + await AuthLoginAsync(username, _options.Password, cancellationToken).ConfigureAwait(false); + IsAuthenticated = true; + return; + } + + throw new SmtpException( + SmtpFailureKind.Authentication, + "SMTP AUTH is configured but the server advertised neither PLAIN nor LOGIN."); + } + + public Task MailFromAsync(string mailbox, CancellationToken cancellationToken) => + CommandAsync("MAIL", "FROM:<" + mailbox + ">", cancellationToken); + + public Task RcptToAsync(string mailbox, CancellationToken cancellationToken) => + CommandAsync("RCPT", "TO:<" + mailbox + ">", cancellationToken); + + public async Task DataAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + var intro = await CommandAsync("DATA", argument: null, cancellationToken).ConfigureAwait(false); + if (!intro.IsPositiveIntermediate) + { + throw Classify(intro, "DATA was rejected."); + } + + await WriteDataAsync(message, cancellationToken).ConfigureAwait(false); + return await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task QuitAsync(CancellationToken cancellationToken) + { + if (_quit || _stream is null) + { + return; + } + + _quit = true; + try + { + _ = await CommandAsync("QUIT", argument: null, cancellationToken).ConfigureAwait(false); + } + catch (SmtpException) + { + // Best-effort close. + } + } + + public async ValueTask DisposeAsync() + { + await DisposeSocketAsync().ConfigureAwait(false); + } + + private async Task UpgradeTlsAsync(CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP TLS upgrade has no transport stream."); + } + + var ssl = new SslStream(_stream, leaveInnerStreamOpen: false); + try + { + using var tlsCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + tlsCts.CancelAfter(_options.ConnectTimeout); + await ssl.AuthenticateAsClientAsync(CreateClientTlsOptions(), tlsCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw new SmtpException(SmtpFailureKind.Tls, "SMTP TLS handshake timed out."); + } + catch (OperationCanceledException) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is AuthenticationException or IOException) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw new SmtpException(SmtpFailureKind.Tls, "SMTP TLS handshake failed.", ex); + } + + _stream = ssl; + _reader?.ReturnBuffer(); + _reader = new SmtpResponseReader(ssl); + _tls = true; + } + + private async Task AuthPlainAsync(string username, string password, CancellationToken cancellationToken) + { + var payload = Convert.ToBase64String(Encoding.UTF8.GetBytes("\0" + username + "\0" + password)); + var response = await CommandAsync("AUTH", "PLAIN " + payload, cancellationToken, redact: true) + .ConfigureAwait(false); + if (!response.IsPositiveCompletion) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH PLAIN failed.") + { + StatusCode = response.Code, + }; + } + } + + private async Task AuthLoginAsync(string username, string password, CancellationToken cancellationToken) + { + var intro = await CommandAsync("AUTH", "LOGIN", cancellationToken, redact: true).ConfigureAwait(false); + if (!intro.IsPositiveIntermediate) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN was rejected.") + { + StatusCode = intro.Code, + }; + } + + var userResponse = await CommandAsync( + Convert.ToBase64String(Encoding.UTF8.GetBytes(username)), + argument: null, + cancellationToken, + redact: true) + .ConfigureAwait(false); + if (!userResponse.IsPositiveIntermediate) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN username was rejected.") + { + StatusCode = userResponse.Code, + }; + } + + var passResponse = await CommandAsync( + Convert.ToBase64String(Encoding.UTF8.GetBytes(password)), + argument: null, + cancellationToken, + redact: true) + .ConfigureAwait(false); + if (!passResponse.IsPositiveCompletion) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN failed.") + { + StatusCode = passResponse.Code, + }; + } + } + + private async Task CommandAsync( + string verb, + string? argument, + CancellationToken cancellationToken, + bool redact = false) + { + _ = redact; + var line = argument is null ? verb : verb + " " + argument; + if (line.IndexOf('\r') >= 0 || line.IndexOf('\n') >= 0) + { + throw new SmtpException(SmtpFailureKind.Message, "SMTP command contained CR or LF."); + } + + await WriteLineAsync(line, cancellationToken).ConfigureAwait(false); + return await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task WriteLineAsync(string line, CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + var encoded = Encoding.ASCII.GetBytes(line); + using var writeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + writeCts.CancelAfter(_options.CommandTimeout); + try + { + await _stream.WriteAsync(encoded, writeCts.Token).ConfigureAwait(false); + await _stream.WriteAsync(Crlf, writeCts.Token).ConfigureAwait(false); + await _stream.FlushAsync(writeCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP write timed out."); + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP write failed.", ex); + } + } + + private async Task WriteDataAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + using var writeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + writeCts.CancelAfter(_options.CommandTimeout); + try + { + await SmtpDataEncoder.WriteStuffedAsync(_stream, message, writeCts.Token).ConfigureAwait(false); + await _stream.FlushAsync(writeCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP DATA write timed out."); + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP DATA write failed.", ex); + } + } + + private async Task ReadResponseAsync(CancellationToken cancellationToken) + { + if (_reader is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + using var readCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readCts.CancelAfter(_options.CommandTimeout); + try + { + return await _reader.ReadAsync(readCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP read timed out."); + } + catch (SmtpException) + { + throw; + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP read failed.", ex); + } + } + + private async Task DisposeSocketAsync() + { + _reader?.ReturnBuffer(); + _reader = null; + if (_stream is not null) + { + await _stream.DisposeAsync().ConfigureAwait(false); + _stream = null; + } + + _tcp?.Dispose(); + _tcp = null; + _tls = false; + IsAuthenticated = false; + _quit = false; + } + + internal static SmtpException Classify(SmtpResponse response, string message) + { + var kind = response.IsTransientNegative + ? SmtpFailureKind.Transient + : response.IsPermanentNegative + ? SmtpFailureKind.Permanent + : SmtpFailureKind.Protocol; + return new SmtpException(kind, message) { StatusCode = response.Code }; + } + + private SslClientAuthenticationOptions CreateClientTlsOptions() + { + var options = new SslClientAuthenticationOptions + { + TargetHost = _options.Host.Trim(), + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }; + + if (_extraTrustedRoots is { Count: > 0 }) + { + var policy = new X509ChainPolicy + { + TrustMode = X509ChainTrustMode.CustomRootTrust, + RevocationMode = X509RevocationMode.NoCheck, + }; + foreach (var root in _extraTrustedRoots) + { + policy.CustomTrustStore.Add(root); + } + + options.CertificateChainPolicy = policy; + } + + return options; + } + + private static string SanitizeEhlo(string hostname) + { + var trimmed = hostname.Trim(); + if (trimmed.Length == 0 || trimmed.IndexOfAny(['\r', '\n', ' ']) >= 0) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP EHLO hostname is missing or invalid."); + } + + return trimmed; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpDataEncoder.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpDataEncoder.cs new file mode 100644 index 00000000..c1ce20a8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpDataEncoder.cs @@ -0,0 +1,80 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Applies RFC 5321 transparency (dot-stuffing) and the DATA terminator. + internal static class SmtpDataEncoder + { + /// Writes with dot-stuffing and a final CRLF.CRLF. + public static async Task WriteStuffedAsync( + Stream stream, + ReadOnlyMemory message, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + var stuffed = Stuff(message.Span); + await stream.WriteAsync(stuffed, cancellationToken).ConfigureAwait(false); + } + + internal static byte[] Stuff(ReadOnlySpan span) + { + if (span.Length == 0) + { + return ".\r\n"u8.ToArray(); + } + + var output = new List(span.Length + 8); + var lineStart = true; + var lastWasCr = false; + var offset = 0; + while (offset < span.Length) + { + var remaining = span[offset..]; + if (lineStart && remaining[0] == (byte)'.') + { + output.Add((byte)'.'); + } + + var cr = remaining.IndexOf((byte)'\r'); + var lf = remaining.IndexOf((byte)'\n'); + int take; + var nextLineStart = false; + if (cr >= 0 && (lf < 0 || cr <= lf)) + { + take = cr + 1; + if (cr + 1 < remaining.Length && remaining[cr + 1] == (byte)'\n') + { + take++; + nextLineStart = true; + } + + lastWasCr = remaining[cr] == (byte)'\r' && !nextLineStart; + } + else if (lf >= 0) + { + take = lf + 1; + nextLineStart = true; + lastWasCr = false; + } + else + { + take = remaining.Length; + lastWasCr = remaining[^1] == (byte)'\r'; + } + + output.AddRange(remaining[..take].ToArray()); + offset += take; + lineStart = nextLineStart; + } + + if (!lastWasCr && !(span.Length >= 2 && span[^2] == (byte)'\r' && span[^1] == (byte)'\n')) + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + + output.Add((byte)'.'); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + return [.. output]; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpException.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpException.cs new file mode 100644 index 00000000..858002ed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpException.cs @@ -0,0 +1,22 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// SMTP transport failure with a classified . + public sealed class SmtpException : Exception + { + /// Initializes a new SMTP failure. + public SmtpException(SmtpFailureKind kind, string message, Exception? inner = null) + : base(message, inner) + { + Kind = kind; + } + + /// Gets the failure classification. + public SmtpFailureKind Kind { get; } + + /// Gets the last SMTP status code when one was received; otherwise 0. + public int StatusCode { get; init; } + + /// Gets whether retry is appropriate. + public bool IsTransient => Kind is SmtpFailureKind.Transient or SmtpFailureKind.Network; + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpFailureKind.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpFailureKind.cs new file mode 100644 index 00000000..75ee9cd5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpFailureKind.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Classifies an SMTP delivery failure for retry policy. + public enum SmtpFailureKind + { + /// Transient SMTP 4xx or temporary network/DNS/TLS issue. May retry. + Transient = 0, + + /// Permanent SMTP 5xx recipient or mailbox rejection. Do not retry. + Permanent = 1, + + /// SMTP protocol violation or malformed response. Do not retry indefinitely. + Protocol = 2, + + /// TCP/DNS connect or disconnect failure. May retry when transient. + Network = 3, + + /// TLS handshake or certificate validation failure. + Tls = 4, + + /// SMTP AUTH failed or is unavailable. Do not retry. + Authentication = 5, + + /// The operation was cancelled. + Cancelled = 6, + + /// Message is malformed or exceeds the server SIZE limit. Do not retry. + Message = 7, + + /// Some recipients were accepted and others rejected. Do not retry (would duplicate). + Partial = 8, + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpRecipientResult.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpRecipientResult.cs new file mode 100644 index 00000000..27fe9e57 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpRecipientResult.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Per-recipient SMTP RCPT outcome. + /// Envelope recipient. + /// Whether the server returned a 2xx RCPT reply. + /// SMTP status code. + /// Reply text (never a credential). + public readonly record struct SmtpRecipientResult( + EmailAddress Address, + bool Accepted, + int StatusCode, + string Text); +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponse.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponse.cs new file mode 100644 index 00000000..df3daa58 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponse.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// A complete SMTP reply (RFC 5321), including continuation lines. + public sealed class SmtpResponse + { + /// Initializes a parsed SMTP reply. + public SmtpResponse(int code, IReadOnlyList lines, string? enhancedStatus, string text) + { + Code = code; + Lines = lines; + EnhancedStatus = enhancedStatus; + Text = text; + } + + /// Gets the three-digit status code. + public int Code { get; } + + /// Gets the optional enhanced status code (RFC 2034) from the last line. + public string? EnhancedStatus { get; } + + /// Gets concatenated reply text without status prefixes. + public string Text { get; } + + /// Gets raw text portions of each reply line. + public IReadOnlyList Lines { get; } + + /// Gets whether the code is 2xx. + public bool IsPositiveCompletion => Code is >= 200 and <= 299; + + /// Gets whether the code is 3xx. + public bool IsPositiveIntermediate => Code is >= 300 and <= 399; + + /// Gets whether the code is 4xx. + public bool IsTransientNegative => Code is >= 400 and <= 499; + + /// Gets whether the code is 5xx. + public bool IsPermanentNegative => Code is >= 500 and <= 599; + + /// + public override string ToString() => Code + " " + Text; + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponseReader.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponseReader.cs new file mode 100644 index 00000000..a1bb65c2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpResponseReader.cs @@ -0,0 +1,199 @@ +using System.Buffers; +using System.Text; + +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Reads RFC 5321 multiline SMTP replies from a stream. + internal sealed class SmtpResponseReader + { + internal const int MaxLineLength = 512; + internal const int MaxReplyLines = 64; + + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _buffered; + private int _consumed; + + public SmtpResponseReader(Stream stream, int bufferSize = 1024) + { + ArgumentNullException.ThrowIfNull(stream); + _stream = stream; + _buffer = ArrayPool.Shared.Rent(bufferSize); + } + + public async Task ReadAsync(CancellationToken cancellationToken) + { + var lines = new List(4); + int? code = null; + while (true) + { + var line = await ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (!TryParseReplyLine(line.Span, out var lineCode, out var continuation, out var text)) + { + throw new SmtpException(SmtpFailureKind.Protocol, "Malformed SMTP reply line."); + } + + if (code is null) + { + code = lineCode; + } + else if (code.Value != lineCode) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply code changed during continuation."); + } + + lines.Add(text); + if (lines.Count > MaxReplyLines) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply exceeded the maximum number of lines."); + } + + if (!continuation) + { + var last = lines[^1]; + TrySplitEnhanced(last, out var enhanced, out var remainder); + var joined = string.Join("\n", lines); + return new SmtpResponse(code.Value, lines, enhanced, joined); + } + } + } + + internal static bool TryParseReplyLine( + ReadOnlySpan line, + out int code, + out bool continuation, + out string text) + { + code = 0; + continuation = false; + text = string.Empty; + if (line.Length < 3) + { + return false; + } + + for (var i = 0; i < 3; i++) + { + var digit = line[i]; + if (digit is < (byte)'0' or > (byte)'9') + { + return false; + } + + code = (code * 10) + (digit - (byte)'0'); + } + + if (line.Length == 3) + { + return true; + } + + var separator = line[3]; + if (separator == (byte)'-') + { + continuation = true; + } + else if (separator != (byte)' ') + { + return false; + } + + text = line.Length > 4 + ? Encoding.ASCII.GetString(line[4..]) + : string.Empty; + return true; + } + + internal static bool TrySplitEnhanced(string text, out string? enhanced, out string remainder) + { + enhanced = null; + remainder = text; + var span = text.AsSpan().TrimStart(); + if (span.Length < 5) + { + return false; + } + + var classDigit = span[0]; + if (classDigit is < '2' or > '5' || span[1] != '.') + { + return false; + } + + var end = 2; + while (end < span.Length && (char.IsAsciiDigit(span[end]) || span[end] == '.')) + { + end++; + } + + if (end >= span.Length || span[end] != ' ') + { + return false; + } + + enhanced = span[..end].ToString(); + remainder = span[(end + 1)..].ToString(); + return true; + } + + private async Task> ReadLineAsync(CancellationToken cancellationToken) + { + var line = new ArrayBufferWriter(64); + while (true) + { + if (_consumed >= _buffered) + { + _consumed = 0; + _buffered = await _stream.ReadAsync(_buffer, cancellationToken).ConfigureAwait(false); + if (_buffered == 0) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP server closed the connection."); + } + } + + var span = _buffer.AsSpan(_consumed, _buffered - _consumed); + var cr = span.IndexOf((byte)'\r'); + var lf = span.IndexOf((byte)'\n'); + if (cr >= 0 && (lf < 0 || cr < lf)) + { + Append(line, span[..cr]); + _consumed += cr + 1; + if (_consumed < _buffered && _buffer[_consumed] == (byte)'\n') + { + _consumed++; + } + + return FinishLine(line); + } + + if (lf >= 0) + { + Append(line, span[..lf]); + _consumed += lf + 1; + return FinishLine(line); + } + + Append(line, span); + _consumed = _buffered; + } + } + + private static void Append(ArrayBufferWriter line, ReadOnlySpan span) + { + if (line.WrittenCount + span.Length > MaxLineLength) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply line exceeded 512 octets."); + } + + line.Write(span); + } + + private static ReadOnlyMemory FinishLine(ArrayBufferWriter line) => + line.WrittenMemory.ToArray(); + + public void ReturnBuffer() + { + ArrayPool.Shared.Return(_buffer); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpSendResult.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpSendResult.cs new file mode 100644 index 00000000..c6327c14 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpSendResult.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// Outcome of one SMTP transaction (MAIL/RCPT/DATA). + public sealed class SmtpSendResult + { + /// Gets whether every recipient was accepted and DATA completed with 2xx. + public required bool Succeeded { get; init; } + + /// Gets per-recipient RCPT results. + public required IReadOnlyList Recipients { get; init; } + + /// Gets the DATA status code when DATA was issued; otherwise 0. + public int DataStatusCode { get; init; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpTransport.cs b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpTransport.cs new file mode 100644 index 00000000..ee50249c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Email/Smtp/SmtpTransport.cs @@ -0,0 +1,170 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Email.Smtp +{ + /// + /// Native SMTP client using and + /// . One connection per delivery. + /// + /// + /// Does not use System.Net.Mail.SmtpClient or a third-party SMTP library. + /// Connection reuse across messages is supported by + /// but this transport opens a fresh session per send so the delivery worker is not + /// coupled to a single remote connection. + /// EHLO/HELO uses the application FQDN from . + /// Certificate validation is mandatory. Tests may supply extra trusted roots; + /// there is no production bypass switch. + /// + public sealed class SmtpTransport : ISmtpTransport + { + private readonly IOptions _options; + private readonly IOptions _nntpd; + private readonly ILogger _logger; + private readonly IReadOnlyCollection? _extraTrustedRoots; + + /// Initializes a new SMTP transport. + public SmtpTransport( + IOptions options, + IOptions nntpd, + ILogger logger) + : this(options, nntpd, logger, extraTrustedRoots: null) + { + } + + /// + /// Initializes a transport that trusts additional roots for certificate + /// validation (tests only). Production DI uses the three-argument constructor. + /// + public SmtpTransport( + IOptions options, + IOptions nntpd, + ILogger logger, + IReadOnlyCollection? extraTrustedRoots) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(nntpd); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _nntpd = nntpd; + _logger = logger; + _extraTrustedRoots = extraTrustedRoots; + } + + /// + public async Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + var smtp = _options.Value.Smtp ?? new SmtpOptions(); + var ehlo = _nntpd.Value.Fqdn; + if (string.IsNullOrWhiteSpace(ehlo)) + { + throw new SmtpException(SmtpFailureKind.Protocol, "Application FQDN is required for SMTP EHLO."); + } + + EmailLogMessages.SmtpSessionStarting( + _logger, + smtp.Host, + smtp.Port, + smtp.Security.ToString()); + + await using var connection = new SmtpConnection(smtp, ehlo, _extraTrustedRoots); + await connection.ConnectAsync(cancellationToken).ConfigureAwait(false); + await connection.EhloAsync(cancellationToken).ConfigureAwait(false); + + if (smtp.Security == SmtpSecurityMode.StartTls) + { + await connection.StartTlsAsync(cancellationToken).ConfigureAwait(false); + } + + if (connection.Capabilities.MaxSize is { } maxSize + && item.EncodedMessage.Length > maxSize) + { + throw new SmtpException( + SmtpFailureKind.Message, + "Encoded message exceeds the server SIZE limit."); + } + + await connection.AuthenticateAsync(cancellationToken).ConfigureAwait(false); + + var mail = await connection + .MailFromAsync(item.EnvelopeSender.Address, cancellationToken) + .ConfigureAwait(false); + if (!mail.IsPositiveCompletion) + { + throw SmtpConnection.Classify(mail, "MAIL FROM was rejected."); + } + + var recipients = new List(item.Recipients.Count); + var accepted = 0; + var transient = 0; + var permanent = 0; + foreach (var recipient in item.Recipients) + { + var rcpt = await connection.RcptToAsync(recipient.Address, cancellationToken).ConfigureAwait(false); + var ok = rcpt.IsPositiveCompletion; + if (ok) + { + accepted++; + } + else if (rcpt.IsTransientNegative) + { + transient++; + } + else + { + permanent++; + } + + recipients.Add(new SmtpRecipientResult(recipient, ok, rcpt.Code, rcpt.Text)); + } + + if (accepted == 0) + { + if (transient > 0 && permanent == 0) + { + throw new SmtpException(SmtpFailureKind.Transient, "All recipients were deferred (4xx).") + { + StatusCode = recipients[0].StatusCode, + }; + } + + throw new SmtpException(SmtpFailureKind.Permanent, "All recipients were rejected.") + { + StatusCode = recipients[0].StatusCode, + }; + } + + var data = await connection.DataAsync(item.EncodedMessage, cancellationToken).ConfigureAwait(false); + await connection.QuitAsync(cancellationToken).ConfigureAwait(false); + + if (!data.IsPositiveCompletion) + { + throw SmtpConnection.Classify(data, "DATA was not accepted."); + } + + if (accepted != item.Recipients.Count) + { + throw new SmtpException( + SmtpFailureKind.Partial, + "Some recipients were rejected after others were accepted.") + { + StatusCode = data.Code, + }; + } + + EmailLogMessages.SmtpSessionCompleted( + _logger, + smtp.Host, + smtp.Port, + accepted); + return new SmtpSendResult + { + Succeeded = true, + Recipients = recipients, + DataStatusCode = data.Code, + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/GlobalUsings.cs b/src/VectorNNTPD.ReaderService/GlobalUsings.cs new file mode 100644 index 00000000..ca57c645 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/GlobalUsings.cs @@ -0,0 +1,5 @@ +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; diff --git a/src/VectorNNTPD.ReaderService/History/HistoryDb.cs b/src/VectorNNTPD.ReaderService/History/HistoryDb.cs new file mode 100644 index 00000000..d8e576b1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryDb.cs @@ -0,0 +1,227 @@ +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.History +{ + /// + /// HistoryDB consumer of : local memory first, then Redis. + /// + public sealed class HistoryDb : IHistoryDb, IHistoryLookupMetrics + { + private readonly LocalHistoryStore _local; + private readonly IRedisService _redis; + private readonly HistoryWriteQueue _writes; + private readonly TimeSpan _retention; + private readonly ILogger _logger; + private long _lookups; + private long _hits; + private long _misses; + private long _errors; + private long _waitTicks; + + /// Initializes a new instance of the class. + public HistoryDb( + IRedisService redis, + IOptions options, + ILogger logger) + : this( + redis, + options?.Value.HistoryTime ?? throw new ArgumentNullException(nameof(options)), + logger, + TimeProvider.System, + new HistoryWriteQueue()) + { + } + + /// Initializes a new instance with explicit retention (tests). + internal HistoryDb( + IRedisService redis, + TimeSpan retention, + ILogger logger, + TimeProvider timeProvider, + HistoryWriteQueue writeQueue, + int maxEntries = LocalHistoryStore.MaxEntries) + { + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(writeQueue); + _redis = redis; + _retention = retention; + _logger = logger; + _local = new LocalHistoryStore(retention, timeProvider, maxEntries); + _writes = writeQueue; + } + + /// Gets the Redis write queue drained by . + internal HistoryWriteQueue Writes => _writes; + + /// Gets the configured retention period. + internal TimeSpan Retention => _retention; + + /// Gets the local store (tests and maintenance). + internal LocalHistoryStore Local => _local; + + /// + public bool ContainsLocal(in HistoryDigest digest) => _local.Contains(digest); + + /// Removes expired local entries. Invoked by only. + internal int Maintain() => _local.Maintain(); + + /// + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + var started = Stopwatch.GetTimestamp(); + var digest = HistoryDigest.FromMessageId(messageId.Span); + if (_local.Contains(digest)) + { + RecordLookup(HistoryLookupResult.Seen, started); + return new ValueTask(HistoryLookupResult.Seen); + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + RecordLookup(HistoryLookupResult.Unavailable, started); + return new ValueTask(HistoryLookupResult.Unavailable); + } + + return AwaitLookupAsync( + LookupRedisAsync(digest, isRecoveryProbe, recordOnMiss: true, cancellationToken), + started); + } + + /// + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + var started = Stopwatch.GetTimestamp(); + var digest = HistoryDigest.FromMessageId(messageId.Span); + if (_local.Contains(digest)) + { + RecordLookup(HistoryLookupResult.Seen, started); + return new ValueTask(HistoryLookupResult.Seen); + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + RecordLookup(HistoryLookupResult.Unavailable, started); + return new ValueTask(HistoryLookupResult.Unavailable); + } + + return AwaitLookupAsync( + LookupRedisAsync(digest, isRecoveryProbe, recordOnMiss: false, cancellationToken), + started); + } + + /// + public HistoryLookupMetricsSnapshot Capture() => + new( + Volatile.Read(ref _lookups), + Volatile.Read(ref _hits), + Volatile.Read(ref _misses), + Volatile.Read(ref _errors), + Volatile.Read(ref _waitTicks)); + + /// + public void Remember(ReadOnlyMemory messageId) + { + var digest = HistoryDigest.FromMessageId(messageId.Span); + _local.Add(digest); + if (!_writes.TryEnqueue(digest)) + { + HistoryLogMessages.WriteQueueFull(_logger); + } + } + + private async ValueTask LookupRedisAsync( + HistoryDigest digest, + bool isRecoveryProbe, + bool recordOnMiss, + CancellationToken cancellationToken) + { + try + { + var key = HistoryRedisKeys.Create(digest); + var exists = await _redis.KeyExistsAsync(key, cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + + if (exists) + { + _local.Add(digest); + return HistoryLookupResult.Seen; + } + + if (recordOnMiss) + { + _local.Add(digest); + if (!_writes.TryEnqueue(digest)) + { + HistoryLogMessages.WriteQueueFull(_logger); + } + } + + return HistoryLookupResult.Unseen; + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + throw; + } + } + + private async ValueTask AwaitLookupAsync( + ValueTask pending, + long started) + { + try + { + var result = await pending.ConfigureAwait(false); + RecordLookup(result, started); + return result; + } + catch (OperationCanceledException) + { + throw; + } + } + + private void RecordLookup(HistoryLookupResult result, long started) + { + Interlocked.Increment(ref _lookups); + switch (result) + { + case HistoryLookupResult.Seen: + Interlocked.Increment(ref _hits); + break; + case HistoryLookupResult.Unseen: + Interlocked.Increment(ref _misses); + break; + default: + Interlocked.Increment(ref _errors); + break; + } + + var ticks = Stopwatch.GetTimestamp() - started; + if (ticks > 0) + { + Interlocked.Add(ref _waitTicks, ticks); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryDigest.cs b/src/VectorNNTPD.ReaderService/History/HistoryDigest.cs new file mode 100644 index 00000000..6be0fd34 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryDigest.cs @@ -0,0 +1,63 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.History +{ + /// 32-byte BLAKE3 digest of a Message-ID's protocol bytes. + /// + /// Delegates to Common so History and ArticleRecord share + /// the same BLAKE3(Message-ID) identity bytes. + /// + public readonly struct HistoryDigest : IEquatable + { + /// BLAKE3 output length in bytes. + public const int Length = ArticleId.Length; + + private readonly ArticleId _id; + + private HistoryDigest(ArticleId id) + { + _id = id; + } + + /// Computes the HistoryDB digest from Message-ID wire octets. + public static HistoryDigest FromMessageId(ReadOnlySpan messageId) + => new(ArticleId.FromMessageId(messageId)); + + /// Creates a digest from an existing 32-byte BLAKE3 output. + public static HistoryDigest FromSpan(ReadOnlySpan digest) + { + if (digest.Length != Length) + { + throw new ArgumentException($"History digest must be {Length} bytes.", nameof(digest)); + } + + return new HistoryDigest(ArticleId.FromSpan(digest)); + } + + /// Copies the digest bytes into . + public void CopyTo(Span destination) + { + if (destination.Length < Length) + { + throw new ArgumentException("Destination is shorter than the digest.", nameof(destination)); + } + + _id.CopyTo(destination); + } + + /// + public bool Equals(HistoryDigest other) => _id.Equals(other._id); + + /// + public override bool Equals(object? obj) => obj is HistoryDigest other && Equals(other); + + /// + public override int GetHashCode() => _id.GetHashCode(); + + /// Equality operator. + public static bool operator ==(HistoryDigest left, HistoryDigest right) => left.Equals(right); + + /// Inequality operator. + public static bool operator !=(HistoryDigest left, HistoryDigest right) => !left.Equals(right); + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryLogMessages.cs b/src/VectorNNTPD.ReaderService/History/HistoryLogMessages.cs new file mode 100644 index 00000000..2ae47dfb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryLogMessages.cs @@ -0,0 +1,54 @@ +namespace VectorNNTPD.ReaderService.History +{ + /// Source-generated HistoryDB log messages. + internal static partial class HistoryLogMessages + { + [LoggerMessage( + EventId = 2200, + Level = LogLevel.Warning, + Message = "HistoryDB Redis write queue is full; local history was recorded and the Redis write was dropped")] + public static partial void WriteQueueFull(ILogger logger); + + [LoggerMessage( + EventId = 2201, + Level = LogLevel.Error, + Message = "HistoryDB Redis write failed")] + public static partial void RedisWriteFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2202, + Level = LogLevel.Warning, + Message = "HistoryDB Redis lookup failed; returning 431")] + public static partial void RedisLookupFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2203, + Level = LogLevel.Information, + Message = "HistoryDB write worker started (queue capacity {Capacity}, retention {Retention})")] + public static partial void WriterStarted(ILogger logger, int Capacity, TimeSpan Retention); + + [LoggerMessage( + EventId = 2204, + Level = LogLevel.Information, + Message = "HistoryDB write worker stopped")] + public static partial void WriterStopped(ILogger logger); + + [LoggerMessage( + EventId = 2205, + Level = LogLevel.Warning, + Message = "HistoryDB write worker stopped with {Queued} queued write(s) remaining")] + public static partial void WriterStoppedWithQueued(ILogger logger, int Queued); + + [LoggerMessage( + EventId = 2206, + Level = LogLevel.Information, + Message = "HistoryDB maintenance worker started (interval {Interval})")] + public static partial void MaintenanceStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2207, + Level = LogLevel.Information, + Message = "HistoryDB maintenance worker stopped")] + public static partial void MaintenanceStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryLookupMetrics.cs b/src/VectorNNTPD.ReaderService/History/HistoryLookupMetrics.cs new file mode 100644 index 00000000..d1173d61 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryLookupMetrics.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.History +{ + /// Lifetime HistoryDB lookup counters (hits, misses, errors, wait). + public readonly struct HistoryLookupMetricsSnapshot + { + /// Initializes a snapshot. + public HistoryLookupMetricsSnapshot(long lookups, long hits, long misses, long errors, long waitTicks) + { + Lookups = lookups; + Hits = hits; + Misses = misses; + Errors = errors; + WaitTicks = waitTicks; + } + + /// Gets completed lookups. + public long Lookups { get; } + + /// Gets outcomes. + public long Hits { get; } + + /// Gets outcomes. + public long Misses { get; } + + /// Gets outcomes. + public long Errors { get; } + + /// Gets accumulated lookup wait ticks (). + public long WaitTicks { get; } + } + + /// Authoritative process-wide HistoryDB lookup metrics. + public interface IHistoryLookupMetrics + { + /// Captures lifetime lookup counters. + HistoryLookupMetricsSnapshot Capture(); + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryLookupResult.cs b/src/VectorNNTPD.ReaderService/History/HistoryLookupResult.cs new file mode 100644 index 00000000..d4231d08 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryLookupResult.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.ReaderService.History +{ + /// Outcome of a HistoryDB lookup for CHECK. + public enum HistoryLookupResult : byte + { + /// Message-ID has not been seen; CHECK should return 238. + Unseen = 0, + + /// Message-ID is already in history. CHECK and IHAVE map this result to a reply. + Seen = 1, + + /// Redis infrastructure failed; CHECK should return 431, not a false 238. + Unavailable = 2, + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryMaintenanceService.cs b/src/VectorNNTPD.ReaderService/History/HistoryMaintenanceService.cs new file mode 100644 index 00000000..27e4813a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryMaintenanceService.cs @@ -0,0 +1,103 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.History +{ + /// + /// Background worker that removes expired local HistoryDB entries off the CHECK path. + /// + public sealed class HistoryMaintenanceService : IApplicationService + { + /// Default interval between dictionary maintenance passes. + internal static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(30); + + private readonly HistoryDb _history; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public HistoryMaintenanceService( + HistoryDb history, + ILogger logger) + : this(history, logger, TimeProvider.System, DefaultInterval) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal HistoryMaintenanceService( + HistoryDb history, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _history = history; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "HistoryMaintenance"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + HistoryLogMessages.MaintenanceStopped(_logger); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + HistoryLogMessages.MaintenanceStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + _ = _history.Maintain(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + HistoryLogMessages.MaintenanceStopped(_logger); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryRedisKeys.cs b/src/VectorNNTPD.ReaderService/History/HistoryRedisKeys.cs new file mode 100644 index 00000000..602b73de --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryRedisKeys.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.History +{ + /// + /// Binary Redis keys for HistoryDB: ASCII namespace prefix plus the 32-byte BLAKE3 digest. + /// + /// + /// Format: nntpd:hist: (11 bytes) concatenated with the digest. The digest is not + /// hex- or base64-encoded. Conversion to byte[] happens at the Redis API boundary. + /// + public static class HistoryRedisKeys + { + /// ASCII namespace isolating HistoryDB keys from other Redis consumers. + public static ReadOnlySpan NamespacePrefix => "nntpd:hist:"u8; + + /// Total Redis key length in bytes. + public static int KeyLength => NamespacePrefix.Length + HistoryDigest.Length; + + /// Builds the deterministic HistoryDB Redis key for . + public static byte[] Create(in HistoryDigest digest) + { + var key = new byte[KeyLength]; + NamespacePrefix.CopyTo(key); + digest.CopyTo(key.AsSpan(NamespacePrefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryWriteQueue.cs b/src/VectorNNTPD.ReaderService/History/HistoryWriteQueue.cs new file mode 100644 index 00000000..eb243bd5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryWriteQueue.cs @@ -0,0 +1,64 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.ReaderService.History +{ + /// Bounded queue of HistoryDB Redis writes. Full writes are dropped, not unbounded. + internal sealed class HistoryWriteQueue + { + /// Default bound for pending Redis history writes. + public const int DefaultCapacity = 4096; + + private readonly Channel _channel; + private int _count; + + /// Initializes a new instance of the class. + public HistoryWriteQueue(int capacity = DefaultCapacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + Capacity = capacity; + _channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// Gets the queue capacity. + public int Capacity { get; } + + /// Gets the approximate number of queued writes. + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// Attempts to enqueue a Redis write. Returns when full or completed. + public bool TryEnqueue(in HistoryDigest digest) + { + if (!_channel.Writer.TryWrite(digest)) + { + return false; + } + + Interlocked.Increment(ref _count); + return true; + } + + /// Completes the writer so the consumer can drain and exit. + public void Complete() => _channel.Writer.TryComplete(); + + /// Dequeues the next write, or when completed and empty. + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var digest)) + { + Interlocked.Decrement(ref _count); + return digest; + } + } + + return null; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/History/HistoryWriteService.cs b/src/VectorNNTPD.ReaderService/History/HistoryWriteService.cs new file mode 100644 index 00000000..5b166508 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/HistoryWriteService.cs @@ -0,0 +1,125 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.History +{ + /// Background consumer that persists HistoryDB markers to Redis. + public sealed class HistoryWriteService : IApplicationService + { + private readonly HistoryDb _history; + private readonly IRedisService _redis; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public HistoryWriteService( + HistoryDb history, + IRedisService redis, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + _history = history; + _redis = redis; + _logger = logger; + } + + /// + public string Name => "HistoryWrite"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _history.Writes.Complete(); + await _runCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + HistoryLogMessages.WriterStoppedWithQueued(_logger, _history.Writes.Count); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + HistoryLogMessages.WriterStarted(_logger, _history.Writes.Capacity, _history.Retention); + while (true) + { + HistoryDigest? digest; + try + { + digest = await _history.Writes.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (digest is null) + { + break; + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + continue; + } + + try + { + var key = HistoryRedisKeys.Create(digest.Value); + await _redis.SetAsync(key, HistoryDbMarker.Value, _history.Retention, CancellationToken.None) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + HistoryLogMessages.RedisWriteFailed(_logger, ex); + } + + if (cancellationToken.IsCancellationRequested && _history.Writes.Count == 0) + { + break; + } + } + + HistoryLogMessages.WriterStopped(_logger); + } + } + + /// Single-byte Redis value stored for a HistoryDB marker. + internal static class HistoryDbMarker + { + /// Presence marker. Existence, not payload, is the history signal. + internal static readonly byte[] Value = [1]; + } +} diff --git a/src/VectorNNTPD.ReaderService/History/IHistoryDb.cs b/src/VectorNNTPD.ReaderService/History/IHistoryDb.cs new file mode 100644 index 00000000..cb996e86 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/IHistoryDb.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.ReaderService.History +{ + /// + /// Shared HistoryDB used by CHECK (peek), IHAVE, and TAKETHIS (peek + remember). + /// + public interface IHistoryDb + { + /// + /// Looks up wire octets. Memory hits do not touch Redis. + /// A miss records the identifier locally and queues a Redis write. + /// CHECK uses instead so a query does not make the identifier known. + /// + ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default); + + /// + /// Looks up without recording a miss. + /// Used by CHECK, IHAVE admission, and TAKETHIS so a query or failed transfer + /// does not make the identifier known. TAKETHIS starts this lookup as soon as + /// the Message-ID is parsed so it overlaps article receive. + /// + ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default); + + /// + /// Records locally and queues a Redis write after a + /// successful IHAVE or TAKETHIS accept. Does not perform a Redis EXISTS. + /// + void Remember(ReadOnlyMemory messageId); + + /// Returns whether the digest is currently present in the local store (tests). + bool ContainsLocal(in HistoryDigest digest); + } +} diff --git a/src/VectorNNTPD.ReaderService/History/LocalHistoryStore.cs b/src/VectorNNTPD.ReaderService/History/LocalHistoryStore.cs new file mode 100644 index 00000000..89c92aa9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/History/LocalHistoryStore.cs @@ -0,0 +1,150 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.ReaderService.History +{ + /// + /// Hard-capped expiring in-memory HistoryDB. + /// + /// + /// CHECK never walks the dictionary. A local hit is one bucket lookup plus a tick + /// comparison. Insert reserves a slot with an interlocked compare-exchange + /// then — never + /// . Expired keys are removed + /// only when the observed expiry still matches (compare/remove), on that key's + /// lookup or by on . + /// When the cap is reached, new digests are not inserted (no false hits); Redis + /// remains the authority for those IDs until maintenance frees a slot. Unexpired + /// entries are not evicted to make room. + /// + internal sealed class LocalHistoryStore + { + /// Hard cap on live local entries. Not a CHECK-path compaction trigger. + internal const int MaxEntries = 1_048_576; + + private readonly ConcurrentDictionary _entries = new(); + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _retention; + private readonly int _maxEntries; + private int _reservedCount; + + /// Initializes a new instance of the class. + public LocalHistoryStore(TimeSpan retention, TimeProvider timeProvider, int maxEntries = MaxEntries) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(retention, TimeSpan.Zero); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThan(maxEntries, 1); + _retention = retention; + _timeProvider = timeProvider; + _maxEntries = maxEntries; + } + + /// Gets the reserved slot count (includes not-yet-scanned expired keys). + public int ApproximateCount => Math.Max(0, Volatile.Read(ref _reservedCount)); + + /// Gets the dictionary entry count (tests). Not used on the CHECK path. + internal int EntryCountForTests => _entries.Count; + + /// Gets how many times has walked the dictionary. + internal int MaintenanceScanCount { get; private set; } + + /// Returns whether is present and unexpired. + public bool Contains(in HistoryDigest digest) + { + if (!_entries.TryGetValue(digest, out var expiryTicks)) + { + return false; + } + + if (_timeProvider.GetUtcNow().UtcTicks <= expiryTicks) + { + return true; + } + + TryRemoveObservedExpiry(digest, expiryTicks); + return false; + } + + /// + /// Records with the configured retention when a slot is available. + /// + public void Add(in HistoryDigest digest) + { + var expiry = _timeProvider.GetUtcNow().Add(_retention).UtcTicks; + while (true) + { + if (_entries.TryGetValue(digest, out var observed) + && (_entries.TryUpdate(digest, expiry, observed) || _entries.ContainsKey(digest))) + { + return; + } + + if (!TryReserveSlot()) + { + return; + } + + if (_entries.TryAdd(digest, expiry)) + { + return; + } + + ReleaseSlot(); + } + } + + /// + /// Removes expired entries. Must run off the CHECK thread (maintenance worker). + /// + /// The number of entries removed. + public int Maintain() + { + MaintenanceScanCount++; + var now = _timeProvider.GetUtcNow().UtcTicks; + var removed = 0; + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _entries) + { + if (pair.Value < now && TryRemoveObservedExpiry(pair.Key, pair.Value)) + { + removed++; + } + } + + return removed; + } + + /// + /// Removes only if its stored expiry still equals + /// . Used by lookup, maintenance, and tests. + /// + internal bool TryRemoveObservedExpiry(in HistoryDigest digest, long observedExpiryTicks) + { + if (_entries.TryRemove(new KeyValuePair(digest, observedExpiryTicks))) + { + ReleaseSlot(); + return true; + } + + return false; + } + + private bool TryReserveSlot() + { + while (true) + { + var current = Volatile.Read(ref _reservedCount); + if (current >= _maxEntries) + { + return false; + } + + if (Interlocked.CompareExchange(ref _reservedCount, current + 1, current) == current) + { + return true; + } + } + } + + private void ReleaseSlot() => Interlocked.Decrement(ref _reservedCount); + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/NntpServerRole.cs b/src/VectorNNTPD.ReaderService/Hosting/NntpServerRole.cs new file mode 100644 index 00000000..441f3cdd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/NntpServerRole.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.ReaderService.Hosting +{ + /// Operational role of this NNTP host process (intrinsic; not a runtime switch). + public enum NntpServerRole + { + /// NNTP reader-server role (MODE READER, article retrieval, POST). + Reader = 1, + + /// NNTP transit-server role (streaming peers, CHECK/TAKETHIS/IHAVE). + Transit = 2, + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/NntpdHostLifetime.cs b/src/VectorNNTPD.ReaderService/Hosting/NntpdHostLifetime.cs new file mode 100644 index 00000000..a2386000 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/NntpdHostLifetime.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting +{ + /// + /// Ensures application shutdown is initiated exactly once and coordinates with the generic host lifetime. + /// + public sealed class NntpdHostLifetime + { + private readonly ApplicationLifecycle _lifecycle; + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly object _sync = new(); + private Task? _shutdownTask; + private int _shutdownRequested; + + /// + /// Initializes a new instance of the class. + /// + public NntpdHostLifetime( + ApplicationLifecycle lifecycle, + IHostApplicationLifetime hostApplicationLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(hostApplicationLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _hostApplicationLifetime = hostApplicationLifetime; + _options = options; + _logger = logger; + } + + /// + /// Requests a single graceful application shutdown. Subsequent calls await the same operation. + /// + /// Host-provided shutdown token. + public Task RequestShutdownAsync(CancellationToken cancellationToken) + { + lock (_sync) + { + if (_shutdownTask is not null) + { + return _shutdownTask; + } + + if (Interlocked.Exchange(ref _shutdownRequested, 1) == 0) + { + HostingLogMessages.ShutdownRequestedOnce(_logger, _options.Value.ApplicationName); + } + + _shutdownTask = ShutdownCoreAsync(cancellationToken); + return _shutdownTask; + } + } + + /// + /// Stops the generic host after an unexpected application-service termination. + /// + public void NotifyUnexpectedTermination() + { + if (!_options.Value.StopHostOnUnexpectedServiceTermination) + { + HostingLogMessages.UnexpectedTerminationOptionDisabled( + _logger, + nameof(NntpdOptions.StopHostOnUnexpectedServiceTermination)); + return; + } + + HostingLogMessages.RequestingHostStop(_logger); + + _hostApplicationLifetime.StopApplication(); + } + + private async Task ShutdownCoreAsync(CancellationToken cancellationToken) + { + try + { + await _lifecycle.StopAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + HostingLogMessages.ShutdownCompletedWithFailure(_logger, ex); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/NntpdHostedService.cs b/src/VectorNNTPD.ReaderService/Hosting/NntpdHostedService.cs new file mode 100644 index 00000000..cfd4104f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/NntpdHostedService.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting +{ + /// + /// Integrates with the .NET Generic Host. + /// + /// + /// Startup runs during so a failed initialization prevents the host from + /// reporting successful start. While running, the background execution waits for host shutdown or + /// unexpected service termination. Fatal failures stop the host via + /// . + /// + public sealed class NntpdHostedService : BackgroundService + { + private readonly ApplicationLifecycle _lifecycle; + private readonly NntpdHostLifetime _hostLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public NntpdHostedService( + ApplicationLifecycle lifecycle, + NntpdHostLifetime hostLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(hostLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _hostLifetime = hostLifetime; + _options = options; + _logger = logger; + } + + /// + public override async Task StartAsync(CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + HostingLogMessages.HostStartingApplication(_logger, _options.Value.ApplicationName); + + await _lifecycle.StartAsync(cancellationToken).ConfigureAwait(false); + + HostingLogMessages.ApplicationEnteredRunning(_logger, sw.ElapsedMilliseconds); + + await base.StartAsync(cancellationToken).ConfigureAwait(false); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await _lifecycle.WaitAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + HostingLogMessages.HostedServiceExecutionCanceled(_logger); + } + catch (InvalidOperationException ex) + { + HostingLogMessages.UnexpectedApplicationServiceTermination(_logger, ex); + _hostLifetime.NotifyUnexpectedTermination(); + throw; + } + } + + /// + public override async Task StopAsync(CancellationToken cancellationToken) + { + HostingLogMessages.HostStoppingApplication(_logger, _options.Value.ApplicationName); + + try + { + await _hostLifetime.RequestShutdownAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await base.StopAsync(cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/NntpdServiceCollectionExtensions.cs b/src/VectorNNTPD.ReaderService/Hosting/NntpdServiceCollectionExtensions.cs new file mode 100644 index 00000000..35e71adb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/NntpdServiceCollectionExtensions.cs @@ -0,0 +1,767 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Hosting; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Acme; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.RabbitMq.Management; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting.Systemd; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Session.SpeedTest; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Email.Smtp; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Ninpaths; +using VectorNNTPD.ReaderService.Telemetry; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Transport.Vatp; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Hosting +{ + /// + /// Extension methods for registering VectorNNTPD.ReaderService hosting and lifecycle services. + /// + public static class NntpdServiceCollectionExtensions + { + + /// + /// Registers the Reader-role NNTP server host (Core + reader authentication/POST stack). + /// + /// The service collection. + /// Optional additional options configuration. + /// When true, registers a no-op placeholder application service. + /// The same instance. + public static IServiceCollection AddNntpReaderHosting( + this IServiceCollection services, + Action? configure = null, + bool includePlaceholderService = false) => + AddNntpdHosting(services, configure, includePlaceholderService); + + /// + /// Registers the Transit-role NNTP server host (Core + transit peer/streaming stack). + /// + /// The service collection. + /// Optional additional options configuration. + /// When true, registers a no-op placeholder application service. + /// The same instance. + public static IServiceCollection AddNntpTransitHosting( + this IServiceCollection services, + Action? configure = null, + bool includePlaceholderService = false) => throw new NotSupportedException("Transit hosting is not registered in ReaderService."); + + /// + /// Adds NNTP server lifecycle, options, and hosted service integration for this host role. + /// + /// The service collection. + /// Optional additional options configuration. + /// + /// When , registers a no-op placeholder . + /// Defaults to now that NNTP listeners are registered. + /// + /// The same instance. + public static IServiceCollection AddNntpdHosting(this IServiceCollection services, Action? configure = null, bool includePlaceholderService = false) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(typeof(NntpServerRole), NntpServerRole.Reader); + var applicationPrefix = NntpdOptions.ReaderApplicationPrefix; + bool includeReader = true; + bool includeTransit = false; + + services.TryAddSingleton(); + services.TryAddSingleton(static _ => SocketListenBinder.Instance); + + if (includeReader) + { + // Newsmaster (when configured) then MySQL nntpusers. Reader-authority only. + // Transit AUTHINFO is MODE STREAM / Transit authority and never enters this provider. + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + new CachedNntpUserRecordStore( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new DistributedSessionStateTracker( + sp.GetRequiredService(), + sp.GetRequiredService>(), + nodeId, + TimeProvider.System, + bytes: sp.GetRequiredService(), + rates: sp.GetRequiredService()); + }); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new CompositeNntpAuthenticationProvider( + NewsmasterNntpAuthenticationProvider.Create(options), + options.NewsmasterUser, + sp.GetRequiredService()); + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + } + else + { + // Transit-only host: deny reader AUTHINFO; peer identity comes from AllowFrom + Transit AUTHINFO. + services.TryAddSingleton(_ => + DenyAllNntpAuthenticationProvider.Instance); + } + + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new DistributedTransitPeerStateTracker( + sp.GetRequiredService(), + sp.GetRequiredService>(), + nodeId); + }); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return FeedDiagnosticsOptions.ResolveEnabled(options) + ? ActivatorUtilities.CreateInstance(sp) + : NullFeedDiagnostics.Instance; + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + } + else + { + // Reader-only hosts never grant transit/streaming peer privileges on connect. + services.TryAddSingleton(_ => TransitPeerAuthorization.Disabled); + services.TryAddSingleton(_ => TransitInboundConnectionLimiter.Disabled); + services.TryAddSingleton(_ => NullFeedDiagnostics.Instance); + } + + var optionsBuilder = services + .AddOptions() + .BindConfiguration(NntpdOptions.SectionName) + .Configure(static (options, configuration) => + { + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + }) + .ValidateDataAnnotations() + .ValidateOnStart() + .PostConfigure(options => + { + options.HostApplicationPrefix = applicationPrefix; + options.Systemd ??= new SystemdOptions(); + options.ArticleIngestion ??= new ArticleIngestionOptions(); + options.Transit ??= new TransitOptions(); + options.SpeedTest ??= new SpeedTestOptions(); + options.FeedDiagnostics ??= new FeedDiagnosticsOptions(); + NormalizeBindAddresses(options); + NormalizeProxyHosts(options); + options.Transit ??= new TransitOptions(); + options.AcmeStateDir = ApplicationLocalPath.ResolveApplicationLocalPath( + options.AcmeStateDir, + AppContext.BaseDirectory); + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + options.CloudFlareApiKey = string.Empty; + }); + + services.AddSingleton, NntpdOptionsValidator>(); + services.AddSingleton, PostFilterLeftoverConfigurationValidator>(); + services.AddSingleton, TransitLeftoverConfigurationValidator>(); + services.AddSingleton>(static sp => + Options.Create(sp.GetRequiredService>().Value)); + services.AddSingleton(); + services.AddAcmeCloudflareInfrastructure(); + + services + .AddOptions() + .BindConfiguration(RedisOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, RedisOptionsValidator>(); + + services + .AddOptions() + .BindConfiguration(RabbitMqOptions.SectionName) + .PostConfigure(static options => + { + options.Management ??= new RabbitMqManagementOptions(); + options.Username = null; + options.Password = null; + }) + .ValidateOnStart(); + services.AddSingleton, NntpdRabbitMqOptionsValidator>(); + + services.AddHttpClient(RabbitMqManagementHttpClient.HttpClientName, static (sp, client) => + { + var options = sp.GetRequiredService>().Value; + var baseUrl = options.Management?.BaseUrl?.Trim().TrimEnd('/'); + if (string.IsNullOrWhiteSpace(baseUrl)) + { + throw new InvalidOperationException("RabbitMQ:Management:BaseUrl is required."); + } + + client.BaseAddress = new Uri(baseUrl + "/", UriKind.Absolute); + // Stall protection comes from per-request CancelAfter on the Management client. + client.Timeout = Timeout.InfiniteTimeSpan; + }); + services.TryAddSingleton(static sp => + { + var http = sp.GetRequiredService() + .CreateClient(RabbitMqManagementHttpClient.HttpClientName); + return new RabbitMqManagementHttpClient( + http, + sp.GetRequiredService>()); + }); + + services + .AddOptions() + .BindConfiguration(TransitPeersOptions.SectionName); + // Validation is applied when building the immutable snapshot (startup throw / + // reload ignore). IValidateOptions is not registered so an invalid reload + // cannot throw from IOptionsMonitor before the last valid snapshot is kept. + + services + .AddOptions() + .BindConfiguration(ControlOptions.SectionName) + .PostConfigure(static options => + { + options.PgpAuthorities ??= new PgpAuthoritiesOptions(); + options.PgpAuthorities.Source ??= new PgpAuthoritySourceOptions(); + options.PgpAuthorities.Authorities ??= []; + foreach (var authority in options.PgpAuthorities.Authorities) + { + if (authority is null) + { + continue; + } + + authority.Authorizations ??= []; + if (PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out var normalized)) + { + authority.KeyFingerprint = normalized; + } + } + }); + services.AddSingleton, ControlOptionsValidator>(); + // Catalogue validation is available for tests and explicit IOptions resolution. + // ValidateOnStart is not used: an empty or omitted Control section must not + // prevent NNTPD startup, and this catalogue is not a runtime dependency. + + if (includeReader) + { + services + .AddOptions() + .BindConfiguration(ModerationOptions.SectionName) + .PostConfigure(static options => + { + options.Source ??= new ModerationSourceOptions(); + options.Moderators ??= []; + }) + .ValidateOnStart(); + services.AddSingleton, ModerationOptionsValidator>(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services + .AddOptions() + .BindConfiguration(EmailOptions.SectionName) + .PostConfigure(static options => + { + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + }) + .ValidateOnStart(); + services.AddSingleton, EmailOptionsValidator>(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + } + + services.AddNntpDbOptions(); + + if (configure is not null) + { + optionsBuilder.Configure(configure); + } + + // Startup order (sequential ApplicationServiceManager): + // NntpDB (hard dep; MySqlConnector pool) → + // nntpsharedconfig (initial snapshot, including RabbitMQ, ACME, and Cloudflare credentials, before those consumers) → + // Cloudflare DNS → Redis → RabbitMQ (hard dep; connection lifecycle only) → + // RabbitMQ topology (hard dep; cache.requests fanout + cache.broadcast + overviewdb.queue) → + // StorageServer fleet consumer (hard dep; cache. + in-memory registry) → + // StorageServer article lookup (hard dep; cache.requests publish + reply consumer) → + // RabbitMQ article-work RPC (hard dep; reply consumer + request orchestration) → + // newsgroup catalogue (initial snapshot before RUNNING) → + // moderator catalogue (nntpmoderators snapshot before RUNNING) → + // PostFilter policy → PostFilter rejection evidence → + // HistoryDB writer → HistoryDB maintenance → incoming spool writer → + // Email delivery (lazy SMTP) → ninpaths (completed inpaths; Top1000) → + // Transit AllowFrom DNS refresh → SessionState lease renewal → + // Transit inbound-ownership renewal → + // plain NNTP listener → ACME → TLS NNTP listener → + // optional feed-diagnostics reporter → always-on application telemetry. + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(static sp => + new DelegateRabbitMqConnectionNameProvider(() => + RabbitMqRuntimeOptions.GetDefaultConnectionName( + ApplicationJsonConfiguration.EntryAssemblyName, + sp.GetRequiredService>().Value.Fqdn))); + services.AddRabbitMqInfrastructure(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton( + static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static provider => + { + var ingestion = provider.GetRequiredService>().Value.ArticleIngestion + ?? new ArticleIngestionOptions(); + return new ArticlePlacementClient( + provider.GetRequiredService>(), + ingestion.MaxOutstandingStoresPerConnection); + }); + + services.TryAddSingleton(); + services.TryAddSingleton( + static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton( + static sp => sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + if (includeReader) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new PostFilterReservationIdentity(nodeId); + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + new SpamdCheckClient( + sp.GetRequiredService(), + sp.GetService>())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(static sp => + AesGcmPostingTraceProtector.Create(sp.GetRequiredService>())); + } + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(IngestionPipelineMetrics.Shared); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includeReader) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + { + _ = sp.GetRequiredService(); + return sp.GetRequiredService(); + })); + } + + // SessionState lease renewal follows Transit DNS refresh when both roles are hosted + // (Combined tests). Reader-only registers it after ninpaths; Transit-only omits it. + if (includeReader) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + if (includeTransit) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includePlaceholderService) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + } + + services.AddSingleton(static sp => + sp.GetRequiredService>().Value); + services.AddSingleton(); + services.AddSingleton(static sp => new ApplicationLifecycle( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + services.AddSingleton(); + services.AddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + // Register systemd lifecycle notifier before the main hosted service so it is + // constructed and subscribed before application startup transitions occur. + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddHostedService(); + services.AddHostedService(); + + return services; + } + + /// + /// Configures platform hosting integrations for Windows Service and Linux systemd without + /// preventing interactive console execution on other platforms. + /// + /// The host application builder. + /// The same instance. + /// + /// + /// AddSystemd() is context-aware: it activates systemd lifetime and notify support only + /// when the process is detected as a systemd service or NOTIFY_SOCKET is set. + /// Interactive console and Windows Service execution remain supported. + /// + /// + /// Console log formatting is owned exclusively by Serilog. Any Microsoft console formatter + /// options registered by AddSystemd() are removed immediately afterward because they + /// are unused under the Serilog-only logging pipeline. + /// + /// + /// Signal handling (SIGTERM/SIGINT) is provided by the Generic Host / + /// SystemdLifetime. This application does not register competing signal handlers. + /// + /// + public static HostApplicationBuilder ConfigureNntpdPlatformHosting(this HostApplicationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Configuration.AddVectorEnvironmentVariables(); + + builder.Services.AddWindowsService(options => + { + options.ServiceName = ApplicationJsonConfiguration.EntryAssemblyName; + }); + + builder.Services.AddSystemd(); + RemoveObsoleteMicrosoftConsoleFormatterConfiguration(builder.Services); + + builder.Services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + + var shutdown = builder.Configuration.GetSection(NntpdOptions.SectionName) + .GetValue("GracefulShutdownTimeout"); + if (shutdown is { } timeout) + { + options.ShutdownTimeout = timeout; + } + }); + + return builder; + } + + /// + /// Adds the canonical VECTOR__ environment-variable source. + /// + /// The configuration builder. + /// The same instance. + public static IConfigurationBuilder AddNntpdPrefixedEnvironmentVariables(this IConfigurationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + return builder.AddVectorEnvironmentVariables(); + } + + /// + /// Removes Microsoft console formatter options that AddSystemd() may register. + /// + /// + /// Microsoft.Extensions.Hosting.Systemd configures + /// to the systemd MEL formatter when it + /// detects a systemd service. That configuration is obsolete after Phase 0.2 because Serilog + /// owns console output. Stripping these registrations prevents a misleading unused MEL + /// formatter from remaining in DI while preserving systemd lifetime and notify registration. + /// + public static void RemoveObsoleteMicrosoftConsoleFormatterConfiguration(IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + for (var i = services.Count - 1; i >= 0; i--) + { + if (IsMicrosoftConsoleLoggerOptionsConfiguration(services[i])) + { + services.RemoveAt(i); + } + } + } + + /// + /// Returns whether a service descriptor configures Microsoft console logger options. + /// + public static bool IsMicrosoftConsoleLoggerOptionsConfiguration(ServiceDescriptor descriptor) + { + ArgumentNullException.ThrowIfNull(descriptor); + + var serviceType = descriptor.ServiceType; + if (!serviceType.IsGenericType) + { + return false; + } + + var definition = serviceType.GetGenericTypeDefinition(); + if (definition != typeof(IConfigureOptions<>) + && definition != typeof(IPostConfigureOptions<>) + && definition != typeof(IValidateOptions<>)) + { + return false; + } + + return serviceType.GenericTypeArguments[0] == typeof(ConsoleLoggerOptions); + } + + private static void NormalizeBindAddresses(NntpdOptions options) + { + if (options.BindAddress is null || options.BindAddress.Length == 0) + { + // Code default when configuration omits BindAddress: listen on all interfaces. + options.BindAddress = ["*"]; + return; + } + + for (var i = 0; i < options.BindAddress.Length; i++) + { + options.BindAddress[i] = options.BindAddress[i]?.Trim() ?? string.Empty; + } + } + + private static void NormalizeProxyHosts(NntpdOptions options) + { + if (options.ProxyHosts is null) + { + options.ProxyHosts = []; + return; + } + + for (var i = 0; i < options.ProxyHosts.Length; i++) + { + options.ProxyHosts[i] = options.ProxyHosts[i]?.Trim() ?? string.Empty; + } + } + + } + + /// + /// Phase-0 placeholder application service so the host has a deterministic no-op service to manage. + /// Replaced by real NNTP services in later phases. + /// + internal sealed class PlaceholderApplicationService : IApplicationService + { + public string Name => "Placeholder"; + + public Task? Execution => null; + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/ReaderStartupSmoke.cs b/src/VectorNNTPD.ReaderService/Hosting/ReaderStartupSmoke.cs new file mode 100644 index 00000000..5b41110e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/ReaderStartupSmoke.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Hosting +{ + /// + /// Builds the real Reader host and stops after configuration binding, without starting hosted services. + /// + /// + /// Set to and + /// to an isolated JSON file. Does not call + /// IHost.RunAsync, so Redis, MySQL, RabbitMQ, DNS, ACME, and listeners are not started. + /// + internal static class ReaderStartupSmoke + { + /// Environment variable that selects startup smoke. The value must be . + internal const string EnvironmentVariable = "VECTORNNTP_READER_STARTUP_SMOKE"; + + /// The only value of that selects startup smoke. + internal const string EnabledValue = "1"; + + /// Environment variable naming the isolated JSON file that replaces application file configuration. + internal const string ConfigurationPathVariable = "VECTORNNTP_READER_STARTUP_SMOKE_CONFIG"; + + /// Stdout prefix written after configuration binding succeeds. + private const string CheckpointPrefix = "ReaderService startup smoke: configuration"; + + /// Returns whether requests startup smoke. + internal static bool IsRequested(string? value) => + string.Equals(value, EnabledValue, StringComparison.Ordinal); + + /// + /// Removes file configuration sources and appends the JSON file named by . + /// + internal static void UseIsolatedConfiguration(IConfigurationBuilder configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var path = Environment.GetEnvironmentVariable(ConfigurationPathVariable); + if (string.IsNullOrWhiteSpace(path)) + { + throw new InvalidOperationException( + ConfigurationPathVariable + " must name the isolated startup smoke configuration file."); + } + + if (!File.Exists(path)) + { + throw new FileNotFoundException("Startup smoke configuration file was not found.", path); + } + + for (var index = configuration.Sources.Count - 1; index >= 0; index--) + { + if (configuration.Sources[index] is FileConfigurationSource) + { + configuration.Sources.RemoveAt(index); + } + } + + configuration.AddJsonFile(Path.GetFullPath(path), optional: false, reloadOnChange: false); + } + + /// Resolves bound Nntpd options and writes the checkpoint. + internal static int Complete(IHost host, TextWriter output) + { + ArgumentNullException.ThrowIfNull(host); + ArgumentNullException.ThrowIfNull(output); + using (host) + { + var options = host.Services.GetRequiredService>().Value; + if (options.ServerId is not { } serverId || string.IsNullOrWhiteSpace(options.Fqdn)) + { + throw new InvalidOperationException( + "Startup smoke did not bind a usable Nntpd ServerId and Fqdn."); + } + + if (!string.Equals(options.HostApplicationPrefix, NntpdOptions.ReaderApplicationPrefix, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "Startup smoke expected HostApplicationPrefix '" + NntpdOptions.ReaderApplicationPrefix + + "' and found '" + options.HostApplicationPrefix + "'."); + } + + output.WriteLine( + "{0} serverId={1} fqdn={2}", + CheckpointPrefix, + serverId, + options.Fqdn); + output.Flush(); + return 0; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/ApplicationHealth.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ApplicationHealth.cs new file mode 100644 index 00000000..96acb4b6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ApplicationHealth.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Lifecycle-based application health used for systemd watchdog decisions. + /// + public sealed class ApplicationHealth : IApplicationHealth + { + private readonly ApplicationLifecycle _lifecycle; + + /// + /// Initializes a new instance of the class. + /// + public ApplicationHealth(ApplicationLifecycle lifecycle) + { + ArgumentNullException.ThrowIfNull(lifecycle); + _lifecycle = lifecycle; + } + + /// + public bool IsHealthyForWatchdog => + _lifecycle is { State: ApplicationState.Running, UnexpectedTermination.IsCompleted: false, ShutdownRequested: false }; + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/IApplicationHealth.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/IApplicationHealth.cs new file mode 100644 index 00000000..e3d6ac81 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/IApplicationHealth.cs @@ -0,0 +1,20 @@ +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Application health view used for systemd watchdog keep-alives. + /// + /// + /// Phase 0.1 health is lifecycle-based only. NNTP/data-plane checks are intentionally deferred. + /// + public interface IApplicationHealth + { + /// + /// Gets a value indicating whether the application is healthy enough to send watchdog keep-alives. + /// + /// + /// Healthy means: lifecycle state is Running, required services have initialized, + /// no fatal supervised service failure has been observed, and shutdown has not started. + /// + bool IsHealthyForWatchdog { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdNotifyBridge.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdNotifyBridge.cs new file mode 100644 index 00000000..dfb4b26f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdNotifyBridge.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Testable abstraction over systemd service notifications. + /// + public interface ISystemdNotifyBridge + { + /// Gets a value indicating whether notifications are delivered to systemd. + bool IsEnabled { get; } + + /// Sends READY=1. + void NotifyReady(); + + /// Sends STOPPING=1. + void NotifyStopping(); + + /// Sends STATUS= with the specified status text. + /// Human-readable status (must not contain newlines). + void NotifyStatus(string status); + + /// Sends WATCHDOG=1. + void NotifyWatchdog(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdRuntime.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdRuntime.cs new file mode 100644 index 00000000..dcc79c14 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/ISystemdRuntime.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Describes the process relationship to systemd for hosting decisions. + /// + public interface ISystemdRuntime + { + /// Gets a value indicating whether the process is running on Linux. + bool IsLinux { get; } + + /// + /// Gets a value indicating whether the process appears to be managed by systemd + /// (direct systemd service / compatible notify environment). + /// + bool IsSystemdService { get; } + + /// + /// Gets a value indicating whether systemd notify is available for this process + /// (typically NOTIFY_SOCKET was present when the host notifier was constructed). + /// + bool IsNotifyEnabled { get; } + + /// + /// Gets the systemd watchdog deadline when configured and applicable to this process; + /// otherwise . + /// + TimeSpan? WatchdogTimeout { get; } + + /// + /// Gets a value indicating whether systemd expects watchdog keep-alives from this process. + /// + bool IsWatchdogConfigured { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdLifecycleNotifier.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdLifecycleNotifier.cs new file mode 100644 index 00000000..3ed8fbed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdLifecycleNotifier.cs @@ -0,0 +1,200 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Publishes systemd readiness and status notifications from application lifecycle transitions. + /// + /// + /// Readiness is reported at most once, and only after a successful transition to + /// . Startup failure never reports ready. Once shutdown + /// begins, further readiness notifications are suppressed. + /// + public sealed class SystemdLifecycleNotifier : IHostedService, IDisposable + { + private readonly ApplicationLifecycle _lifecycle; + private readonly ISystemdNotifyBridge _notify; + private readonly ISystemdRuntime _runtime; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly object _sync = new(); + private int _readySent; + private int _stoppingSent; + private string? _lastStatus; + private bool _subscribed; + + /// + /// Initializes a new instance of the class. + /// + public SystemdLifecycleNotifier( + ApplicationLifecycle lifecycle, + ISystemdNotifyBridge notify, + ISystemdRuntime runtime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(notify); + ArgumentNullException.ThrowIfNull(runtime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _notify = notify; + _runtime = runtime; + _options = options; + _logger = logger; + + // Subscribe in the constructor so transitions are observed even if this hosted + // service's StartAsync runs after ApplicationLifecycle has begun starting. + _lifecycle.StateChanged += OnStateChanged; + _subscribed = true; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (_runtime.IsLinux && (_runtime.IsSystemdService || _notify.IsEnabled)) + { + SystemdLogMessages.LifecycleNotificationsActivated( + _logger, + _notify.IsEnabled, + _runtime.IsSystemdService); + } + + // Synchronize with current state in case transitions already occurred. + var current = _lifecycle.State; + OnStateChanged(_lifecycle, new ApplicationStateChangedEventArgs(current, current)); + return Task.CompletedTask; + } + + /// + public Task StopAsync(CancellationToken cancellationToken) + { + Unsubscribe(); + return Task.CompletedTask; + } + + /// + public void Dispose() => Unsubscribe(); + + private void Unsubscribe() + { + if (!_subscribed) + { + return; + } + + _lifecycle.StateChanged -= OnStateChanged; + _subscribed = false; + } + + private void OnStateChanged(object? sender, ApplicationStateChangedEventArgs e) + { + // When called for synchronization, FromState == ToState == current. + var state = e.ToState; + switch (state) + { + case ApplicationState.Starting: + TryNotifyStatus("Starting: initializing application services"); + break; + + case ApplicationState.Running: + TryNotifyStatus("Running: application initialization completed"); + TryNotifyReady(); + break; + + case ApplicationState.Stopping: + TryNotifyStopping(); + TryNotifyStatus("Stopping: graceful shutdown in progress"); + break; + + case ApplicationState.Stopped: + TryNotifyStopping(); + TryNotifyStatus("Stopped"); + break; + + case ApplicationState.Created: + break; + + default: + TryNotifyStatus($"State={state}"); + break; + } + } + + private void TryNotifyReady() + { + if (!_options.Value.Systemd.NotifyReadyOnApplicationRunning) + { + return; + } + + if (!_notify.IsEnabled) + { + return; + } + + if (_lifecycle.ShutdownRequested || _lifecycle.State != ApplicationState.Running) + { + return; + } + + if (Interlocked.Exchange(ref _readySent, 1) != 0) + { + return; + } + + _notify.NotifyReady(); + SystemdLogMessages.ReadinessReported(_logger); + } + + private void TryNotifyStopping() + { + if (!_options.Value.Systemd.NotifyStoppingOnApplicationShutdown) + { + return; + } + + if (!_notify.IsEnabled) + { + return; + } + + if (Interlocked.Exchange(ref _stoppingSent, 1) != 0) + { + return; + } + + // Once stopping, readiness must not be reported later. + Interlocked.Exchange(ref _readySent, 1); + + _notify.NotifyStopping(); + SystemdLogMessages.StoppingReported(_logger); + } + + private void TryNotifyStatus(string status) + { + if (!_options.Value.Systemd.ReportLifecycleStatus || !_notify.IsEnabled) + { + return; + } + + lock (_sync) + { + if (string.Equals(_lastStatus, status, StringComparison.Ordinal)) + { + return; + } + + _lastStatus = status; + } + + _notify.NotifyStatus(status); + SystemdLogMessages.StatusReported(_logger, status); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdNotifyBridge.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdNotifyBridge.cs new file mode 100644 index 00000000..187432f0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdNotifyBridge.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.Hosting.Systemd; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Forwards notifications to the official when registered and enabled. + /// + public sealed class SystemdNotifyBridge : ISystemdNotifyBridge + { + private static readonly ServiceState Ready = ServiceState.Ready; + private static readonly ServiceState Stopping = ServiceState.Stopping; + private static readonly ServiceState Watchdog = new("WATCHDOG=1"); + + private readonly ISystemdNotifier? _notifier; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public SystemdNotifyBridge( + IEnumerable systemdNotifiers, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(systemdNotifiers); + ArgumentNullException.ThrowIfNull(logger); + + _notifier = systemdNotifiers.FirstOrDefault(); + _logger = logger; + } + + /// + public bool IsEnabled => _notifier?.IsEnabled == true; + + /// + public void NotifyReady() => Notify(Ready, "READY=1"); + + /// + public void NotifyStopping() => Notify(Stopping, "STOPPING=1"); + + /// + public void NotifyStatus(string status) + { + ArgumentException.ThrowIfNullOrWhiteSpace(status); + if (status.Contains('\n', StringComparison.Ordinal) || status.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException("Status must not contain newlines or NUL characters.", nameof(status)); + } + + Notify(new ServiceState("STATUS=" + status), "STATUS"); + } + + /// + public void NotifyWatchdog() => Notify(Watchdog, "WATCHDOG=1"); + + private void Notify(ServiceState state, string label) + { + if (_notifier is null || !_notifier.IsEnabled) + { + return; + } + + try + { + _notifier.Notify(state); + SystemdLogMessages.NotificationSent(_logger, label); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.NotificationFailed(_logger, ex, label); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdRuntime.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdRuntime.cs new file mode 100644 index 00000000..5f73fa7c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdRuntime.cs @@ -0,0 +1,133 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Hosting.Systemd; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Default systemd environment detection based on OS platform checks and systemd variables. + /// + /// + /// + /// Detection distinguishes Linux from other OSes, and systemd-managed execution from ordinary + /// Linux console sessions. is preferred for + /// service detection; notify enablement uses the registered when + /// available (the hosting package clears NOTIFY_SOCKET after constructing the notifier). + /// + /// + /// Watchdog configuration is read from WATCHDOG_USEC and optionally WATCHDOG_PID. + /// Missing or malformed values disable watchdog behaviour rather than inventing an interval. + /// + /// + public sealed class SystemdRuntime : ISystemdRuntime + { + /// Environment variable containing the watchdog timeout in microseconds. + public const string WatchdogUsecVariable = "WATCHDOG_USEC"; + + /// Environment variable containing the PID expected to send watchdog keep-alives. + public const string WatchdogPidVariable = "WATCHDOG_PID"; + + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public SystemdRuntime( + IEnumerable systemdNotifiers, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(systemdNotifiers); + ArgumentNullException.ThrowIfNull(logger); + + _logger = logger; + + IsLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + IsSystemdService = IsLinux && SystemdHelpers.IsSystemdService(); + + var notifier = systemdNotifiers.FirstOrDefault(); + IsNotifyEnabled = notifier?.IsEnabled == true; + + WatchdogTimeout = TryReadWatchdogTimeout(out var reason); + IsWatchdogConfigured = WatchdogTimeout is not null; + + if (IsLinux) + { + SystemdLogMessages.RuntimeDetection( + _logger, + IsSystemdService, + IsNotifyEnabled, + IsWatchdogConfigured, + WatchdogTimeout, + reason); + } + } + + /// + public bool IsLinux { get; } + + /// + public bool IsSystemdService { get; } + + /// + public bool IsNotifyEnabled { get; } + + /// + public TimeSpan? WatchdogTimeout { get; } + + /// + public bool IsWatchdogConfigured { get; } + + private TimeSpan? TryReadWatchdogTimeout(out string reason) + { + if (!IsLinux) + { + reason = "non-Linux platform"; + return null; + } + + var usecText = Environment.GetEnvironmentVariable(WatchdogUsecVariable); + if (string.IsNullOrWhiteSpace(usecText)) + { + reason = "WATCHDOG_USEC not set"; + return null; + } + + if (!ulong.TryParse(usecText, NumberStyles.None, CultureInfo.InvariantCulture, out var usec) || usec == 0) + { + reason = "WATCHDOG_USEC missing or invalid"; + SystemdLogMessages.MalformedWatchdogUsec(_logger); + return null; + } + + var watchdogPidText = Environment.GetEnvironmentVariable(WatchdogPidVariable); + if (!string.IsNullOrWhiteSpace(watchdogPidText)) + { + if (!int.TryParse(watchdogPidText, NumberStyles.None, CultureInfo.InvariantCulture, out var watchdogPid)) + { + reason = "WATCHDOG_PID malformed"; + SystemdLogMessages.MalformedWatchdogPid(_logger); + return null; + } + + if (watchdogPid != Environment.ProcessId) + { + reason = "WATCHDOG_PID does not match current process"; + SystemdLogMessages.WatchdogConfiguredForDifferentPid(_logger); + return null; + } + } + + // TimeSpan ticks are 100ns; microseconds * 10 = ticks. + if (usec > (ulong)(TimeSpan.MaxValue.Ticks / 10)) + { + reason = "WATCHDOG_USEC out of range"; + SystemdLogMessages.WatchdogUsecOutOfRange(_logger); + return null; + } + + reason = "WATCHDOG_USEC accepted"; + return TimeSpan.FromTicks((long)usec * 10); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogInterval.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogInterval.cs new file mode 100644 index 00000000..06c7d1c7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogInterval.cs @@ -0,0 +1,43 @@ +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Calculates systemd watchdog heartbeat intervals from the systemd-provided deadline. + /// + public static class SystemdWatchdogInterval + { + /// + /// Derives a heartbeat interval that leaves margin before the systemd watchdog deadline. + /// + /// Deadline supplied by systemd (WATCHDOG_USEC). + /// Fraction of the deadline between heartbeats; must be in (0, 1). + /// The heartbeat interval. + /// Thrown when inputs are invalid. + public static TimeSpan Calculate(TimeSpan watchdogTimeout, double fraction = 0.5) + { + if (watchdogTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(watchdogTimeout), watchdogTimeout, "Watchdog timeout must be positive."); + } + + if (double.IsNaN(fraction) || fraction is <= 0 or >= 1) + { + throw new ArgumentOutOfRangeException(nameof(fraction), fraction, "Fraction must be in the open interval (0, 1)."); + } + + var ticks = (long)Math.Floor(watchdogTimeout.Ticks * fraction); + if (ticks < TimeSpan.TicksPerMillisecond) + { + ticks = TimeSpan.TicksPerMillisecond; + } + + // Always leave at least 1ms margin before the deadline. + var maxTicks = Math.Max(TimeSpan.TicksPerMillisecond, watchdogTimeout.Ticks - TimeSpan.TicksPerMillisecond); + if (ticks > maxTicks) + { + ticks = maxTicks; + } + + return TimeSpan.FromTicks(ticks); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogService.cs b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogService.cs new file mode 100644 index 00000000..10dae182 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Hosting/Systemd/SystemdWatchdogService.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Hosting.Systemd +{ + /// + /// Sends systemd watchdog keep-alives while the application is healthy. + /// + /// + /// The watchdog is disabled unless systemd configured WATCHDOG_USEC for this process and + /// is . Heartbeats stop when + /// shutdown begins or the application becomes unhealthy. Unexpected failures stop the host. + /// + public sealed class SystemdWatchdogService : BackgroundService + { + private readonly ISystemdRuntime _runtime; + private readonly ISystemdNotifyBridge _notify; + private readonly IApplicationHealth _health; + private readonly IHostApplicationLifetime _hostLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + private TimeSpan _interval; + private int _active; + + /// + /// Initializes a new instance of the class. + /// + public SystemdWatchdogService( + ISystemdRuntime runtime, + ISystemdNotifyBridge notify, + IApplicationHealth health, + IHostApplicationLifetime hostLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(runtime); + ArgumentNullException.ThrowIfNull(notify); + ArgumentNullException.ThrowIfNull(health); + ArgumentNullException.ThrowIfNull(hostLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _runtime = runtime; + _notify = notify; + _health = health; + _hostLifetime = hostLifetime; + _options = options; + _logger = logger; + } + + /// Gets a value indicating whether the watchdog loop is active. + public bool IsActive => Volatile.Read(ref _active) != 0; + + /// Gets the heartbeat interval when active; otherwise . + public TimeSpan HeartbeatInterval => _interval; + + /// + public override Task StartAsync(CancellationToken cancellationToken) + { + if (!ShouldActivate(out var reason)) + { + SystemdLogMessages.WatchdogRemainDisabled(_logger, reason); + return Task.CompletedTask; + } + + var timeout = _runtime.WatchdogTimeout + ?? throw new InvalidOperationException("Watchdog timeout unexpectedly unavailable."); + + _interval = SystemdWatchdogInterval.Calculate( + timeout, + _options.Value.Systemd.WatchdogIntervalFraction); + + Interlocked.Exchange(ref _active, 1); + SystemdLogMessages.WatchdogActivated(_logger, timeout, _interval); + + return base.StartAsync(cancellationToken); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (Volatile.Read(ref _active) == 0) + { + return; + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + stoppingToken, + _hostLifetime.ApplicationStopping); + + try + { + using var timer = new PeriodicTimer(_interval); + while (await timer.WaitForNextTickAsync(linked.Token).ConfigureAwait(false)) + { + if (!_health.IsHealthyForWatchdog) + { + SystemdLogMessages.WatchdogKeepAliveSkippedUnhealthy(_logger); + continue; + } + + try + { + _notify.NotifyWatchdog(); + SystemdLogMessages.WatchdogKeepAliveSent(_logger); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.WatchdogNotificationFailed(_logger, ex); + throw; + } + } + } + catch (OperationCanceledException) when (linked.IsCancellationRequested) + { + SystemdLogMessages.WatchdogDeactivated(_logger); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.WatchdogLoopFailed(_logger, ex); + Interlocked.Exchange(ref _active, 0); + _hostLifetime.StopApplication(); + throw; + } + finally + { + Interlocked.Exchange(ref _active, 0); + } + } + + /// + public override async Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Exchange(ref _active, 0); + SystemdLogMessages.WatchdogStopping(_logger); + await base.StopAsync(cancellationToken).ConfigureAwait(false); + } + + private bool ShouldActivate(out string reason) + { + if (!_options.Value.Systemd.EnableWatchdog) + { + reason = "EnableWatchdog is false"; + return false; + } + + if (!_runtime.IsLinux) + { + reason = "non-Linux platform"; + return false; + } + + if (!_runtime.IsWatchdogConfigured || _runtime.WatchdogTimeout is null) + { + reason = "systemd did not configure WATCHDOG_USEC for this process"; + return false; + } + + if (!_notify.IsEnabled) + { + reason = "systemd notify is not enabled"; + return false; + } + + reason = "ok"; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/HostingLogMessages.cs b/src/VectorNNTPD.ReaderService/Logging/HostingLogMessages.cs new file mode 100644 index 00000000..d33f8762 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/HostingLogMessages.cs @@ -0,0 +1,74 @@ +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Source-generated structured log messages for Generic Host integration events. + /// + internal static partial class HostingLogMessages + { + [LoggerMessage( + EventId = 1200, + Level = LogLevel.Information, + Message = "Host starting application {ApplicationName}")] + public static partial void HostStartingApplication(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1201, + Level = LogLevel.Information, + Message = "Application entered Running state after {ElapsedMs} ms. Host startup continuing")] + public static partial void ApplicationEnteredRunning(ILogger logger, long ElapsedMs); + + [LoggerMessage( + EventId = 1202, + Level = LogLevel.Information, + Message = "Hosted service execution canceled due to host shutdown")] + public static partial void HostedServiceExecutionCanceled(ILogger logger); + + [LoggerMessage( + EventId = 1203, + Level = LogLevel.Critical, + Message = "Background hosted service observed unexpected application-service termination")] + public static partial void UnexpectedApplicationServiceTermination(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1204, + Level = LogLevel.Information, + Message = "Host stopping application {ApplicationName}")] + public static partial void HostStoppingApplication(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1205, + Level = LogLevel.Information, + Message = "Application shutdown requested exactly once for {ApplicationName}")] + public static partial void ShutdownRequestedOnce(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1206, + Level = LogLevel.Warning, + Message = "Unexpected service termination observed, but {Option} is disabled")] + public static partial void UnexpectedTerminationOptionDisabled(ILogger logger, string Option); + + [LoggerMessage( + EventId = 1207, + Level = LogLevel.Critical, + Message = "Requesting host stop due to unexpected application-service termination")] + public static partial void RequestingHostStop(ILogger logger); + + [LoggerMessage( + EventId = 1208, + Level = LogLevel.Error, + Message = "Application shutdown completed with failure")] + public static partial void ShutdownCompletedWithFailure(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1209, + Level = LogLevel.Information, + Message = "Application logging initialized Application={Application} Provider={Provider} Category={Category} Environment={Environment} ContentRoot={ContentRoot}")] + public static partial void LoggingInitialized( + ILogger logger, + string Application, + string Provider, + string Category, + string Environment, + string ContentRoot); + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/ICompletedPathSurveyFileHandler.cs b/src/VectorNNTPD.ReaderService/Logging/ICompletedPathSurveyFileHandler.cs new file mode 100644 index 00000000..54c9a6a8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/ICompletedPathSurveyFileHandler.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Receives a completed uncompressed Path-survey file before gzip and deletion. + /// + /// + /// Serilog's File sink invokes file-lifecycle delete before gzip-archiving a + /// rolled uncompressed file. The Path-survey archive chain calls this handler + /// first, then gzip-archives the same path, then returns so Serilog can delete + /// the uncompressed original. + /// Production registers NinpathsCompletedFileHandler, which opens the + /// file and enqueues it for background ninpaths processing. The handler must + /// return without parsing Path records. + /// + public interface ICompletedPathSurveyFileHandler + { + /// + /// Called with the still-uncompressed completed Path-survey file. + /// + /// Absolute or sink-supplied path of the rolled inpaths file. + /// + /// The file is readable and must not be deleted by the handler. Gzip and + /// Serilog deletion run after this method returns. Implementations should + /// not retain Path observations in memory beyond processing the file. + /// + void OnCompletedFile(string completedFilePath); + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/InnNewsTextFormatter.cs b/src/VectorNNTPD.ReaderService/Logging/InnNewsTextFormatter.cs new file mode 100644 index 00000000..272e6696 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/InnNewsTextFormatter.cs @@ -0,0 +1,107 @@ +using System.Text; +using Serilog.Events; +using Serilog.Formatting; +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Application-owned Serilog formatter for the INN news line. + /// + /// + /// The INN format is an application invariant. This type ignores the log + /// event message template and any configured outputTemplate. Operators + /// cannot change field order, timestamp representation, or delimiters through + /// appsettings. + /// + public sealed class InnNewsTextFormatter : ITextFormatter + { + /// + public void Format(LogEvent logEvent, TextWriter output) + { + ArgumentNullException.ThrowIfNull(logEvent); + ArgumentNullException.ThrowIfNull(output); + + var disposition = ReadDisposition(logEvent); + var messageId = ReadUtf8Property(logEvent, NntpdNewsLogging.MessageIdProperty); + var feed = ReadUtf8Property(logEvent, NntpdNewsLogging.FeedProperty); + var sites = ReadUtf8Property(logEvent, NntpdNewsLogging.SitesProperty); + var reason = ReadUtf8Property(logEvent, NntpdNewsLogging.ReasonProperty); + var evt = new NewsLogEvent( + disposition, + messageId, + feed, + sites, + logEvent.Timestamp, + ReadResponseCode(logEvent), + reason, + ReadSize(logEvent)); + var required = NewsLogLineFormatter.RequiredLength(in evt); + Span buffer = required <= 512 ? stackalloc byte[required] : new byte[required]; + var written = NewsLogLineFormatter.Write(buffer, in evt, logEvent.Timestamp); + output.Write(Encoding.ASCII.GetString(buffer[..written])); + } + + private static NewsLogDisposition ReadDisposition(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.DispositionProperty, out var value) + || value is not ScalarValue { Value: char ch }) + { + return NewsLogDisposition.Accepted; + } + + return ch switch + { + (char)NewsLogDisposition.Junk => NewsLogDisposition.Junk, + (char)NewsLogDisposition.Rejected => NewsLogDisposition.Rejected, + (char)NewsLogDisposition.Moderated => NewsLogDisposition.Moderated, + (char)NewsLogDisposition.Accepted => NewsLogDisposition.Accepted, + _ => NewsLogDisposition.Accepted, + }; + } + + private static int ReadResponseCode(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.ResponseCodeProperty, out var value) + || value is not ScalarValue scalar) + { + return 0; + } + + return scalar.Value switch + { + int code => code, + long code => (int)code, + _ => 0, + }; + } + + private static int ReadSize(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.SizeProperty, out var value) + || value is not ScalarValue scalar) + { + return 0; + } + + return scalar.Value switch + { + int size => size, + long size => (int)size, + _ => 0, + }; + } + + private static ReadOnlyMemory ReadUtf8Property(LogEvent logEvent, string name) + { + if (!logEvent.Properties.TryGetValue(name, out var value) + || value is not ScalarValue { Value: string text } + || text.Length == 0) + { + return default; + } + + return Encoding.ASCII.GetBytes(text); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/InnPathSurveyTextFormatter.cs b/src/VectorNNTPD.ReaderService/Logging/InnPathSurveyTextFormatter.cs new file mode 100644 index 00000000..e8242d97 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/InnPathSurveyTextFormatter.cs @@ -0,0 +1,52 @@ +using System.Text; +using Serilog.Events; +using Serilog.Formatting; +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Application-owned Serilog formatter for the INN-style Path-survey line. + /// + /// + /// The Path-survey format is an application invariant. This type ignores the + /// log event message template and any configured outputTemplate. + /// Operators cannot change the Path: prefix, field contents, or + /// terminator through appsettings. + /// + public sealed class InnPathSurveyTextFormatter : ITextFormatter + { + /// + public void Format(LogEvent logEvent, TextWriter output) + { + ArgumentNullException.ThrowIfNull(logEvent); + ArgumentNullException.ThrowIfNull(output); + + var path = ReadPath(logEvent); + var required = PathSurveyLineFormatter.RequiredLength(path.Span); + Span buffer = required <= 1024 ? stackalloc byte[required] : new byte[required]; + var written = PathSurveyLineFormatter.Write(buffer, path.Span); + output.Write(Encoding.Latin1.GetString(buffer[..written])); + } + + private static ReadOnlyMemory ReadPath(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdPathSurveyLogging.PathProperty, out var value) + || value is not ScalarValue scalar + || scalar.Value is null) + { + return default; + } + + switch (scalar.Value) + { + case string text: + return text.Length == 0 ? default : Encoding.Latin1.GetBytes(text); + case byte[] bytes: + return bytes; + default: + return default; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NntpdConfiguredSinks.cs b/src/VectorNNTPD.ReaderService/Logging/NntpdConfiguredSinks.cs new file mode 100644 index 00000000..996a08b5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NntpdConfiguredSinks.cs @@ -0,0 +1,421 @@ +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Configuration; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Registers the application Console, File, and Async File sinks with direct Serilog calls. + /// + /// + /// ReadFrom.Configuration discovers sink extension methods by reflection. Native AOT does + /// not retain that metadata, so a configured + /// Serilog:WriteTo section produced a logger with no sinks. Argument values still come from + /// configuration. Omitted arguments use the same defaults as the Serilog extension methods. + /// A sink this type cannot represent exactly stays on ReadFrom.Configuration. + /// + internal static class NntpdConfiguredSinks + { + /// + /// Default File template used by Serilog when outputTemplate is omitted. + /// + private const string DefaultFileTemplate = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}"; + + /// + /// Default Console template used by Serilog when outputTemplate is omitted. + /// + private const string DefaultConsoleTemplate = + "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}"; + + /// + /// Serilog File default when fileSizeLimitBytes is absent. An explicit JSON null stays unlimited. + /// + private const long DefaultFileSizeLimitBytes = 1L * 1024 * 1024 * 1024; + + private static readonly string[] ConsoleArgumentNames = ["restrictedToMinimumLevel", "outputTemplate"]; + + private static readonly string[] AsyncArgumentNames = ["bufferSize", "blockWhenFull", "configure"]; + + private static readonly string[] FileArgumentNames = + [ + "path", + "restrictedToMinimumLevel", + "outputTemplate", + "fileSizeLimitBytes", + "buffered", + "flushToDiskInterval", + "rollingInterval", + "rollOnFileSizeLimit", + "retainedFileCountLimit" + ]; + + /// + /// Adds each Serilog:WriteTo entry. An empty section keeps the direct console fallback. + /// + /// The host logger being built. + /// The application configuration after the File path has been resolved. + public static void Apply(LoggerConfiguration logger, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(configuration); + + var children = configuration.GetSection("Serilog:WriteTo").GetChildren().ToList(); + if (children.Count == 0) + { + logger.WriteTo.Console( + restrictedToMinimumLevel: NntpdFileLogging.ConsoleMinimumLevel, + outputTemplate: NntpdLoggingExtensions.ConsoleOutputTemplate); + return; + } + + // codeql[cs/linq/missed-where] TryApply predicate mutates logger; not a pure filter. + foreach (var sink in children) + { + if (!TryApply(logger, sink)) + ApplyRemainingSink(logger, configuration, sink); + } + } + + /// + /// Returns the Serilog sections that do not construct sinks, so + /// ReadFrom.Configuration can still apply minimum level and enrichment. + /// + /// The application configuration. + /// A configuration whose Serilog section has no WriteTo entries. + public static IConfiguration WithoutWriteTo(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + CopySection(configuration.GetSection("Serilog:MinimumLevel"), values); + CopySection(configuration.GetSection("Serilog:Properties"), values); + CopySection(configuration.GetSection("Serilog:LevelSwitches"), values); + CopySection(configuration.GetSection("Serilog:FilterSwitches"), values); + CopySection(configuration.GetSection("Serilog:Enrich"), values); + CopySection(configuration.GetSection("Serilog:Filter"), values); + CopySection(configuration.GetSection("Serilog:Destructure"), values); + CopySection(configuration.GetSection("Serilog:AuditTo"), values); + if (RequiresSinkAssemblies(configuration)) + CopySection(configuration.GetSection("Serilog:Using"), values); + + return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + } + + /// + /// Applies one sink by name. Returns false when the entry needs the configuration binder. + /// + private static bool TryApply(LoggerConfiguration logger, IConfigurationSection sink) + { + var name = sink["Name"]; + var args = sink.GetSection("Args"); + if (name is null || !HasOnlyArguments(args, name.Equals("Console", StringComparison.OrdinalIgnoreCase) + ? ConsoleArgumentNames + : name.Equals("File", StringComparison.OrdinalIgnoreCase) + ? FileArgumentNames + : name.Equals("Async", StringComparison.OrdinalIgnoreCase) + ? AsyncArgumentNames + : [])) + { + return false; + } + + if (name.Equals("Console", StringComparison.OrdinalIgnoreCase)) + { + logger.WriteTo.Console( + restrictedToMinimumLevel: ReadLevel(args, LogEventLevel.Verbose), + outputTemplate: ReadText(args, "outputTemplate", DefaultConsoleTemplate)); + return true; + } + + if (name.Equals("File", StringComparison.OrdinalIgnoreCase)) + { + WriteFile(logger.WriteTo, args); + return true; + } + + if (name.Equals("Async", StringComparison.OrdinalIgnoreCase) + && TryReadSingleFile(args, out var fileArgs)) + { + logger.WriteTo.Async( + wrapped => WriteFile(wrapped, fileArgs), + bufferSize: ReadInt(args, "bufferSize", 10_000), + blockWhenFull: ReadBool(args, "blockWhenFull", false)); + return true; + } + + return false; + } + + /// + /// Passes one unsupported sink through the configuration binder without the sinks already applied. + /// + private static void ApplyRemainingSink( + LoggerConfiguration logger, + IConfiguration configuration, + IConfigurationSection sink) + { + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + CopySection(configuration.GetSection("Serilog:Using"), values); + CopySection(sink, values, "Serilog:WriteTo:0"); + var remaining = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + logger.ReadFrom.Configuration(remaining); + } + + /// + /// Reads the single nested File sink inside an Async wrapper. + /// + private static bool TryReadSingleFile( + IConfigurationSection args, + [NotNullWhen(true)] out IConfigurationSection? fileArgs) + { + fileArgs = null; + var nested = args.GetSection("configure").GetChildren().ToList(); + if (nested.Count != 1 + || !string.Equals(nested[0]["Name"], "File", StringComparison.OrdinalIgnoreCase) + || !HasOnlyArguments(nested[0].GetSection("Args"), FileArgumentNames)) + { + return false; + } + + fileArgs = nested[0].GetSection("Args"); + return true; + } + + /// + /// Adds a File sink from the same argument names Serilog.Settings.Configuration binds. + /// + private static void WriteFile(LoggerSinkConfiguration sink, IConfigurationSection args) + { + var path = args["path"]; + if (string.IsNullOrWhiteSpace(path)) + throw new InvalidOperationException("The Serilog File sink requires Args:path."); + + sink.DailyGzipFile( + path, + restrictedToMinimumLevel: ReadLevel(args, LogEventLevel.Verbose), + outputTemplate: ReadText(args, "outputTemplate", DefaultFileTemplate), + fileSizeLimitBytes: ReadNullableLong(args, "fileSizeLimitBytes", DefaultFileSizeLimitBytes), + buffered: ReadBool(args, "buffered", false), + flushToDiskInterval: ReadTimeSpan(args, "flushToDiskInterval"), + rollingInterval: ReadRollingInterval(args), + rollOnFileSizeLimit: ReadBool(args, "rollOnFileSizeLimit", true), + retainedFileCountLimit: ReadNullableInt(args, "retainedFileCountLimit", 31)); + } + + /// + /// True when a copied section discovers methods and therefore needs Serilog:Using. + /// + private static bool RequiresSinkAssemblies(IConfiguration configuration) + { + return configuration.GetSection("Serilog:Enrich").GetChildren().Any() + || configuration.GetSection("Serilog:Filter").GetChildren().Any() + || configuration.GetSection("Serilog:Destructure").GetChildren().Any() + || configuration.GetSection("Serilog:AuditTo").GetChildren().Any(); + } + + /// + /// True when every argument name is one this type binds itself. + /// + private static bool HasOnlyArguments(IConfigurationSection args, string[] allowed) + { + if (allowed.Length == 0 && args.GetChildren().Any()) + return false; + + foreach (var child in args.GetChildren()) + { + if (!allowed.Contains(child.Key, StringComparer.OrdinalIgnoreCase)) + return false; + } + + return true; + } + + /// + /// Copies a configuration section, optionally placing it at a different key. + /// + private static void CopySection( + IConfigurationSection section, + Dictionary values, + string? destinationPath = null) + { + if (!section.Exists()) + return; + + var path = destinationPath ?? section.Path; + if (section.Value is not null || !section.GetChildren().Any()) + values[path] = section.Value; + + foreach (var child in section.GetChildren()) + CopySection(child, values, path + ":" + child.Key); + } + + /// + /// Reads a minimum level, using when the argument is absent. + /// + private static LogEventLevel ReadLevel(IConfigurationSection args, LogEventLevel fallback) + { + var text = args["restrictedToMinimumLevel"]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!Enum.TryParse(text, ignoreCase: true, out LogEventLevel level)) + { + throw new InvalidOperationException( + $"Serilog level '{text}' at '{args.Path}:restrictedToMinimumLevel' is not a log event level."); + } + + return level; + } + + /// + /// Reads text, using when the argument is absent. + /// + private static string ReadText(IConfigurationSection args, string name, string fallback) + { + var text = args[name]; + return string.IsNullOrWhiteSpace(text) ? fallback : text; + } + + /// + /// Reads a boolean, using when the argument is absent. + /// + private static bool ReadBool(IConfigurationSection args, string name, bool fallback) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!bool.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not a boolean."); + } + + return value; + } + + /// + /// Reads an integer, using when the argument is absent. + /// + private static int ReadInt(IConfigurationSection args, string name, int fallback) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!int.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not an integer."); + } + + return value; + } + + /// + /// Reads a nullable long. A missing key uses ; an explicit null is unlimited. + /// + /// + /// is false for a JSON null, + /// so presence is taken from . + /// + private static long? ReadNullableLong(IConfigurationSection args, string name, long missing) + { + if (!TryGetArgument(args, name, out var argument)) + return missing; + + if (string.IsNullOrWhiteSpace(argument.Value)) + return null; + + if (!long.TryParse(argument.Value, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{argument.Value}' at '{argument.Path}' is not an integer."); + } + + return value; + } + + /// + /// Reads a nullable integer. A missing key uses ; an explicit null stays null. + /// + private static int? ReadNullableInt(IConfigurationSection args, string name, int missing) + { + if (!TryGetArgument(args, name, out var argument)) + return missing; + + if (string.IsNullOrWhiteSpace(argument.Value)) + return null; + + if (!int.TryParse(argument.Value, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{argument.Value}' at '{argument.Path}' is not an integer."); + } + + return value; + } + + /// + /// Finds an argument that is present even when its value is JSON null. + /// + private static bool TryGetArgument( + IConfigurationSection args, + string name, + [NotNullWhen(true)] out IConfigurationSection? argument) + { + foreach (var child in args.GetChildren()) + { + if (child.Key.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + argument = child; + return true; + } + } + + argument = null; + return false; + } + + /// + /// Reads a flush interval. A missing key leaves flushing to the File sink default. + /// + private static TimeSpan? ReadTimeSpan(IConfigurationSection args, string name) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return null; + + if (!TimeSpan.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not a time span."); + } + + return value; + } + + /// + /// Reads the rolling interval. A missing key uses . + /// + private static RollingInterval ReadRollingInterval(IConfigurationSection args) + { + var text = args["rollingInterval"]; + if (string.IsNullOrWhiteSpace(text)) + return RollingInterval.Infinite; + + if (!Enum.TryParse(text, ignoreCase: true, out RollingInterval interval)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:rollingInterval' is not a rolling interval."); + } + + return interval; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NntpdFileLogging.cs b/src/VectorNNTPD.ReaderService/Logging/NntpdFileLogging.cs new file mode 100644 index 00000000..d3bdb3fd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NntpdFileLogging.cs @@ -0,0 +1,130 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Resolves into the Serilog File path before the File sink + /// is constructed. Operational File/Async settings live in VectorNNTPD.ReaderService.json. + /// + /// + /// Serilog.Settings.Configuration cannot expand Nntpd:LogDir into path. + /// This type creates the directory and overwrites the File sink path so the JSON + /// placeholder is never the runtime directory. It does not add a second File sink. + /// + internal static class NntpdFileLogging + { + /// Serilog rolling path token: {ApplicationName}-yyyyMMdd.log. + public const string RollingPathSuffix = "-.log"; + + /// + /// Suffix appended when a completed log is gzip-compressed: {original-file-name}.gz. + /// + public const string GzipArchiveSuffix = ".gz"; + + /// Console-sink fallback minimum when Serilog:WriteTo is omitted. + public const Serilog.Events.LogEventLevel ConsoleMinimumLevel = Serilog.Events.LogEventLevel.Information; + + /// + /// Resolves through Common + /// against + /// (default + /// ). Absolute paths stay absolute. + /// + public static string ResolveDirectory(string? logDir, string? applicationBaseDirectory = null) + { + var configured = string.IsNullOrWhiteSpace(logDir) + ? NntpdOptions.DefaultLogDir + : logDir.Trim(); + return ApplicationLocalPath.ResolveApplicationLocalPath( + configured, + applicationBaseDirectory ?? AppContext.BaseDirectory); + } + + /// + /// Builds the Serilog rolling path {logDirectory}/{applicationName}-.log. + /// + public static string RollingFilePath(string logDirectory, string applicationName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + ArgumentException.ThrowIfNullOrWhiteSpace(applicationName); + return Path.Join(logDirectory, applicationName.Trim() + RollingPathSuffix); + } + + /// + /// Archive file name produced when a completed log is gzip-compressed. + /// + public static string GzipArchiveFileName(string rolledLogPath) => + Path.GetFileName(rolledLogPath) + GzipArchiveSuffix; + + /// + /// Creates and binds the resolved File path over the JSON placeholder. + /// + public static void BindResolvedFilePath( + ConfigurationManager configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + + var logDir = ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + + var path = RollingFilePath(logDir, ApplicationJsonConfiguration.EntryAssemblyName); + var pathKey = FindConfiguredFilePathKey(configuration); + if (pathKey is null) + { + return; + } + + configuration.AddInMemoryCollection(new Dictionary { [pathKey] = path }); + NntpdNewsLogging.BindResolvedNewsPath(configuration, applicationBaseDirectory); + NntpdPathSurveyLogging.BindResolvedPathSurveyPath(configuration, applicationBaseDirectory); + } + + private static string? FindConfiguredFilePathKey(IConfiguration configuration) + { + foreach (var writeTo in configuration.GetSection("Serilog:WriteTo").GetChildren()) + { + if (!string.Equals(writeTo["Name"], "Async", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + foreach (var inner in writeTo.GetSection("Args:configure").GetChildren()) + { + if (string.Equals(inner["Name"], "File", StringComparison.OrdinalIgnoreCase)) + { + return $"{inner.Path}:Args:path"; + } + } + } + + return null; + } + + /// + /// In-memory Serilog File/Async keys matching production VectorNNTPD.ReaderService.json (path is a placeholder). + /// + public static Dictionary AsyncFileWriteToKeys(int writeToIndex = 1) => new() + { + [$"Serilog:WriteTo:{writeToIndex}:Name"] = "Async", + [$"Serilog:WriteTo:{writeToIndex}:Args:bufferSize"] = "50000", + [$"Serilog:WriteTo:{writeToIndex}:Args:blockWhenFull"] = "true", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Name"] = "File", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:path"] = + "logs/" + ApplicationJsonConfiguration.EntryAssemblyName + "-.log", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:restrictedToMinimumLevel"] = "Debug", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:outputTemplate"] = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:fileSizeLimitBytes"] = null, + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:buffered"] = "true", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:flushToDiskInterval"] = "00:00:01", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:rollingInterval"] = "Day", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:rollOnFileSizeLimit"] = "false", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:retainedFileCountLimit"] = "14", + }; + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NntpdLoggingExtensions.cs b/src/VectorNNTPD.ReaderService/Logging/NntpdLoggingExtensions.cs new file mode 100644 index 00000000..df6084fc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NntpdLoggingExtensions.cs @@ -0,0 +1,157 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using Serilog; +using Serilog.Events; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Configures Serilog as the exclusive logging implementation for VectorNNTPD.ReaderService. + /// + public static class NntpdLoggingExtensions + { + /// + /// Single-line console template suitable for interactive terminals and journald collection. + /// + public const string ConsoleOutputTemplate = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}"; + + /// + /// Creates an early bootstrap logger used before the Generic Host is built. + /// + /// A bootstrap logger assigned to . + public static Serilog.ILogger CreateBootstrapLogger() + { + return new LoggerConfiguration() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", ApplicationJsonConfiguration.EntryAssemblyName) + .WriteTo.Console(outputTemplate: ConsoleOutputTemplate) + .CreateBootstrapLogger(); + } + + /// + /// Makes flush every write. + /// + /// + /// When stdout is a console, the runtime already flushes often enough that Information + /// lines appear immediately. When stdout is a pipe (IDE, redirected capture), + /// is block-buffered: a startup burst fills the buffer and + /// becomes visible, then later Information events sit unseen until the buffer fills + /// again or the process exits. Serilog's Console sink writes to + /// and does not Flush. This must run before the first Serilog Console sink is created. + /// + public static void UseAutoFlushConsoleOutput() + { + // codeql[cs/local-not-disposed] Owning Console.Out; disposing would close process stdout. + var writer = new StreamWriter(Console.OpenStandardOutput(), Console.OutputEncoding) + { + AutoFlush = true, + }; + Console.SetOut(writer); + } + + /// + /// Removes Microsoft default logging providers and registers Serilog as the sole provider. + /// + /// The host application builder. + /// + /// Optional additional Serilog configuration applied after reading appsettings. + /// Used by tests to attach in-memory sinks without reintroducing MEL providers. + /// + /// The same instance. + /// + /// + /// Call order: clear MEL providers, remove default registrations, + /// then AddSerilog with writeToProviders: false so framework and application + /// traffic flows only through Serilog sinks. + /// + /// + /// Console formatting is owned by Serilog. journald collects Serilog console stdout under + /// systemd. Microsoft console formatter options that AddSystemd() may register are + /// stripped by ConfigureNntpdPlatformHosting and are not part of this logging pipeline. + /// + /// + /// Console, File, and Async File sinks are registered with direct Serilog calls from the + /// Serilog:WriteTo argument values. ReadFrom.Configuration still applies + /// minimum level, enrichment, and any sink entry those direct calls do not represent. + /// Native AOT does not retain the reflection metadata sink discovery requires. + /// + /// + public static HostApplicationBuilder ConfigureNntpdLogging( + this HostApplicationBuilder builder, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Logging.ClearProviders(); + + // Host.CreateApplicationBuilder registers a default LoggerFactory; remove it so + // SerilogLoggerFactory is the only ILoggerFactory in the container. + builder.Services.RemoveAll(); + builder.Services.RemoveAll(); + + NntpdFileLogging.BindResolvedFilePath(builder.Configuration); + + builder.Services.AddSerilog( + (services, loggerConfiguration) => + { + loggerConfiguration + .ReadFrom.Configuration(NntpdConfiguredSinks.WithoutWriteTo(builder.Configuration)) + .ReadFrom.Services(services) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", ApplicationJsonConfiguration.EntryAssemblyName); + + NntpdConfiguredSinks.Apply(loggerConfiguration, builder.Configuration); + configure?.Invoke(loggerConfiguration); + }, + preserveStaticLogger: false, + writeToProviders: false); + + return builder; + } + + /// + /// Emits one Information event through the host after + /// Serilog has replaced the bootstrap logger. + /// + /// The built host service provider. + /// Host environment name (no secrets). + /// Resolved content root used for VectorNNTPD.ReaderService.json. + public static void WriteLoggingInitialized( + IServiceProvider services, + string environmentName, + string contentRootPath) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(environmentName); + ArgumentException.ThrowIfNullOrWhiteSpace(contentRootPath); + + var factory = services.GetRequiredService(); + var logger = factory.CreateLogger(NntpdLogCategories.Hosting); + HostingLogMessages.LoggingInitialized( + logger, + ApplicationJsonConfiguration.EntryAssemblyName, + factory.GetType().Name, + NntpdLogCategories.Hosting, + environmentName, + contentRootPath); + } + + /// + /// Returns registered implementations for diagnostics and tests. + /// + /// + /// When Serilog is configured via AddSerilog, logging is provided by replacing + /// with SerilogLoggerFactory. In that mode the provider + /// list is typically empty, which confirms Microsoft default providers were not retained. + /// + public static IReadOnlyList GetLoggerProviders(this IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetServices().ToArray(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NntpdNewsLogging.cs b/src/VectorNNTPD.ReaderService/Logging/NntpdNewsLogging.cs new file mode 100644 index 00000000..474279ef --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NntpdNewsLogging.cs @@ -0,0 +1,180 @@ +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Dedicated Serilog pipeline for the INN news file. + /// + /// + /// + /// Operational File/Async settings bind from Serilog:News using the same + /// argument names as the application File sink (path, rolling, retention, + /// buffering, flushToDiskInterval). The INN line format is not read from configuration; the + /// sink is always constructed with . + /// + /// + /// This small bridge exists because Serilog.Settings.Configuration cannot attach + /// a code-owned formatter without also exposing a user-editable + /// outputTemplate or formatter string. Path resolution still uses + /// via . + /// + /// + public static class NntpdNewsLogging + { + /// Serilog source context for news events. Not used by the application File/Console sinks. + public const string SourceContext = "VectorNNTPD.ReaderService.News"; + + /// Configuration section that holds news File/Async operational arguments. + public const string SectionName = "Serilog:News"; + + /// Canonical rolling path token: news-yyyyMMdd.log. + public const string RollingFileName = "news"; + + /// Log event property for the INN disposition character. + public const string DispositionProperty = "NewsDisposition"; + + /// Log event property for the INN feed token. + public const string FeedProperty = "NewsFeed"; + + /// Log event property for the article Message-ID. + public const string MessageIdProperty = "NewsMessageId"; + + /// Log event property for optional future SITE tokens. + public const string SitesProperty = "NewsSites"; + + /// Log event property for the INN article size in bytes. + public const string SizeProperty = "NewsSize"; + + /// Log event property for the NNTP rejection response code. + public const string ResponseCodeProperty = "NewsResponseCode"; + + /// Log event property for the already-decided rejection reason. + public const string ReasonProperty = "NewsReason"; + + /// + /// Overwrites Serilog:News:path from + /// so the JSON placeholder is never the runtime path. + /// + public static void BindResolvedNewsPath( + IConfiguration configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration[SectionName + ":path"] is null) + { + return; + } + + var logDir = NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + var path = NewsRollingFilePath(logDir); + if (configuration is IConfigurationBuilder builder) + { + builder.AddInMemoryCollection(new Dictionary { [SectionName + ":path"] = path }); + } + } + + /// Builds {logDirectory}/news-.log for Serilog daily rolling. + public static string NewsRollingFilePath(string logDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + return Path.Join(logDirectory, RollingFileName + NntpdFileLogging.RollingPathSuffix); + } + + /// + /// Reads File/Async operational settings from Serilog:News. + /// The INN formatter is never taken from configuration. + /// + public static NewsFileSinkSettings ReadSettings(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var news = configuration.GetSection(SectionName); + var path = news["path"]; + if (string.IsNullOrWhiteSpace(path)) + { + path = NewsRollingFilePath( + NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"])); + } + + return new NewsFileSinkSettings( + path, + ParseRolling(news["rollingInterval"]), + news.GetValue("retainedFileCountLimit") ?? 1, + news.GetValue("buffered", true), + news.GetValue("rollOnFileSizeLimit", false), + news.GetValue("fileSizeLimitBytes"), + news.GetValue("bufferSize", 50000), + news.GetValue("blockWhenFull", true), + news.GetValue("flushToDiskInterval")); + } + + /// + /// Creates a dedicated Serilog logger that writes only INN news lines. + /// + public static Serilog.Core.Logger CreateLogger(IConfiguration configuration) + { + var settings = ReadSettings(configuration); + Directory.CreateDirectory( + Path.GetDirectoryName(Path.GetFullPath(settings.Path)) ?? NntpdOptions.DefaultLogDir); + + return new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.Async( + a => a.DailyGzipFile( + new InnNewsTextFormatter(), + settings.Path, + restrictedToMinimumLevel: LogEventLevel.Information, + fileSizeLimitBytes: settings.FileSizeLimitBytes, + buffered: settings.Buffered, + flushToDiskInterval: settings.FlushToDiskInterval, + rollingInterval: settings.RollingInterval, + rollOnFileSizeLimit: settings.RollOnFileSizeLimit, + retainedFileCountLimit: settings.RetainedFileCountLimit), + bufferSize: settings.BufferSize, + blockWhenFull: settings.BlockWhenFull) + .CreateLogger(); + } + + private static RollingInterval ParseRolling(string? value) => + Enum.TryParse(value, ignoreCase: true, out var interval) + ? interval + : RollingInterval.Day; + } + + /// + /// Operational File/Async settings for the dedicated news sink. + /// The INN line format is not represented here. + /// + /// Resolved news file path, including Serilog rolling token. + /// Serilog File rolling interval. + /// + /// Daily files to keep, including the active file. .log and .log.gz for the same day count as one. + /// + /// Whether the File sink buffers writes. + /// Whether size-based rolling is enabled. + /// Serilog File size cap; is unlimited. + /// Serilog.Sinks.Async buffer size. + /// Whether Async blocks when the buffer is full. + /// + /// Serilog File periodic flush. leaves buffering until dispose. + /// Production Serilog:News sets one second. + /// + public readonly record struct NewsFileSinkSettings( + string Path, + RollingInterval RollingInterval, + int RetainedFileCountLimit, + bool Buffered, + bool RollOnFileSizeLimit, + long? FileSizeLimitBytes, + int BufferSize, + bool BlockWhenFull, + TimeSpan? FlushToDiskInterval); +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NntpdPathSurveyLogging.cs b/src/VectorNNTPD.ReaderService/Logging/NntpdPathSurveyLogging.cs new file mode 100644 index 00000000..28d6fac6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NntpdPathSurveyLogging.cs @@ -0,0 +1,197 @@ +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Dedicated Serilog pipeline for the INN-style Path-survey (inpaths) file. + /// + /// + /// + /// Operational File/Async settings bind from Serilog:Inpaths using the same + /// argument names as the application File sink (path, rolling, retention, + /// buffering, flushToDiskInterval). The Path-survey line format is not read from configuration; + /// the sink is always constructed with . + /// + /// + /// This small bridge exists because Serilog.Settings.Configuration cannot attach + /// a code-owned formatter without also exposing a user-editable + /// outputTemplate or formatter string. Path resolution still uses + /// via . + /// + /// + /// Production is 1, + /// so only the active day is kept. When the day rolls, the completed file is handed + /// to and then gzip-compressed. + /// Retention then deletes that compressed day because it is outside the one-file limit. + /// The handler opens the file with share-read/write/delete before compression, so it + /// can keep reading after the uncompressed file is replaced. Handler failure is logged + /// and does not skip gzip. + /// + /// + public static class NntpdPathSurveyLogging + { + /// Serilog source context for Path-survey events. Not used by the application File/Console sinks. + public const string SourceContext = "VectorNNTPD.ReaderService.Inpaths"; + + /// Configuration section that holds Path-survey File/Async operational arguments. + public const string SectionName = "Serilog:Inpaths"; + + /// Canonical rolling path token: inpaths-yyyyMMdd.log. + public const string RollingFileName = "inpaths"; + + /// Log event property for the canonical Path value. + public const string PathProperty = "PathSurveyPath"; + + /// + /// Overwrites Serilog:Inpaths:path from + /// so the JSON placeholder is never the runtime path. + /// + public static void BindResolvedPathSurveyPath( + IConfiguration configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration[SectionName + ":path"] is null) + { + return; + } + + var logDir = NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + var path = PathSurveyRollingFilePath(logDir); + if (configuration is IConfigurationBuilder builder) + { + builder.AddInMemoryCollection(new Dictionary { [SectionName + ":path"] = path }); + } + } + + /// Builds {logDirectory}/inpaths-.log for Serilog daily rolling. + public static string PathSurveyRollingFilePath(string logDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + return Path.Join(logDirectory, RollingFileName + NntpdFileLogging.RollingPathSuffix); + } + + /// + /// Reads File/Async operational settings from Serilog:Inpaths. + /// The Path-survey formatter is never taken from configuration. + /// + public static PathSurveyFileSinkSettings ReadSettings(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var inpaths = configuration.GetSection(SectionName); + var path = inpaths["path"]; + if (string.IsNullOrWhiteSpace(path)) + { + path = PathSurveyRollingFilePath( + NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"])); + } + + return new PathSurveyFileSinkSettings( + path, + ParseRolling(inpaths["rollingInterval"]), + inpaths.GetValue("retainedFileCountLimit") ?? 1, + inpaths.GetValue("buffered", true), + inpaths.GetValue("rollOnFileSizeLimit", false), + inpaths.GetValue("fileSizeLimitBytes"), + inpaths.GetValue("bufferSize", 50000), + inpaths.GetValue("blockWhenFull", true), + inpaths.GetValue("flushToDiskInterval")); + } + + /// + /// Creates a dedicated Serilog logger that writes only Path-survey lines. + /// + /// Application configuration containing Serilog:Inpaths. + /// + /// Invoked with the uncompressed rolled file before gzip. Production + /// enqueues the file for background ninpaths processing. + /// + /// Logger for completed-file handler failures; must not be the Path-survey logger. + public static Serilog.Core.Logger CreateLogger( + IConfiguration configuration, + ICompletedPathSurveyFileHandler completedFileHandler, + Microsoft.Extensions.Logging.ILogger hookLogger) + { + ArgumentNullException.ThrowIfNull(completedFileHandler); + ArgumentNullException.ThrowIfNull(hookLogger); + var settings = ReadSettings(configuration); + Directory.CreateDirectory( + Path.GetDirectoryName(Path.GetFullPath(settings.Path)) ?? NntpdOptions.DefaultLogDir); + + return new LoggerConfiguration() + .MinimumLevel.Information() + .Enrich.WithProperty(Serilog.Core.Constants.SourceContextPropertyName, SourceContext) + .Filter.ByIncludingOnly(static e => e.Properties.ContainsKey(PathProperty)) + .WriteTo.Async( + a => a.DailyGzipFile( + new InnPathSurveyTextFormatter(), + settings.Path, + restrictedToMinimumLevel: LogEventLevel.Information, + fileSizeLimitBytes: settings.FileSizeLimitBytes, + buffered: settings.Buffered, + flushToDiskInterval: settings.FlushToDiskInterval, + rollingInterval: settings.RollingInterval, + rollOnFileSizeLimit: settings.RollOnFileSizeLimit, + retainedFileCountLimit: settings.RetainedFileCountLimit, + onCompletedFile: path => + { + try + { + completedFileHandler.OnCompletedFile(path); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PathSurveyLogMessages.CompletedFileHandlerFailed(hookLogger, ex, path); + } + }), + bufferSize: settings.BufferSize, + blockWhenFull: settings.BlockWhenFull) + .CreateLogger(); + } + + private static RollingInterval ParseRolling(string? value) => + Enum.TryParse(value, ignoreCase: true, out var interval) + ? interval + : RollingInterval.Day; + } + + /// + /// Operational File/Async settings for the dedicated Path-survey sink. + /// The Path-survey line format is not represented here. + /// + /// Resolved Path-survey file path, including Serilog rolling token. + /// Serilog File rolling interval. + /// + /// Daily files to keep, including the active file. .log and .log.gz for the same day count as one. + /// Production uses 1 so a completed day is removed after the ninpaths handoff and gzip. + /// + /// Whether the File sink buffers writes. + /// Whether size-based rolling is enabled. + /// Serilog File size cap; is unlimited. + /// Serilog.Sinks.Async buffer size. + /// Whether Async blocks when the buffer is full. + /// + /// Serilog File periodic flush. leaves buffering until dispose. + /// Production Serilog:Inpaths sets one second. + /// + public readonly record struct PathSurveyFileSinkSettings( + string Path, + RollingInterval RollingInterval, + int RetainedFileCountLimit, + bool Buffered, + bool RollOnFileSizeLimit, + long? FileSizeLimitBytes, + int BufferSize, + bool BlockWhenFull, + TimeSpan? FlushToDiskInterval); +} diff --git a/src/VectorNNTPD.ReaderService/Logging/NullCompletedPathSurveyFileHandler.cs b/src/VectorNNTPD.ReaderService/Logging/NullCompletedPathSurveyFileHandler.cs new file mode 100644 index 00000000..4af8f814 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/NullCompletedPathSurveyFileHandler.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// No-op completed-file handler. Path-survey files are still gzip-archived. + /// Production DI registers NinpathsCompletedFileHandler instead; this + /// type remains for tests that opt out of ninpaths mail. + /// + public sealed class NullCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler + { + /// Gets the shared no-op instance. + public static NullCompletedPathSurveyFileHandler Instance { get; } = new(); + + /// + public void OnCompletedFile(string completedFilePath) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/PathSurveyLogMessages.cs b/src/VectorNNTPD.ReaderService/Logging/PathSurveyLogMessages.cs new file mode 100644 index 00000000..324a0f5c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/PathSurveyLogMessages.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.ReaderService.Logging +{ + /// Source-generated Path-survey lifecycle log messages. + internal static partial class PathSurveyLogMessages + { + [LoggerMessage( + EventId = 2010, + Level = LogLevel.Error, + Message = "Path-survey completed-file handler failed for {CompletedFile}; gzip archive will still run")] + public static partial void CompletedFileHandlerFailed(ILogger logger, Exception exception, string CompletedFile); + } +} diff --git a/src/VectorNNTPD.ReaderService/Logging/SystemdLogMessages.cs b/src/VectorNNTPD.ReaderService/Logging/SystemdLogMessages.cs new file mode 100644 index 00000000..bc0fe740 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Logging/SystemdLogMessages.cs @@ -0,0 +1,134 @@ +namespace VectorNNTPD.ReaderService.Logging +{ + /// + /// Source-generated structured log messages for systemd notify and watchdog events. + /// + internal static partial class SystemdLogMessages + { + [LoggerMessage( + EventId = 1300, + Level = LogLevel.Information, + Message = "systemd lifecycle notifications activated (notifyEnabled={NotifyEnabled}, isSystemdService={IsSystemdService}")] + public static partial void LifecycleNotificationsActivated( + ILogger logger, + bool NotifyEnabled, + bool IsSystemdService); + + [LoggerMessage( + EventId = 1301, + Level = LogLevel.Information, + Message = "Reported systemd readiness after successful application initialization")] + public static partial void ReadinessReported(ILogger logger); + + [LoggerMessage( + EventId = 1302, + Level = LogLevel.Information, + Message = "Reported systemd STOPPING notification for graceful shutdown")] + public static partial void StoppingReported(ILogger logger); + + [LoggerMessage( + EventId = 1303, + Level = LogLevel.Information, + Message = "Reported systemd status: {Status}")] + public static partial void StatusReported(ILogger logger, string Status); + + [LoggerMessage( + EventId = 1304, + Level = LogLevel.Debug, + Message = "Sent systemd notification {Notification}")] + public static partial void NotificationSent(ILogger logger, string Notification); + + [LoggerMessage( + EventId = 1305, + Level = LogLevel.Error, + Message = "Failed to send systemd notification {Notification}")] + public static partial void NotificationFailed(ILogger logger, Exception exception, string Notification); + + [LoggerMessage( + EventId = 1306, + Level = LogLevel.Information, + Message = "systemd runtime detection: isSystemdService={IsSystemdService}, notifyEnabled={NotifyEnabled}, watchdogConfigured={WatchdogConfigured}, watchdogTimeout={WatchdogTimeout}, detail={Detail}")] + public static partial void RuntimeDetection( + ILogger logger, + bool IsSystemdService, + bool NotifyEnabled, + bool WatchdogConfigured, + TimeSpan? WatchdogTimeout, + string Detail); + + [LoggerMessage( + EventId = 1307, + Level = LogLevel.Warning, + Message = "Ignoring malformed systemd watchdog configuration (WATCHDOG_USEC is not a positive integer)")] + public static partial void MalformedWatchdogUsec(ILogger logger); + + [LoggerMessage( + EventId = 1308, + Level = LogLevel.Warning, + Message = "Ignoring systemd watchdog configuration because WATCHDOG_PID is malformed")] + public static partial void MalformedWatchdogPid(ILogger logger); + + [LoggerMessage( + EventId = 1309, + Level = LogLevel.Information, + Message = "systemd watchdog is configured for a different PID; watchdog keep-alives will remain disabled")] + public static partial void WatchdogConfiguredForDifferentPid(ILogger logger); + + [LoggerMessage( + EventId = 1310, + Level = LogLevel.Warning, + Message = "Ignoring systemd watchdog configuration because WATCHDOG_USEC is out of range")] + public static partial void WatchdogUsecOutOfRange(ILogger logger); + + [LoggerMessage( + EventId = 1311, + Level = LogLevel.Information, + Message = "systemd watchdog remain disabled ({Reason})")] + public static partial void WatchdogRemainDisabled(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 1312, + Level = LogLevel.Information, + Message = "systemd watchdog activated. Deadline={WatchdogTimeout}, heartbeatInterval={HeartbeatInterval}")] + public static partial void WatchdogActivated( + ILogger logger, + TimeSpan WatchdogTimeout, + TimeSpan HeartbeatInterval); + + [LoggerMessage( + EventId = 1313, + Level = LogLevel.Debug, + Message = "Skipping systemd watchdog keep-alive because the application is unhealthy")] + public static partial void WatchdogKeepAliveSkippedUnhealthy(ILogger logger); + + [LoggerMessage( + EventId = 1314, + Level = LogLevel.Trace, + Message = "Sent systemd watchdog keep-alive")] + public static partial void WatchdogKeepAliveSent(ILogger logger); + + [LoggerMessage( + EventId = 1315, + Level = LogLevel.Error, + Message = "systemd watchdog notification failed")] + public static partial void WatchdogNotificationFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1316, + Level = LogLevel.Information, + Message = "systemd watchdog deactivated due to shutdown")] + public static partial void WatchdogDeactivated(ILogger logger); + + [LoggerMessage( + EventId = 1317, + Level = LogLevel.Critical, + Message = "systemd watchdog loop failed unexpectedly; requesting host stop")] + public static partial void WatchdogLoopFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1318, + Level = LogLevel.Information, + Message = "systemd watchdog stopping")] + public static partial void WatchdogStopping(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/EmailModerationSubmissionService.cs b/src/VectorNNTPD.ReaderService/Moderation/EmailModerationSubmissionService.cs new file mode 100644 index 00000000..ad791145 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/EmailModerationSubmissionService.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Forwards a proto-article to the moderator by enqueueing outbound email. + /// + /// + /// + /// This type knows email messages, not SMTP. + /// acceptance means the complete message was durably written to the local + /// filesystem spool. It is not proof that a remote SMTP server or moderator + /// mailbox received the message. SMTP delivery is asynchronous. + /// + /// + /// POST maps to 240 and + /// other statuses to 441. + /// + /// + public sealed class EmailModerationSubmissionService : IModerationSubmissionService + { + private readonly IEmailService _email; + private readonly ModerationEmailComposer _composer; + private readonly IOptions _emailOptions; + + /// Initializes a new email-backed submission service. + public EmailModerationSubmissionService( + IEmailService email, + ModerationEmailComposer composer, + IOptions emailOptions) + { + ArgumentNullException.ThrowIfNull(email); + ArgumentNullException.ThrowIfNull(composer); + ArgumentNullException.ThrowIfNull(emailOptions); + _email = email; + _composer = composer; + _emailOptions = emailOptions; + } + + /// + public async ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + if (!_emailOptions.Value.Enabled) + { + return new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + "email subsystem is disabled"); + } + + EmailMessage message; + try + { + message = _composer.Compose(submission); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + "moderation email could not be constructed"); + } + + var enqueue = await _email.SendAsync(message, cancellationToken).ConfigureAwait(false); + return enqueue.Status switch + { + EmailEnqueueStatus.Accepted => new ModerationSubmissionResult( + ModerationSubmissionStatus.Accepted, + "durably accepted into the local email spool"), + EmailEnqueueStatus.Disabled or EmailEnqueueStatus.Unavailable => new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + enqueue.Detail), + EmailEnqueueStatus.Rejected => new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + enqueue.Detail), + _ => new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + enqueue.Detail), + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/EmptyModeratorAuthorization.cs b/src/VectorNNTPD.ReaderService/Moderation/EmptyModeratorAuthorization.cs new file mode 100644 index 00000000..33b572d4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/EmptyModeratorAuthorization.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Moderator authorization with no configured routes. Every resolve and approval fails. + /// + public sealed class EmptyModeratorAuthorization : IModeratorAuthorization + { + /// Shared instance. + public static EmptyModeratorAuthorization Instance { get; } = new(); + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) + { + _ = authenticatedUsername; + return false; + } + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) + { + _ = newsgroup; + identity = default; + return false; + } + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) + { + _ = authenticatedUsername; + _ = approvedIdentity; + _ = newsgroup; + return false; + } + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) + { + _ = authenticatedUsername; + _ = approvedIdentities; + failureDetail = moderatedGroups.Length == 0 + ? "no moderated groups" + : "unresolved moderator"; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/IModerationSubmissionService.cs b/src/VectorNNTPD.ReaderService/Moderation/IModerationSubmissionService.cs new file mode 100644 index 00000000..e6dd3ae1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/IModerationSubmissionService.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Durable boundary that accepts a proto-article for moderator forwarding. + /// + /// + /// POST policy calls this abstraction; it does not send email or speak SMTP. + /// The production implementation composes a moderator email and calls + /// IEmailService.SendAsync. Queue acceptance is not remote SMTP delivery. + /// + public interface IModerationSubmissionService + { + /// + /// Offers for moderator delivery. + /// + /// Proto-article and resolved moderator route. + /// Cancels the attempt. Must not be ignored by implementations that wait. + /// Accepted, unavailable, or failed. Does not inject the article. + ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/IModeratorAuthorization.cs b/src/VectorNNTPD.ReaderService/Moderation/IModeratorAuthorization.cs new file mode 100644 index 00000000..fec7d095 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/IModeratorAuthorization.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Authorizes moderated-newsgroup approval from the authenticated NNTP principal, + /// the configured moderator route, and the claimed Approved: identity. + /// + /// + /// An Approved: header is an assertion. It is trusted only when the + /// authenticated AUTHINFO principal is configured as the moderator for every + /// moderated target. Presence of Approved: alone is never sufficient. + /// This contract is not Control:PgpAuthorities. + /// + public interface IModeratorAuthorization + { + /// + /// Returns whether is configured as a + /// moderator for at least one mapping. This is not group authorization. + /// + bool IsAuthenticatedModerator(string? authenticatedUsername); + + /// + /// Resolves the first-match moderator route for . + /// + bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity); + + /// + /// Returns whether the authenticated principal may approve + /// using . + /// + bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup); + + /// + /// Authorizes every moderated target against the collected Approved: identities. + /// + /// AUTHINFO principal, or when unauthenticated. + /// Distinct mailbox identities parsed from Approved:. + /// Moderated Newsgroups: targets in header order. + /// Internal reject detail when authorization fails. + /// only when every moderated target is authorized. + bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/IModeratorCatalogue.cs b/src/VectorNNTPD.ReaderService/Moderation/IModeratorCatalogue.cs new file mode 100644 index 00000000..d45cdeef --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/IModeratorCatalogue.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Process-wide moderator catalogue. Command handlers capture once. + /// + public interface IModeratorCatalogue + { + /// + /// Gets the last successfully published immutable snapshot. + /// + /// + /// After a successful start this is never null. Readers need no lock. + /// Capture the reference once per POST; a concurrent refresh may replace + /// the published snapshot while that command continues on the captured instance. + /// + ModeratorSnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/INntpModeratorRepository.cs b/src/VectorNNTPD.ReaderService/Moderation/INntpModeratorRepository.cs new file mode 100644 index 00000000..5c0e9366 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/INntpModeratorRepository.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// Loads enabled moderator rows from the NntpDB catalogue source. + public interface INntpModeratorRepository + { + /// + /// Returns enabled rows in moderator_id order. Callers must not query this + /// from the POST hot path. + /// + ValueTask> GetEnabledAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModerationEmailComposer.cs b/src/VectorNNTPD.ReaderService/Moderation/ModerationEmailComposer.cs new file mode 100644 index 00000000..57460bc0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModerationEmailComposer.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Builds a standards-defined moderator email from a proto-article. + /// + /// + /// RFC 5537 §3.5.1 / §4.1: the proto-article is encapsulated as + /// application/news-transmission; usage=moderate. Injection-Info, + /// Injection-Date, Path, and X-Trace are not added here — forwarding happens + /// before those headers are written. + /// + public sealed class ModerationEmailComposer + { + private readonly IOptions _emailOptions; + + /// Initializes a new composer. + public ModerationEmailComposer(IOptions emailOptions) + { + ArgumentNullException.ThrowIfNull(emailOptions); + _emailOptions = emailOptions; + } + + /// Creates the outbound moderation email. Does not send it. + public EmailMessage Compose(ModerationSubmission submission) + { + ArgumentNullException.ThrowIfNull(submission); + if (!EmailOptionsValidator.TryValidateMailbox(submission.ModeratorAddress, out var moderator)) + { + throw new ArgumentException("Moderator address is not a mailbox.", nameof(submission)); + } + + var options = _emailOptions.Value; + if (!EmailOptionsValidator.TryValidateMailbox(options.DefaultFrom, out var fromMailbox) + && !EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out fromMailbox)) + { + throw new InvalidOperationException("Email:DefaultFrom must be a mailbox when composing moderation mail."); + } + + EmailAddress? envelope = null; + if (EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out var envelopeMailbox)) + { + envelope = new EmailAddress(envelopeMailbox); + } + + var proto = DestuffProtoArticle(submission.ProtoArticle.Span); + var subject = submission.TargetModeratedGroup + " " + submission.MessageId; + return new EmailMessage + { + From = new EmailAddress(fromMailbox), + To = [new EmailAddress(moderator)], + EnvelopeSender = envelope, + Subject = subject, + Body = proto, + ContentType = "application/news-transmission; usage=moderate", + Charset = "us-ascii", + CorrelationId = submission.MessageId, + Headers = + [ + new EmailHeader("X-Moderated-Newsgroup", submission.TargetModeratedGroup), + new EmailHeader("X-Original-Message-ID", submission.MessageId), + new EmailHeader( + "X-Submitting-Sender", + string.IsNullOrEmpty(submission.AuthenticatedUsername) + ? "unauthenticated" + : submission.AuthenticatedUsername), + ], + }; + } + + /// Destuffs NNTP proto-article wire into a complete article (CRLF lines). + internal static byte[] DestuffProtoArticle(ReadOnlySpan stuffed) + { + if (stuffed.IsEmpty) + { + return []; + } + + var output = new List(stuffed.Length); + var offset = 0; + while (offset < stuffed.Length) + { + var remaining = stuffed[offset..]; + var crlf = remaining.IndexOf("\r\n"u8); + ReadOnlySpan line; + if (crlf < 0) + { + line = remaining; + offset = stuffed.Length; + } + else + { + line = remaining[..crlf]; + offset += crlf + 2; + } + + if (line.Length > 0 && line[0] == (byte)'.') + { + line = line[1..]; + } + + output.AddRange(line.ToArray()); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + + return [.. output]; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModerationSubmission.cs b/src/VectorNNTPD.ReaderService/Moderation/ModerationSubmission.cs new file mode 100644 index 00000000..71e311d7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModerationSubmission.cs @@ -0,0 +1,58 @@ +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Proto-article offered to a moderator before Injection-Info / Injection-Date are added. + /// + /// + /// RFC 5537 §3.5 step 7 and §3.5.1: forwarding happens after Message-ID/Date are present + /// and before injection trace headers. The payload is the stuffed proto-article (IHAVE + /// wire representation without the terminator), not a finalized injected article. + /// + public sealed class ModerationSubmission + { + /// Gets the stuffed proto-article (terminator omitted). + public required ReadOnlyMemory ProtoArticle { get; init; } + + /// Gets the article Message-ID (supplied or synthesized). + public required string MessageId { get; init; } + + /// Gets the syntax-validated Newsgroups: targets. + public required string[] Newsgroups { get; init; } + + /// + /// Gets the leftmost moderated group — the RFC 5537 §3.5.1 forwarding target. + /// + public required string TargetModeratedGroup { get; init; } + + /// Gets the resolved moderator mailbox for . + public required string ModeratorAddress { get; init; } + + /// Gets the submitting AUTHINFO principal, or when unauthenticated. + public string? AuthenticatedUsername { get; init; } + + /// Gets the session client identity of the submitting poster. + public required ConnectionClientIdentity Sender { get; init; } + } + + /// Outcome of . + public enum ModerationSubmissionStatus + { + /// The proto-article was accepted for moderator delivery. + Accepted = 0, + + /// No delivery mechanism is configured or available. + Unavailable = 1, + + /// Delivery was attempted and failed. + Failed = 2, + } + + /// Result of one moderation submission attempt. + /// Acceptance / availability / failure. + /// Internal detail (not sent on the NNTP wire). + public readonly record struct ModerationSubmissionResult( + ModerationSubmissionStatus Status, + string Detail); +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModeratorAddressTemplate.cs b/src/VectorNNTPD.ReaderService/Moderation/ModeratorAddressTemplate.cs new file mode 100644 index 00000000..43d99661 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModeratorAddressTemplate.cs @@ -0,0 +1,102 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// INN samples/moderators address-template expansion. + /// + /// + /// %s is replaced with the matched newsgroup name after . is + /// changed to -. The result is a routing mailbox for unapproved + /// submission. Expansion does not authorize an NNTP principal. + /// + public static class ModeratorAddressTemplate + { + /// INN substitution token. + public const string Token = "%s"; + + /// + /// Returns whether contains the INN %s token. + /// + public static bool IsTemplate(ReadOnlySpan address) => + address.IndexOf(Token, StringComparison.Ordinal) >= 0; + + /// + /// Expands a static mailbox or an INN %s template against + /// . + /// + public static string Expand(string address, ReadOnlySpan newsgroup) + { + ArgumentException.ThrowIfNullOrWhiteSpace(address); + if (!IsTemplate(address)) + { + return address; + } + + var name = Encoding.ASCII.GetString(newsgroup); + var substituted = name.Replace('.', '-'); + return address.Replace(Token, substituted, StringComparison.Ordinal); + } + + /// + /// Validates a static mailbox or a single-%s INN template. + /// + public static bool TryValidate(string address) + { + if (string.IsNullOrWhiteSpace(address) || !IsAscii(address)) + { + return false; + } + + var tokenCount = 0; + var i = 0; + while (i < address.Length) + { + if (address[i] == '%') + { + if (i + 1 >= address.Length || address[i + 1] != 's') + { + return false; + } + + tokenCount++; + i += 2; + continue; + } + + i++; + } + + if (tokenCount > 1) + { + return false; + } + + var candidate = tokenCount == 0 + ? address + : address.Replace(Token, "x", StringComparison.Ordinal); + return IsMailboxIdentity(candidate); + } + + private static bool IsMailboxIdentity(string value) + { + var at = value.IndexOf('@'); + return at > 0 + && at == value.LastIndexOf('@') + && at < value.Length - 1; + } + + private static bool IsAscii(string value) + { + foreach (var c in value) + { + if (c > 127) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueLogMessages.cs b/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueLogMessages.cs new file mode 100644 index 00000000..c7aef285 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// Source-generated moderator catalogue lifecycle log messages. + internal static partial class ModeratorCatalogueLogMessages + { + [LoggerMessage( + EventId = 2510, + Level = LogLevel.Information, + Message = "Loading initial moderator catalogue")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2511, + Level = LogLevel.Information, + Message = "Published moderator catalogue snapshot with {RuleCount} rules")] + public static partial void SnapshotPublished(ILogger logger, int RuleCount); + + [LoggerMessage( + EventId = 2512, + Level = LogLevel.Error, + Message = "Initial moderator catalogue load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2513, + Level = LogLevel.Error, + Message = "Moderator catalogue refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2514, + Level = LogLevel.Information, + Message = "Moderator catalogue refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2515, + Level = LogLevel.Information, + Message = "Moderator catalogue refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + + [LoggerMessage( + EventId = 2516, + Level = LogLevel.Error, + Message = "nntpmoderators lookup failed")] + public static partial void RepositoryFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueService.cs b/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueService.cs new file mode 100644 index 00000000..e14510e9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModeratorCatalogueService.cs @@ -0,0 +1,237 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Loads the moderator catalogue from NntpDB, publishes an immutable snapshot, + /// and refreshes it on the same interval as the newsgroup catalogue. + /// + /// + /// Initial population runs during and fails startup when + /// the query cannot be completed. Subsequent refresh failures keep the last + /// known-good snapshot. POST captures once and never queries MySQL. + /// + public sealed class ModeratorCatalogueService : IModeratorCatalogue, IModeratorAuthorization, IApplicationService, IAsyncDisposable + { + /// Catalogue refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5); + + private readonly INntpModeratorRepository _repository; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private ModeratorSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + public ModeratorCatalogueService( + INntpModeratorRepository repository, + ILogger logger) + : this(repository, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + internal ModeratorCatalogueService( + INntpModeratorRepository repository, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(repository); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _repository = repository; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "ModeratorCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public ModeratorSnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + throw new InvalidOperationException("Moderator catalogue has not been published."); + } + + return snapshot; + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) => + Current.IsAuthenticatedModerator(authenticatedUsername); + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) => + Current.TryResolve(newsgroup, out identity); + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) => + Current.CanApprove(authenticatedUsername, approvedIdentity, newsgroup); + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) => + Current.TryAuthorizeApproval(authenticatedUsername, approvedIdentities, moderatedGroups, out failureDetail); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + ModeratorCatalogueLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + ModeratorCatalogueLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + ModeratorCatalogueLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + /// Replaces the published snapshot (tests). + internal void PublishForTests(ModeratorSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Publish(snapshot); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + + ModeratorCatalogueLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ModeratorCatalogueLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + var rows = await _repository.GetEnabledAsync(cancellationToken).ConfigureAwait(false); + return ModeratorSnapshot.Create(rows); + } + + private void Publish(ModeratorSnapshot snapshot) + { + Interlocked.Exchange(ref _current, snapshot); + ModeratorCatalogueLogMessages.SnapshotPublished(_logger, snapshot.Count); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModeratorIdentity.cs b/src/VectorNNTPD.ReaderService/Moderation/ModeratorIdentity.cs new file mode 100644 index 00000000..4b106f89 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModeratorIdentity.cs @@ -0,0 +1,11 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Resolved moderator route for one newsgroup: expected Approved: identity and + /// authorized AUTHINFO principal. + /// + /// Wildmat that matched the newsgroup. + /// Resolved routing mailbox (static or INN %s expansion). + /// AUTHINFO username authorized to inject that approval, or empty for routing-only. + public readonly record struct ModeratorIdentity(string Pattern, string Address, string Username); +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/ModeratorSnapshot.cs b/src/VectorNNTPD.ReaderService/Moderation/ModeratorSnapshot.cs new file mode 100644 index 00000000..9a4c22eb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/ModeratorSnapshot.cs @@ -0,0 +1,314 @@ +using System.Text; +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Immutable first-match moderator catalogue compiled from nntpmoderators. + /// + /// + /// Lookups do not query MySQL. The compiled table is safe for concurrent POST use. + /// Username comparison is ordinal (same as AUTHINFO). Approved identities compare + /// ASCII case-insensitively after mailbox extraction. + /// + public sealed class ModeratorSnapshot : IModeratorAuthorization + { + /// Empty catalogue. Every resolve and approval fails. + public static ModeratorSnapshot Empty { get; } = new([]); + + private readonly ModeratorRule[] _rules; + + private ModeratorSnapshot(ModeratorRule[] rules) + { + _rules = rules; + } + + /// Gets the number of compiled rules (tests). + public int Count => _rules.Length; + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) + { + if (string.IsNullOrEmpty(authenticatedUsername)) + { + return false; + } + + for (var i = 0; i < _rules.Length; i++) + { + if (!string.IsNullOrEmpty(_rules[i].Username) + && string.Equals(_rules[i].Username, authenticatedUsername, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) + { + for (var i = 0; i < _rules.Length; i++) + { + var rule = _rules[i]; + if (NntpWildmat.IsMatchValidated(newsgroup, rule.PatternBytes)) + { + identity = new ModeratorIdentity( + rule.Pattern, + ModeratorAddressTemplate.Expand(rule.Address, newsgroup), + rule.Username); + return true; + } + } + + identity = default; + return false; + } + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) + { + if (string.IsNullOrEmpty(authenticatedUsername) + || !TryResolve(newsgroup, out var identity) + || string.IsNullOrEmpty(identity.Username) + || !string.Equals(authenticatedUsername, identity.Username, StringComparison.Ordinal)) + { + return false; + } + + return MailboxesEqual(approvedIdentity, identity.Address); + } + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) + { + if (string.IsNullOrEmpty(authenticatedUsername)) + { + failureDetail = "unauthenticated approval"; + return false; + } + + if (approvedIdentities.IsEmpty) + { + failureDetail = "missing Approved"; + return false; + } + + if (moderatedGroups.IsEmpty) + { + failureDetail = "no moderated groups"; + return false; + } + + Span groupBytes = stackalloc byte[128]; + for (var i = 0; i < moderatedGroups.Length; i++) + { + var group = moderatedGroups[i]; + if (group.Length is < 1 or > 128) + { + failureDetail = "malformed newsgroup name"; + return false; + } + + var written = Encoding.ASCII.GetBytes(group, groupBytes); + if (!TryResolve(groupBytes[..written], out var identity)) + { + failureDetail = "unresolved moderator"; + return false; + } + + if (string.IsNullOrEmpty(identity.Username) + || !string.Equals(authenticatedUsername, identity.Username, StringComparison.Ordinal)) + { + failureDetail = "unauthorized moderator principal"; + return false; + } + + if (!ContainsIdentity(approvedIdentities, identity.Address)) + { + failureDetail = "Approved identity not authorized for group"; + return false; + } + } + + for (var i = 0; i < approvedIdentities.Length; i++) + { + if (!IdentityCoversAModeratedGroup(approvedIdentities[i], moderatedGroups)) + { + failureDetail = "conflicting Approved"; + return false; + } + } + + failureDetail = null; + return true; + } + + /// Compiles enabled rows in the supplied order (must already be moderator_id ASC). + public static ModeratorSnapshot Create(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + if (rows.Count == 0) + { + return Empty; + } + + var rules = new List(rows.Count); + for (var i = 0; i < rows.Count; i++) + { + var row = rows[i]; + if (row is null) + { + continue; + } + + var pattern = row.GroupPattern.Trim(); + var address = row.ModeratorAddress.Trim(); + var username = row.AccountName.Trim(); + if (pattern.Length == 0 || address.Length == 0) + { + continue; + } + + var patternBytes = Encoding.ASCII.GetBytes(pattern); + if (!NntpWildmat.TryValidate(patternBytes) || !ModeratorAddressTemplate.TryValidate(address)) + { + continue; + } + + rules.Add(new ModeratorRule(pattern, patternBytes, address, username)); + } + + return rules.Count == 0 ? Empty : new ModeratorSnapshot([.. rules]); + } + + private bool IdentityCoversAModeratedGroup(string approvedIdentity, ReadOnlySpan moderatedGroups) + { + Span groupBytes = stackalloc byte[128]; + for (var i = 0; i < moderatedGroups.Length; i++) + { + var group = moderatedGroups[i]; + if (group.Length is < 1 or > 128) + { + continue; + } + + var written = Encoding.ASCII.GetBytes(group, groupBytes); + if (TryResolve(groupBytes[..written], out var identity) + && MailboxesEqual(approvedIdentity, identity.Address)) + { + return true; + } + } + + return false; + } + + private static bool ContainsIdentity(ReadOnlySpan identities, string expected) + { + for (var i = 0; i < identities.Length; i++) + { + if (MailboxesEqual(identities[i], expected)) + { + return true; + } + } + + return false; + } + + private static bool MailboxesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + var a = Trim(left); + var b = Trim(right); + if (a.Length != b.Length) + { + return false; + } + + for (var i = 0; i < a.Length; i++) + { + if (FoldChar(a[i]) != FoldChar(b[i])) + { + return false; + } + } + + return true; + } + + private static bool MailboxesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + var a = Trim(left); + var b = Trim(right); + if (a.Length != b.Length) + { + return false; + } + + for (var i = 0; i < a.Length; i++) + { + if (FoldByte(a[i]) != FoldChar(b[i])) + { + return false; + } + } + + return true; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && char.IsWhiteSpace(value[start])) + { + start++; + } + + while (end > start && char.IsWhiteSpace(value[end - 1])) + { + end--; + } + + return value[start..end]; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && value[start] is (byte)' ' or (byte)'\t') + { + start++; + } + + while (end > start && value[end - 1] is (byte)' ' or (byte)'\t') + { + end--; + } + + return value[start..end]; + } + + private static char FoldChar(char c) => + c is >= 'a' and <= 'z' ? (char)(c - 32) : c; + + private static char FoldByte(byte b) => + (char)(b is >= (byte)'a' and <= (byte)'z' ? b - 32 : b); + + private readonly record struct ModeratorRule( + string Pattern, + byte[] PatternBytes, + string Address, + string Username); + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/MySqlNntpModeratorRepository.cs b/src/VectorNNTPD.ReaderService/Moderation/MySqlNntpModeratorRepository.cs new file mode 100644 index 00000000..0528e984 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/MySqlNntpModeratorRepository.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Moderation +{ + /// Loads nntpmoderators through the existing NntpDB connection pool. + public sealed class MySqlNntpModeratorRepository : INntpModeratorRepository + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlNntpModeratorRepository(NntpDbService nntpDb, ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask> GetEnabledAsync( + CancellationToken cancellationToken = default) + { + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryEnabledModeratorsAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ModeratorCatalogueLogMessages.RepositoryFailed(_logger, ex); + throw new NntpDbUnavailableException("MySQL nntpmoderators lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/NntpModeratorRow.cs b/src/VectorNNTPD.ReaderService/Moderation/NntpModeratorRow.cs new file mode 100644 index 00000000..7c7ef3a7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/NntpModeratorRow.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// One enabled nntpmoderators row after mapping. + public sealed class NntpModeratorRow + { + /// Initializes a mapped moderator row. + public NntpModeratorRow(long moderatorId, string groupPattern, string moderatorAddress, string accountName) + { + ArgumentNullException.ThrowIfNull(groupPattern); + ArgumentNullException.ThrowIfNull(moderatorAddress); + ModeratorId = moderatorId; + GroupPattern = groupPattern; + ModeratorAddress = moderatorAddress; + AccountName = accountName ?? string.Empty; + } + + /// Gets moderator_id. Snapshot order follows this value ascending. + public long ModeratorId { get; } + + /// Gets the RFC 3977 wildmat matched against the newsgroup name. + public string GroupPattern { get; } + + /// Gets the routing / Approved mailbox or INN %s template. + public string ModeratorAddress { get; } + + /// Gets the AUTHINFO username principal, or empty for routing-only. + public string AccountName { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Moderation/UnavailableModerationSubmissionService.cs b/src/VectorNNTPD.ReaderService/Moderation/UnavailableModerationSubmissionService.cs new file mode 100644 index 00000000..8f7914df --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Moderation/UnavailableModerationSubmissionService.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.Moderation +{ + /// + /// Production default when no mail or durable moderator-delivery backend is registered. + /// + /// + /// RFC 5537 §3.5 step 7 requires the injecting agent to reject the proto-article when + /// forwarding is not possible. POST maps + /// to 441. + /// + public sealed class UnavailableModerationSubmissionService : IModerationSubmissionService + { + /// Shared instance. + public static UnavailableModerationSubmissionService Instance { get; } = new(); + + /// + public ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + _ = cancellationToken; + return ValueTask.FromResult( + new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + "moderation delivery is not configured")); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Listeners/ConnectionAcceptanceLogging.cs b/src/VectorNNTPD.ReaderService/Networking/Listeners/ConnectionAcceptanceLogging.cs new file mode 100644 index 00000000..9a5c0a3b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Listeners/ConnectionAcceptanceLogging.cs @@ -0,0 +1,94 @@ +using System.Net; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Networking.Listeners +{ + /// Formats connection-acceptance Information logs (TCP peer vs PROXY effective client). + internal static class ConnectionAcceptanceLogging + { + /// Logs a successful plain (or plain+PROXY) connection acceptance. + public static void LogPlainAccepted(ILogger logger, ConnectionClientIdentity identity) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(identity); + + if (identity.UsedProxyHeader) + { + NetworkingLogMessages.ProxyConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + FormatEndpoint(identity.Client)); + } + else + { + NetworkingLogMessages.PlainConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer)); + } + } + + /// Logs a successful TLS (or TLS+PROXY) connection acceptance after handshake. + public static void LogTlsAccepted( + ILogger logger, + ConnectionClientIdentity identity, + string tlsVersion, + string cipher) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(identity); + ArgumentException.ThrowIfNullOrWhiteSpace(tlsVersion); + ArgumentException.ThrowIfNullOrWhiteSpace(cipher); + + if (identity.UsedProxyHeader) + { + NetworkingLogMessages.TlsProxyConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + FormatEndpoint(identity.Client), + tlsVersion, + cipher); + } + else + { + NetworkingLogMessages.TlsConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + tlsVersion, + cipher); + } + } + + /// Formats an endpoint as ip:port for connection acceptance logs. + public static string FormatEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return $"{endpoint.Address}:{endpoint.Port}"; + } + + /// Formats a TCP endpoint for accept/disconnect logs, or unknown. + public static string FormatEndpoint(EndPoint? endpoint) => + endpoint switch + { + IPEndPoint ip => FormatEndpoint(ip), + not null => endpoint.ToString() ?? "unknown", + _ => "unknown", + }; + + /// Logs the single Information disconnect summary for an established TCP connection. + public static void LogDisconnected( + ILogger logger, + EndPoint? remote, + EndPoint? local, + string reason) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + NetworkingLogMessages.TcpConnectionDisconnected( + logger, + FormatEndpoint(remote), + FormatEndpoint(local), + reason); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Listeners/IListenSocketBinder.cs b/src/VectorNNTPD.ReaderService/Networking/Listeners/IListenSocketBinder.cs new file mode 100644 index 00000000..0b7bd954 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Listeners/IListenSocketBinder.cs @@ -0,0 +1,41 @@ +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.ReaderService.Networking.Listeners +{ + /// + /// Binds and listens on an already-created TCP socket. + /// + /// + /// This is the startup ownership seam for a listening endpoint. A successful + /// means the socket accepted bind/listen under the + /// application's socket options. Accept-loop failures after that point are not + /// startup failures. + /// + public interface IListenSocketBinder + { + /// + /// Binds to and starts listening. + /// + /// The listen socket created for . + /// The local endpoint that must be owned after this call. + /// The listen backlog. + void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog); + } + + /// Production / implementation. + internal sealed class SocketListenBinder : IListenSocketBinder + { + /// Gets the shared production binder. + public static SocketListenBinder Instance { get; } = new(); + + /// + public void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(endpoint); + socket.Bind(endpoint); + socket.Listen(backlog); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpPlainListenerService.cs b/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpPlainListenerService.cs new file mode 100644 index 00000000..62e8201c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpPlainListenerService.cs @@ -0,0 +1,506 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Session.SpeedTest; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Transport.Vatp; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.ReaderService.Networking.Listeners +{ + /// + /// Application service that binds cleartext NNTP TCP listeners and owns accepted plain connections. + /// + /// + /// Starts after Cloudflare DNS reconciliation and before ACME. Does not wait for certificates. + /// succeeds only after every planned endpoint has bound; a bind + /// failure disposes already-bound endpoints and fails startup. Accept-loop failures after a + /// successful bind do not fail startup. + /// + public sealed class NntpPlainListenerService : IApplicationService, IAsyncDisposable + { + private readonly IOptions _options; + private readonly ITrustedProxyHosts _trustedProxyHosts; + private readonly ITlsCertificateContextProvider _certificateProvider; + private readonly INntpAuthenticationProvider _authenticationProvider; + private readonly IArticleIngestionQueue _articleIngestion; + private readonly IHistoryDb? _historyDb; + private readonly ISpeedTestCoordinator? _speedTest; + private readonly ITransitPeerAuthorization _transitPeerAuthorization; + private readonly ITransitInboundConnectionLimiter _inboundConnectionLimiter; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly INntpSessionCensus? _sessionCensus; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IPostingTraceProtector? _postingTraceProtector; + private readonly INewsgroupCatalogue? _newsgroupCatalogue; + private readonly INntpArticlePolicySource? _sharedConfiguration; + private readonly IModeratorCatalogue? _moderatorCatalogue; + private readonly IModeratorAuthorization _moderatorAuthorization; + private readonly IModerationSubmissionService _moderationSubmission; + private readonly ISessionStateTracker? _sessionAdmission; + private readonly NntpSaslService? _saslService; + private readonly IAccountByteAccountant _accountBytes; + private readonly IAccountRateAllocator _accountRates; + private readonly IArticleWorkRpcClient? _articleWorkRpc; + private readonly IVatpArticleClient? _vatpArticleClient; + private readonly IStorageArticleLookupClient? _storageArticleLookup; + private readonly IPostFilter _postFilter; + private readonly PostFilterMetrics _postFilterMetrics; + private readonly IPostFilterRejectionEvidenceQueue _postFilterEvidence; + private readonly INewsLogWriter _newsLog; + private readonly IListenSocketBinder _listenBinder; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly NntpServerRole _serverRole; + + /// + /// Publishes DNS after every plain socket is bound, and only when the TLS listener is disabled. + /// When TLS is enabled, publishes after its own bind. + /// Null skips publication. A failure rolls back the sockets bound by this start. + /// + private readonly IListenerDnsPublication? _publication; + private readonly ConcurrentDictionary _connections = new(); + private readonly List _listeners = []; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public NntpPlainListenerService( + IOptions options, + ITrustedProxyHosts trustedProxyHosts, + ITlsCertificateContextProvider certificateProvider, + INntpAuthenticationProvider authenticationProvider, + IArticleIngestionQueue articleIngestion, + ITransitPeerAuthorization transitPeerAuthorization, + ILoggerFactory loggerFactory, + ILogger logger, + ITransitInboundConnectionLimiter? inboundConnectionLimiter = null, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + IFeedDiagnostics? feedDiagnostics = null, + IListenSocketBinder? listenBinder = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + INntpArticlePolicySource? sharedConfiguration = null, + IListenerDnsPublication? publication = null, + NntpServerRole serverRole = NntpServerRole.Reader) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(authenticationProvider); + ArgumentNullException.ThrowIfNull(articleIngestion); + ArgumentNullException.ThrowIfNull(transitPeerAuthorization); + ArgumentNullException.ThrowIfNull(loggerFactory); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _trustedProxyHosts = trustedProxyHosts; + _certificateProvider = certificateProvider; + _authenticationProvider = authenticationProvider; + _articleIngestion = articleIngestion; + _transitPeerAuthorization = transitPeerAuthorization; + _inboundConnectionLimiter = inboundConnectionLimiter ?? TransitInboundConnectionLimiter.Disabled; + _historyDb = historyDb; + _speedTest = speedTest; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _sessionCensus = sessionCensus; + _peerMetrics = peerMetrics; + _postingTraceProtector = postingTraceProtector; + _newsgroupCatalogue = newsgroupCatalogue; + _sharedConfiguration = sharedConfiguration; + _moderatorCatalogue = moderatorCatalogue; + _moderatorAuthorization = moderatorAuthorization ?? EmptyModeratorAuthorization.Instance; + _moderationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + _sessionAdmission = sessionAdmission; + _saslService = saslService; + _accountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + _accountRates = accountRates ?? NullAccountRateAllocator.Instance; + _articleWorkRpc = articleWorkRpc; + _vatpArticleClient = vatpArticleClient; + _storageArticleLookup = storageArticleLookup; + _postFilter = postFilter ?? DisabledPostFilter.Instance; + _postFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + _postFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _listenBinder = listenBinder ?? SocketListenBinder.Instance; + _loggerFactory = loggerFactory; + _logger = logger; + _publication = publication; + _serverRole = serverRole; + } + + /// + public string Name => "NntpPlainListener"; + + /// + public Task? Execution => _execution; + + /// Gets the number of active plain connections (tests). + internal int ActiveConnectionCount => _connections.Count; + + /// Gets bound local endpoints after start (tests). + internal IReadOnlyList LocalEndPoints => + [.. _listeners.Select(static l => l.LocalEndPoint)]; + + /// Gets whether any accept loop is still running (tests). + internal bool HasActiveAcceptLoops => _listeners.Exists(static l => l.AcceptLoopActive); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.Value; + var bindings = ListenEndpointPlanner.Plan(options.BindAddress, options.BindPort); + if (bindings.Count == 0) + { + throw new InvalidOperationException("No listen bindings were produced for the plain NNTP port."); + } + + foreach (var binding in bindings) + { + cancellationToken.ThrowIfCancellationRequested(); + var listener = new SocketAcceptListener( + binding, + OnAcceptedAsync, + _loggerFactory.CreateLogger($"{nameof(SocketAcceptListener)}.Plain"), + _listenBinder); + try + { + listener.Start(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.ListenerBindFailed( + _logger, + ex, + "Plain", + binding.EndPoint.ToString(), + options.BindPort); + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + + _listeners.Add(listener); + } + + if (_publication is not null && !_options.Value.IsTlsListenerEnabled) + { + await _publication.PublishAfterListenerBoundAsync(cancellationToken).ConfigureAwait(false); + } + + _execution = WaitUntilStoppedAsync(_runCts.Token); + NetworkingLogMessages.PlainListenersStarted(_logger, _listeners.Count, options.BindPort); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await RollbackPartialStartupAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + + foreach (var listener in _listeners) + { + await listener.StopAsync().ConfigureAwait(false); + } + + var connections = _connections.Keys.ToArray(); + foreach (var connection in connections) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(TcpDisconnectReason.Shutdown); + } + + await connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.PlainConnectionCompleteError(_logger, ex); + } + } + + foreach (var connection in connections) + { + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _connections.TryRemove(connection, out _); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort dispose. + } + + foreach (var listener in _listeners) + { + await listener.DisposeAsync().ConfigureAwait(false); + } + + _listeners.Clear(); + _runCts.Dispose(); + } + + private async Task RollbackPartialStartupAsync() + { + foreach (var listener in _listeners) + { + try + { + await listener.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort rollback of already-bound endpoints. + } + } + + _listeners.Clear(); + Interlocked.Exchange(ref _started, 0); + } + + // codeql[cs/missed-using-statement] Accepted socket ownership transfers to NntpConnection on success; disposed in the method finally only when no connection was started. A using on the parameter would dispose under the live session. + private async ValueTask OnAcceptedAsync(Socket socket, CancellationToken cancellationToken) + { + NntpConnection? connection = null; + try + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + NetworkingLogMessages.PlainAcceptDiscarded(_logger); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, _trustedProxyHosts, cancellationToken) + .ConfigureAwait(false); + } + catch (ProxyProtocolException ex) + { + NetworkingLogMessages.PlainProxyPreambleRejected(_logger, ex, tcpPeer); + return; + } + + connection = NntpConnection.StartPlain( + socket, + preamble.Identity, + _loggerFactory.CreateLogger(), + preamble.Leftover, + _feedDiagnostics); + _connections[connection] = 0; + + // Session owns NNTP greeting/command loop; transport owns the socket lifecycle. + var session = new NntpSession( + connection, + _loggerFactory.CreateLogger(), + certificateProvider: _certificateProvider, + authenticationProvider: _authenticationProvider, + allowCleartextAuth: _options.Value.AllowCleartextAuth, + loggerFactory: _loggerFactory, + articleIngestion: _articleIngestion, + transitPeerAuthorization: _transitPeerAuthorization, + streamOutstandingArticleDepth: _options.Value.Transit.StreamOutstandingArticleDepth, + historyDb: _historyDb, + speedTest: _speedTest, + sessionCensus: _sessionCensus, + peerMetrics: _peerMetrics, + commandIdleTimeout: TimeSpan.FromSeconds(_options.Value.IdleTime), + maxArticleSize: _options.Value.MaxArticleSize, + injectionIdentity: _options.Value.Fqdn, + mailComplaintsTo: _options.Value.MailComplaintsTo, + postingTraceProtector: _postingTraceProtector, + newsgroupCatalogue: _newsgroupCatalogue, + moderatorCatalogue: _moderatorCatalogue, + moderatorAuthorization: _moderatorAuthorization, + moderationSubmission: _moderationSubmission, + sessionAdmission: _sessionAdmission, + saslService: _saslService, + accountBytes: _accountBytes, + accountRates: _accountRates, + articleWorkRpc: _articleWorkRpc, + vatpArticleClient: _vatpArticleClient, + storageArticleLookup: _storageArticleLookup, + postFilter: _postFilter, + postFilterMetrics: _postFilterMetrics, + postFilterEvidence: _postFilterEvidence, + newsLog: _newsLog, + transit: _options.Value.Transit, + serverRole: _serverRole) + { + SharedConfiguration = _sharedConfiguration, + }; + ConnectionAcceptanceLogging.LogPlainAccepted(_logger, connection.ClientIdentity); + + TransitInboundAdmitResult admission; + try + { + admission = await TransitConnectionAdmission + .TryAdmitAsync(_inboundConnectionLimiter, session, _runCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (_runCts.IsCancellationRequested) + { + return; + } + + if (!admission.Admitted) + { + session.RecordPeerRejected(); + _feedDiagnostics.OnRejected( + session.Authorization.TransitPeerName ?? "none", + ConnectionAcceptanceLogging.FormatEndpoint(session.ClientIdentity.Client)); + await TransitConnectionAdmission + .WriteUnavailableAsync(connection, _runCts.Token) + .ConfigureAwait(false); + return; + } + + session.AttachTransitAdmission(admission.Lease); + session.RecordPeerAccepted(); + session.FeedProbe = _feedDiagnostics.OnAccepted(session); + try + { + await session.RunAsync(_runCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_runCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Listener stopping. + } + finally + { + _feedDiagnostics.OnReleased(session.FeedProbe); + await admission.Lease.DisposeAsync().ConfigureAwait(false); + } + } + finally + { + if (connection is not null) + { + _connections.TryRemove(connection, out _); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + else + { + socket.Dispose(); + } + } + } + + private static async Task WaitUntilStoppedAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpTlsListenerService.cs b/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpTlsListenerService.cs new file mode 100644 index 00000000..cfa7e2f1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Listeners/NntpTlsListenerService.cs @@ -0,0 +1,549 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Session.SpeedTest; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Transport.Vatp; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.ReaderService.Networking.Listeners +{ + /// + /// Application service that binds implicit-TLS NNTP listeners after a usable certificate context exists. + /// + /// + /// Idle when is . + /// Does not rebind or restart on certificate rotation; new handshakes observe the latest context. + /// PROXY preamble (when required) is consumed on the cleartext socket before TLS. + /// When TLS is enabled, succeeds only after every planned endpoint has + /// bound; a bind failure disposes already-bound endpoints and fails startup. Accept-loop + /// failures after a successful bind do not fail startup. + /// + public sealed class NntpTlsListenerService : IApplicationService, IAsyncDisposable + { + private readonly IOptions _options; + private readonly ITlsCertificateContextProvider _certificateProvider; + private readonly ITrustedProxyHosts _trustedProxyHosts; + private readonly INntpAuthenticationProvider _authenticationProvider; + private readonly IArticleIngestionQueue _articleIngestion; + private readonly IHistoryDb? _historyDb; + private readonly ISpeedTestCoordinator? _speedTest; + private readonly ITransitPeerAuthorization _transitPeerAuthorization; + private readonly ITransitInboundConnectionLimiter _inboundConnectionLimiter; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly INntpSessionCensus? _sessionCensus; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IPostingTraceProtector? _postingTraceProtector; + private readonly INewsgroupCatalogue? _newsgroupCatalogue; + private readonly INntpArticlePolicySource? _sharedConfiguration; + private readonly IModeratorCatalogue? _moderatorCatalogue; + private readonly IModeratorAuthorization _moderatorAuthorization; + private readonly IModerationSubmissionService _moderationSubmission; + private readonly ISessionStateTracker? _sessionAdmission; + private readonly NntpSaslService? _saslService; + private readonly IAccountByteAccountant _accountBytes; + private readonly IAccountRateAllocator _accountRates; + private readonly IArticleWorkRpcClient? _articleWorkRpc; + private readonly IVatpArticleClient? _vatpArticleClient; + private readonly IStorageArticleLookupClient? _storageArticleLookup; + private readonly IPostFilter _postFilter; + private readonly PostFilterMetrics _postFilterMetrics; + private readonly IPostFilterRejectionEvidenceQueue _postFilterEvidence; + private readonly INewsLogWriter _newsLog; + private readonly IListenSocketBinder _listenBinder; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly NntpServerRole _serverRole; + + /// + /// Publishes DNS after every TLS socket is bound. Null skips publication. + /// A failure rolls back the sockets bound by this start. The plain listener publishes instead when TLS is disabled. + /// + private readonly IListenerDnsPublication? _publication; + private readonly ConcurrentDictionary _connections = new(); + private readonly List _listeners = []; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public NntpTlsListenerService( + IOptions options, + ITlsCertificateContextProvider certificateProvider, + ITrustedProxyHosts trustedProxyHosts, + INntpAuthenticationProvider authenticationProvider, + IArticleIngestionQueue articleIngestion, + ITransitPeerAuthorization transitPeerAuthorization, + ILoggerFactory loggerFactory, + ILogger logger, + ITransitInboundConnectionLimiter? inboundConnectionLimiter = null, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + IFeedDiagnostics? feedDiagnostics = null, + IListenSocketBinder? listenBinder = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + INntpArticlePolicySource? sharedConfiguration = null, + IListenerDnsPublication? publication = null, + NntpServerRole serverRole = NntpServerRole.Reader) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + ArgumentNullException.ThrowIfNull(authenticationProvider); + ArgumentNullException.ThrowIfNull(articleIngestion); + ArgumentNullException.ThrowIfNull(transitPeerAuthorization); + ArgumentNullException.ThrowIfNull(loggerFactory); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _authenticationProvider = authenticationProvider; + _certificateProvider = certificateProvider; + _trustedProxyHosts = trustedProxyHosts; + _articleIngestion = articleIngestion; + _transitPeerAuthorization = transitPeerAuthorization; + _inboundConnectionLimiter = inboundConnectionLimiter ?? TransitInboundConnectionLimiter.Disabled; + _historyDb = historyDb; + _speedTest = speedTest; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _sessionCensus = sessionCensus; + _peerMetrics = peerMetrics; + _postingTraceProtector = postingTraceProtector; + _newsgroupCatalogue = newsgroupCatalogue; + _sharedConfiguration = sharedConfiguration; + _moderatorCatalogue = moderatorCatalogue; + _moderatorAuthorization = moderatorAuthorization ?? EmptyModeratorAuthorization.Instance; + _moderationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + _sessionAdmission = sessionAdmission; + _saslService = saslService; + _accountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + _accountRates = accountRates ?? NullAccountRateAllocator.Instance; + _articleWorkRpc = articleWorkRpc; + _vatpArticleClient = vatpArticleClient; + _storageArticleLookup = storageArticleLookup; + _postFilter = postFilter ?? DisabledPostFilter.Instance; + _postFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + _postFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _listenBinder = listenBinder ?? SocketListenBinder.Instance; + _loggerFactory = loggerFactory; + _logger = logger; + _publication = publication; + _serverRole = serverRole; + } + + /// + public string Name => "NntpTlsListener"; + + /// + public Task? Execution => _execution; + + /// Gets whether TLS listeners were bound (tests). + internal bool ListenersBound => _listeners.Count > 0; + + /// Gets the number of active TLS connections (tests). + internal int ActiveConnectionCount => _connections.Count; + + /// Gets bound local endpoints after start (tests). + internal IReadOnlyList LocalEndPoints => + [.. _listeners.Select(static l => l.LocalEndPoint)]; + + /// Gets whether any accept loop is still running (tests). + internal bool HasActiveAcceptLoops => _listeners.Exists(static l => l.AcceptLoopActive); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var options = _options.Value; + if (!options.IsTlsListenerEnabled) + { + NetworkingLogMessages.TlsListenerIdle(_logger); + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!_certificateProvider.IsAvailable) + { + throw new InvalidOperationException( + "TLS listener cannot start: no usable TLS certificate context is published."); + } + + var bindings = ListenEndpointPlanner.Plan(options.BindAddress, options.BindPortTls); + if (bindings.Count == 0) + { + throw new InvalidOperationException("No listen bindings were produced for the TLS NNTP port."); + } + + foreach (var binding in bindings) + { + cancellationToken.ThrowIfCancellationRequested(); + var listener = new SocketAcceptListener( + binding, + OnAcceptedAsync, + _loggerFactory.CreateLogger($"{nameof(SocketAcceptListener)}.Tls"), + _listenBinder); + try + { + listener.Start(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.ListenerBindFailed( + _logger, + ex, + "TLS", + binding.EndPoint.ToString(), + options.BindPortTls); + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + + _listeners.Add(listener); + } + + if (_publication is not null) + { + await _publication.PublishAfterListenerBoundAsync(cancellationToken).ConfigureAwait(false); + } + + _execution = WaitUntilStoppedAsync(_runCts.Token); + NetworkingLogMessages.TlsListenersStarted(_logger, _listeners.Count, options.BindPortTls); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await RollbackPartialStartupAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + + foreach (var listener in _listeners) + { + await listener.StopAsync().ConfigureAwait(false); + } + + var connections = _connections.Keys.ToArray(); + foreach (var connection in connections) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(TcpDisconnectReason.Shutdown); + } + + await connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.TlsConnectionCompleteError(_logger, ex); + } + } + + foreach (var connection in connections) + { + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _connections.TryRemove(connection, out _); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort dispose. + } + + foreach (var listener in _listeners) + { + await listener.DisposeAsync().ConfigureAwait(false); + } + + _listeners.Clear(); + _runCts.Dispose(); + } + + private async Task RollbackPartialStartupAsync() + { + foreach (var listener in _listeners) + { + try + { + await listener.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort rollback of already-bound endpoints. + } + } + + _listeners.Clear(); + Interlocked.Exchange(ref _started, 0); + } + + // codeql[cs/missed-using-statement] Accepted socket ownership transfers to NntpConnection on success; disposed in the method finally only when no connection was started. A using on the parameter would dispose under the live session. + private async ValueTask OnAcceptedAsync(Socket socket, CancellationToken cancellationToken) + { + NntpConnection? connection = null; + try + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + NetworkingLogMessages.TlsAcceptDiscarded(_logger); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, _trustedProxyHosts, cancellationToken) + .ConfigureAwait(false); + } + catch (ProxyProtocolException ex) + { + NetworkingLogMessages.TlsProxyPreambleRejected(_logger, ex, tcpPeer); + return; + } + + connection = await NntpConnection.StartTlsAsync( + socket, + _certificateProvider, + preamble.Identity, + _loggerFactory.CreateLogger(), + cancellationToken, + preamble.Leftover, + _feedDiagnostics) + .ConfigureAwait(false); + _connections[connection] = 0; + + var session = new NntpSession( + connection, + _loggerFactory.CreateLogger(), + certificateProvider: _certificateProvider, + authenticationProvider: _authenticationProvider, + allowCleartextAuth: _options.Value.AllowCleartextAuth, + loggerFactory: _loggerFactory, + articleIngestion: _articleIngestion, + transitPeerAuthorization: _transitPeerAuthorization, + streamOutstandingArticleDepth: _options.Value.Transit.StreamOutstandingArticleDepth, + historyDb: _historyDb, + speedTest: _speedTest, + sessionCensus: _sessionCensus, + peerMetrics: _peerMetrics, + commandIdleTimeout: TimeSpan.FromSeconds(_options.Value.IdleTime), + maxArticleSize: _options.Value.MaxArticleSize, + injectionIdentity: _options.Value.Fqdn, + mailComplaintsTo: _options.Value.MailComplaintsTo, + postingTraceProtector: _postingTraceProtector, + newsgroupCatalogue: _newsgroupCatalogue, + moderatorCatalogue: _moderatorCatalogue, + moderatorAuthorization: _moderatorAuthorization, + moderationSubmission: _moderationSubmission, + sessionAdmission: _sessionAdmission, + saslService: _saslService, + accountBytes: _accountBytes, + accountRates: _accountRates, + articleWorkRpc: _articleWorkRpc, + vatpArticleClient: _vatpArticleClient, + storageArticleLookup: _storageArticleLookup, + postFilter: _postFilter, + postFilterMetrics: _postFilterMetrics, + postFilterEvidence: _postFilterEvidence, + newsLog: _newsLog, + transit: _options.Value.Transit, + serverRole: _serverRole) + { + SharedConfiguration = _sharedConfiguration, + }; + + if (!connection.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)) + { + throw new InvalidOperationException("TLS connection missing negotiated parameters after handshake."); + } + + ConnectionAcceptanceLogging.LogTlsAccepted( + _logger, + connection.ClientIdentity, + tlsVersion, + cipher); + + TransitInboundAdmitResult admission; + try + { + admission = await TransitConnectionAdmission + .TryAdmitAsync(_inboundConnectionLimiter, session, _runCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (_runCts.IsCancellationRequested) + { + return; + } + + if (!admission.Admitted) + { + session.RecordPeerRejected(); + _feedDiagnostics.OnRejected( + session.Authorization.TransitPeerName ?? "none", + ConnectionAcceptanceLogging.FormatEndpoint(session.ClientIdentity.Client)); + await TransitConnectionAdmission + .WriteUnavailableAsync(connection, _runCts.Token) + .ConfigureAwait(false); + return; + } + + session.AttachTransitAdmission(admission.Lease); + session.RecordPeerAccepted(); + session.FeedProbe = _feedDiagnostics.OnAccepted(session); + try + { + await session.RunAsync(_runCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_runCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Listener stopping. + } + finally + { + _feedDiagnostics.OnReleased(session.FeedProbe); + await admission.Lease.DisposeAsync().ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + NetworkingLogMessages.TlsHandshakeOrSetupFailed(_logger, ex); + } + finally + { + if (connection is not null) + { + _connections.TryRemove(connection, out _); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + else + { + try + { + socket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + } + + private static async Task WaitUntilStoppedAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Listeners/SocketAcceptListener.cs b/src/VectorNNTPD.ReaderService/Networking/Listeners/SocketAcceptListener.cs new file mode 100644 index 00000000..dcba5b03 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Listeners/SocketAcceptListener.cs @@ -0,0 +1,217 @@ +using System.Net; +using System.Net.Sockets; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Listeners; + +namespace VectorNNTPD.ReaderService.Networking.Listeners +{ + using VectorNNTPD.Common.Core; + /// + /// Owns one listening and runs an allocation-efficient asynchronous accept loop. + /// + /// + /// A single failed accepted connection must not stop this listener. The listen socket remains open + /// until / stop. + /// + public sealed class SocketAcceptListener : IAsyncDisposable + { + private readonly ListenBinding _binding; + private readonly Func _onAccepted; + private readonly ILogger _logger; + private readonly IListenSocketBinder _binder; + private readonly Socket _listenSocket; + private readonly CancellationTokenSource _cts = new(); + private Task? _acceptLoop; + private int _stopped; + private int _disposed; + + /// Initializes a new instance of the class. + public SocketAcceptListener( + ListenBinding binding, + Func onAccepted, + ILogger logger, + IListenSocketBinder? listenBinder = null) + { + ArgumentNullException.ThrowIfNull(onAccepted); + ArgumentNullException.ThrowIfNull(logger); + _binding = binding; + _onAccepted = onAccepted; + _logger = logger; + _binder = listenBinder ?? SocketListenBinder.Instance; + + var family = binding.Address.AddressFamily; + _listenSocket = new Socket(family, SocketType.Stream, ProtocolType.Tcp); + if (family == AddressFamily.InterNetworkV6) + { + _listenSocket.DualMode = binding.DualMode; + } + + // SO_REUSEADDR is retained: Bind success is ownership under these socket options. + // On Windows it can allow another process to bind the same endpoint; that is not a + // Bind failure and is not treated as exclusive port ownership. + try + { + _listenSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Best-effort; some platforms differ. + } + } + + /// Gets the local endpoint after (port may be ephemeral in tests). + public IPEndPoint LocalEndPoint => + _listenSocket.LocalEndPoint as IPEndPoint ?? _binding.EndPoint; + + /// Gets the planned binding. + public ListenBinding Binding => _binding; + + /// Gets whether the accept loop is still running (tests). + internal bool AcceptLoopActive => _acceptLoop is { IsCompleted: false }; + + /// Binds, listens, and starts the accept loop. + /// + /// Bind/listen exceptions propagate to the caller and are startup failures. + /// The accept loop is started only after bind/listen succeed; later accept + /// failures are logged and the loop continues. + /// + public void Start(int backlog = 512) + { + _binder.BindAndListen(_listenSocket, _binding.EndPoint, backlog); + _acceptLoop = AcceptLoopAsync(_cts.Token); + NetworkingLogMessages.ListenerStarted(_logger, LocalEndPoint, _binding.DualMode); + } + + /// Stops accepting and closes the listen socket. + public async Task StopAsync() + { + if (Interlocked.Exchange(ref _stopped, 1) == 1) + { + return; + } + + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Already disposed. + } + + try + { + _listenSocket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + if (_acceptLoop is not null) + { + try + { + await _acceptLoop.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptLoopStopError(_logger, ex); + } + } + + NetworkingLogMessages.ListenerStopped(_logger, _binding.Address, _binding.Port); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await StopAsync().ConfigureAwait(false); + _cts.Dispose(); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + Socket accepted; + try + { + accepted = await _listenSocket.AcceptAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested || _stopped != 0) + { + break; + } + catch (SocketException ex) when (cancellationToken.IsCancellationRequested || _stopped != 0) + { + NetworkingLogMessages.AcceptInterruptedDuringShutdown(_logger, ex); + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptFailed(_logger, ex, _binding.EndPoint); + continue; + } + + _ = HandleAcceptedAsync(accepted, cancellationToken); + } + } + + private async Task HandleAcceptedAsync(Socket accepted, CancellationToken cancellationToken) + { + try + { + await _onAccepted(accepted, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + try + { + accepted.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptedHandlerFailed(_logger, ex); + try + { + accepted.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/ConnectionClientIdentity.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/ConnectionClientIdentity.cs new file mode 100644 index 00000000..ec264653 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/ConnectionClientIdentity.cs @@ -0,0 +1,94 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// + /// Immutable connection-level client identity established once at accept time. + /// + /// + /// + /// is always the accepted socket's remote endpoint. + /// is the effective client endpoint exposed to the session layer + /// (equal to for direct and untrusted peers; the PROXY-reported + /// source for trusted HAProxy peers that supplied a proxied address). + /// + /// + /// Trust and PROXY parsing are completed before this value is published; the session layer + /// must not reparse PROXY headers or inspect the socket to obtain the client endpoint. + /// + /// + public sealed class ConnectionClientIdentity + { + private ConnectionClientIdentity( + IPEndPoint tcpPeer, + IPEndPoint client, + bool isTrustedProxy, + bool usedProxyHeader, + int? proxyProtocolVersion) + { + TcpPeer = tcpPeer; + Client = client; + IsTrustedProxy = isTrustedProxy; + UsedProxyHeader = usedProxyHeader; + ProxyProtocolVersion = proxyProtocolVersion; + } + + /// Gets the actual TCP peer endpoint of the accepted socket. + public IPEndPoint TcpPeer { get; } + + /// Gets the effective client endpoint for the session layer. + public IPEndPoint Client { get; } + + /// Gets the effective client IP address. + public IPAddress ClientAddress => Client.Address; + + /// Gets the effective client TCP source port. + public int ClientPort => Client.Port; + + /// Gets whether the TCP peer matched a configured trusted PROXY host. + public bool IsTrustedProxy { get; } + + /// Gets whether a PROXY protocol header was consumed for this connection. + public bool UsedProxyHeader { get; } + + /// Gets the PROXY protocol version when a header was consumed; otherwise . + public int? ProxyProtocolVersion { get; } + + /// Creates a direct (non-proxy) identity where client equals TCP peer. + /// + /// Publishes independent instances so later mutation of the + /// caller's endpoint (or of one published property) cannot alias into the other. + /// + public static ConnectionClientIdentity Direct(IPEndPoint tcpPeer) + { + ArgumentNullException.ThrowIfNull(tcpPeer); + var peer = CloneEndpoint(tcpPeer); + var client = CloneEndpoint(tcpPeer); + return new ConnectionClientIdentity( + peer, + client, + isTrustedProxy: false, + usedProxyHeader: false, + proxyProtocolVersion: null); + } + + /// Creates an identity for a trusted proxy peer with an optional PROXY-reported client. + public static ConnectionClientIdentity FromTrustedProxy( + IPEndPoint tcpPeer, + IPEndPoint client, + int proxyProtocolVersion) + { + ArgumentNullException.ThrowIfNull(tcpPeer); + ArgumentNullException.ThrowIfNull(client); + return new ConnectionClientIdentity( + CloneEndpoint(tcpPeer), + CloneEndpoint(client), + isTrustedProxy: true, + usedProxyHeader: true, + proxyProtocolVersion); + } + + private static IPEndPoint CloneEndpoint(IPEndPoint endpoint) => + new(endpoint.Address, endpoint.Port); + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/Crc32C.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/Crc32C.cs new file mode 100644 index 00000000..89c3ba37 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/Crc32C.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// + /// CRC32c (Castagnoli) for PROXY v2 PP2_TYPE_CRC32C (RFC 4960 Appendix B). + /// + internal static class Crc32C + { + // Reflected Castagnoli polynomial 0x1EDC6F41 → 0x82F63B78. + private const uint Polynomial = 0x82F63B78u; + private static readonly uint[] Table = BuildTable(); + + /// Computes CRC32c of (final XOR applied). + public static uint Compute(ReadOnlySpan data) + { + var crc = 0xFFFFFFFFu; + foreach (var b in data) + { + crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8); + } + + return crc ^ 0xFFFFFFFFu; + } + + private static uint[] BuildTable() + { + var table = new uint[256]; + for (uint i = 0; i < 256; i++) + { + var crc = i; + for (var bit = 0; bit < 8; bit++) + { + crc = (crc & 1) != 0 ? (crc >> 1) ^ Polynomial : crc >> 1; + } + + table[i] = crc; + } + + return table; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/ITrustedProxyHosts.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/ITrustedProxyHosts.cs new file mode 100644 index 00000000..7566f256 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/ITrustedProxyHosts.cs @@ -0,0 +1,21 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// + /// Canonical set of trusted HAProxy PROXY-protocol peer addresses from configuration. + /// + public interface ITrustedProxyHosts + { + /// Gets whether any trusted PROXY peers are configured. + bool IsEnabled { get; } + + /// + /// Returns whether matches a configured trusted proxy host. + /// + /// + /// Comparison uses canonical forms: IPv4-mapped IPv6 addresses match their IPv4 equivalents. + /// + bool IsTrusted(IPAddress peerAddress); + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/PrefixedStream.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/PrefixedStream.cs new file mode 100644 index 00000000..898bda82 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/PrefixedStream.cs @@ -0,0 +1,137 @@ +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// + /// Stream that yields a fixed prefix before reading from an inner stream (TLS leftover after PROXY). + /// + internal sealed class PrefixedStream : Stream + { + private readonly Stream _inner; + private readonly bool _leaveInnerOpen; + private byte[]? _prefix; + private int _prefixOffset; + + public PrefixedStream(Stream inner, ReadOnlyMemory prefix, bool leaveInnerOpen = false) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + _leaveInnerOpen = leaveInnerOpen; + if (!prefix.IsEmpty) + { + _prefix = prefix.ToArray(); + } + } + + /// + public override bool CanRead => true; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => _inner.CanWrite; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public override void Flush() => _inner.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _inner.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) + { + ArgumentNullException.ThrowIfNull(buffer); + return Read(buffer.AsSpan(offset, count)); + } + + /// + public override int Read(Span buffer) + { + if (_prefix is { } prefix && _prefixOffset < prefix.Length) + { + var available = prefix.Length - _prefixOffset; + var take = Math.Min(available, buffer.Length); + prefix.AsSpan(_prefixOffset, take).CopyTo(buffer); + _prefixOffset += take; + if (_prefixOffset >= prefix.Length) + { + _prefix = null; + } + + return take; + } + + return _inner.Read(buffer); + } + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (_prefix is { } prefix && _prefixOffset < prefix.Length) + { + var available = prefix.Length - _prefixOffset; + var take = Math.Min(available, buffer.Length); + prefix.AsSpan(_prefixOffset, take).CopyTo(buffer.Span); + _prefixOffset += take; + if (_prefixOffset >= prefix.Length) + { + _prefix = null; + } + + return ValueTask.FromResult(take); + } + + return _inner.ReadAsync(buffer, cancellationToken); + } + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + /// + public override void Write(byte[] buffer, int offset, int count) => + _inner.Write(buffer, offset, count); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => + _inner.WriteAsync(buffer, cancellationToken); + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (disposing && !_leaveInnerOpen) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (!_leaveInnerOpen) + { + await _inner.DisposeAsync().ConfigureAwait(false); + } + + await base.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyPreambleResolver.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyPreambleResolver.cs new file mode 100644 index 00000000..d06e3f06 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyPreambleResolver.cs @@ -0,0 +1,146 @@ +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// Result of resolving connection identity and optional PROXY leftover octets. + public readonly struct ProxyPreambleResolution + { + /// Initializes a new instance of the struct. + public ProxyPreambleResolution(ConnectionClientIdentity identity, ReadOnlyMemory leftover) + { + Identity = identity; + Leftover = leftover; + } + + /// Gets the established client identity. + public ConnectionClientIdentity Identity { get; } + + /// + /// Gets octets that followed a consumed PROXY header and must be processed as application/TLS data. + /// + public ReadOnlyMemory Leftover { get; } + } + + /// + /// Resolves effective client identity from the TCP peer and, when trusted, a PROXY preamble. + /// + public static class ProxyPreambleResolver + { + /// + /// Establishes connection identity for an accepted socket according to trusted PROXY hosts. + /// + /// + /// + /// Empty trusted set: PROXY disabled; client = TCP peer; no socket reads. + /// Untrusted peer: client = TCP peer; PROXY bytes are not interpreted. + /// Trusted peer: PROXY v1/v2 required; malformed/timeout fails the connection. + /// + /// + public static async ValueTask ResolveAsync( + Socket socket, + IPEndPoint tcpPeer, + ITrustedProxyHosts trustedProxyHosts, + CancellationToken cancellationToken, + TimeSpan? preambleTimeout = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(tcpPeer); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + + if (!trustedProxyHosts.IsEnabled || !trustedProxyHosts.IsTrusted(tcpPeer.Address)) + { + return new ProxyPreambleResolution(ConnectionClientIdentity.Direct(tcpPeer), ReadOnlyMemory.Empty); + } + + var timeout = preambleTimeout ?? ProxyProtocolParser.DefaultPreambleTimeout; + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + + try + { + return await ReadRequiredPreambleAsync(socket, tcpPeer, timeoutCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new ProxyProtocolTimeoutException("PROXY preamble timed out"); + } + } + + private static async ValueTask ReadRequiredPreambleAsync( + Socket socket, + IPEndPoint tcpPeer, + CancellationToken cancellationToken) + { + var buffer = new byte[Math.Min(4096, ProxyProtocolParser.MaxV2HeaderOctets)]; + var gathered = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + var gatheredLength = 0; + + while (true) + { + var status = ProxyProtocolParser.TryParse( + gathered.AsSpan(0, gatheredLength), + tcpPeer, + out var parsed, + out var error); + + if (status == ProxyParseStatus.Ok) + { + var leftoverLength = gatheredLength - parsed.HeaderLength; + ReadOnlyMemory leftover = leftoverLength > 0 + ? gathered.AsMemory(parsed.HeaderLength, leftoverLength).ToArray() + : ReadOnlyMemory.Empty; + return new ProxyPreambleResolution(parsed.Identity, leftover); + } + + if (status == ProxyParseStatus.Error) + { + throw new ProxyProtocolMalformedException(error); + } + + if (gatheredLength >= ProxyProtocolParser.MaxV2HeaderOctets) + { + throw new ProxyProtocolMalformedException("PROXY preamble exceeds maximum size"); + } + + var room = Math.Min(buffer.Length, ProxyProtocolParser.MaxV2HeaderOctets - gatheredLength); + var read = await socket + .ReceiveAsync(buffer.AsMemory(0, room), SocketFlags.None, cancellationToken) + .ConfigureAwait(false); + if (read == 0) + { + throw new ProxyProtocolMalformedException( + gatheredLength == 0 ? "PROXY header required" : "EOF during PROXY preamble"); + } + + buffer.AsSpan(0, read).CopyTo(gathered.AsSpan(gatheredLength)); + gatheredLength += read; + } + } + + /// Extracts an from a socket remote endpoint. + public static bool TryGetTcpPeer(Socket socket, out IPEndPoint tcpPeer) + { + ArgumentNullException.ThrowIfNull(socket); + try + { + if (socket.RemoteEndPoint is IPEndPoint ep) + { + tcpPeer = ep; + return true; + } + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + tcpPeer = null!; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolException.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolException.cs new file mode 100644 index 00000000..b6448e4b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolException.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// Base failure while consuming a required PROXY protocol preamble. + public class ProxyProtocolException : IOException + { + /// Initializes a new instance of the class. + public ProxyProtocolException(string message) + : base(message) + { + } + + /// Initializes a new instance of the class. + public ProxyProtocolException(string message, Exception innerException) + : base(message, innerException) + { + } + } + + /// Malformed, incomplete, or otherwise invalid PROXY protocol header. + public sealed class ProxyProtocolMalformedException : ProxyProtocolException + { + /// Initializes a new instance of the class. + public ProxyProtocolMalformedException(string message) + : base(message) + { + } + } + + /// Timed out waiting for a complete PROXY protocol preamble. + public sealed class ProxyProtocolTimeoutException : ProxyProtocolException + { + /// Initializes a new instance of the class. + public ProxyProtocolTimeoutException(string message) + : base(message) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolParser.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolParser.cs new file mode 100644 index 00000000..63202e1c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/ProxyProtocolParser.cs @@ -0,0 +1,610 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// Incremental parse status for a PROXY v1/v2 preamble buffer. + internal enum ProxyParseStatus + { + NeedMore, + Ok, + Error, + } + + /// Successful PROXY preamble parse result (before leftover slicing). + internal readonly struct ProxyParseResult + { + public ProxyParseResult( + ConnectionClientIdentity identity, + int headerLength, + int version, + bool localCommand) + { + Identity = identity; + HeaderLength = headerLength; + Version = version; + LocalCommand = localCommand; + } + + public ConnectionClientIdentity Identity { get; } + + public int HeaderLength { get; } + + public int Version { get; } + + public bool LocalCommand { get; } + } + + /// + /// Byte-oriented PROXY protocol v1/v2 parser (HAProxy PROXY protocol specification). + /// + /// + /// Normative: docs/standards/haproxy/proxy-protocol.txt. Does not perform trust checks; + /// callers must only invoke this for TCP peers already classified as trusted proxies. + /// + internal static class ProxyProtocolParser + { + /// Spec §2.1: worst-case v1 line including CRLF. + public const int MaxV1LineOctets = 107; + + /// Spec §2.2: address length is uint16 — full protocol maximum. + public const int MaxV2AddressOctets = 65_535; + + /// Maximum gather buffer for a v2 header (signature + length). + public const int MaxV2HeaderOctets = 16 + MaxV2AddressOctets; + + /// Default preamble timeout (spec recommends at least 3 seconds). + public static readonly TimeSpan DefaultPreambleTimeout = TimeSpan.FromSeconds(5); + + private static readonly byte[] V2Signature = + [ + 0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A, + ]; + + private static readonly byte[] V1Prefix = "PROXY "u8.ToArray(); + + private const byte Pp2TypeCrc32C = 0x03; + + /// + /// Attempts to parse as a complete PROXY preamble. + /// + public static ProxyParseStatus TryParse( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.IsEmpty) + { + return ProxyParseStatus.NeedMore; + } + + if (buffer[0] == 0x0D) + { + return TryParseV2(buffer, tcpPeer, out result, out error); + } + + if (buffer[0] == (byte)'P') + { + return TryParseV1(buffer, tcpPeer, out result, out error); + } + + error = "PROXY header required"; + return ProxyParseStatus.Error; + } + + private static ProxyParseStatus TryParseV1( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.Length < V1Prefix.Length) + { + if (V1Prefix.AsSpan().StartsWith(buffer)) + { + return ProxyParseStatus.NeedMore; + } + + error = "PROXY v1 invalid signature"; + return ProxyParseStatus.Error; + } + + if (!buffer.StartsWith(V1Prefix)) + { + error = "PROXY v1 invalid signature"; + return ProxyParseStatus.Error; + } + + var crlf = buffer.IndexOf("\r\n"u8); + if (crlf < 0) + { + if (buffer.IndexOf((byte)'\n') >= 0) + { + error = "PROXY v1 requires CRLF terminator"; + return ProxyParseStatus.Error; + } + + var cr = buffer.IndexOf((byte)'\r'); + if (cr >= 0 && cr + 1 < buffer.Length) + { + error = "PROXY v1 requires CRLF terminator"; + return ProxyParseStatus.Error; + } + + if (buffer.Length >= MaxV1LineOctets) + { + error = "PROXY v1 missing CRLF within 107 octets"; + return ProxyParseStatus.Error; + } + + return ProxyParseStatus.NeedMore; + } + + if (crlf + 2 > MaxV1LineOctets) + { + error = "PROXY v1 line exceeds 107 octets"; + return ProxyParseStatus.Error; + } + + var line = buffer[..crlf]; + var headerLength = crlf + 2; + var parts = SplitAsciiSpaces(line); + if (parts.Count < 2 || !parts[0].Span.SequenceEqual("PROXY"u8) || parts[1].IsEmpty) + { + error = "PROXY v1 malformed signature"; + return ProxyParseStatus.Error; + } + + var family = parts[1].Span; + if (family.SequenceEqual("UNKNOWN"u8)) + { + // Spec §2.1: ignore remainder before CRLF; use real connection endpoints. + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 1), + headerLength, + version: 1, + localCommand: false); + return ProxyParseStatus.Ok; + } + + foreach (var part in parts) + { + if (part.IsEmpty) + { + error = "PROXY v1 malformed spacing"; + return ProxyParseStatus.Error; + } + } + + if (!family.SequenceEqual("TCP4"u8) && !family.SequenceEqual("TCP6"u8)) + { + error = "PROXY v1 unsupported family"; + return ProxyParseStatus.Error; + } + + if (parts.Count != 6) + { + error = "PROXY v1 expected 6 fields"; + return ProxyParseStatus.Error; + } + + var isV4 = family.SequenceEqual("TCP4"u8); + if (!TryParseV1Address(parts[2].Span, isV4, out var src) || !TryParseV1Address(parts[3].Span, isV4, out _)) + { + error = "PROXY v1 invalid address"; + return ProxyParseStatus.Error; + } + + if (!TryParseV1Port(parts[4].Span, out var sport) || !TryParseV1Port(parts[5].Span, out _)) + { + error = "PROXY v1 invalid port"; + return ProxyParseStatus.Error; + } + + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 1), + headerLength, + version: 1, + localCommand: false); + return ProxyParseStatus.Ok; + } + + private static ProxyParseStatus TryParseV2( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.Length < V2Signature.Length) + { + if (V2Signature.AsSpan().StartsWith(buffer)) + { + return ProxyParseStatus.NeedMore; + } + + error = "PROXY v2 invalid signature"; + return ProxyParseStatus.Error; + } + + if (!buffer.StartsWith(V2Signature)) + { + error = "PROXY v2 invalid signature"; + return ProxyParseStatus.Error; + } + + if (buffer.Length < 16) + { + return ProxyParseStatus.NeedMore; + } + + var verCmd = buffer[12]; + var version = verCmd >> 4; + var command = verCmd & 0x0F; + if (version != 0x2) + { + error = "PROXY v2 invalid version"; + return ProxyParseStatus.Error; + } + + var famProto = buffer[13]; + var family = famProto >> 4; + var protocol = famProto & 0x0F; + var addrLen = BinaryPrimitives.ReadUInt16BigEndian(buffer.Slice(14, 2)); + var total = 16 + addrLen; + if (buffer.Length < total) + { + return ProxyParseStatus.NeedMore; + } + + var fullHeader = buffer[..total]; + var payload = buffer.Slice(16, addrLen); + + if (command == 0x0) + { + if (family is >= 0 and <= 3 && protocol is >= 0 and <= 2) + { + var localMin = AddressBlockMinimum(famProto); + var tlvOff = 16 + (payload.Length >= localMin ? localMin : payload.Length); + if (!TryValidateV2Tlvs(fullHeader, tlvOff, out error)) + { + return ProxyParseStatus.Error; + } + } + + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: true); + return ProxyParseStatus.Ok; + } + + if (command != 0x1) + { + error = "PROXY v2 unsupported command"; + return ProxyParseStatus.Error; + } + + if (family is < 0 or > 3 || protocol is < 0 or > 2) + { + error = "PROXY v2 unsupported address family/protocol"; + return ProxyParseStatus.Error; + } + + var listedMin = AddressBlockMinimumOrNull(famProto); + if (listedMin is null) + { + // Valid nibbles but unlisted combination — UNSPEC fallback. + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + var tlvOffset = 16 + (payload.Length >= listedMin.Value ? listedMin.Value : payload.Length); + if (!TryValidateV2Tlvs(fullHeader, tlvOffset, out error)) + { + return ProxyParseStatus.Error; + } + + if (famProto == 0x00) + { + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + if (famProto == 0x11) + { + if (payload.Length < 12) + { + error = "PROXY v2 address block truncated"; + return ProxyParseStatus.Error; + } + + var src = new IPAddress(payload[..4]); + var sport = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(8, 2)); + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + if (famProto == 0x21) + { + if (payload.Length < 36) + { + error = "PROXY v2 address block truncated"; + return ProxyParseStatus.Error; + } + + var src = new IPAddress(payload[..16]); + var sport = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(32, 2)); + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + // UDP/UNIX listed combinations: fall back to real endpoints (spec §2.2). + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + private static int AddressBlockMinimum(int famProto) => AddressBlockMinimumOrNull(famProto) ?? 0; + + private static int? AddressBlockMinimumOrNull(int famProto) => famProto switch + { + 0x00 => 0, + 0x11 => 12, + 0x12 => 12, + 0x21 => 36, + 0x22 => 36, + 0x31 => 216, + 0x32 => 216, + _ => null, + }; + + private static bool TryValidateV2Tlvs(ReadOnlySpan fullHeader, int tlvOffset, out string error) + { + error = string.Empty; + var end = fullHeader.Length; + var pos = tlvOffset; + uint? crcValue = null; + int? crcValueOffset = null; + + while (pos < end) + { + if (end - pos < 3) + { + error = "PROXY v2 truncated TLV header"; + return false; + } + + var tlvType = fullHeader[pos]; + var tlvLen = (fullHeader[pos + 1] << 8) | fullHeader[pos + 2]; + var valueOff = pos + 3; + var valueEnd = valueOff + tlvLen; + if (valueEnd > end) + { + error = "PROXY v2 TLV length exceeds header"; + return false; + } + + if (tlvType == Pp2TypeCrc32C) + { + if (tlvLen != 4) + { + error = "PROXY v2 CRC32C TLV must be 4 octets"; + return false; + } + + if (crcValue is not null) + { + error = "PROXY v2 duplicate CRC32C TLV"; + return false; + } + + crcValue = BinaryPrimitives.ReadUInt32BigEndian(fullHeader.Slice(valueOff, 4)); + crcValueOffset = valueOff; + } + + pos = valueEnd; + } + + if (crcValue is not null && crcValueOffset is not null) + { + var mutable = fullHeader.ToArray(); + mutable.AsSpan(crcValueOffset.Value, 4).Clear(); + var calculated = Crc32C.Compute(mutable); + if (calculated != crcValue.Value) + { + error = "PROXY v2 CRC32C mismatch"; + return false; + } + } + + return true; + } + + private static bool TryParseV1Address(ReadOnlySpan token, bool ipv4, out IPAddress address) + { + address = IPAddress.None; + if (ipv4) + { + return TryParseV1Ipv4(token, out address); + } + + return TryParseV1Ipv6(token, out address); + } + + private static bool TryParseV1Ipv4(ReadOnlySpan token, out IPAddress address) + { + address = IPAddress.None; + Span octets = stackalloc byte[4]; + var octetIndex = 0; + var start = 0; + for (var i = 0; i <= token.Length; i++) + { + if (i == token.Length || token[i] == (byte)'.') + { + if (octetIndex >= 4) + { + return false; + } + + if (!TryParseDecimalNoLeadingZero(token[start..i], out var value) || value > 255) + { + return false; + } + + octets[octetIndex++] = (byte)value; + start = i + 1; + } + } + + if (octetIndex != 4) + { + return false; + } + + address = new IPAddress(octets); + return true; + } + + private static bool TryParseV1Ipv6(ReadOnlySpan token, out IPAddress address) + { + address = IPAddress.None; + if (!Ascii.IsValid(token)) + { + return false; + } + + var text = Encoding.ASCII.GetString(token); + if (text.Contains('.') || text.Split("::").Length > 2) + { + return false; + } + + string[] groups; + if (text.Contains("::", StringComparison.Ordinal)) + { + var parts = text.Split("::", 2); + var left = parts[0].Length == 0 ? [] : parts[0].Split(':', StringSplitOptions.RemoveEmptyEntries); + var right = parts[1].Length == 0 ? [] : parts[1].Split(':', StringSplitOptions.RemoveEmptyEntries); + groups = [.. left, .. right]; + } + else + { + groups = text.Split(':'); + if (groups.Length != 8) + { + return false; + } + } + + foreach (var group in groups) + { + if (group.Length is < 1 or > 4) + { + return false; + } + + foreach (var ch in group) + { + if (!Uri.IsHexDigit(ch)) + { + return false; + } + } + } + + return IPAddress.TryParse(text, out address!) + && address.AddressFamily == AddressFamily.InterNetworkV6; + } + + private static bool TryParseV1Port(ReadOnlySpan token, out int port) + { + port = 0; + if (!TryParseDecimalNoLeadingZero(token, out var value) || value > 65535) + { + return false; + } + + port = value; + return true; + } + + private static bool TryParseDecimalNoLeadingZero(ReadOnlySpan token, out int value) + { + value = 0; + if (token.IsEmpty) + { + return false; + } + + if (token.Length > 1 && token[0] == (byte)'0') + { + return false; + } + + foreach (var b in token) + { + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + + value = (value * 10) + (b - '0'); + } + + return true; + } + + private static List> SplitAsciiSpaces(ReadOnlySpan line) + { + // Materialize into owned segments for field indexing (v1 lines are ≤107 octets). + var owned = line.ToArray(); + var parts = new List>(6); + var start = 0; + for (var i = 0; i <= owned.Length; i++) + { + if (i == owned.Length || owned[i] == (byte)' ') + { + parts.Add(owned.AsMemory(start, i - start)); + start = i + 1; + } + } + + return parts; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Proxy/TrustedProxyHosts.cs b/src/VectorNNTPD.ReaderService/Networking/Proxy/TrustedProxyHosts.cs new file mode 100644 index 00000000..d01bf2aa --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Proxy/TrustedProxyHosts.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Networking.Proxy +{ + /// + /// Parses and caches once for hot-path peer matching. + /// + public sealed class TrustedProxyHosts : ITrustedProxyHosts + { + private readonly HashSet _hosts; + + /// Initializes a new instance of the class. + public TrustedProxyHosts(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _hosts = BuildSet(options.Value.ProxyHosts); + } + + /// Creates an instance from an already-validated address list (tests). + internal TrustedProxyHosts(IEnumerable hosts) + { + ArgumentNullException.ThrowIfNull(hosts); + _hosts = new HashSet(hosts.Select(Canonicalize), IPAddressEqualityComparer.Instance); + } + + /// + public bool IsEnabled => _hosts.Count > 0; + + /// + public bool IsTrusted(IPAddress peerAddress) + { + ArgumentNullException.ThrowIfNull(peerAddress); + if (_hosts.Count == 0) + { + return false; + } + + return _hosts.Contains(Canonicalize(peerAddress)); + } + + /// Normalizes an address for PROXY peer trust comparison. + public static IPAddress Canonicalize(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return address is { AddressFamily: AddressFamily.InterNetworkV6, IsIPv4MappedToIPv6: true } ? address.MapToIPv4() : address; + } + + private static HashSet BuildSet(string[]? entries) + { + var set = new HashSet(IPAddressEqualityComparer.Instance); + if (entries is null) + { + return set; + } + + foreach (var entry in entries) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + if (!IPAddress.TryParse(entry.Trim(), out var address)) + { + continue; + } + + set.Add(Canonicalize(address)); + } + + return set; + } + + private sealed class IPAddressEqualityComparer : IEqualityComparer + { + public static readonly IPAddressEqualityComparer Instance = new(); + + public bool Equals(IPAddress? x, IPAddress? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return x.Equals(y); + } + + public int GetHashCode(IPAddress obj) => obj.GetHashCode(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/ConnectionByteTransport.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/ConnectionByteTransport.cs new file mode 100644 index 00000000..f0538eaf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/ConnectionByteTransport.cs @@ -0,0 +1,748 @@ +using System.Diagnostics; +using System.Net.Security; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Single owner of byte-stream I/O under pumps. + /// + /// + /// + /// Pumps call / only. They never touch the socket + /// or directly. The active stream may be replaced during an in-place TLS + /// upgrade after guarantees no outstanding stream I/O. + /// + /// + /// Quiescence blocks new application reads/writes at a gate, cancels in-flight reads + /// (freeing the socket for TLS handshake or DEFLATE wrap), and waits for in-flight writes + /// to finish under the caller token. Admission of an I/O operation and registration of that + /// operation as outstanding are atomic with respect to the quiescence state transition under + /// _gate. + /// + /// + internal sealed class ConnectionByteTransport : IAsyncDisposable + { + private const int StateActive = 0; + private const int StateReadsPaused = 1; + private const int StateQuiesced = 2; + private const int StateAbandoned = 3; + private const int StateDisposed = 4; + + private readonly object _gate = new(); + private Stream _stream; + private CancellationTokenSource _readCts = new(); + private TaskCompletionSource _resumeTcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource? _idleTcs; + private TaskCompletionSource? _readIdleTcs; + private int _activeReads; + private int _activeWrites; + private int _state; + private bool _isTls; + private bool _isCompressed; + private byte[]? _pendingUpgradePrefix; + private readonly OutboundRateLimiter _rateLimiter; + + public ConnectionByteTransport(Stream stream, bool isTls, OutboundRateLimiter? rateLimiter = null) + { + ArgumentNullException.ThrowIfNull(stream); + if (rateLimiter is not null) + { + _rateLimiter = rateLimiter; + _stream = stream; + } + else if (stream is OutboundRateLimiter existing) + { + _rateLimiter = existing; + _stream = existing; + } + else + { + _rateLimiter = new OutboundRateLimiter(stream, initialMaxSendBytesPerSecond: 0, leaveInnerOpen: false); + _stream = _rateLimiter; + } + + _isTls = isTls; + _resumeTcs.SetResult(); + } + + /// + /// Wire-level outbound cap. Sits under TLS/DEFLATE so throttled bytes are + /// the octets written toward the socket. + /// + internal OutboundRateLimiter RateLimiter => _rateLimiter; + + public bool IsTls + { + get + { + lock (_gate) + { + return _isTls; + } + } + } + + public bool IsCompressed + { + get + { + lock (_gate) + { + return _isCompressed; + } + } + } + + /// + /// Test-only: awaited after / writable gate returns and before admission under + /// _gate. Used to reproduce the historical Active-observed / not-yet-counted window. + /// + internal Func? AfterActiveBeforeAdmitProbe { get; set; } + + /// + /// Test-only: awaited inside after write admission and before the stream write. + /// Used to hold SendAsync between ReadAsync and AdvanceTo. + /// + internal Func? BeforeStreamWriteProbe { get; set; } + + /// + /// Test-only: awaited inside after read admission and before the stream read. + /// + internal Func? BeforeStreamReadProbe { get; set; } + + /// + /// Test-only: awaited after a successful stream read returns and before the count is + /// handed back to ReceiveAsync for Advance. + /// + internal Func? AfterStreamReadProbe { get; set; } + + /// DIAGNOSTIC-ONLY session attached when is enabled. + internal TransportIoSession? Io { get; set; } + + public async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilReadableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + CancellationToken readToken; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateActive) + { + continue; + } + + _activeReads++; + stream = _stream; + readToken = _readCts.Token; + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(readToken, cancellationToken); + try + { + if (BeforeStreamReadProbe is { } readProbe) + { + await readProbe().ConfigureAwait(false); + } + + var received = await ReadStreamAsync(stream, buffer, linked.Token).ConfigureAwait(false); + if (AfterStreamReadProbe is { } afterRead) + { + await afterRead().ConfigureAwait(false); + } + + return received; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + continue; + } + finally + { + CompleteOperation(isRead: true); + } + } + } + + public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilWritableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateActive or StateReadsPaused)) + { + continue; + } + + _activeWrites++; + stream = _stream; + } + + try + { + if (BeforeStreamWriteProbe is { } writeProbe) + { + await writeProbe().ConfigureAwait(false); + } + + await WriteStreamAsync(stream, buffer, cancellationToken).ConfigureAwait(false); + return; + } + finally + { + CompleteOperation(isRead: false); + } + } + } + + /// + /// Flushes the active stream (required for DEFLATE sync-flush so compressed bytes reach the peer). + /// + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilWritableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateActive or StateReadsPaused)) + { + continue; + } + + _activeWrites++; + stream = _stream; + } + + try + { + await FlushStreamAsync(stream, cancellationToken).ConfigureAwait(false); + return; + } + finally + { + CompleteOperation(isRead: false); + } + } + } + + /// + /// Gets a value indicating whether the transport is in the reads-paused upgrade drain state. + /// + public bool IsReadsPaused + { + get + { + lock (_gate) + { + return _state is StateReadsPaused or StateQuiesced; + } + } + } + + /// + /// After a successful , decides whether bytes may enter the application input pipe. + /// + /// + /// When quiesced for TLS/DEFLATE upgrade, socket octets already read must not be injected into + /// — they are retained as a prefix for the upgraded stream + /// (ClientHello race after a STARTTLS 382 response). + /// + /// + /// if the caller should write to the application pipe; + /// if the bytes were claimed for the pending upgrade prefix. + /// + public bool TryCommitReadToApplication(ReadOnlySpan data) + { + lock (_gate) + { + if (_state is not (StateReadsPaused or StateQuiesced or StateAbandoned)) + { + return true; + } + + if (data.IsEmpty) + { + return false; + } + + if (_pendingUpgradePrefix is null) + { + _pendingUpgradePrefix = data.ToArray(); + } + else + { + var combined = new byte[_pendingUpgradePrefix.Length + data.Length]; + _pendingUpgradePrefix.CopyTo(combined, 0); + data.CopyTo(combined.AsSpan(_pendingUpgradePrefix.Length)); + _pendingUpgradePrefix = combined; + } + + return false; + } + } + + /// Takes any socket octets retained during quiescence for the upgraded stream wrap. + public ReadOnlyMemory TakePendingUpgradePrefix() + { + lock (_gate) + { + var prefix = _pendingUpgradePrefix; + _pendingUpgradePrefix = null; + return prefix ?? ReadOnlyMemory.Empty; + } + } + + /// + /// Restores Active I/O after a post-quiescence precondition failure (no stream replacement). + /// + public void ResumeFromQuiesceWithoutUpgrade() + { + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateQuiesced or StateReadsPaused)) + { + throw new InvalidOperationException("Transport is not quiesced."); + } + + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + _pendingUpgradePrefix = null; + } + + resume.TrySetResult(); + } + + /// + /// Cancels in-flight reads and blocks new reads while still allowing writes (outbound drain). + /// + public async Task PauseReadsAsync(CancellationToken cancellationToken) + { + CancellationTokenSource oldReadCts; + TaskCompletionSource? readIdle; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is StateReadsPaused or StateQuiesced) + { + throw new InvalidOperationException("Transport reads are already paused."); + } + + _state = StateReadsPaused; + _resumeTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + oldReadCts = _readCts; + _readCts = new CancellationTokenSource(); + if (_activeReads == 0) + { + readIdle = null; + } + else + { + readIdle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _readIdleTcs = readIdle; + } + } + + await oldReadCts.CancelAsync().ConfigureAwait(false); + oldReadCts.Dispose(); + + if (readIdle is not null) + { + await readIdle.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// After , blocks writes and waits until no stream I/O is outstanding. + /// + public async Task QuiesceWritesAsync(CancellationToken cancellationToken) + { + TaskCompletionSource idle; + bool alreadyIdle; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state == StateQuiesced) + { + throw new InvalidOperationException("Transport is already quiesced."); + } + + if (_state != StateReadsPaused) + { + throw new InvalidOperationException("Pause reads before quiescing writes."); + } + + _state = StateQuiesced; + idle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _idleTcs = idle; + alreadyIdle = _activeReads == 0 && _activeWrites == 0; + } + + if (alreadyIdle) + { + idle.TrySetResult(); + } + + await idle.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Stops application stream I/O and waits until the underlying stream has no outstanding operations. + /// + public async Task QuiesceAsync(CancellationToken cancellationToken) + { + await PauseReadsAsync(cancellationToken).ConfigureAwait(false); + await QuiesceWritesAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Returns the quiesced plain stream so the caller can wrap it in . + /// + public Stream TakeQuiescedStreamForTlsWrap() + { + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before TLS wrap."); + } + + if (_isTls) + { + throw new InvalidOperationException("Transport is already TLS-protected."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + return _stream; + } + } + + /// + /// Returns the quiesced stream (plain or TLS) so the caller can wrap it in . + /// + public Stream TakeQuiescedStreamForDeflateWrap() + { + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before DEFLATE wrap."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Transport is already DEFLATE-compressed."); + } + + return _stream; + } + } + + /// + /// Publishes an authenticated and resumes pump I/O. + /// + public void PublishTlsAndResume(SslStream sslStream) + { + ArgumentNullException.ThrowIfNull(sslStream); + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before publishing TLS."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Cannot publish TLS after DEFLATE is active."); + } + + _stream = sslStream; + _isTls = true; + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + } + + resume.TrySetResult(); + } + + /// + /// Publishes a bidirectional raw-DEFLATE stream and resumes pump I/O. + /// + public void PublishDeflateAndResume(NntpDeflateStream deflateStream) + { + ArgumentNullException.ThrowIfNull(deflateStream); + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before publishing DEFLATE."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Transport is already DEFLATE-compressed."); + } + + _stream = deflateStream; + _isCompressed = true; + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + } + + resume.TrySetResult(); + } + + /// + /// Marks the transport abandoned after a failed upgrade and wakes waiters for connection teardown. + /// Does not restore plaintext I/O. + /// + public void AbortQuiesceForConnectionTeardown() + { + TaskCompletionSource? resume; + TaskCompletionSource? idle; + lock (_gate) + { + if (_state != StateQuiesced) + { + return; + } + + _state = StateAbandoned; + resume = _resumeTcs; + idle = _idleTcs; + _idleTcs = null; + } + + idle?.TrySetResult(); + resume.TrySetResult(); + } + + public async ValueTask DisposeAsync() + { + Stream stream; + CancellationTokenSource readCts; + TaskCompletionSource resume; + TaskCompletionSource? idle; + lock (_gate) + { + if (_state == StateDisposed) + { + return; + } + + _state = StateDisposed; + stream = _stream; + readCts = _readCts; + resume = _resumeTcs; + idle = _idleTcs; + _idleTcs = null; + } + + idle?.TrySetResult(); + resume.TrySetResult(); + await readCts.CancelAsync().ConfigureAwait(false); + readCts.Dispose(); + await stream.DisposeAsync().ConfigureAwait(false); + } + + private void ThrowIfUnavailable() + { + if (_state == StateAbandoned) + { + throw new ObjectDisposedException(nameof(ConnectionByteTransport), "Transport abandoned after failed TLS upgrade."); + } + + ObjectDisposedException.ThrowIf(_state == StateDisposed, this); + } + + private void CompleteOperation(bool isRead) + { + TaskCompletionSource? idle = null; + TaskCompletionSource? readIdle = null; + lock (_gate) + { + if (isRead) + { + _activeReads--; + } + else + { + _activeWrites--; + } + + if (isRead + && _state == StateReadsPaused + && _readIdleTcs is not null + && _activeReads == 0) + { + readIdle = _readIdleTcs; + _readIdleTcs = null; + } + + if (_state == StateQuiesced + && _idleTcs is not null + && _activeReads == 0 + && _activeWrites == 0) + { + idle = _idleTcs; + } + } + + readIdle?.TrySetResult(); + idle?.TrySetResult(); + } + + private async ValueTask ReadStreamAsync(Stream stream, Memory buffer, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + return await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.ReadAsync(buffer, cancellationToken); + if (vt.IsCompletedSuccessfully) + { + var n = vt.Result; + io.RecordReceive(buffer.Length, n, started, started, sync: true); + return n; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var received = await vt.ConfigureAwait(false); + io.RecordReceive(buffer.Length, received, started, awaitStart, sync: false); + return received; + } + + private async ValueTask WriteStreamAsync(Stream stream, ReadOnlyMemory buffer, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + await stream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + return; + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.WriteAsync(buffer, cancellationToken); + if (vt.IsCompletedSuccessfully) + { + io.RecordSend(buffer.Length, buffer.Length, started, started, sync: true); + return; + } + + var awaitStart = Stopwatch.GetTimestamp(); + await vt.ConfigureAwait(false); + io.RecordSend(buffer.Length, buffer.Length, started, awaitStart, sync: false); + } + + private async ValueTask FlushStreamAsync(Stream stream, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + return; + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.FlushAsync(cancellationToken); + if (vt.IsCompletedSuccessfully) + { + io.RecordFlush(started, started, sync: true); + return; + } + + var awaitStart = Stopwatch.GetTimestamp(); + await vt.ConfigureAwait(false); + io.RecordFlush(started, awaitStart, sync: false); + } + + private async Task WaitUntilReadableAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state == StateActive) + { + return; + } + + wait = _resumeTcs.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private async Task WaitUntilWritableAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is StateActive or StateReadsPaused) + { + return; + } + + wait = _resumeTcs.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/INntpConnection.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/INntpConnection.cs new file mode 100644 index 00000000..ba2918ad --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/INntpConnection.cs @@ -0,0 +1,187 @@ +using System.IO.Pipelines; +using System.Net; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Byte-oriented NNTP connection transport exposed to a future session/protocol layer. + /// + /// + /// + /// carries octets received from the peer (after TLS decryption when applicable). + /// carries octets to send to the peer (before TLS encryption when applicable). + /// + /// + /// No NNTP greeting or command semantics are implemented. The transport does not keep an unused + /// connection alive beyond normal TCP/TLS lifetime and pipeline backpressure. + /// + /// + /// upgrades an established plaintext connection to TLS on the same + /// TCP socket. activates bidirectional raw DEFLATE (RFC 8054) + /// above the current byte stream (plain or TLS). The NNTP STARTTLS and COMPRESS commands live in the + /// session layer and invoke these transport upgrades when appropriate. + /// + /// + public interface INntpConnection : IAsyncDisposable + { + /// + /// Wire-level outbound rate cap, or when the connection + /// does not expose a limiter (test doubles). + /// + IOutboundRateCap? OutboundRate => null; + + /// Gets octets received from the peer (after TLS decryption / DEFLATE inflate when applicable). + PipeReader Input { get; } + + /// Gets the writer used by the application to send octets to the network. + PipeWriter Output { get; } + + /// Gets the remote TCP peer endpoint when known. + EndPoint? RemoteEndPoint { get; } + + /// Gets the local endpoint when known. + EndPoint? LocalEndPoint { get; } + + /// + /// Gets the immutable effective client identity established at connection initialization. + /// + ConnectionClientIdentity ClientIdentity { get; } + + /// Gets a value indicating whether the connection is TLS-protected. + bool IsTls { get; } + + /// + /// When TLS is active, returns the negotiated protocol and cipher suite labels for logging. + /// + /// Operational TLS version label (for example TLSv1.3). + /// Negotiated cipher suite name from . + /// when TLS parameters are available. + bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher); + + /// Gets a value indicating whether bidirectional raw DEFLATE is active. + bool IsCompressed { get; } + + /// Gets a token that is cancelled when the connection transport is shutting down. + CancellationToken ConnectionClosed { get; } + + /// + /// Gets a value indicating whether has been entered (transport is terminal). + /// + bool IsCompleted { get; } + + /// + /// Completes the transport (stops pumps, closes the socket). Safe to call multiple times. + /// + /// Optional exception indicating abortive completion. + Task CompleteAsync(Exception? exception = null); + + /// + /// Gets a monotonic generation bumped each time the send pump becomes idle (awaiting more Output). + /// + /// + /// For STARTTLS: first, capture this version, write 382, then + /// until the generation advances. + /// + long OutboundIdleVersion { get; } + + /// + /// Pauses application socket reads while still allowing writes (STARTTLS handoff ownership). + /// + /// + /// After this returns, successful receive completions are retained for the TLS upgrade prefix + /// instead of being committed to . Writes remain admitted so a 382 + /// response can still reach the peer. Must be called before writing 382. + /// + /// Cancellation token. + Task PauseReadsAsync(CancellationToken cancellationToken = default); + + /// + /// Waits until the send pump has become idle after . + /// + /// + /// captured before flushing pre-upgrade plaintext to . + /// + /// Cancellation token. + Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default); + + /// + /// Pauses application reads, then waits until the send pump has become idle after + /// . + /// + /// + /// Prefer before writing 382, then + /// . This combined helper is for callers that already + /// wrote the pre-upgrade response under an existing pause, or that pause immediately after write + /// when no peer ClientHello can race (test helpers). STARTTLS must pause before 382. + /// + Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default); + + /// + /// Performs an in-place server TLS handshake on the existing TCP socket. + /// + /// Provider used to acquire a connection-lifetime certificate lease. + /// Token that cancels the handshake (and aborts the connection on failure). + /// + /// + /// Preconditions: the connection must be plaintext and not DEFLATE-compressed; the caller must not + /// hold outstanding reads or writes; any plaintext that must + /// be visible to the peer before TLS (for example a STARTTLS 382 response) must already be + /// flushed to (STARTTLS pauses reads before writing 382). + /// The upgrade quiesces writes before inspecting ; any application plaintext + /// that still remains in is a precondition failure (connection stays plaintext). + /// Pipelined NNTP after STARTTLS must be discarded by the session (RFC 8143) before calling this + /// method. Reads may already be paused by . + /// + /// + /// On success, subsequent / traffic is TLS application data + /// on the same socket. is unchanged. On failure, the connection is + /// completed and does not fall back to plaintext. + /// + /// + /// + /// Already TLS, already compressed, upgrade in progress, or unconsumed plaintext remains in + /// (precondition failure does not complete the connection). + /// + /// The connection is closed or disposed. + Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default); + + /// + /// Activates bidirectional raw DEFLATE on the existing connection byte stream. + /// + /// Token that cancels activation (and aborts the connection on failure). + /// + /// + /// Layering (RFC 8054): DEFLATE sits above TLS when both are active — + /// NNTP → DEFLATE → TLS → TCP. TLS must be negotiated first when both are used; this method + /// rejects activation when DEFLATE is already active, and rejects + /// TLS after DEFLATE. + /// + /// + /// Preconditions: DEFLATE not already active; no outstanding reads / + /// writes; application data that belongs before compression must already be + /// consumed from ; any bytes that must reach the peer uncompressed (for example + /// a future 206 COMPRESS response) must already be flushed to . + /// + /// + /// On success, subsequent pipe traffic is compressed in both directions on the same socket and + /// pipes. is unchanged. Post-quiescence failure completes the + /// connection with no uncompressed fallback. + /// + /// + /// + /// Already compressed, upgrade in progress, or unconsumed application data remains in + /// (precondition failure does not complete the connection). + /// + /// The connection is closed or disposed. + Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/NntpConnection.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpConnection.cs new file mode 100644 index 00000000..2e0308d3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpConnection.cs @@ -0,0 +1,1522 @@ +using System.Buffers; +using System.Diagnostics; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + using VectorNNTPD.Common.Core; + /// + /// Duplex pipe transport over a replaceable byte-stream I/O owner (). + /// + /// + /// + /// Ownership: this type owns the accepted socket and the (plain + /// or authenticated ). The application owns + /// consumption of / production to . + /// + /// + /// Plain connections may call to authenticate TLS on the same socket. + /// Plain or TLS connections may call to activate bidirectional raw + /// DEFLATE above the current byte stream (RFC 8054 layering: NNTP → DEFLATE → TLS → TCP). + /// Pumps keep running across upgrades; exclusive stream ownership is enforced by transport quiescence. + /// The NNTP STARTTLS and COMPRESS commands live in the session layer and invoke these upgrades. + /// + /// + /// NNTPD performs server-side TLS authentication only. Client certificates are never requested or validated. + /// + /// + public sealed class NntpConnection : INntpConnection + { + private const int ModePlain = 0; + private const int ModeUpgrading = 1; + private const int ModeTls = 2; + + private const int CompressionOff = 0; + private const int CompressionUpgrading = 1; + private const int CompressionOn = 2; + + private readonly ILogger _logger; + private readonly Pipe _inputPipe; + private readonly Pipe _outputPipe; + private readonly CancellationTokenSource _connectionCts = new(); + private readonly object _completeGate = new(); + private readonly ConnectionClientIdentity _clientIdentity; + private readonly IFeedDiagnostics? _feedDiagnostics; + private Socket? _socket; + private ConnectionByteTransport? _transport; + private TlsCertificateLease? _certificateLease; + private byte[]? _receivePrefix; + private Task? _receiveTask; + private Task? _sendTask; + private int _mode; + private int _compression; + private int _inplaceUpgradeBusy; + private int _completed; + private int _outputWriterCompleted; + private int _outputReaderCompleted; + private int _inputReaderCompleted; + private int _inputWriterCompleted; + private int _inputReaderConsumer; + private int _disconnectReason; + private int _disconnectedLogged; + private int _disposed; + private int _sendPumpAwaitingOutput; + private long _outboundIdleVersion; + private TaskCompletionSource? _outboundIdleWaiter; + private string? _negotiatedTlsVersion; + private string? _negotiatedCipher; + + private NntpConnection( + Socket socket, + ConnectionByteTransport transport, + EndPoint? remoteEndPoint, + EndPoint? localEndPoint, + bool isTls, + ConnectionClientIdentity clientIdentity, + ILogger logger, + IFeedDiagnostics? feedDiagnostics = null) + { + _socket = socket; + _transport = transport; + RemoteEndPoint = remoteEndPoint; + LocalEndPoint = localEndPoint; + _mode = isTls ? ModeTls : ModePlain; + _clientIdentity = clientIdentity; + _logger = logger; + _feedDiagnostics = feedDiagnostics; + var inputOptions = NntpPipeOptions.Create(); + var outputOptions = NntpPipeOptions.CreateOutput(); + _inputPipe = new Pipe(inputOptions); + _outputPipe = new Pipe(outputOptions); + if (transport.Io is { } io) + { + io.TxPipePauseBytes = outputOptions.PauseWriterThreshold; + io.TxPipeResumeBytes = outputOptions.ResumeWriterThreshold; + } + } + + /// + public PipeReader Input => _inputPipe.Reader; + + /// + public PipeWriter Output => _outputPipe.Writer; + + /// + public IOutboundRateCap? OutboundRate => _transport?.RateLimiter; + + /// + public EndPoint? RemoteEndPoint { get; } + + /// + public EndPoint? LocalEndPoint { get; } + + /// + public ConnectionClientIdentity ClientIdentity => _clientIdentity; + + /// + public bool IsTls => Volatile.Read(ref _mode) == ModeTls; + + /// + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + var version = Volatile.Read(ref _negotiatedTlsVersion); + var suite = Volatile.Read(ref _negotiatedCipher); + if (version is null || suite is null) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + tlsVersion = version; + cipher = suite; + return true; + } + + /// + public bool IsCompressed => Volatile.Read(ref _compression) == CompressionOn; + + /// + public long OutboundIdleVersion => Volatile.Read(ref _outboundIdleVersion); + + /// + public bool IsCompleted => Volatile.Read(ref _completed) == 1; + + /// Test-only: current byte transport (null after teardown closes it). + internal ConnectionByteTransport? ByteTransportForTests => _transport; + + /// Test-only: send pump task started by . + internal Task? SendPumpTaskForTests => _sendTask; + + /// Test-only: whether Output.Writer has been completed. + internal bool OutputWriterCompletedForTests => Volatile.Read(ref _outputWriterCompleted) == 1; + + /// Test-only: whether Output.Reader has been completed exactly once. + internal bool OutputReaderCompletedForTests => Volatile.Read(ref _outputReaderCompleted) == 1; + + /// Test-only: whether Input.Reader has been completed exactly once. + internal bool InputReaderCompletedForTests => Volatile.Read(ref _inputReaderCompleted) == 1; + + /// Test-only: whether Input.Writer has been completed exactly once. + internal bool InputWriterCompletedForTests => Volatile.Read(ref _inputWriterCompleted) == 1; + + /// Test-only: receive pump task started by . + internal Task? ReceivePumpTaskForTests => _receiveTask; + + /// Test-only: 1 when the send pump is awaiting more . + internal int SendPumpAwaitingOutputForTests => Volatile.Read(ref _sendPumpAwaitingOutput); + + /// + public Task PauseReadsAsync(CancellationToken cancellationToken = default) + { + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + return transport.PauseReadsAsync(cancellationToken); + } + + /// + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + WaitForOutboundIdleAfterAsync(outboundIdleVersionBeforeFlush, cancellationToken); + + /// + public async Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) + { + await PauseReadsAsync(cancellationToken).ConfigureAwait(false); + await WaitForOutboundIdleAfterAsync(outboundIdleVersionBeforeFlush, cancellationToken) + .ConfigureAwait(false); + } + + /// + public CancellationToken ConnectionClosed => _connectionCts.Token; + + /// Creates and starts a plain TCP connection transport. + public static NntpConnection StartPlain( + Socket socket, + ConnectionClientIdentity clientIdentity, + ILogger logger, + ReadOnlyMemory receivePrefix = default, + IFeedDiagnostics? feedDiagnostics = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentNullException.ThrowIfNull(logger); + + ConfigureAcceptedSocket(socket); + var network = new NetworkStream(socket, ownsSocket: false); + var transport = new ConnectionByteTransport(network, isTls: false) + { + Io = TransportIoProbe.CreateSession(), + }; + var connection = new NntpConnection( + socket, + transport, + TryGetRemote(socket), + TryGetLocal(socket), + isTls: false, + clientIdentity, + logger, + feedDiagnostics); + if (!receivePrefix.IsEmpty) + { + connection._receivePrefix = receivePrefix.ToArray(); + } + + connection.StartPumps(); + return connection; + } + + /// + /// Performs a server TLS handshake then starts the duplex pipe pumps over the encrypted stream. + /// + public static async Task StartTlsAsync( + Socket socket, + ITlsCertificateContextProvider certificateProvider, + ConnectionClientIdentity clientIdentity, + ILogger logger, + CancellationToken cancellationToken, + ReadOnlyMemory tlsPrefix = default, + IFeedDiagnostics? feedDiagnostics = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentNullException.ThrowIfNull(logger); + + ConfigureAcceptedSocket(socket); + var remote = TryGetRemote(socket); + var local = TryGetLocal(socket); + var lease = certificateProvider.Acquire(); + + var networkStream = new NetworkStream(socket, ownsSocket: false); + Stream sslInner = tlsPrefix.IsEmpty + ? networkStream + : new PrefixedStream(networkStream, tlsPrefix, leaveInnerOpen: false); + var limiter = new OutboundRateLimiter(sslInner, initialMaxSendBytesPerSecond: 0, leaveInnerOpen: false); + var sslStream = new SslStream(limiter, leaveInnerStreamOpen: false); + + try + { + await sslStream + .AuthenticateAsServerAsync(CreateServerAuthenticationOptions(lease.Context), cancellationToken) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await sslStream.DisposeAsync().ConfigureAwait(false); + lease.Dispose(); + socket.Dispose(); + throw; + } + + TlsNegotiationLogging.Capture(sslStream, out var tlsVersion, out var cipher); + var transport = new ConnectionByteTransport(sslStream, isTls: true, limiter) + { + Io = TransportIoProbe.CreateSession(), + }; + var connection = new NntpConnection(socket, transport, remote, local, isTls: true, clientIdentity, logger, feedDiagnostics) + { + _certificateLease = lease, + _negotiatedTlsVersion = tlsVersion, + _negotiatedCipher = cipher, + }; + + connection.StartPumps(); + TransportLogMessages.TlsHandshakeCompleted(logger, remote); + return connection; + } + + /// + public async Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(certificateProvider); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + if (Volatile.Read(ref _completed) == 1) + { + throw new ObjectDisposedException(nameof(NntpConnection), "Connection is closed."); + } + + if (Volatile.Read(ref _mode) == ModeTls) + { + throw new InvalidOperationException("Connection is already TLS-protected."); + } + + if (Volatile.Read(ref _compression) != CompressionOff) + { + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + if (Interlocked.CompareExchange(ref _inplaceUpgradeBusy, 1, 0) != 0) + { + throw new InvalidOperationException("A transport layer upgrade is already in progress."); + } + + var prior = Interlocked.CompareExchange(ref _mode, ModeUpgrading, ModePlain); + if (prior != ModePlain) + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + if (prior == ModeUpgrading) + { + throw new InvalidOperationException("A TLS upgrade is already in progress."); + } + + if (prior == ModeTls) + { + throw new InvalidOperationException("Connection is already TLS-protected."); + } + + throw new InvalidOperationException("Connection cannot be upgraded in its current state."); + } + + if (Volatile.Read(ref _compression) != CompressionOff) + { + Volatile.Write(ref _mode, ModePlain); + Volatile.Write(ref _inplaceUpgradeBusy, 0); + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + Exception? failure = null; + var quiesced = false; + var preconditionFailed = false; + + try + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _connectionCts.Token); + + // STARTTLS pauses reads before writing 382; other callers may still need a pause here. + if (!transport.IsReadsPaused) + { + await transport.PauseReadsAsync(linked.Token).ConfigureAwait(false); + } + + await WaitForOutboundIdleAsync(linked.Token).ConfigureAwait(false); + await transport.QuiesceWritesAsync(linked.Token).ConfigureAwait(false); + quiesced = true; + + try + { + EnsureApplicationInputDrainedForUpgrade(); + } + catch (InvalidOperationException) + { + preconditionFailed = true; + transport.ResumeFromQuiesceWithoutUpgrade(); + quiesced = false; + Volatile.Write(ref _mode, ModePlain); + throw; + } + + var lease = certificateProvider.Acquire(); + var plainStream = transport.TakeQuiescedStreamForTlsWrap(); + var upgradePrefix = transport.TakePendingUpgradePrefix(); + Stream sslInner = upgradePrefix.IsEmpty + ? plainStream + : new PrefixedStream(plainStream, upgradePrefix, leaveInnerOpen: false); + var sslStream = new SslStream(sslInner, leaveInnerStreamOpen: false); + + try + { + await sslStream + .AuthenticateAsServerAsync(CreateServerAuthenticationOptions(lease.Context), linked.Token) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + try + { + await sslStream.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; disposes plain NetworkStream as well. + } + + lease.Dispose(); + failure = ex; + throw; + } + + _certificateLease = lease; + TlsNegotiationLogging.Capture(sslStream, out var tlsVersion, out var cipher); + Volatile.Write(ref _negotiatedTlsVersion, tlsVersion); + Volatile.Write(ref _negotiatedCipher, cipher); + transport.PublishTlsAndResume(sslStream); + Volatile.Write(ref _mode, ModeTls); + TransportLogMessages.InPlaceTlsUpgradeCompleted(_logger, RemoteEndPoint); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (failure is null && !preconditionFailed) + { + failure = ex; + throw; + } + finally + { + if (failure is not null) + { + // Mark terminal and cancel ConnectionClosed before waking pumps so the session + // dispatcher cannot write a secondary NNTP status onto a dying transport. + if (Volatile.Read(ref _completed) == 0) + { + await CompleteAsync(failure).ConfigureAwait(false); + } + + if (quiesced) + { + transport.AbortQuiesceForConnectionTeardown(); + } + else if (Volatile.Read(ref _mode) == ModeUpgrading) + { + Volatile.Write(ref _mode, ModePlain); + } + + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + else + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + } + } + + /// + public async Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + if (Volatile.Read(ref _completed) == 1) + { + throw new ObjectDisposedException(nameof(NntpConnection), "Connection is closed."); + } + + if (Volatile.Read(ref _compression) == CompressionOn) + { + throw new InvalidOperationException("Connection is already DEFLATE-compressed."); + } + + if (Interlocked.CompareExchange(ref _inplaceUpgradeBusy, 1, 0) != 0) + { + throw new InvalidOperationException("A transport layer upgrade is already in progress."); + } + + var prior = Interlocked.CompareExchange(ref _compression, CompressionUpgrading, CompressionOff); + if (prior != CompressionOff) + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + if (prior == CompressionUpgrading) + { + throw new InvalidOperationException("A DEFLATE upgrade is already in progress."); + } + + throw new InvalidOperationException("Connection is already DEFLATE-compressed."); + } + + if (Volatile.Read(ref _mode) == ModeUpgrading) + { + Volatile.Write(ref _compression, CompressionOff); + Volatile.Write(ref _inplaceUpgradeBusy, 0); + throw new InvalidOperationException("A TLS upgrade is already in progress."); + } + + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + Exception? failure = null; + var quiesced = false; + var preconditionFailed = false; + + try + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _connectionCts.Token); + + if (!transport.IsReadsPaused) + { + await transport.PauseReadsAsync(linked.Token).ConfigureAwait(false); + } + + await WaitForOutboundIdleAsync(linked.Token).ConfigureAwait(false); + await transport.QuiesceWritesAsync(linked.Token).ConfigureAwait(false); + quiesced = true; + + try + { + EnsureApplicationInputDrainedForUpgrade(); + } + catch (InvalidOperationException) + { + preconditionFailed = true; + transport.ResumeFromQuiesceWithoutUpgrade(); + quiesced = false; + Volatile.Write(ref _compression, CompressionOff); + throw; + } + + var inner = transport.TakeQuiescedStreamForDeflateWrap(); + var upgradePrefix = transport.TakePendingUpgradePrefix(); + Stream deflateInner = upgradePrefix.IsEmpty + ? inner + : new PrefixedStream(inner, upgradePrefix, leaveInnerOpen: false); + var deflate = new NntpDeflateStream(deflateInner); + transport.PublishDeflateAndResume(deflate); + Volatile.Write(ref _compression, CompressionOn); + TransportLogMessages.InPlaceDeflateActivated(_logger, RemoteEndPoint); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (!preconditionFailed) + { + failure = ex; + throw; + } + finally + { + if (failure is not null) + { + if (Volatile.Read(ref _completed) == 0) + { + await CompleteAsync(failure).ConfigureAwait(false); + } + + if (quiesced) + { + transport.AbortQuiesceForConnectionTeardown(); + } + else if (Volatile.Read(ref _compression) == CompressionUpgrading) + { + Volatile.Write(ref _compression, CompressionOff); + } + } + + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + } + + /// + /// + /// + /// writer completion and connection cancellation may unblock + /// SendAsync. Output.Reader is owned exclusively by the send pump: this + /// method must not complete that reader while SendAsync can still + /// AdvanceTo an examined buffer. After the send pump returns, a best-effort + /// reader complete runs only when the pump did not complete the reader itself. + /// + /// + /// Connection cancellation unblocks the receive pump. Input.Writer is owned + /// exclusively by ReceiveAsync: this method must not complete the writer while + /// GetMemory / Advance can still run. The receive pump completes the writer + /// in finally after it exits. Completing the writer here while the session has + /// already completed the reader would CompletePipe and invalidate the writing head. + /// + /// + /// Input.Reader is owned exclusively by the session RX consumer + /// (NntpSession / NntpContinuousRxReader): this method must not complete + /// the reader while ReadAsync / AdvanceTo can still run. The session + /// completes the reader after the RX loop has stopped. + /// + /// + public async Task CompleteAsync(Exception? exception = null) + { + if (Interlocked.Exchange(ref _completed, 1) == 1) + { + return; + } + + NoteDisconnectReason(ClassifyCompleteReason(exception)); + LogTcpDisconnectedOnce(); + + try + { + await _connectionCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Already disposed. + } + + try + { + await _outputPipe.Writer.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + finally + { + Interlocked.Exchange(ref _outputWriterCompleted, 1); + } + + var receive = _receiveTask; + var send = _sendTask; + if (receive is not null) + { + try + { + await receive.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Observed via pump logging. + } + } + else + { + await CompleteInputWriterAsync(exception).ConfigureAwait(false); + } + + if (send is not null) + { + try + { + await send.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Observed via pump logging. + } + + if (!send.IsCompletedSuccessfully) + { + await CompleteOutputReaderBestEffortAsync(exception).ConfigureAwait(false); + } + } + else + { + await CompleteOutputReaderBestEffortAsync(exception).ConfigureAwait(false); + } + + WriteTransportIoDump(); + await CloseTransportSocketAsync().ConfigureAwait(false); + } + + private void WriteTransportIoDump() + { + var io = _transport?.Io; + if (io is null || !TransportIoProbe.IsEnabled || TransportIoProbe.OutputDirectory is not { } dir) + { + return; + } + + try + { + var label = RemoteEndPoint?.ToString() ?? "connection"; + io.Write(dir, label); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Diagnostic dump must not affect teardown. + } + } + + /// + /// Completes Output.Reader only when the send pump did not already do so. + /// + private async Task CompleteOutputReaderBestEffortAsync(Exception? exception) + { + if (Interlocked.Exchange(ref _outputReaderCompleted, 1) == 1) + { + return; + } + + try + { + await _outputPipe.Reader.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort after a faulted send pump. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await CompleteAsync().ConfigureAwait(false); + if (Volatile.Read(ref _inputReaderConsumer) == 0) + { + await CompleteInputReaderAsync().ConfigureAwait(false); + } + + _connectionCts.Dispose(); + _certificateLease?.Dispose(); + _certificateLease = null; + } + + /// + /// Records the first known disconnect cause. Later notes do not overwrite. + /// + internal void NoteDisconnectReason(TcpDisconnectReason reason) + { + if (reason == TcpDisconnectReason.Unspecified) + { + return; + } + + Interlocked.CompareExchange(ref _disconnectReason, (int)reason, (int)TcpDisconnectReason.Unspecified); + } + + /// Test-only: first-wins disconnect reason recorded for this connection. + internal TcpDisconnectReason DisconnectReasonForTests => + (TcpDisconnectReason)Volatile.Read(ref _disconnectReason); + + private static TcpDisconnectReason ClassifyCompleteReason(Exception? exception) => + exception switch + { + null => TcpDisconnectReason.LocalClose, + OperationCanceledException => TcpDisconnectReason.Cancellation, + _ => TcpDisconnectReason.ConnectionClosed, + }; + + private void LogTcpDisconnectedOnce() + { + if (Interlocked.Exchange(ref _disconnectedLogged, 1) == 1) + { + return; + } + + var reason = (TcpDisconnectReason)Volatile.Read(ref _disconnectReason); + if (reason == TcpDisconnectReason.Unspecified) + { + reason = TcpDisconnectReason.ConnectionClosed; + } + + ConnectionAcceptanceLogging.LogDisconnected( + _logger, + RemoteEndPoint, + LocalEndPoint, + reason.ToString()); + } + + /// + /// Marks the session RX loop as the exclusive reader owner. + /// + internal void AttachInputReaderConsumer() => Interlocked.Exchange(ref _inputReaderConsumer, 1); + + /// + /// Completes after the application consumer has stopped reading. + /// Safe to call multiple times. + /// + internal async Task CompleteInputReaderAsync() + { + Interlocked.Exchange(ref _inputReaderConsumer, 0); + if (Interlocked.Exchange(ref _inputReaderCompleted, 1) == 1) + { + return; + } + + try + { + await _inputPipe.Reader.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; reader may already be completed by a test host. + } + } + + /// + /// Completes Input.Writer exactly once. Owned by ReceiveAsync; + /// calls this only when no receive pump was started. + /// + private async Task CompleteInputWriterAsync(Exception? exception) + { + if (Interlocked.Exchange(ref _inputWriterCompleted, 1) == 1) + { + return; + } + + try + { + await _inputPipe.Writer.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + /// Shared server TLS options (server auth only; no client certificates). + internal static SslServerAuthenticationOptions CreateServerAuthenticationOptions( + SslStreamCertificateContext certificateContext) => + new() + { + ServerCertificateContext = certificateContext, + ClientCertificateRequired = false, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }; + + private void StartPumps() + { + _receiveTask = ReceiveAsync(); + _sendTask = SendAsync(); + _ = ObservePumpAsync(_receiveTask, "receive"); + _ = ObservePumpAsync(_sendTask, "send"); + } + + private void EnsureApplicationInputDrainedForUpgrade() + { + if (!_inputPipe.Reader.TryRead(out var result)) + { + return; + } + + try + { + if (!result.Buffer.IsEmpty) + { + throw new InvalidOperationException( + "Cannot upgrade to TLS while unconsumed plaintext remains in Input. " + + "Drain application data before calling UpgradeToTlsAsync."); + } + } + finally + { + // Leave unconsumed bytes in place for the caller (do not examine past start on failure). + _inputPipe.Reader.AdvanceTo(result.Buffer.Start, result.Buffer.Start); + } + } + + private async Task ObservePumpAsync(Task pump, string name) + { + Exception? error = null; + try + { + await pump.ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_connectionCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + error = ex; + TransportLogMessages.PumpEndedWithError(_logger, ex, name); + NoteDisconnectReason( + string.Equals(name, "send", StringComparison.Ordinal) + ? TcpDisconnectReason.SendError + : TcpDisconnectReason.ReceiveError); + } + finally + { + if (Volatile.Read(ref _completed) == 0) + { + _ = CompleteAsync(error); + } + } + } + + private async Task WaitForOutboundIdleAsync(CancellationToken cancellationToken) => + await WaitForOutboundIdleAfterAsync(Volatile.Read(ref _outboundIdleVersion) - 1, cancellationToken) + .ConfigureAwait(false); + + private async Task WaitForOutboundIdleAfterAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (Volatile.Read(ref _sendPumpAwaitingOutput) == 1 + && Volatile.Read(ref _outboundIdleVersion) > outboundIdleVersionBeforeFlush) + { + return; + } + + var waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Volatile.Write(ref _outboundIdleWaiter, waiter); + + if (Volatile.Read(ref _sendPumpAwaitingOutput) == 1 + && Volatile.Read(ref _outboundIdleVersion) > outboundIdleVersionBeforeFlush) + { + Volatile.Write(ref _outboundIdleWaiter, null); + waiter.TrySetResult(); + return; + } + + await waiter.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private void SignalOutboundIdle() + { + Interlocked.Increment(ref _outboundIdleVersion); + Volatile.Write(ref _sendPumpAwaitingOutput, 1); + var waiter = Interlocked.Exchange(ref _outboundIdleWaiter, null); + waiter?.TrySetResult(); + } + + private void SignalOutboundBusy() => Volatile.Write(ref _sendPumpAwaitingOutput, 0); + + private static async ValueTask FlushInputAsync( + PipeWriter writer, + CancellationToken token, + TransportIoSession? io) + { + if (io is null) + { + return await writer.FlushAsync(token).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = writer.FlushAsync(token); + if (vt.IsCompletedSuccessfully) + { + io.RecordRxPipeFlush(started, started, sync: true); + return vt.Result; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var result = await vt.ConfigureAwait(false); + io.RecordRxPipeFlush(started, awaitStart, sync: false); + return result; + } + + private static async ValueTask ReadOutputAsync( + PipeReader reader, + CancellationToken token, + TransportIoSession? io) + { + if (io is null) + { + return await reader.ReadAsync(token).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = reader.ReadAsync(token); + if (vt.IsCompletedSuccessfully) + { + var completed = vt.Result; + io.RecordTxPipeWait(started, started, sync: true); + io.RecordTxReadShape(completed.Buffer); + return completed; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var result = await vt.ConfigureAwait(false); + io.RecordTxPipeWait(started, awaitStart, sync: false); + io.RecordTxReadShape(result.Buffer); + return result; + } + + private async Task ReceiveAsync() + { + Exception? error = null; + try + { + var transport = _transport ?? throw new InvalidOperationException("Transport missing."); + var writer = _inputPipe.Writer; + var token = _connectionCts.Token; + + var prefix = Interlocked.Exchange(ref _receivePrefix, null); + if (prefix is { Length: > 0 }) + { + var memory = writer.GetMemory(prefix.Length); + prefix.CopyTo(memory); + writer.Advance(prefix.Length); + RecordReceivedBytes(prefix.Length); + var prefixFlush = await FlushInputAsync(writer, token, transport.Io).ConfigureAwait(false); + if (prefixFlush.IsCompleted || prefixFlush.IsCanceled) + { + return; + } + } + + while (!token.IsCancellationRequested) + { + var memory = writer.GetMemory(NntpPipeOptions.MinimumSegmentSize); + var bytes = await transport.ReadAsync(memory, token).ConfigureAwait(false); + if (bytes == 0) + { + if (!token.IsCancellationRequested) + { + NoteDisconnectReason(TcpDisconnectReason.RemoteClosed); + } + + break; + } + + // If Quiesce completed while this read returned, octets belong to the upgrade wrap + // (e.g. TLS ClientHello), not the application Input pipe. + if (!transport.TryCommitReadToApplication(memory.Span[..bytes])) + { + continue; + } + + RecordReceivedBytes(bytes); + writer.Advance(bytes); + var flush = await FlushInputAsync(writer, token, transport.Io).ConfigureAwait(false); + if (flush.IsCompleted || flush.IsCanceled) + { + break; + } + } + } + catch (OperationCanceledException ex) when (_connectionCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown; complete the writer without faulting the reader. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + await CompleteInputWriterAsync(error).ConfigureAwait(false); + } + } + + private void RecordReceivedBytes(int bytes) + { + if (bytes > 0 && _feedDiagnostics is { IsEnabled: true }) + { + _feedDiagnostics.RecordTcpBytes(bytes); + } + } + + private async Task SendAsync() + { + var transport = _transport ?? throw new InvalidOperationException("Transport missing."); + var reader = _outputPipe.Reader; + var token = _connectionCts.Token; + var socket = _socket; + var io = transport.Io; + io?.RecordSendLoopStart(); + var iterationEndTs = 0L; + TxWriteCoalescer? coalescer = null; + + while (!token.IsCancellationRequested) + { + SignalOutboundIdle(); + var readIssuedTs = 0L; + if (io is not null) + { + readIssuedTs = Stopwatch.GetTimestamp(); + if (iterationEndTs != 0) + { + io.RecordAdvanceToNextRead(iterationEndTs, readIssuedTs); + } + } + + var result = await ReadOutputAsync(reader, token, io).ConfigureAwait(false); + var readDoneTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + SignalOutboundBusy(); + var buffer = result.Buffer; + var consumed = buffer.Start; + var firstWriteTs = 0L; + var lastWriteEndTs = 0L; + try + { + if (buffer.IsEmpty && result.IsCompleted) + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + break; + } + + if (TxWriteGranularityExperiment.TryResolveTarget(transport, out var targetBytes)) + { + if (io is not null) + { + io.WriteGranularityTargetBytes = targetBytes; + } + + if (coalescer is null || coalescer.TargetBytes != targetBytes) + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + firstWriteTs = lastWriteEndTs; + } + + coalescer = new TxWriteCoalescer(targetBytes); + } + + lastWriteEndTs = await CopyAndWriteAggregatesAsync( + transport, + coalescer, + buffer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + else + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + coalescer.Clear(); + coalescer = null; + } + + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var writeStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + if (firstWriteTs == 0) + { + firstWriteTs = writeStartTs; + io.RecordReadToFirstWrite(readDoneTs, firstWriteTs); + } + else + { + io.RecordBetweenWrites(lastWriteEndTs, writeStartTs); + } + } + + await transport.WriteAsync(segment, token).ConfigureAwait(false); + if (io is not null) + { + lastWriteEndTs = Stopwatch.GetTimestamp(); + io.RecordTransportWrite(writeStartTs, lastWriteEndTs); + } + } + + // Required for DEFLATE: sync-flush compressed bytes to the peer without ending the stream. + // No-op / cheap for NetworkStream and SslStream over NetworkStream. + await transport.FlushAsync(token).ConfigureAwait(false); + } + + consumed = buffer.End; + } + finally + { + var advanceStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null && lastWriteEndTs != 0) + { + io.RecordLastWriteToAdvance(lastWriteEndTs, advanceStartTs); + } + + reader.AdvanceTo(consumed); + if (io is not null) + { + var advanceEndTs = Stopwatch.GetTimestamp(); + io.RecordAdvanceTo(advanceStartTs, advanceEndTs); + iterationEndTs = advanceEndTs; + } + } + + if (coalescer is not null && !token.IsCancellationRequested) + { + var drained = await DrainImmediateReadsAsync( + reader, + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + lastWriteEndTs = drained.LastWriteEndTs; + + if (coalescer.PendingCount > 0) + { + _ = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + if (drained.Completed) + { + break; + } + } + + if (result.IsCompleted) + { + break; + } + } + + io?.RecordSendLoopEnd(); + await reader.CompleteAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _outputReaderCompleted, 1); + + // Half-close TCP send only for plaintext end-of-stream; TLS shutdown is via SslStream dispose. + if (socket is not null && !IsTls) + { + try + { + socket.Shutdown(SocketShutdown.Send); + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private static async ValueTask<(long LastWriteEndTs, bool Completed)> DrainImmediateReadsAsync( + PipeReader reader, + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + var completed = false; + while (reader.TryRead(out var extra)) + { + var extraConsumed = extra.Buffer.Start; + try + { + if (extra.Buffer.IsEmpty) + { + completed = extra.IsCompleted; + break; + } + + io?.RecordTxReadShape(extra.Buffer); + lastWriteEndTs = await CopyAndWriteAggregatesAsync( + transport, + coalescer, + extra.Buffer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + extraConsumed = extra.Buffer.End; + completed = extra.IsCompleted; + } + finally + { + reader.AdvanceTo(extraConsumed); + } + + if (completed) + { + break; + } + } + + return (lastWriteEndTs, completed); + } + + private static async ValueTask CopyAndWriteAggregatesAsync( + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + ReadOnlySequence buffer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var offset = 0; + while (offset < segment.Length) + { + if (coalescer.RemainingCapacity == 0) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + firstWriteTs = firstWriteTs == 0 ? lastWriteEndTs : firstWriteTs; + } + + offset += coalescer.Copy(segment.Span[offset..], io); + } + } + + if (coalescer.IsFull) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + return lastWriteEndTs; + } + + private static async ValueTask WriteCoalescedPendingAsync( + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + if (coalescer.PendingCount == 0) + { + return lastWriteEndTs; + } + + var writeStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + if (firstWriteTs == 0 && lastWriteEndTs == 0) + { + io.RecordReadToFirstWrite(readDoneTs, writeStartTs); + } + else + { + io.RecordBetweenWrites(lastWriteEndTs, writeStartTs); + } + } + + await transport.WriteAsync(coalescer.PendingMemory, token).ConfigureAwait(false); + var writeEndTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + io.RecordTransportWrite(writeStartTs, writeEndTs); + } + + await transport.FlushAsync(token).ConfigureAwait(false); + coalescer.Clear(); + return writeEndTs; + } + + private async Task CloseTransportSocketAsync() + { + ConnectionByteTransport? transport; + Socket? socket; + lock (_completeGate) + { + transport = _transport; + _transport = null; + socket = _socket; + _socket = null; + } + + if (transport is not null) + { + try + { + await transport.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + if (socket is not null) + { + try + { + socket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + + private static void ConfigureAcceptedSocket(Socket socket) + { + try + { + socket.NoDelay = true; + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private static EndPoint? TryGetRemote(Socket socket) + { + try + { + return socket.RemoteEndPoint; + } + catch (SocketException) + { + return null; + } + catch (ObjectDisposedException) + { + return null; + } + } + + private static EndPoint? TryGetLocal(Socket socket) + { + try + { + return socket.LocalEndPoint; + } + catch (SocketException) + { + return null; + } + catch (ObjectDisposedException) + { + return null; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/NntpDeflateStream.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpDeflateStream.cs new file mode 100644 index 00000000..50c24304 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpDeflateStream.cs @@ -0,0 +1,148 @@ +using System.IO.Compression; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Bidirectional raw DEFLATE transform for an NNTP connection byte stream (RFC 8054 / RFC 1951). + /// + /// + /// + /// Wraps an underlying duplex stream (plain or + /// ) with independent compressor and decompressor state. + /// Application writes are compressed toward the peer; application reads are decompressed from the peer. + /// + /// + /// Wire format is raw DEFLATE (zlib windowBits in the negative range), matching + /// RFC 8054 §4. This uses , which does not emit a zlib (RFC 1950) + /// wrapper or a gzip wrapper. Do not substitute or . + /// + /// + /// forwards to + /// . On .NET 10 that + /// implementation uses DEFLATE sync-flush semantics so pending compressed bytes become visible on + /// the underlying stream while retaining the sliding dictionary. Finalization (end-of-stream + /// trailer) occurs only when this stream is disposed. + /// + /// + /// Concurrent and + /// are supported + /// (full-duplex): the inflater only reads the inner stream; the deflater only writes it. + /// + /// + internal sealed class NntpDeflateStream : Stream + { + private readonly Stream _inner; + private readonly DeflateStream _compressor; + private readonly DeflateStream _decompressor; + private bool _disposed; + + /// + /// Creates a duplex raw-DEFLATE layer over . + /// + /// Underlying duplex byte stream; ownership transfers to this instance. + /// zlib-compatible compression level for the outbound compressor. + public NntpDeflateStream(Stream inner, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + ArgumentNullException.ThrowIfNull(inner); + if (!inner.CanRead || !inner.CanWrite) + { + throw new ArgumentException("Inner stream must be readable and writable.", nameof(inner)); + } + + _inner = inner; + // leaveOpen: true — this type owns and disposes the inner stream exactly once. + _compressor = new DeflateStream(inner, compressionLevel, leaveOpen: true); + _decompressor = new DeflateStream(inner, CompressionMode.Decompress, leaveOpen: true); + } + + /// + public override bool CanRead => !_disposed; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => !_disposed; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public override void Flush() => _compressor.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _compressor.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) => + _decompressor.Read(buffer, offset, count); + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + _decompressor.ReadAsync(buffer, offset, count, cancellationToken); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + _decompressor.ReadAsync(buffer, cancellationToken); + + /// + public override void Write(byte[] buffer, int offset, int count) => + _compressor.Write(buffer, offset, count); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + _compressor.WriteAsync(buffer, offset, count, cancellationToken); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => + _compressor.WriteAsync(buffer, cancellationToken); + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + // Compressor dispose finalizes the DEFLATE stream (writes remaining bits / end marker). + _compressor.Dispose(); + _decompressor.Dispose(); + _inner.Dispose(); + } + + _disposed = true; + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + await _compressor.DisposeAsync().ConfigureAwait(false); + await _decompressor.DisposeAsync().ConfigureAwait(false); + await _inner.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/NntpPipeOptions.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpPipeOptions.cs new file mode 100644 index 00000000..392fd101 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/NntpPipeOptions.cs @@ -0,0 +1,57 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Centralized defaults for NNTP connection transports. + /// + /// + /// Per-connection thresholds (not global). Values are engineering defaults for bounded + /// memory under many concurrent clients; they are not a claim about 10/40 Gbps throughput. + /// + public static class NntpPipeOptions + { + /// Pause the pipe writer (and thus receive) when unread bytes reach this size. + public const long PauseWriterThreshold = 64 * 1024; + + /// Resume the pipe writer after unread bytes fall to this size. + public const long ResumeWriterThreshold = 32 * 1024; + + /// Minimum pooled segment size for pipe buffers. + public const int MinimumSegmentSize = 4 * 1024; + + /// Creates reader/writer options for the input (RX) side of a duplex connection transport. + public static PipeOptions Create() => + CreateCore(PauseWriterThreshold, ResumeWriterThreshold, PipeScheduler.ThreadPool, PipeScheduler.ThreadPool); + + /// + /// TX output-pipe options. Same thresholds as unless + /// is configured. Reader and writer schedulers + /// are . Input/RX pipes must keep using + /// . + /// + internal static PipeOptions CreateOutput() + { + if (TxPipePauseExperiment.TryGetEffective(out var pause, out var resume)) + { + return CreateCore(pause, resume, PipeScheduler.Inline, PipeScheduler.Inline); + } + + return CreateCore(PauseWriterThreshold, ResumeWriterThreshold, PipeScheduler.Inline, PipeScheduler.Inline); + } + + private static PipeOptions CreateCore( + long pauseWriterThreshold, + long resumeWriterThreshold, + PipeScheduler readerScheduler, + PipeScheduler writerScheduler) => + new( + pool: null, + readerScheduler: readerScheduler, + writerScheduler: writerScheduler, + pauseWriterThreshold: pauseWriterThreshold, + resumeWriterThreshold: resumeWriterThreshold, + minimumSegmentSize: MinimumSegmentSize, + useSynchronizationContext: false); + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/SocketPayloadSender.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/SocketPayloadSender.cs new file mode 100644 index 00000000..4394d201 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/SocketPayloadSender.cs @@ -0,0 +1,62 @@ +using System.Net.Sockets; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Completes a TCP send of an entire buffer, looping on partial sends. + /// + internal static class SocketPayloadSender + { + /// + /// Sends on until every byte is accepted or the + /// operation fails / is cancelled. + /// + public static ValueTask SendAllAsync( + Socket socket, + ReadOnlyMemory buffer, + CancellationToken cancellationToken) => + SendAllAsync( + static (memory, token, state) => + ((Socket)state!).SendAsync(memory, SocketFlags.None, token), + buffer, + socket, + cancellationToken); + + /// + /// Testable send loop over an arbitrary send delegate (no heap copy of payload). + /// + /// + /// Sends as many bytes as possible from the supplied memory; return value is the count accepted. + /// + /// Payload to transmit completely. + /// Opaque state passed to . + /// Cancellation token. + public static async ValueTask SendAllAsync( + Func, CancellationToken, object?, ValueTask> sendAsync, + ReadOnlyMemory buffer, + object? state, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(sendAsync); + + var remaining = buffer; + while (!remaining.IsEmpty) + { + cancellationToken.ThrowIfCancellationRequested(); + var sent = await sendAsync(remaining, cancellationToken, state).ConfigureAwait(false); + if (sent == 0) + { + throw new IOException("Socket send transferred zero bytes."); + } + + if ((uint)sent > (uint)remaining.Length) + { + throw new IOException( + $"Socket send reported {sent} bytes but only {remaining.Length} were requested."); + } + + remaining = remaining[sent..]; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TcpDisconnectReason.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TcpDisconnectReason.cs new file mode 100644 index 00000000..84d47d41 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TcpDisconnectReason.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// Why an established TCP connection completed. Values are only those the existing + /// teardown paths can set from known cause, not inferred from timing. + /// + internal enum TcpDisconnectReason + { + /// No path has recorded a cause yet. + Unspecified = 0, + + /// Receive pump observed a 0-byte read (peer FIN) before local cancellation. + RemoteClosed = 1, + + /// was invoked without an error or prior cause. + LocalClose = 2, + + /// The receive pump faulted. + ReceiveError = 3, + + /// The send pump faulted. + SendError = 4, + + /// was invoked with . + Cancellation = 5, + + /// Listener stop completed remaining connections. + Shutdown = 6, + + /// Session RequestClose (QUIT and other protocol-initiated close). + ProtocolClose = 7, + + /// Completed with an exception that is not a receive/send pump fault. + ConnectionClosed = 8, + + /// No NNTP command was executed within Nntpd:IdleTime. + IdleTimeout = 9, + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TlsNegotiationLogging.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TlsNegotiationLogging.cs new file mode 100644 index 00000000..077a8d79 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TlsNegotiationLogging.cs @@ -0,0 +1,33 @@ +using System.Net.Security; +using System.Security.Authentication; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// Formats negotiated parameters for operational logs. + internal static class TlsNegotiationLogging + { + /// Builds the STARTTLS / TLS acceptance detail fragment. + public static string FormatDetail(string tlsVersion, string cipher) => + $"TlsVersion={tlsVersion}, Cipher={cipher}"; + + /// Maps to the operational TLSv1.x label. + public static string FormatSslProtocol(SslProtocols protocol) => + protocol switch + { + SslProtocols.Tls13 => "TLSv1.3", + SslProtocols.Tls12 => "TLSv1.2", + _ => protocol.ToString(), + }; + + /// Formats the negotiated cipher suite name. + public static string FormatCipherSuite(TlsCipherSuite cipherSuite) => cipherSuite.ToString(); + + /// Captures version and cipher labels from an authenticated . + public static void Capture(SslStream sslStream, out string tlsVersion, out string cipher) + { + ArgumentNullException.ThrowIfNull(sslStream); + tlsVersion = FormatSslProtocol(sslStream.SslProtocol); + cipher = FormatCipherSuite(sslStream.NegotiatedCipherSuite); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoProbe.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoProbe.cs new file mode 100644 index 00000000..e87e37a8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoProbe.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default ConnectionByteTransport socket/Pipe cadence sampler. + /// + /// + /// Disabled unless VECTORNNTP_TRANSPORT_IO is set or is called. + /// When disabled, no session state is allocated and transport I/O is unchanged. + /// This is not a production architecture change. + /// + internal static class TransportIoProbe + { + /// Environment variable that enables transport I/O cadence sampling. + public const string EnvironmentVariableName = "VECTORNNTP_TRANSPORT_IO"; + + private static ProbeState? _state; + + static TransportIoProbe() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var dir = value.Equals("1", StringComparison.OrdinalIgnoreCase) || + value.Equals("true", StringComparison.OrdinalIgnoreCase) + ? DefaultDirectory() + : value; + Enable(dir); + } + + /// Gets whether cadence timestamps should be recorded. + public static bool IsEnabled => Volatile.Read(ref _state) is not null; + + /// Gets the directory that receives connection dumps, or . + public static string? OutputDirectory => Volatile.Read(ref _state)?.Directory; + + /// Enables sampling and writes dumps under . + public static void Enable(string outputDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(outputDirectory); + Directory.CreateDirectory(outputDirectory); + Volatile.Write(ref _state, new ProbeState(outputDirectory)); + } + + /// Disables sampling (tests). + public static void Disable() => Volatile.Write(ref _state, null); + + /// Creates a per-connection log when enabled; otherwise . + public static TransportIoSession? CreateSession() => + IsEnabled ? new TransportIoSession() : null; + + /// + /// Repo-local diagnostic dump directory. + /// + /// + /// Deliberately walks from the process working directory toward a folder that + /// contains VectorNNTPD.ReaderService.sln or .artifacts/. This is + /// development-only output, off unless is set. + /// It is not application-local runtime state and must not be used for certs/logs. + /// + internal static string DefaultDirectory() + { + var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.ReaderService.sln")) || + Directory.Exists(Path.Join(dir.FullName, ".artifacts"))) + { + return Path.Join(dir.FullName, ".artifacts", "transport-io"); + } + + dir = dir.Parent; + } + + return Path.Join(Directory.GetCurrentDirectory(), ".artifacts", "transport-io"); + } + + private sealed class ProbeState(string directory) + { + public string Directory { get; } = directory; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoReport.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoReport.cs new file mode 100644 index 00000000..7456e665 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoReport.cs @@ -0,0 +1,294 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// DIAGNOSTIC-ONLY formatter for . + internal static class TransportIoReport + { + /// Formats a human-readable cadence report. + public static string Format(in TransportIoSnapshot snapshot, string sessionLabel) + { + var sb = new StringBuilder(); + sb.AppendLine("CONNECTION BYTE TRANSPORT I/O CADENCE — DIAGNOSTIC"); + sb.AppendLine(); + sb.AppendLine("Off-by-default. Production path is unchanged when VECTORNNTP_TRANSPORT_IO is unset."); + sb.AppendLine("Clock: Stopwatch.GetTimestamp (monotonic). Durations in microseconds."); + sb.AppendLine("Stream ops are NetworkStream/SslStream ReadAsync/WriteAsync/FlushAsync, not raw Socket APIs."); + sb.AppendLine("Sync = ValueTask.IsCompletedSuccessfully before await. Await duration is 0 when sync."); + sb.AppendLine("RX pipe flush is PipeWriter.FlushAsync after Advance (may include backpressure wait)."); + sb.AppendLine("TX pipe wait is PipeReader.ReadAsync and includes waiting for the application producer."); + sb.AppendLine(); + sb.AppendLine($"Session: {sessionLabel}"); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"Connection elapsed: {snapshot.ConnectionElapsed.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"TX pipe pause: {snapshot.TxPipePauseBytes:N0} bytes")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"TX pipe resume: {snapshot.TxPipeResumeBytes:N0} bytes")); + var experiment = TxPipePauseExperiment.IsConfigured ? "active (VECTORNNTP_TX_PIPE_PAUSE)" : "inactive (production 64 KiB)"; + sb.AppendLine($"TX pipe experiment: {experiment}"); + var coalesce = snapshot.Coalesce.TargetBytes > 0 + ? $"active (VECTORNNTP_TX_WRITE_GRANULARITY {snapshot.Coalesce.TargetBytes:N0} bytes)" + : "inactive (one WriteAsync per Pipe segment)"; + sb.AppendLine($"TX write coalesce: {coalesce}"); + sb.AppendLine(); + + WriteDirection(sb, "TX (stream WriteAsync)", snapshot.Tx, snapshot.TxWindow); + WriteDirection(sb, "RX (stream ReadAsync)", snapshot.Rx, snapshot.RxWindow); + + sb.AppendLine("STREAM FlushAsync (not included in TX send stats)"); + WriteSimple(sb, snapshot.FlushOps, snapshot.FlushSync, snapshot.FlushAsync, snapshot.FlushUs); + sb.AppendLine(); + + sb.AppendLine("RX PipeWriter.FlushAsync (pipe processing / backpressure)"); + WriteSimple(sb, snapshot.RxPipeFlushes, snapshot.RxPipeSync, snapshot.RxPipeAsync, snapshot.RxPipeFlushUs); + sb.AppendLine(); + + sb.AppendLine("TX PipeReader.ReadAsync (includes producer wait; not socket time)"); + WriteSimple(sb, snapshot.TxPipeWaits, snapshot.TxPipeSync, snapshot.TxPipeAsync, snapshot.TxPipeWaitUs); + sb.AppendLine(); + + WriteReadShape(sb, snapshot.TxRead, snapshot.Tx.Ops); + sb.AppendLine(); + + WriteTxLoop(sb, snapshot); + sb.AppendLine(); + WriteWriteCoalesce(sb, snapshot.Coalesce); + sb.AppendLine(); + + if (snapshot.Tx.Ops > 0 && snapshot.Tx.Bytes > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"TX ops/MiB: {snapshot.Tx.Ops / (snapshot.Tx.Bytes / (1024.0 * 1024.0)):F2}")); + } + + if (snapshot.Rx.Ops > 0 && snapshot.Rx.Bytes > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"RX ops/MiB: {snapshot.Rx.Ops / (snapshot.Rx.Bytes / (1024.0 * 1024.0)):F2}")); + } + + return sb.ToString(); + } + + /// Nearest-rank percentile of a sorted sample (microseconds). + internal static double Percentile(int[] sorted, double p) + { + if (sorted.Length == 0) + { + return 0; + } + + if (sorted.Length == 1) + { + return sorted[0]; + } + + var position = p * (sorted.Length - 1); + var lo = (int)position; + var hi = Math.Min(lo + 1, sorted.Length - 1); + var frac = position - lo; + return sorted[lo] + ((sorted[hi] - sorted[lo]) * frac); + } + + internal static (double Mean, double P50, double P95, double P99, int Max) Stats(int[] samples) + { + if (samples.Length == 0) + { + return (0, 0, 0, 0, 0); + } + + var copy = (int[])samples.Clone(); + Array.Sort(copy); + long sum = 0; + for (var i = 0; i < copy.Length; i++) + { + sum += copy[i]; + } + + return (sum / (double)copy.Length, Percentile(copy, 0.50), Percentile(copy, 0.95), Percentile(copy, 0.99), copy[^1]); + } + + private static void WriteDirection(StringBuilder sb, string title, in TransportIoDirection d, TimeSpan window) + { + sb.AppendLine(title); + sb.AppendLine($" operations: {d.Ops:N0}"); + sb.AppendLine($" bytes: {d.Bytes:N0}"); + sb.AppendLine($" requested bytes: {d.Requested:N0}"); + if (d.Ops > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/op: {d.Bytes / (double)d.Ops:F1}")); + sb.AppendLine($" min bytes/op: {d.MinBytes:N0}"); + sb.AppendLine($" max bytes/op: {d.MaxBytes:N0}"); + } + + var gbit = window.TotalSeconds > 0 && d.Bytes > 0 + ? d.Bytes * 8d / window.TotalSeconds / 1_000_000_000d + : 0; + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" I/O window: {window.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" throughput: {gbit:F6} Gbit/s (bytes / I/O window)")); + var total = d.Sync + d.Async; + var syncPct = total > 0 ? 100.0 * d.Sync / total : 0; + var asyncPct = total > 0 ? 100.0 * d.Async / total : 0; + sb.AppendLine($" sync complete: {d.Sync:N0} ({syncPct:F1}%)"); + sb.AppendLine($" async complete: {d.Async:N0} ({asyncPct:F1}%)"); + WriteDuration(sb, " op duration us", d.OpUs); + WriteDuration(sb, " await duration us", d.AwaitUs); + sb.AppendLine(); + } + + private static void WriteSimple(StringBuilder sb, long ops, long sync, long async, int[] samples) + { + sb.AppendLine($" operations: {ops:N0}"); + var total = sync + async; + var syncPct = total > 0 ? 100.0 * sync / total : 0; + var asyncPct = total > 0 ? 100.0 * async / total : 0; + sb.AppendLine($" sync complete: {sync:N0} ({syncPct:F1}%)"); + sb.AppendLine($" async complete: {async:N0} ({asyncPct:F1}%)"); + WriteDuration(sb, " duration us", samples); + } + + private static void WriteReadShape(StringBuilder sb, in TransportIoReadShape shape, long writeOps) + { + sb.AppendLine("TX Pipe ReadResult shape (non-empty; DIAGNOSTIC)"); + sb.AppendLine($" reads: {shape.Reads:N0}"); + sb.AppendLine($" empty reads: {shape.EmptyReads:N0}"); + sb.AppendLine($" bytes: {shape.Bytes:N0}"); + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/read: {shape.Bytes / (double)shape.Reads:F1}")); + sb.AppendLine($" min bytes/read: {shape.MinBytes:N0}"); + sb.AppendLine($" max bytes/read: {shape.MaxBytes:N0}"); + } + + sb.AppendLine($" segments: {shape.Segments:N0} (non-empty)"); + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg segments/read: {shape.Segments / (double)shape.Reads:F2}")); + sb.AppendLine($" min segments/read: {shape.MinSegments:N0}"); + sb.AppendLine($" max segments/read: {shape.MaxSegments:N0}"); + } + + if (shape.Segments > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/segment: {shape.Bytes / (double)shape.Segments:F1}")); + sb.AppendLine($" min bytes/segment: {shape.MinSegmentBytes:N0}"); + sb.AppendLine($" max bytes/segment: {shape.MaxSegmentBytes:N0}"); + } + + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" writes/read: {writeOps / (double)shape.Reads:F2} (stream WriteAsync / non-empty ReadResult)")); + } + + sb.AppendLine($" WriteAsync ops: {writeOps:N0}"); + sb.AppendLine($" segments vs writes: {shape.Segments:N0} segments -> {writeOps:N0} WriteAsync"); + } + + private static void WriteTxLoop(StringBuilder sb, in TransportIoSnapshot snapshot) + { + var loop = snapshot.Loop; + sb.AppendLine("TX SendAsync loop stages (DIAGNOSTIC; NntpConnection.SendAsync)"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" SendAsync elapsed: {loop.SendAsyncElapsed.TotalMilliseconds:F3} ms (pump lifetime, includes idle)")); + sb.AppendLine($" AdvanceTo calls: {loop.AdvanceCount:N0}"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" AdvanceTo total: {loop.AdvanceTo.TotalMilliseconds:F3} ms")); + sb.AppendLine($" Transport writes: {loop.TransportWriteCount:N0} (ConnectionByteTransport.WriteAsync)"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Transport write: {loop.TransportWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine($" Between-write gaps: {loop.BetweenWriteCount:N0}"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Between writes: {loop.BetweenWrites.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Read→first write: {loop.ReadToFirstWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Last write→Advance: {loop.LastWriteToAdvance.TotalMilliseconds:F3} ms (includes FlushAsync)")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Advance→next Read: {loop.AdvanceToNextRead.TotalMilliseconds:F3} ms")); + WriteDuration(sb, " read→write us", loop.ReadToFirstWriteUs); + WriteDuration(sb, " between write us", loop.BetweenWriteUs); + WriteDuration(sb, " write→advance us", loop.LastWriteToAdvanceUs); + WriteDuration(sb, " AdvanceTo us", loop.AdvanceUs); + WriteDuration(sb, " advance→read us", loop.AdvanceToNextReadUs); + WriteDuration(sb, " transport wr us", loop.TransportWriteUs); + + var readWait = SumUs(snapshot.TxPipeWaitUs); + var streamWrite = SumUs(snapshot.Tx.OpUs); + var accounted = loop.ReadToFirstWrite + loop.BetweenWrites + loop.LastWriteToAdvance + + loop.AdvanceTo + loop.AdvanceToNextRead + loop.TransportWrite + + TimeSpan.FromMilliseconds(readWait / 1000.0); + sb.AppendLine(); + sb.AppendLine(" Accounting (accumulated stage totals; ReadAsync wait uses sampled us sum)"); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" ReadAsync waiting: {readWait / 1000.0:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Pipe → first Write: {loop.ReadToFirstWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Transport WriteAsync: {loop.TransportWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" NetworkStream write: {streamWrite / 1000.0:F3} ms (subset of transport write)")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Between writes: {loop.BetweenWrites.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Last write → Advance: {loop.LastWriteToAdvance.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" AdvanceTo: {loop.AdvanceTo.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Advance → next Read: {loop.AdvanceToNextRead.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Copy (in read→write): {snapshot.Coalesce.CopyTime.TotalMilliseconds:F3} ms (subset of Pipe → first Write / between writes)")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Accounted stages: {accounted.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" SendAsync elapsed: {loop.SendAsyncElapsed.TotalMilliseconds:F3} ms")); + if (loop.SendAsyncElapsed > TimeSpan.Zero) + { + var unaccounted = loop.SendAsyncElapsed - accounted; + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Other/unaccounted: {unaccounted.TotalMilliseconds:F3} ms (idle between commands is expected)")); + } + } + + private static void WriteWriteCoalesce(StringBuilder sb, in TransportIoWriteCoalesce coalesce) + { + sb.AppendLine("TX write granularity (DIAGNOSTIC; VECTORNNTP_TX_WRITE_GRANULARITY)"); + if (coalesce.TargetBytes <= 0) + { + sb.AppendLine(" target: inactive"); + return; + } + + sb.AppendLine($" target: {coalesce.TargetBytes:N0} bytes"); + sb.AppendLine($" copy operations: {coalesce.CopyCount:N0}"); + sb.AppendLine($" copy bytes: {coalesce.CopyBytes:N0}"); + if (coalesce.CopyCount > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" avg copy bytes: {coalesce.CopyBytes / (double)coalesce.CopyCount:F1}")); + } + + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" copy time: {coalesce.CopyTime.TotalMilliseconds:F3} ms")); + WriteDuration(sb, " copy us", coalesce.CopyUs); + } + + private static long SumUs(int[] samples) + { + long sum = 0; + for (var i = 0; i < samples.Length; i++) + { + sum += samples[i]; + } + + return sum; + } + + private static void WriteDuration(StringBuilder sb, string label, int[] samples) + { + var s = Stats(samples); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"{label,-20} n={samples.Length,-7:N0} mean={s.Mean,10:F1} p50={s.P50,10:F1} p95={s.P95,10:F1} p99={s.P99,10:F1} max={s.Max,10:N0}")); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoSession.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoSession.cs new file mode 100644 index 00000000..6332fa02 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportIoSession.cs @@ -0,0 +1,680 @@ +using System.Buffers; +using System.Diagnostics; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY per-connection aggregate of stream Read/Write/Flush and pipe wait times. + /// + /// + /// No per-operation objects. Samples are stored in preallocated arrays + /// (microseconds). Excess operations update counters only. + /// + internal sealed class TransportIoSession + { + /// Maximum duration samples retained per series. + public const int SampleCapacity = 32768; + + private readonly int[] _rxOpUs = new int[SampleCapacity]; + private readonly int[] _rxAwaitUs = new int[SampleCapacity]; + private readonly int[] _txOpUs = new int[SampleCapacity]; + private readonly int[] _txAwaitUs = new int[SampleCapacity]; + private readonly int[] _rxPipeFlushUs = new int[SampleCapacity]; + private readonly int[] _txPipeWaitUs = new int[SampleCapacity]; + private readonly int[] _flushUs = new int[SampleCapacity]; + + private int _rxOpSamples; + private int _txOpSamples; + private int _rxPipeSamples; + private int _txPipeSamples; + private int _flushSamples; + + private long _rxOps; + private long _txOps; + private long _flushOps; + private long _rxPipeFlushes; + private long _txPipeWaits; + private long _txReadShapes; + private long _txReadEmpty; + private long _txReadBytes; + private long _txReadSegments; + private int _txReadMinBytes = int.MaxValue; + private int _txReadMaxBytes; + private int _txReadMinSegments = int.MaxValue; + private int _txReadMaxSegments; + private int _txReadMinSegBytes = int.MaxValue; + private int _txReadMaxSegBytes; + + private long _rxBytes; + private long _txBytes; + private long _rxRequested; + private long _txRequested; + + private int _rxMinBytes = int.MaxValue; + private int _rxMaxBytes; + private int _txMinBytes = int.MaxValue; + private int _txMaxBytes; + + private long _rxSync; + private long _rxAsync; + private long _txSync; + private long _txAsync; + private long _flushSync; + private long _flushAsync; + private long _rxPipeSync; + private long _rxPipeAsync; + private long _txPipeSync; + private long _txPipeAsync; + + private long _firstRxTs; + private long _lastRxTs; + private long _firstTxTs; + private long _lastTxTs; + private readonly long _createdTs = Stopwatch.GetTimestamp(); + + private readonly int[] _readToFirstWriteUs = new int[SampleCapacity]; + private readonly int[] _betweenWritesUs = new int[SampleCapacity]; + private readonly int[] _lastWriteToAdvanceUs = new int[SampleCapacity]; + private readonly int[] _advanceUs = new int[SampleCapacity]; + private readonly int[] _advanceToNextReadUs = new int[SampleCapacity]; + private readonly int[] _transportWriteUs = new int[SampleCapacity]; + private readonly int[] _copyUs = new int[SampleCapacity]; + + private int _readToFirstWriteSamples; + private int _betweenWriteSamples; + private int _lastWriteToAdvanceSamples; + private int _advanceSamples; + private int _advanceToNextReadSamples; + private int _transportWriteSamples; + private int _copySamples; + + private long _sendLoopStartTs; + private long _sendLoopEndTs; + private long _readToFirstWriteTicks; + private long _betweenWriteTicks; + private long _lastWriteToAdvanceTicks; + private long _advanceTicks; + private long _advanceToNextReadTicks; + private long _transportWriteTicks; + private long _copyTicks; + private long _advanceCount; + private long _betweenWriteCount; + private long _transportWriteCount; + private long _copyCount; + private long _copyBytes; + + /// TX output-pipe pause threshold actually used by this connection. + public long TxPipePauseBytes { get; set; } = NntpPipeOptions.PauseWriterThreshold; + + /// TX output-pipe resume threshold actually used by this connection. + public long TxPipeResumeBytes { get; set; } = NntpPipeOptions.ResumeWriterThreshold; + + /// DIAGNOSTIC write-aggregate target, or 0 when the experiment is inactive. + public int WriteGranularityTargetBytes { get; set; } + + /// Records one stream ReadAsync. + public void RecordReceive(int requested, int received, long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + var opUs = ToUs(startedTs, endTs); + var awaitUs = sync ? 0 : ToUs(awaitStartTs, endTs); + Interlocked.Increment(ref _rxOps); + Interlocked.Add(ref _rxBytes, received); + Interlocked.Add(ref _rxRequested, requested); + UpdateMin(ref _rxMinBytes, received); + UpdateMax(ref _rxMaxBytes, received); + if (sync) + { + Interlocked.Increment(ref _rxSync); + } + else + { + Interlocked.Increment(ref _rxAsync); + } + + StampWindow(ref _firstRxTs, ref _lastRxTs, startedTs, endTs); + var index = Interlocked.Increment(ref _rxOpSamples) - 1; + if ((uint)index < SampleCapacity) + { + _rxOpUs[index] = opUs; + _rxAwaitUs[index] = awaitUs; + } + } + + /// Records one stream WriteAsync. + public void RecordSend(int requested, int sent, long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + var opUs = ToUs(startedTs, endTs); + var awaitUs = sync ? 0 : ToUs(awaitStartTs, endTs); + Interlocked.Increment(ref _txOps); + Interlocked.Add(ref _txBytes, sent); + Interlocked.Add(ref _txRequested, requested); + UpdateMin(ref _txMinBytes, sent); + UpdateMax(ref _txMaxBytes, sent); + if (sync) + { + Interlocked.Increment(ref _txSync); + } + else + { + Interlocked.Increment(ref _txAsync); + } + + StampWindow(ref _firstTxTs, ref _lastTxTs, startedTs, endTs); + var index = Interlocked.Increment(ref _txOpSamples) - 1; + if ((uint)index < SampleCapacity) + { + _txOpUs[index] = opUs; + _txAwaitUs[index] = awaitUs; + } + } + + /// Records one stream FlushAsync (not mixed into send stats). + public void RecordFlush(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _flushOps); + if (sync) + { + Interlocked.Increment(ref _flushSync); + } + else + { + Interlocked.Increment(ref _flushAsync); + } + + var index = Interlocked.Increment(ref _flushSamples) - 1; + if ((uint)index < SampleCapacity) + { + _flushUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// Records one application-input PipeWriter.FlushAsync (RX backpressure). + public void RecordRxPipeFlush(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _rxPipeFlushes); + if (sync) + { + Interlocked.Increment(ref _rxPipeSync); + } + else + { + Interlocked.Increment(ref _rxPipeAsync); + } + + var index = Interlocked.Increment(ref _rxPipeSamples) - 1; + if ((uint)index < SampleCapacity) + { + _rxPipeFlushUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// + /// Records one application-output PipeReader.ReadAsync. + /// Includes waiting for the producer; this is not socket time. + /// + public void RecordTxPipeWait(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _txPipeWaits); + if (sync) + { + Interlocked.Increment(ref _txPipeSync); + } + else + { + Interlocked.Increment(ref _txPipeAsync); + } + + var index = Interlocked.Increment(ref _txPipeSamples) - 1; + if ((uint)index < SampleCapacity) + { + _txPipeWaitUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// + /// Records the shape of one non-empty TX from PipeReader.ReadAsync. + /// Empty buffers are counted separately and do not affect averages. + /// + public void RecordTxReadShape(in ReadOnlySequence buffer) + { + if (buffer.IsEmpty) + { + Interlocked.Increment(ref _txReadEmpty); + return; + } + + var bytesLong = buffer.Length; + var bytes = bytesLong > int.MaxValue ? int.MaxValue : (int)bytesLong; + var segments = 0; + var minSeg = int.MaxValue; + var maxSeg = 0; + foreach (var memory in buffer) + { + var length = memory.Length; + if (length == 0) + { + continue; + } + + segments++; + if (length < minSeg) + { + minSeg = length; + } + + if (length > maxSeg) + { + maxSeg = length; + } + } + + Interlocked.Increment(ref _txReadShapes); + Interlocked.Add(ref _txReadBytes, bytesLong); + Interlocked.Add(ref _txReadSegments, segments); + UpdateMin(ref _txReadMinBytes, bytes); + UpdateMax(ref _txReadMaxBytes, bytes); + UpdateMin(ref _txReadMinSegments, segments); + UpdateMax(ref _txReadMaxSegments, segments); + if (segments > 0) + { + UpdateMin(ref _txReadMinSegBytes, minSeg); + UpdateMax(ref _txReadMaxSegBytes, maxSeg); + } + } + + /// Marks NntpConnection.SendAsync entry (monotonic). + public void RecordSendLoopStart() => + Interlocked.CompareExchange(ref _sendLoopStartTs, Stopwatch.GetTimestamp(), 0); + + /// Marks NntpConnection.SendAsync exit after the read loop. + public void RecordSendLoopEnd() => + Volatile.Write(ref _sendLoopEndTs, Stopwatch.GetTimestamp()); + + /// ReadAsync completion → first transport WriteAsync of that iteration. + public void RecordReadToFirstWrite(long readDoneTs, long firstWriteTs) => + RecordSpan( + readDoneTs, + firstWriteTs, + ref _readToFirstWriteTicks, + ref _readToFirstWriteSamples, + _readToFirstWriteUs); + + /// Gap between consecutive transport WriteAsync calls in one ReadResult. + public void RecordBetweenWrites(long previousWriteEndTs, long nextWriteStartTs) + { + Interlocked.Increment(ref _betweenWriteCount); + RecordSpan( + previousWriteEndTs, + nextWriteStartTs, + ref _betweenWriteTicks, + ref _betweenWriteSamples, + _betweenWritesUs); + } + + /// Last WriteAsync completion → AdvanceTo (includes FlushAsync in the current loop). + public void RecordLastWriteToAdvance(long lastWriteEndTs, long advanceStartTs) => + RecordSpan( + lastWriteEndTs, + advanceStartTs, + ref _lastWriteToAdvanceTicks, + ref _lastWriteToAdvanceSamples, + _lastWriteToAdvanceUs); + + /// Times one PipeReader.AdvanceTo. + public void RecordAdvanceTo(long startedTs, long endedTs) + { + Interlocked.Increment(ref _advanceCount); + RecordSpan(startedTs, endedTs, ref _advanceTicks, ref _advanceSamples, _advanceUs); + } + + /// AdvanceTo return → next ReadAsync issued. + public void RecordAdvanceToNextRead(long advanceEndTs, long nextReadIssuedTs) => + RecordSpan( + advanceEndTs, + nextReadIssuedTs, + ref _advanceToNextReadTicks, + ref _advanceToNextReadSamples, + _advanceToNextReadUs); + + /// Times one ConnectionByteTransport.WriteAsync (admission + stream write). + public void RecordTransportWrite(long startedTs, long endedTs) + { + Interlocked.Increment(ref _transportWriteCount); + RecordSpan(startedTs, endedTs, ref _transportWriteTicks, ref _transportWriteSamples, _transportWriteUs); + } + + /// Times one copy into the diagnostic write-aggregate buffer. + public void RecordCopy(int bytes, long startedTs, long endedTs) + { + if (bytes < 0) + { + return; + } + + Interlocked.Increment(ref _copyCount); + Interlocked.Add(ref _copyBytes, bytes); + RecordSpan(startedTs, endedTs, ref _copyTicks, ref _copySamples, _copyUs); + } + + /// Snapshots counters and duration samples for a report. + public TransportIoSnapshot Snapshot() + { + var now = Stopwatch.GetTimestamp(); + return new TransportIoSnapshot( + ConnectionElapsed: Stopwatch.GetElapsedTime(_createdTs, now), + RxWindow: Window(_firstRxTs, _lastRxTs), + TxWindow: Window(_firstTxTs, _lastTxTs), + Rx: Direction( + Volatile.Read(ref _rxOps), + Volatile.Read(ref _rxBytes), + Volatile.Read(ref _rxRequested), + Volatile.Read(ref _rxMinBytes), + Volatile.Read(ref _rxMaxBytes), + Volatile.Read(ref _rxSync), + Volatile.Read(ref _rxAsync), + CopySamples(_rxOpUs, Volatile.Read(ref _rxOpSamples)), + CopySamples(_rxAwaitUs, Volatile.Read(ref _rxOpSamples))), + Tx: Direction( + Volatile.Read(ref _txOps), + Volatile.Read(ref _txBytes), + Volatile.Read(ref _txRequested), + Volatile.Read(ref _txMinBytes), + Volatile.Read(ref _txMaxBytes), + Volatile.Read(ref _txSync), + Volatile.Read(ref _txAsync), + CopySamples(_txOpUs, Volatile.Read(ref _txOpSamples)), + CopySamples(_txAwaitUs, Volatile.Read(ref _txOpSamples))), + FlushOps: Volatile.Read(ref _flushOps), + FlushSync: Volatile.Read(ref _flushSync), + FlushAsync: Volatile.Read(ref _flushAsync), + FlushUs: CopySamples(_flushUs, Volatile.Read(ref _flushSamples)), + RxPipeFlushes: Volatile.Read(ref _rxPipeFlushes), + RxPipeSync: Volatile.Read(ref _rxPipeSync), + RxPipeAsync: Volatile.Read(ref _rxPipeAsync), + RxPipeFlushUs: CopySamples(_rxPipeFlushUs, Volatile.Read(ref _rxPipeSamples)), + TxPipeWaits: Volatile.Read(ref _txPipeWaits), + TxPipeSync: Volatile.Read(ref _txPipeSync), + TxPipeAsync: Volatile.Read(ref _txPipeAsync), + TxPipeWaitUs: CopySamples(_txPipeWaitUs, Volatile.Read(ref _txPipeSamples)), + TxPipePauseBytes: TxPipePauseBytes, + TxPipeResumeBytes: TxPipeResumeBytes, + TxRead: new TransportIoReadShape( + Volatile.Read(ref _txReadShapes), + Volatile.Read(ref _txReadEmpty), + Volatile.Read(ref _txReadBytes), + Volatile.Read(ref _txReadSegments), + MinOrZero(_txReadMinBytes), + Volatile.Read(ref _txReadMaxBytes), + MinOrZero(_txReadMinSegments), + Volatile.Read(ref _txReadMaxSegments), + MinOrZero(_txReadMinSegBytes), + Volatile.Read(ref _txReadMaxSegBytes)), + Loop: new TransportIoTxLoop( + SendAsyncElapsed: Window( + Volatile.Read(ref _sendLoopStartTs), + EndOrNow(Volatile.Read(ref _sendLoopEndTs), now)), + ReadToFirstWrite: TicksToTimeSpan(Volatile.Read(ref _readToFirstWriteTicks)), + BetweenWrites: TicksToTimeSpan(Volatile.Read(ref _betweenWriteTicks)), + LastWriteToAdvance: TicksToTimeSpan(Volatile.Read(ref _lastWriteToAdvanceTicks)), + AdvanceTo: TicksToTimeSpan(Volatile.Read(ref _advanceTicks)), + AdvanceToNextRead: TicksToTimeSpan(Volatile.Read(ref _advanceToNextReadTicks)), + TransportWrite: TicksToTimeSpan(Volatile.Read(ref _transportWriteTicks)), + AdvanceCount: Volatile.Read(ref _advanceCount), + BetweenWriteCount: Volatile.Read(ref _betweenWriteCount), + TransportWriteCount: Volatile.Read(ref _transportWriteCount), + ReadToFirstWriteUs: CopySamples(_readToFirstWriteUs, Volatile.Read(ref _readToFirstWriteSamples)), + BetweenWriteUs: CopySamples(_betweenWritesUs, Volatile.Read(ref _betweenWriteSamples)), + LastWriteToAdvanceUs: CopySamples(_lastWriteToAdvanceUs, Volatile.Read(ref _lastWriteToAdvanceSamples)), + AdvanceUs: CopySamples(_advanceUs, Volatile.Read(ref _advanceSamples)), + AdvanceToNextReadUs: CopySamples(_advanceToNextReadUs, Volatile.Read(ref _advanceToNextReadSamples)), + TransportWriteUs: CopySamples(_transportWriteUs, Volatile.Read(ref _transportWriteSamples))), + Coalesce: new TransportIoWriteCoalesce( + TargetBytes: WriteGranularityTargetBytes, + CopyCount: Volatile.Read(ref _copyCount), + CopyBytes: Volatile.Read(ref _copyBytes), + CopyTime: TicksToTimeSpan(Volatile.Read(ref _copyTicks)), + CopyUs: CopySamples(_copyUs, Volatile.Read(ref _copySamples)))); + } + + /// Writes the connection dump and returns the path. + public string Write(string directory, string sessionLabel) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionLabel); + Directory.CreateDirectory(directory); + var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", System.Globalization.CultureInfo.InvariantCulture); + var safe = Sanitize(sessionLabel); + var path = Path.Join(directory, $"{stamp}-{safe}.txt"); + File.WriteAllText(path, TransportIoReport.Format(Snapshot(), sessionLabel), System.Text.Encoding.UTF8); + return path; + } + + private static TransportIoDirection Direction( + long ops, + long bytes, + long requested, + int minBytes, + int maxBytes, + long sync, + long async, + int[] opUs, + int[] awaitUs) => + new( + ops, + bytes, + requested, + MinOrZero(minBytes), + maxBytes, + sync, + async, + opUs, + awaitUs); + + private static int MinOrZero(int value) => value == int.MaxValue ? 0 : value; + + private static void RecordSpan(long start, long end, ref long ticks, ref int samples, int[] us) + { + if (start <= 0 || end < start) + { + return; + } + + Interlocked.Add(ref ticks, end - start); + var index = Interlocked.Increment(ref samples) - 1; + if ((uint)index < SampleCapacity) + { + us[index] = ToUs(start, end); + } + } + + private static TimeSpan TicksToTimeSpan(long ticks) => + ticks <= 0 ? TimeSpan.Zero : TimeSpan.FromSeconds(ticks / (double)Stopwatch.Frequency); + + private static long EndOrNow(long end, long now) => end > 0 ? end : now; + + private static TimeSpan Window(long first, long last) => + first > 0 && last >= first ? Stopwatch.GetElapsedTime(first, last) : TimeSpan.Zero; + + private static void StampWindow(ref long first, ref long last, long start, long end) + { + if (Interlocked.CompareExchange(ref first, start, 0) == 0) + { + // first write won + } + + long current; + do + { + current = Volatile.Read(ref last); + if (end <= current) + { + return; + } + } + while (Interlocked.CompareExchange(ref last, end, current) != current); + } + + private static int[] CopySamples(int[] source, int written) + { + var count = Math.Min(Math.Max(written, 0), SampleCapacity); + if (count == 0) + { + return []; + } + + var copy = new int[count]; + Array.Copy(source, copy, count); + return copy; + } + + private static void UpdateMin(ref int field, int value) + { + var current = Volatile.Read(ref field); + while (value < current) + { + var original = Interlocked.CompareExchange(ref field, value, current); + if (original == current) + { + return; + } + + current = original; + } + } + + private static void UpdateMax(ref int field, int value) + { + var current = Volatile.Read(ref field); + while (value > current) + { + var original = Interlocked.CompareExchange(ref field, value, current); + if (original == current) + { + return; + } + + current = original; + } + } + + private static int ToUs(long start, long end) + { + if (start <= 0 || end < start) + { + return 0; + } + + var us = Stopwatch.GetElapsedTime(start, end).TotalMicroseconds; + if (us >= int.MaxValue) + { + return int.MaxValue; + } + + if (us <= 0) + { + return 0; + } + + return (int)us; + } + + private static string Sanitize(string value) + { + var chars = value.ToCharArray(); + for (var i = 0; i < chars.Length; i++) + { + if (!char.IsAsciiLetterOrDigit(chars[i]) && chars[i] is not '-' and not '_') + { + chars[i] = '_'; + } + } + + return new string(chars); + } + } + + /// Immutable counters and duration samples for one connection. + internal readonly record struct TransportIoSnapshot( + TimeSpan ConnectionElapsed, + TimeSpan RxWindow, + TimeSpan TxWindow, + TransportIoDirection Rx, + TransportIoDirection Tx, + long FlushOps, + long FlushSync, + long FlushAsync, + int[] FlushUs, + long RxPipeFlushes, + long RxPipeSync, + long RxPipeAsync, + int[] RxPipeFlushUs, + long TxPipeWaits, + long TxPipeSync, + long TxPipeAsync, + int[] TxPipeWaitUs, + long TxPipePauseBytes, + long TxPipeResumeBytes, + TransportIoReadShape TxRead, + TransportIoTxLoop Loop, + TransportIoWriteCoalesce Coalesce); + + /// DIAGNOSTIC write-granularity copy counters (when the experiment is active). + internal readonly record struct TransportIoWriteCoalesce( + int TargetBytes, + long CopyCount, + long CopyBytes, + TimeSpan CopyTime, + int[] CopyUs); + + /// Accumulated NntpConnection.SendAsync loop-stage timings (when probe enabled). + internal readonly record struct TransportIoTxLoop( + TimeSpan SendAsyncElapsed, + TimeSpan ReadToFirstWrite, + TimeSpan BetweenWrites, + TimeSpan LastWriteToAdvance, + TimeSpan AdvanceTo, + TimeSpan AdvanceToNextRead, + TimeSpan TransportWrite, + long AdvanceCount, + long BetweenWriteCount, + long TransportWriteCount, + int[] ReadToFirstWriteUs, + int[] BetweenWriteUs, + int[] LastWriteToAdvanceUs, + int[] AdvanceUs, + int[] AdvanceToNextReadUs, + int[] TransportWriteUs); + + /// Shape of TX PipeReader.ReadAsync buffers (non-empty reads). + internal readonly record struct TransportIoReadShape( + long Reads, + long EmptyReads, + long Bytes, + long Segments, + int MinBytes, + int MaxBytes, + int MinSegments, + int MaxSegments, + int MinSegmentBytes, + int MaxSegmentBytes); + + /// One direction of stream I/O (RX read or TX write). + internal readonly record struct TransportIoDirection( + long Ops, + long Bytes, + long Requested, + int MinBytes, + int MaxBytes, + long Sync, + long Async, + int[] OpUs, + int[] AwaitUs); +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TransportLogMessages.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportLogMessages.cs new file mode 100644 index 00000000..4b350ba1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TransportLogMessages.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// Source-generated NNTP connection transport log messages. + internal static partial class TransportLogMessages + { + [LoggerMessage( + EventId = 1500, + Level = LogLevel.Debug, + Message = "TLS handshake completed for {Remote}.")] + public static partial void TlsHandshakeCompleted(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1501, + Level = LogLevel.Debug, + Message = "In-place TLS upgrade completed for {Remote}.")] + public static partial void InPlaceTlsUpgradeCompleted(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1502, + Level = LogLevel.Debug, + Message = "In-place DEFLATE compression activated for {Remote}.")] + public static partial void InPlaceDeflateActivated(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1503, + Level = LogLevel.Debug, + Message = "NNTP connection {Pump} pump ended with an error")] + public static partial void PumpEndedWithError(ILogger logger, Exception exception, string Pump); + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TxPipePauseExperiment.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TxPipePauseExperiment.cs new file mode 100644 index 00000000..39b545f6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TxPipePauseExperiment.cs @@ -0,0 +1,207 @@ +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default TX output-pipe pause-threshold override. + /// + /// + /// + /// Production default remains (64 KiB). + /// This type never changes the input (RX) pipe. It exists only so a SPEEDTEST sweep can + /// vary how far the TX producer may run ahead before FlushAsync backpressures. + /// + /// + /// Activate only with VECTORNNTP_TX_PIPE_PAUSE set to an allowlisted size + /// (64KiB … 8MiB), or via in tests. Unset / empty / 0 + /// / false leave production options unchanged. Any other value is rejected so a typo + /// cannot silently apply a large buffer. + /// + /// + internal static class TxPipePauseExperiment + { + /// Environment variable that selects an experimental TX pipe pause size. + public const string EnvironmentVariableName = "VECTORNNTP_TX_PIPE_PAUSE"; + + /// Allowlisted pause sizes (bytes). 64 KiB is the production control. + public static readonly long[] AllowedPauseBytes = + [ + 64 * 1024, + 128 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + ]; + + private static OverrideState? _state; + + static TxPipePauseExperiment() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryParseAllowed(value, out var pause)) + { + throw new InvalidOperationException( + EnvironmentVariableName + + " must be an allowlisted size (64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 2MiB, 4MiB, 8MiB) or the equivalent byte count."); + } + + Volatile.Write(ref _state, new OverrideState(pause)); + } + + /// Gets whether an experimental pause size is configured. + public static bool IsConfigured => Volatile.Read(ref _state) is not null; + + /// Gets the configured pause bytes, or when inactive. + public static long? ConfiguredPauseBytes => Volatile.Read(ref _state)?.PauseBytes; + + /// Test/experiment hook. must be allowlisted. + public static void Set(long pauseBytes) + { + if (!IsAllowed(pauseBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(pauseBytes), + pauseBytes, + "TX pipe pause experiment accepts only 64KiB…8MiB allowlisted sizes."); + } + + Volatile.Write(ref _state, new OverrideState(pauseBytes)); + } + + /// Clears the override (tests). Production default applies. + public static void Clear() => Volatile.Write(ref _state, null); + + /// + /// Resolves the TX output-pipe thresholds. When inactive, returns production 64 KiB / 32 KiB + /// and . + /// + public static bool TryGetEffective(out long pauseBytes, out long resumeBytes) + { + var configured = Volatile.Read(ref _state); + if (configured is null) + { + pauseBytes = NntpPipeOptions.PauseWriterThreshold; + resumeBytes = NntpPipeOptions.ResumeWriterThreshold; + return false; + } + + pauseBytes = configured.PauseBytes; + resumeBytes = ResumeFor(configured.PauseBytes); + return true; + } + + /// Resume is half of pause, never below the production 32 KiB resume. + public static long ResumeFor(long pauseBytes) + { + var half = pauseBytes / 2; + return half < NntpPipeOptions.ResumeWriterThreshold + ? NntpPipeOptions.ResumeWriterThreshold + : half; + } + + /// Parses an allowlisted size token or byte count. + public static bool TryParseAllowed(string value, out long pauseBytes) + { + pauseBytes = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (long.TryParse(trimmed, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var raw) && + IsAllowed(raw)) + { + pauseBytes = raw; + return true; + } + + if (TryParseSizeToken(trimmed, out var parsed) && IsAllowed(parsed)) + { + pauseBytes = parsed; + return true; + } + + return false; + } + + /// Returns whether is on the experiment allowlist. + public static bool IsAllowed(long pauseBytes) + { + for (var i = 0; i < AllowedPauseBytes.Length; i++) + { + if (AllowedPauseBytes[i] == pauseBytes) + { + return true; + } + } + + return false; + } + + private static bool TryParseSizeToken(string value, out long bytes) + { + bytes = 0; + var span = value.AsSpan().Trim(); + var suffixLen = 0; + long multiplier; + if (span.EndsWith("KiB", StringComparison.OrdinalIgnoreCase) || + span.EndsWith("kib", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024; + } + else if (span.EndsWith("MiB", StringComparison.OrdinalIgnoreCase) || + span.EndsWith("mib", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024 * 1024; + } + else if (span.Length > 0 && (span[^1] is 'k' or 'K')) + { + suffixLen = 1; + multiplier = 1024; + } + else if (span.Length > 0 && (span[^1] is 'm' or 'M')) + { + suffixLen = 1; + multiplier = 1024 * 1024; + } + else + { + return false; + } + + var number = span[..^suffixLen]; + if (!long.TryParse(number, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var count) || + count <= 0) + { + return false; + } + + try + { + bytes = checked(count * multiplier); + } + catch (OverflowException) + { + return false; + } + + return true; + } + + private sealed class OverrideState(long pauseBytes) + { + public long PauseBytes { get; } = pauseBytes; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteCoalescer.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteCoalescer.cs new file mode 100644 index 00000000..a143f75d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteCoalescer.cs @@ -0,0 +1,70 @@ +using System.Diagnostics; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY reusable contiguous buffer for experimental TX write aggregation. + /// + /// + /// One buffer is allocated for the connection lifetime. Copies are measured when a + /// is supplied. The buffer remains valid until + /// after the transport write has completed. + /// + internal sealed class TxWriteCoalescer + { + private readonly byte[] _buffer; + private int _count; + + /// Creates a coalescer whose buffer is exactly . + public TxWriteCoalescer(int targetBytes) + { + if (!TxWriteGranularityExperiment.IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException(nameof(targetBytes), targetBytes, "Target must be an allowlisted write-granularity size."); + } + + TargetBytes = targetBytes; + _buffer = GC.AllocateUninitializedArray(targetBytes); + } + + /// Configured aggregate size in bytes. + public int TargetBytes { get; } + + /// Bytes currently held in the reusable buffer. + public int PendingCount => _count; + + /// Unused bytes before the next transport write must be issued. + public int RemainingCapacity => TargetBytes - _count; + + /// Gets whether the buffer is full and must be written before more copies. + public bool IsFull => _count >= TargetBytes; + + /// Pending bytes. Valid until . + public ReadOnlyMemory PendingMemory => _buffer.AsMemory(0, _count); + + /// + /// Copies as many bytes as fit from . Returns the number copied. + /// + public int Copy(ReadOnlySpan source, TransportIoSession? io) + { + var take = Math.Min(source.Length, RemainingCapacity); + if (take == 0) + { + return 0; + } + + var started = io is null ? 0L : Stopwatch.GetTimestamp(); + source[..take].CopyTo(_buffer.AsSpan(_count, take)); + _count += take; + if (io is not null) + { + io.RecordCopy(take, started, Stopwatch.GetTimestamp()); + } + + return take; + } + + /// Drops pending bytes after a successful transport write (or a terminal failure). + public void Clear() => _count = 0; + } +} diff --git a/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteGranularityExperiment.cs b/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteGranularityExperiment.cs new file mode 100644 index 00000000..030b7f7b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Networking/Transport/TxWriteGranularityExperiment.cs @@ -0,0 +1,234 @@ +using System.Runtime.CompilerServices; + +namespace VectorNNTPD.ReaderService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default TX write-granularity override. + /// + /// + /// + /// Production default remains one ConnectionByteTransport.WriteAsync per Pipe + /// segment. This type never changes Pipe thresholds, schedulers, Channel, TCS, + /// NetworkStream, or socket options. It exists only so a SPEEDTEST sweep can + /// copy consecutive Pipe segments into one larger transport write. + /// + /// + /// Activate only with VECTORNNTP_TX_WRITE_GRANULARITY set to an allowlisted + /// size (64KiB … 4MiB), or via / + /// in tests. Unset / empty / 0 / false leave the production path + /// unchanged. Any other value is rejected so a typo cannot silently apply a large + /// aggregate. + /// + /// + internal static class TxWriteGranularityExperiment + { + /// Environment variable that selects an experimental TX write aggregate size. + public const string EnvironmentVariableName = "VECTORNNTP_TX_WRITE_GRANULARITY"; + + /// Allowlisted aggregate sizes (bytes). 64 KiB is the instrumented control. + public static readonly int[] AllowedTargetBytes = + [ + 64 * 1024, + 128 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 4 * 1024 * 1024, + ]; + + private static readonly ConditionalWeakTable> PerTransport = []; + + private static OverrideState? _state; + + static TxWriteGranularityExperiment() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryParseAllowed(value, out var target)) + { + throw new InvalidOperationException( + EnvironmentVariableName + + " must be an allowlisted size (64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 4MiB) or the equivalent byte count."); + } + + Volatile.Write(ref _state, new OverrideState(target)); + } + + /// Gets whether a process-wide experimental aggregate size is configured. + public static bool IsConfigured => Volatile.Read(ref _state) is not null; + + /// Gets the process-wide target bytes, or when inactive. + public static int? ConfiguredTargetBytes => Volatile.Read(ref _state)?.TargetBytes; + + /// Test/experiment hook. must be allowlisted. + public static void Set(int targetBytes) + { + if (!IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(targetBytes), + targetBytes, + "TX write-granularity experiment accepts only 64KiB…4MiB allowlisted sizes."); + } + + Volatile.Write(ref _state, new OverrideState(targetBytes)); + } + + /// Clears the process-wide override (tests). Production default applies. + public static void Clear() => Volatile.Write(ref _state, null); + + /// + /// Test-only per-transport override. Does not affect other connections. + /// + public static void SetFor(ConnectionByteTransport transport, int targetBytes) + { + ArgumentNullException.ThrowIfNull(transport); + if (!IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(targetBytes), + targetBytes, + "TX write-granularity experiment accepts only 64KiB…4MiB allowlisted sizes."); + } + + PerTransport.AddOrUpdate(transport, new StrongBox(targetBytes)); + } + + /// Removes a per-transport override (tests). + public static void ClearFor(ConnectionByteTransport transport) + { + ArgumentNullException.ThrowIfNull(transport); + PerTransport.Remove(transport); + } + + /// + /// Resolves the aggregate target for this send pump. Per-transport overrides win + /// over the process-wide setting. Returns when inactive. + /// + public static bool TryResolveTarget(ConnectionByteTransport? transport, out int targetBytes) + { + if (transport is not null && PerTransport.TryGetValue(transport, out var box)) + { + targetBytes = box.Value; + return true; + } + + return TryGetTarget(out targetBytes); + } + + /// Gets the process-wide target, or when inactive. + public static bool TryGetTarget(out int targetBytes) + { + var configured = Volatile.Read(ref _state); + if (configured is null) + { + targetBytes = 0; + return false; + } + + targetBytes = configured.TargetBytes; + return true; + } + + /// Parses an allowlisted size token or byte count. + public static bool TryParseAllowed(string value, out int targetBytes) + { + targetBytes = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (int.TryParse(trimmed, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var raw) && + IsAllowed(raw)) + { + targetBytes = raw; + return true; + } + + if (TryParseSizeToken(trimmed, out var parsed) && IsAllowed(parsed)) + { + targetBytes = parsed; + return true; + } + + return false; + } + + /// Returns whether is on the experiment allowlist. + public static bool IsAllowed(int targetBytes) + { + for (var i = 0; i < AllowedTargetBytes.Length; i++) + { + if (AllowedTargetBytes[i] == targetBytes) + { + return true; + } + } + + return false; + } + + private static bool TryParseSizeToken(string value, out int bytes) + { + bytes = 0; + var span = value.AsSpan().Trim(); + var suffixLen = 0; + int multiplier; + if (span.EndsWith("KiB", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024; + } + else if (span.EndsWith("MiB", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024 * 1024; + } + else if (span.Length > 0 && (span[^1] is 'k' or 'K')) + { + suffixLen = 1; + multiplier = 1024; + } + else if (span.Length > 0 && (span[^1] is 'm' or 'M')) + { + suffixLen = 1; + multiplier = 1024 * 1024; + } + else + { + return false; + } + + var number = span[..^suffixLen]; + if (!int.TryParse(number, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var count) || + count <= 0) + { + return false; + } + + try + { + bytes = checked(count * multiplier); + } + catch (OverflowException) + { + return false; + } + + return true; + } + + private sealed class OverrideState(int targetBytes) + { + public int TargetBytes { get; } = targetBytes; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/INewsgroupCatalogue.cs b/src/VectorNNTPD.ReaderService/Newsgroups/INewsgroupCatalogue.cs new file mode 100644 index 00000000..4d1346da --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/INewsgroupCatalogue.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// Process-wide newsgroup catalogue. Command handlers capture once. + /// + public interface INewsgroupCatalogue + { + /// + /// Gets the last successfully published immutable snapshot. + /// + /// + /// After a successful start this is never null. Readers need no lock. + /// Capture the reference once per command; a concurrent refresh may replace + /// the published snapshot while that command continues on the captured instance. + /// + NewsgroupSnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueException.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueException.cs new file mode 100644 index 00000000..3e60efe2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueException.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// Malformed or incomplete newsgroup catalogue data that must not be published. + public sealed class NewsgroupCatalogueException : Exception + { + /// Initializes a new instance of the class. + public NewsgroupCatalogueException(string message) + : base(message) + { + } + + /// Initializes a new instance of the class. + public NewsgroupCatalogueException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueLogMessages.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueLogMessages.cs new file mode 100644 index 00000000..04878a33 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// Source-generated newsgroup catalogue lifecycle log messages. + internal static partial class NewsgroupCatalogueLogMessages + { + [LoggerMessage( + EventId = 2500, + Level = LogLevel.Information, + Message = "Loading initial newsgroup catalogue")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2501, + Level = LogLevel.Information, + Message = "Published newsgroup catalogue snapshot with {GroupCount} groups")] + public static partial void SnapshotPublished(ILogger logger, int GroupCount); + + [LoggerMessage( + EventId = 2502, + Level = LogLevel.Error, + Message = "Initial newsgroup catalogue load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2503, + Level = LogLevel.Error, + Message = "Newsgroup catalogue refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2504, + Level = LogLevel.Information, + Message = "Newsgroup catalogue refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2505, + Level = LogLevel.Information, + Message = "Newsgroup catalogue refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueService.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueService.cs new file mode 100644 index 00000000..e741d07b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupCatalogueService.cs @@ -0,0 +1,221 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// Loads the newsgroup catalogue from NntpDB, publishes an immutable snapshot, + /// and refreshes it every five minutes. + /// + /// + /// + /// MySqlConnector owns physical connection pooling. Each refresh opens one logical + /// connection through , consumes the result set, and + /// disposes the connection before the snapshot is published. The snapshot retains + /// only managed immutable data. + /// + /// + /// Initial population runs during and fails startup when + /// the query cannot be completed. Subsequent refresh failures keep the last + /// known-good snapshot. A refresh does not start while the previous refresh is + /// still running (single loop). + /// + /// + public sealed class NewsgroupCatalogueService : INewsgroupCatalogue, IApplicationService, IAsyncDisposable + { + /// Catalogue refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5); + + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private NewsgroupSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + public NewsgroupCatalogueService( + NntpDbService nntpDb, + ILogger logger) + : this(nntpDb, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + internal NewsgroupCatalogueService( + NntpDbService nntpDb, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "NewsgroupCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public NewsgroupSnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + return snapshot ?? throw new InvalidOperationException("Newsgroup catalogue has not been published."); + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + NewsgroupCatalogueLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + NewsgroupCatalogueLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + NewsgroupCatalogueLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + /// Replaces the published snapshot (tests). + internal void PublishForTests(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Publish(snapshot); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + + NewsgroupCatalogueLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NewsgroupCatalogueLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryNewsgroupsAsync(cancellationToken).ConfigureAwait(false); + return NewsgroupSnapshot.Create(rows); + } + + private void Publish(NewsgroupSnapshot snapshot) + { + Interlocked.Exchange(ref _current, snapshot); + NewsgroupCatalogueLogMessages.SnapshotPublished(_logger, snapshot.Groups.Count); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupPostingStatus.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupPostingStatus.cs new file mode 100644 index 00000000..be45626c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupPostingStatus.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// LIST ACTIVE posting-status octet as stored in nntpgroups.posting_status. + /// + /// + /// + /// Supported values are the fixed MySQL enum y, n, m, x, + /// j. Meanings follow RFC 3977 §7.6.3 and RFC 6048 §3.1: + /// + /// + /// y — posting is permitted. + /// n — posting is not permitted. + /// m — postings are forwarded to the newsgroup moderator. + /// x — postings and articles from peers are not permitted. + /// j — articles from peers are permitted, but nothing is locally filed. + /// + /// + /// RFC 6048's =<newsgroup> alias form is intentionally unsupported. + /// There is no enum member for =. Unknown octets are rejected; they are + /// never coerced. + /// + /// + public enum NewsgroupPostingStatus : byte + { + /// Posting is permitted (y). RFC 3977 §7.6.3. + Allowed = (byte)'y', + + /// Posting is prohibited (n). RFC 3977 §7.6.3. + Prohibited = (byte)'n', + + /// Postings are forwarded to the moderator (m). RFC 3977 §7.6.3. + Moderated = (byte)'m', + + /// + /// Posting and articles from peers are not permitted (x). RFC 6048 §3.1. + /// Distinct from ; not equivalent to n. + /// + NoPostingOrPeerArticles = (byte)'x', + + /// + /// Articles from peers are permitted, but nothing is locally filed (j). RFC 6048 §3.1. + /// Distinct from and . + /// This is not a catch-all junk group; the octet is the RFC 6048 j status. + /// + PeerOnly = (byte)'j', + } + + /// Validation helpers for octets. + public static class NewsgroupPostingStatusOctets + { + /// + /// Returns whether is one of the five supported status octets. + /// + /// Candidate status octet. + /// for y/n/m/x/j. + public static bool IsSupported(byte value) => + value is (byte)'y' or (byte)'n' or (byte)'m' or (byte)'x' or (byte)'j'; + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupRecord.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupRecord.cs new file mode 100644 index 00000000..2743c0cd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupRecord.cs @@ -0,0 +1,77 @@ +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// One immutable newsgroup entry owned by a . + /// + /// + /// Precomputed UTF-8 wire lines include CRLF and, when required, multiline + /// dot-stuffing. Command handlers must not re-encode these buffers. + /// + public sealed class NewsgroupRecord + { + internal NewsgroupRecord( + string groupName, + byte[] groupNameBytes, + string groupDescription, + ulong countHigh, + ulong countLow, + NewsgroupPostingStatus postingStatus, + ulong estimatedCount, + byte[] listActiveLine, + byte[] listCountsLine, + byte[] listNewsgroupsLine, + byte[] groupSelectedLine) + { + GroupName = groupName; + GroupNameBytes = groupNameBytes; + GroupDescription = groupDescription; + CountHigh = countHigh; + CountLow = countLow; + PostingStatus = postingStatus; + EstimatedCount = estimatedCount; + ListActiveLine = listActiveLine; + ListCountsLine = listCountsLine; + ListNewsgroupsLine = listNewsgroupsLine; + GroupSelectedLine = groupSelectedLine; + } + + /// Gets the original group name as stored in the database. + public string GroupName { get; } + + /// Gets the original group name as UTF-8 octets used for lookup and wire output. + public ReadOnlyMemory GroupNameBytes { get; } + + /// Gets the original group description (may be empty). + public string GroupDescription { get; } + + /// Gets the unsigned high water mark. + public ulong CountHigh { get; } + + /// Gets the unsigned low water mark. + public ulong CountLow { get; } + + /// Gets the database posting status (y/n/m/x/j). + public NewsgroupPostingStatus PostingStatus { get; } + + /// + /// Gets the GROUP article-count estimate derived from the water marks + /// (RFC 3977 §6.1.1: high − low + 1, or 0 when the group is empty). + /// + public ulong EstimatedCount { get; } + + /// Gets the precomputed LIST ACTIVE line including CRLF. + public ReadOnlyMemory ListActiveLine { get; } + + /// + /// Gets the precomputed LIST COUNTS line including CRLF + /// (group high low estimated status). + /// + public ReadOnlyMemory ListCountsLine { get; } + + /// Gets the precomputed LIST NEWSGROUPS line including CRLF. + public ReadOnlyMemory ListNewsgroupsLine { get; } + + /// Gets the precomputed 211 number low high group line including CRLF. + public ReadOnlyMemory GroupSelectedLine { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupSnapshot.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupSnapshot.cs new file mode 100644 index 00000000..0a953d1c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NewsgroupSnapshot.cs @@ -0,0 +1,430 @@ +using System.Buffers.Text; +using System.Text; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// Immutable, fully constructed newsgroup catalogue snapshot. + /// + /// + /// Readers observe either the previous complete snapshot or this complete snapshot. + /// No lock is required. Command handlers must capture + /// once and use only that instance. + /// + public sealed class NewsgroupSnapshot + { + private static readonly NewsgroupNameComparer NameComparer = new(); + + /// Empty published snapshot (zero groups; valid LIST body is empty). + public static NewsgroupSnapshot Empty { get; } = Create(Array.Empty()); + + private readonly NewsgroupRecord[] _groups; + + private NewsgroupSnapshot( + NewsgroupRecord[] groups, + byte[] listActiveBody, + byte[] listActiveComplete, + byte[] listCountsBody, + byte[] listCountsComplete, + byte[] listNewsgroupsBody, + byte[] listNewsgroupsComplete) + { + _groups = groups; + Groups = groups; + ListActiveBody = listActiveBody; + ListActiveComplete = listActiveComplete; + ListCountsBody = listCountsBody; + ListCountsComplete = listCountsComplete; + ListNewsgroupsBody = listNewsgroupsBody; + ListNewsgroupsComplete = listNewsgroupsComplete; + } + + /// Gets groups in deterministic ordinal-ignore-case name order. + public IReadOnlyList Groups { get; } + + /// Gets concatenated precomputed LIST ACTIVE lines (no status or terminator). + public ReadOnlyMemory ListActiveBody { get; } + + /// Gets the complete LIST / LIST ACTIVE multiline wire (status + body + terminator). + public ReadOnlyMemory ListActiveComplete { get; } + + /// Gets concatenated precomputed LIST COUNTS lines (no status or terminator). + public ReadOnlyMemory ListCountsBody { get; } + + /// Gets the complete LIST COUNTS multiline wire (status + body + terminator). + public ReadOnlyMemory ListCountsComplete { get; } + + /// Gets concatenated precomputed LIST NEWSGROUPS lines (no status or terminator). + public ReadOnlyMemory ListNewsgroupsBody { get; } + + /// Gets the complete LIST NEWSGROUPS multiline wire (status + body + terminator). + public ReadOnlyMemory ListNewsgroupsComplete { get; } + + /// Builds a snapshot from raw database rows. Fails the entire build on any malformed row. + public static NewsgroupSnapshot Create(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + var records = new NewsgroupRecord[rows.Count]; + for (var i = 0; i < rows.Count; i++) + { + records[i] = CreateRecord(rows[i], i); + } + + return Publish(records); + } + + /// Builds a snapshot from already-typed definitions (tests and in-process construction). + public static NewsgroupSnapshot Create(IReadOnlyList definitions) + { + ArgumentNullException.ThrowIfNull(definitions); + var records = new NewsgroupRecord[definitions.Count]; + for (var i = 0; i < definitions.Count; i++) + { + var definition = definitions[i]; + records[i] = CreateRecord( + new NntpGroupRow( + definition.GroupName, + definition.GroupDescription, + definition.CountHigh, + definition.CountLow, + (byte)definition.PostingStatus), + i); + } + + return Publish(records); + } + + /// + /// Looks up a group by name using ASCII ordinal case-insensitive comparison. + /// The original stored name is retained on the returned record. + /// + public bool TryGet(ReadOnlySpan groupName, out NewsgroupRecord group) + { + var lo = 0; + var hi = _groups.Length - 1; + while (lo <= hi) + { + var mid = lo + ((hi - lo) / 2); + var cmp = CompareOrdinalIgnoreCase(_groups[mid].GroupNameBytes.Span, groupName); + if (cmp == 0) + { + group = _groups[mid]; + return true; + } + + if (cmp < 0) + { + lo = mid + 1; + } + else + { + hi = mid - 1; + } + } + + group = null!; + return false; + } + + private static NewsgroupSnapshot Publish(NewsgroupRecord[] records) + { + Array.Sort(records, NameComparer); + for (var i = 1; i < records.Length; i++) + { + if (CompareOrdinalIgnoreCase(records[i - 1].GroupNameBytes.Span, records[i].GroupNameBytes.Span) == 0) + { + throw new NewsgroupCatalogueException( + $"Duplicate newsgroup name '{records[i].GroupName}' (case-insensitive)."); + } + } + + var activeBody = ConcatenateLines(records, static r => r.ListActiveLine); + var countsBody = ConcatenateLines(records, static r => r.ListCountsLine); + var newsgroupsBody = ConcatenateLines(records, static r => r.ListNewsgroupsLine); + return new NewsgroupSnapshot( + records, + activeBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, activeBody), + countsBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, countsBody), + newsgroupsBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, newsgroupsBody)); + } + + private static NewsgroupRecord CreateRecord(NntpGroupRow row, int index) + { + if (string.IsNullOrEmpty(row.GroupName)) + { + throw new NewsgroupCatalogueException($"nntpgroups row {index} has an empty group_name."); + } + + if (!NewsgroupPostingStatusOctets.IsSupported(row.PostingStatus)) + { + throw new NewsgroupCatalogueException( + $"nntpgroups row '{row.GroupName}' has invalid posting_status 0x{row.PostingStatus:X2}."); + } + + var nameBytes = Encoding.UTF8.GetBytes(row.GroupName); + var description = row.GroupDescription ?? string.Empty; + var descriptionBytes = Encoding.UTF8.GetBytes(description); + var status = (NewsgroupPostingStatus)row.PostingStatus; + var estimate = EstimateCount(row.CountLow, row.CountHigh); + return new NewsgroupRecord( + row.GroupName, + nameBytes, + description, + row.CountHigh, + row.CountLow, + status, + estimate, + EncodeActiveLine(nameBytes, row.CountHigh, row.CountLow, row.PostingStatus), + EncodeCountsLine(nameBytes, row.CountHigh, row.CountLow, estimate, row.PostingStatus), + EncodeNewsgroupsLine(nameBytes, descriptionBytes), + EncodeGroupSelectedLine(nameBytes, estimate, row.CountLow, row.CountHigh)); + } + + /// RFC 3977 §6.1.1 estimate: 0 when empty; otherwise high − low + 1 without wrapping. + internal static ulong EstimateCount(ulong low, ulong high) + { + if (high < low) + { + return 0; + } + + if (low == 0 && high == 0) + { + return 0; + } + + var span = high - low; + return span == ulong.MaxValue ? ulong.MaxValue : span + 1; + } + + private static byte[] EncodeActiveLine( + ReadOnlySpan name, + ulong high, + ulong low, + byte status) + { + Span highDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(high, highDigits, out var highLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen)) + { + throw new NewsgroupCatalogueException("Failed to format unsigned water marks."); + } + + var length = name.Length + 1 + highLen + 1 + lowLen + 1 + 1 + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + line[offset++] = status; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeCountsLine( + ReadOnlySpan name, + ulong high, + ulong low, + ulong estimate, + byte status) + { + Span highDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + Span estimateDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(high, highDigits, out var highLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen) + || !Utf8Formatter.TryFormat(estimate, estimateDigits, out var estimateLen)) + { + throw new NewsgroupCatalogueException("Failed to format LIST COUNTS water marks."); + } + + var length = name.Length + 1 + highLen + 1 + lowLen + 1 + estimateLen + 1 + 1 + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + estimateDigits[..estimateLen].CopyTo(line.AsSpan(offset)); + offset += estimateLen; + line[offset++] = (byte)' '; + line[offset++] = status; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeNewsgroupsLine(ReadOnlySpan name, ReadOnlySpan description) + { + var length = name.Length + 1 + description.Length + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + description.CopyTo(line.AsSpan(offset)); + offset += description.Length; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeGroupSelectedLine( + ReadOnlySpan name, + ulong estimate, + ulong low, + ulong high) + { + Span estimateDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + Span highDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(estimate, estimateDigits, out var estimateLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen) + || !Utf8Formatter.TryFormat(high, highDigits, out var highLen)) + { + throw new NewsgroupCatalogueException("Failed to format GROUP water marks."); + } + + // "211 " + estimate + " " + low + " " + high + " " + name + CRLF + var length = 4 + estimateLen + 1 + lowLen + 1 + highLen + 1 + name.Length + 2; + var line = new byte[length]; + line[0] = (byte)'2'; + line[1] = (byte)'1'; + line[2] = (byte)'1'; + line[3] = (byte)' '; + var offset = 4; + estimateDigits[..estimateLen].CopyTo(line.AsSpan(offset)); + offset += estimateLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return line; + } + + private static byte[] StuffIfNeeded(byte[] line) + { + if (line.Length == 0 || line[0] != (byte)'.') + { + return line; + } + + var stuffed = new byte[line.Length + 1]; + stuffed[0] = (byte)'.'; + line.CopyTo(stuffed, 1); + return stuffed; + } + + private static byte[] ConcatenateLines( + NewsgroupRecord[] records, + Func> selector) + { + var total = 0; + for (var i = 0; i < records.Length; i++) + { + total = checked(total + selector(records[i]).Length); + } + + var body = new byte[total]; + var offset = 0; + for (var i = 0; i < records.Length; i++) + { + var line = selector(records[i]).Span; + line.CopyTo(body.AsSpan(offset)); + offset += line.Length; + } + + return body; + } + + private static byte[] ConcatenateComplete(ReadOnlyMemory status, ReadOnlySpan body) + { + var total = checked(status.Length + body.Length + NntpResponses.MultilineTerminator.Length); + var complete = new byte[total]; + status.Span.CopyTo(complete); + body.CopyTo(complete.AsSpan(status.Length)); + NntpResponses.MultilineTerminator.Span.CopyTo(complete.AsSpan(status.Length + body.Length)); + return complete; + } + + internal static int CompareOrdinalIgnoreCase(ReadOnlySpan left, ReadOnlySpan right) + { + var n = Math.Min(left.Length, right.Length); + for (var i = 0; i < n; i++) + { + var a = NntpAscii.FoldUpper(left[i]); + var b = NntpAscii.FoldUpper(right[i]); + if (a != b) + { + return a.CompareTo(b); + } + } + + return left.Length.CompareTo(right.Length); + } + + private sealed class NewsgroupNameComparer : IComparer + { + public int Compare(NewsgroupRecord? x, NewsgroupRecord? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var ignoreCase = CompareOrdinalIgnoreCase(x.GroupNameBytes.Span, y.GroupNameBytes.Span); + return ignoreCase != 0 + ? ignoreCase + : x.GroupNameBytes.Span.SequenceCompareTo(y.GroupNameBytes.Span); + } + } + } + + /// Typed newsgroup fields used to construct a snapshot without a database row. + /// Original group name retained for wire output. + /// Group description (UTF-8 on the wire). + /// Unsigned high water mark. + /// Unsigned low water mark. + /// Exactly y, n, m, x, or j. The RFC 6048 =<newsgroup> form is not supported. + public readonly record struct NewsgroupDefinition( + string GroupName, + string GroupDescription, + ulong CountHigh, + ulong CountLow, + NewsgroupPostingStatus PostingStatus); +} diff --git a/src/VectorNNTPD.ReaderService/Newsgroups/NntpWildmat.cs b/src/VectorNNTPD.ReaderService/Newsgroups/NntpWildmat.cs new file mode 100644 index 00000000..4269d658 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Newsgroups/NntpWildmat.cs @@ -0,0 +1,242 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Newsgroups +{ + /// + /// RFC 3977 §4 wildmat matcher for newsgroup names. + /// + /// + /// Operates on UTF-8 octets. Literal matching is ASCII ordinal case-insensitive + /// (NNTP newsgroup names). Comma-separated patterns and rightmost ! negation + /// follow §4.2. Characters \, [, and ] are rejected as syntax + /// errors (RFC 3977 §4.1 / §3.2.1.1). + /// + public static class NntpWildmat + { + /// Returns whether is a syntactically valid RFC 3977 wildmat. + public static bool TryValidate(ReadOnlySpan wildmat) + { + if (wildmat.IsEmpty) + { + return false; + } + + var index = 0; + var first = true; + while (index < wildmat.Length) + { + if (!first) + { + if (wildmat[index] != (byte)',') + { + return false; + } + + index++; + if (index < wildmat.Length && wildmat[index] == (byte)'!') + { + index++; + } + } + + first = false; + if (index >= wildmat.Length) + { + return false; + } + + var patternStart = index; + while (index < wildmat.Length && wildmat[index] != (byte)',') + { + var b = wildmat[index]; + if (b is (byte)'\\' or (byte)'[' or (byte)']') + { + return false; + } + + if (b == (byte)'!' && index == patternStart) + { + return false; + } + + index++; + } + + if (patternStart == index) + { + return false; + } + } + + return true; + } + + /// + /// Returns whether matches + /// using ASCII ordinal case-insensitive comparison. + /// + public static bool IsMatch(ReadOnlySpan text, ReadOnlySpan wildmat) + { + if (!TryValidate(wildmat)) + { + return false; + } + + return IsMatchValidated(text, wildmat); + } + + /// + /// Matches against an already-validated wildmat. + /// + /// + /// LIST command handlers use this after has already + /// succeeded at parse time, so each group is not revalidated. + /// + public static bool IsMatchValidated(ReadOnlySpan text, ReadOnlySpan wildmat) + { + var rightmostMatchNegated = false; + var anyMatch = false; + var index = 0; + var first = true; + while (index < wildmat.Length) + { + var negated = false; + if (!first) + { + index++; + if (index < wildmat.Length && wildmat[index] == (byte)'!') + { + negated = true; + index++; + } + } + + first = false; + var start = index; + while (index < wildmat.Length && wildmat[index] != (byte)',') + { + index++; + } + + if (MatchPattern(text, wildmat[start..index])) + { + anyMatch = true; + rightmostMatchNegated = negated; + } + } + + return anyMatch && !rightmostMatchNegated; + } + + private static bool MatchPattern(ReadOnlySpan text, ReadOnlySpan pattern) + { + return Match(text, pattern); + } + + private static bool Match(ReadOnlySpan text, ReadOnlySpan pattern) + { + while (!pattern.IsEmpty) + { + if (pattern[0] == (byte)'*') + { + while (pattern.Length > 1 && pattern[1] == (byte)'*') + { + pattern = pattern[1..]; + } + + pattern = pattern[1..]; + if (pattern.IsEmpty) + { + return true; + } + + var remaining = text; + while (true) + { + if (Match(remaining, pattern)) + { + return true; + } + + if (remaining.IsEmpty) + { + return false; + } + + remaining = ConsumeCharacter(remaining); + } + } + + if (text.IsEmpty) + { + return false; + } + + if (pattern[0] == (byte)'?') + { + text = ConsumeCharacter(text); + pattern = pattern[1..]; + continue; + } + + if (!TryConsumeLiteral(ref text, pattern[0])) + { + return false; + } + + pattern = pattern[1..]; + } + + return text.IsEmpty; + } + + private static bool TryConsumeLiteral(ref ReadOnlySpan text, byte literal) + { + if (text.IsEmpty) + { + return false; + } + + var foldedText = NntpAscii.FoldUpper(text[0]); + var foldedLiteral = NntpAscii.FoldUpper(literal); + if (foldedText != foldedLiteral) + { + return false; + } + + text = text[1..]; + return true; + } + + private static ReadOnlySpan ConsumeCharacter(ReadOnlySpan text) + { + if (text.IsEmpty) + { + return text; + } + + var width = Utf8CharacterWidth(text[0]); + return width >= text.Length ? ReadOnlySpan.Empty : text[width..]; + } + + private static int Utf8CharacterWidth(byte lead) + { + if (lead < 0x80) + { + return 1; + } + + if (lead < 0xE0) + { + return 2; + } + + if (lead < 0xF0) + { + return 3; + } + + return 4; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFile.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFile.cs new file mode 100644 index 00000000..5ab5c4ee --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFile.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// An opened completed Path-survey file held across gzip/deletion. + internal sealed class NinpathsCompletedFile : IDisposable + { + /// Initializes a handoff of with an already-open . + public NinpathsCompletedFile(string path, FileStream stream) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(stream); + Path = path; + Stream = stream; + } + + /// Gets the completed file path supplied by Serilog. + public string Path { get; } + + /// Gets the sequential reader. Opened with share-read/write/delete. + public FileStream Stream { get; } + + /// + public void Dispose() => Stream.Dispose(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFileHandler.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFileHandler.cs new file mode 100644 index 00000000..f22b8978 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsCompletedFileHandler.cs @@ -0,0 +1,84 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Rotation-hook handoff: opens the completed Path-survey file and enqueues it + /// for background ninpaths processing. Does not parse the file. + /// + /// + /// The stream is opened with and + /// so gzip and deletion of the completed file can proceed + /// while the worker still reads. Failures are logged and never thrown to rotation. + /// + public sealed class NinpathsCompletedFileHandler : ICompletedPathSurveyFileHandler + { + private readonly NinpathsProcessingService _processor; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// Initializes a handler that enqueues completed files onto . + public NinpathsCompletedFileHandler( + NinpathsProcessingService processor, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(processor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _processor = processor; + _options = options; + _logger = logger; + } + + /// + public void OnCompletedFile(string completedFilePath) + { + if (string.IsNullOrWhiteSpace(completedFilePath)) + { + return; + } + + if (!NinpathsTop1000.IsEnabled(_options.Value)) + { + NinpathsLogMessages.Disabled(_logger, completedFilePath); + return; + } + + FileStream stream; + try + { + stream = new FileStream( + completedFilePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + NinpathsConstants.ReadBufferSize, + FileOptions.SequentialScan); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NinpathsLogMessages.OpenFailed(_logger, ex, completedFilePath); + return; + } + + // codeql[cs/missed-using-statement] On successful TryEnqueue ownership transfers to NinpathsProcessingService; using would dispose an enqueued file. finally disposes only when enqueue fails or throws. + var file = new NinpathsCompletedFile(completedFilePath, stream); + var enqueued = false; + try + { + enqueued = _processor.TryEnqueue(file); + } + finally + { + if (!enqueued) + { + file.Dispose(); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsConstants.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsConstants.cs new file mode 100644 index 00000000..a5824fd6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsConstants.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Constants matching current INN backends/ninpaths.c (version 3.1.1). + /// + internal static class NinpathsConstants + { + /// Dump format version written in the !!NINP header. + public const string Version = "3.1.1"; + + /// INN MAXLINE: fgets payload is this size including the terminator slot. + public const int MaxLine = 1024; + + /// INN HASH_TBL. + public const int HashTableSize = 65536; + + /// INN MAXHOST including the C string terminator. + public const int MaxHost = 128; + + /// INN RECLINE dump line soft wrap. + public const int RecordLineSoftLimit = 120; + + /// FNV prime used by INN hash(). + public const ulong HashPrime = 16777619; + + /// Maximum host token length (MAXHOST - 1). + public const int MaxHostChars = MaxHost - 1; + + /// Maximum bytes fgets(buf, MAXLINE) stores before the NUL. + public const int MaxFgetsChars = MaxLine - 1; + + /// Bounded pending completed-file queue (daily files, not Path records). + public const int PendingFileCapacity = 8; + + /// Sequential read buffer for completed Path-survey files. + public const int ReadBufferSize = 64 * 1024; + + /// sendinpaths subject prefix including the trailing space before the pathhost. + public const string EmailSubjectPrefix = "inpaths "; + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogMessages.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogMessages.cs new file mode 100644 index 00000000..5ed852ce --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogMessages.cs @@ -0,0 +1,90 @@ +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// Source-generated ninpaths processing log messages. + internal static partial class NinpathsLogMessages + { + [LoggerMessage( + EventId = 2900, + Level = LogLevel.Debug, + Message = "Ninpaths skipped completed Path-survey file {CompletedFile}; Nntpd:Top1000 has no recipients")] + public static partial void Disabled(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2901, + Level = LogLevel.Error, + Message = "Ninpaths could not open completed Path-survey file {CompletedFile}")] + public static partial void OpenFailed(ILogger logger, Exception exception, string CompletedFile); + + [LoggerMessage( + EventId = 2902, + Level = LogLevel.Debug, + Message = "Ninpaths ignored duplicate completed Path-survey file {CompletedFile}")] + public static partial void Duplicate(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2903, + Level = LogLevel.Error, + Message = "Ninpaths pending-file queue is full; dropping handoff for {CompletedFile}")] + public static partial void QueueFull(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2904, + Level = LogLevel.Information, + Message = "Ninpaths processing completed Path-survey file {CompletedFile}")] + public static partial void Processing(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2905, + Level = LogLevel.Information, + Message = "Ninpaths produced no dump for {CompletedFile}; total articles is zero")] + public static partial void EmptyDump(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2906, + Level = LogLevel.Error, + Message = "Ninpaths cannot compose mail for {CompletedFile}; Email:DefaultFrom is missing or invalid")] + public static partial void MissingFrom(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2907, + Level = LogLevel.Information, + Message = "Ninpaths report for {CompletedFile} not mailed; Email:Enabled is false")] + public static partial void EmailDisabled(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2908, + Level = LogLevel.Error, + Message = "Ninpaths failed to spool report for {CompletedFile} status={Status} detail={Detail}")] + public static partial void EmailFailed(ILogger logger, string CompletedFile, string Status, string Detail); + + [LoggerMessage( + EventId = 2909, + Level = LogLevel.Information, + Message = "Ninpaths spooled report for {CompletedFile} recipients={RecipientCount} bytes={ReportBytes}")] + public static partial void EmailAccepted(ILogger logger, string CompletedFile, int RecipientCount, int ReportBytes); + + [LoggerMessage( + EventId = 2910, + Level = LogLevel.Error, + Message = "Ninpaths processing failed for {CompletedFile}")] + public static partial void ProcessingFailed(ILogger logger, Exception exception, string CompletedFile); + + [LoggerMessage( + EventId = 2911, + Level = LogLevel.Error, + Message = "Ninpaths background worker failed")] + public static partial void WorkerFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2912, + Level = LogLevel.Information, + Message = "Ninpaths processing service started")] + public static partial void Started(ILogger logger); + + [LoggerMessage( + EventId = 2913, + Level = LogLevel.Information, + Message = "Ninpaths processing service stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogReader.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogReader.cs new file mode 100644 index 00000000..711e7226 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsLogReader.cs @@ -0,0 +1,160 @@ +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Streaming Path-survey reader that applies INN procpaths / fgets line semantics. + /// + /// + /// Reads sequentially with a fixed buffer. Does not load the file, does not + /// retain Path strings, and does not allocate per article beyond interned + /// unique sites. Matches INN fgets(buf, MAXLINE), isspace, + /// case-insensitive Path: , last-element chop, over-length token abort, + /// and the one-window bogus-line skip (not skip-until-newline). + /// + public static class NinpathsLogReader + { + private static ReadOnlySpan PathPrefix => "Path: "u8; + + /// + /// Aggregates every valid Path line from into + /// a new instance. + /// + public static NinpathsStatistics Read(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + var stats = new NinpathsStatistics(); + Read(stream, stats); + return stats; + } + + /// Aggregates Path lines from into . + public static void Read(Stream stream, NinpathsStatistics stats) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(stats); + + var window = new byte[NinpathsConstants.MaxFgetsChars]; + var block = new byte[NinpathsConstants.ReadBufferSize]; + var blockLength = 0; + var blockOffset = 0; + var valid = true; + while (true) + { + var n = ReadFgets(stream, window, block, ref blockLength, ref blockOffset); + if (n <= 0) + { + break; + } + + ProcessFgetsWindow(window.AsSpan(0, n), stats, ref valid); + } + } + + /// Applies INN procpaths to one fgets window (newline included when present). + internal static void ProcessFgetsWindow(ReadOnlySpan window, NinpathsStatistics stats, ref bool valid) + { + var body = window; + if (StartsWithPathPrefix(body)) + { + body = body[PathPrefix.Length..]; + } + + var end = 0; + for (; end < body.Length; end++) + { + if (IsSpace(body[end])) + { + break; + } + } + + if (end >= body.Length) + { + valid = false; + return; + } + + if (!valid) + { + valid = true; + return; + } + + var path = body[..end]; + var lastBang = path.LastIndexOf((byte)'!'); + var chopped = lastBang < 0 ? ReadOnlySpan.Empty : path[..lastBang]; + stats.AddChoppedPath(chopped); + stats.AddValidArticle(); + } + + private static int ReadFgets( + Stream stream, + byte[] window, + byte[] block, + ref int blockLength, + ref int blockOffset) + { + var written = 0; + while (written < window.Length) + { + if (blockOffset >= blockLength) + { + blockLength = stream.Read(block, 0, block.Length); + blockOffset = 0; + if (blockLength == 0) + { + return written; + } + } + + var b = block[blockOffset++]; + window[written++] = b; + if (b == (byte)'\n') + { + break; + } + } + + return written; + } + + private static bool StartsWithPathPrefix(ReadOnlySpan line) + { + var prefix = PathPrefix; + if (line.Length < prefix.Length) + { + return false; + } + + for (var i = 0; i < prefix.Length; i++) + { + var a = line[i]; + var b = prefix[i]; + if (a == b) + { + continue; + } + + if (a is >= (byte)'A' and <= (byte)'Z') + { + a += 32; + } + + if (b is >= (byte)'A' and <= (byte)'Z') + { + b += 32; + } + + if (a != b) + { + return false; + } + } + + return true; + } + + /// C isspace on unsigned char (C locale). + internal static bool IsSpace(byte value) => + value is 0x09 or 0x0A or 0x0B or 0x0C or 0x0D or 0x20; + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsProcessingService.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsProcessingService.cs new file mode 100644 index 00000000..c824841a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsProcessingService.cs @@ -0,0 +1,243 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Background ninpaths worker. Consumes completed Path-survey files without + /// blocking rotation or NNTP request processing. + /// + /// + /// The pending queue holds at most + /// completed files (not Path records). Worker exceptions are logged and do not + /// fault . The source file is streamed; raw Path lines + /// are not retained. + /// + public sealed class NinpathsProcessingService : IApplicationService + { + private readonly Channel _queue; + private readonly ConcurrentDictionary _seen = new(StringComparer.OrdinalIgnoreCase); + private readonly IEmailService _email; + private readonly IOptions _nntpdOptions; + private readonly IOptions _emailOptions; + private readonly TimeProvider _time; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new ninpaths processing service. + public NinpathsProcessingService( + IEmailService email, + IOptions nntpdOptions, + IOptions emailOptions, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(email); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(emailOptions); + ArgumentNullException.ThrowIfNull(logger); + _email = email; + _nntpdOptions = nntpdOptions; + _emailOptions = emailOptions; + _logger = logger; + _time = timeProvider ?? TimeProvider.System; + _queue = Channel.CreateBounded( + new BoundedChannelOptions(NinpathsConstants.PendingFileCapacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait, + }); + } + + /// + public string Name => "NinpathsProcessing"; + + /// + public Task? Execution => _execution; + + /// Gets the number of completed files waiting for the worker. + internal int PendingCount => _queue.Reader.Count; + + /// + /// Enqueues an already-open completed file. Returns + /// on duplicate path or a full queue (the caller must dispose ). + /// + internal bool TryEnqueue(NinpathsCompletedFile file) + { + ArgumentNullException.ThrowIfNull(file); + var key = System.IO.Path.GetFullPath(file.Path); + if (!_seen.TryAdd(key, 0)) + { + NinpathsLogMessages.Duplicate(_logger, file.Path); + return false; + } + + if (_queue.Writer.TryWrite(file)) + { + return true; + } + + _seen.TryRemove(key, out _); + NinpathsLogMessages.QueueFull(_logger, file.Path); + return false; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + NinpathsLogMessages.Started(_logger); + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Writer.TryComplete(); + try + { + await _runCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.WorkerFailed(_logger, ex); + } + } + + DrainPending(); + NinpathsLogMessages.Stopped(_logger); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await foreach (var file in _queue.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + using (file) + { + try + { + await ProcessAsync(file, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.ProcessingFailed(_logger, ex, file.Path); + } + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.WorkerFailed(_logger, ex); + } + } + + private async Task ProcessAsync(NinpathsCompletedFile file, CancellationToken cancellationToken) + { + var nntpd = _nntpdOptions.Value; + var recipients = NinpathsTop1000.GetRecipients(nntpd); + if (recipients.Length == 0) + { + NinpathsLogMessages.Disabled(_logger, file.Path); + return; + } + + NinpathsLogMessages.Processing(_logger, file.Path); + var stats = NinpathsLogReader.Read(file.Stream); + if (stats.TotalArticles == 0) + { + NinpathsLogMessages.EmptyDump(_logger, file.Path); + return; + } + + var (start, end, average) = NinpathsReportPeriod.Resolve(file.Path, _time); + var report = NinpathsReportFormatter.Format(stats, start, end, average); + if (report.Length == 0) + { + NinpathsLogMessages.EmptyDump(_logger, file.Path); + return; + } + + if (!EmailOptionsValidator.TryValidateMailbox(_emailOptions.Value.DefaultFrom, out var from) + && !EmailOptionsValidator.TryValidateMailbox(_emailOptions.Value.EnvelopeSender, out from)) + { + NinpathsLogMessages.MissingFrom(_logger, file.Path); + return; + } + + var message = NinpathsReportMail.TryCreate(report, nntpd.Fqdn, from, recipients); + if (message is null) + { + return; + } + + var result = await _email.SendAsync(message, cancellationToken).ConfigureAwait(false); + switch (result.Status) + { + case EmailEnqueueStatus.Accepted: + NinpathsLogMessages.EmailAccepted(_logger, file.Path, recipients.Length, report.Length); + break; + case EmailEnqueueStatus.Disabled: + NinpathsLogMessages.EmailDisabled(_logger, file.Path); + break; + default: + NinpathsLogMessages.EmailFailed( + _logger, + file.Path, + result.Status.ToString(), + result.Detail); + break; + } + } + + private void DrainPending() + { + while (_queue.Reader.TryRead(out var leftover)) + { + leftover.Dispose(); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportFormatter.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportFormatter.cs new file mode 100644 index 00000000..6593dd9a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportFormatter.cs @@ -0,0 +1,148 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Formats compact INN 3.1.x !!NINP dump output from aggregated statistics. + /// + /// + /// Does not reread the Path-survey file. Site numbers are assigned in INN + /// hash-table walk order. Returns an empty buffer when + /// is 0 (INN writedump returns -1). + /// + public static class NinpathsReportFormatter + { + /// + /// Writes the compact dump for . Empty when there is no traffic. + /// + public static byte[] Format( + NinpathsStatistics stats, + long startUnix, + long endUnix, + long averageUnix) + { + ArgumentNullException.ThrowIfNull(stats); + if (stats.TotalArticles == 0) + { + return []; + } + + using var buffer = new MemoryStream(4096); + using (var writer = new StreamWriter(buffer, Encoding.Latin1, 1024, leaveOpen: true)) + { + writer.NewLine = "\n"; + writer.Write("!!NINP "); + writer.Write(NinpathsConstants.Version); + writer.Write(' '); + writer.Write(startUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(endUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(stats.SiteCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(stats.TotalArticles.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(averageUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write('\n'); + + long numbered = 0; + var lineLength = 0; + var buckets = stats.Buckets; + for (var i = 0; i < buckets.Length; i++) + { + for (var site = buckets[i]; site is not null; site = site.BucketNext) + { + site.Number = numbered++; + var written = WriteSiteRecord(writer, site); + lineLength += written; + if (lineLength > NinpathsConstants.RecordLineSoftLimit) + { + writer.Write('\n'); + lineLength = 0; + } + else + { + writer.Write(' '); + } + } + } + + writer.Write('\n'); + writer.Write("!!NLREC\n"); + + long relations = 0; + lineLength = 0; + for (var i = 0; i < buckets.Length; i++) + { + for (var site = buckets[i]; site is not null; site = site.BucketNext) + { + if (site.Relations is null) + { + continue; + } + + lineLength += WriteColonNumber(writer, site.Number); + for (var rel = site.Relations; rel is not null; rel = rel.Next) + { + lineLength += WriteBangNumber(writer, rel.Target.Number); + if (rel.Tally > 1) + { + lineLength += WriteCommaNumber(writer, rel.Tally); + } + + relations++; + } + + if (lineLength > NinpathsConstants.RecordLineSoftLimit) + { + writer.Write('\n'); + lineLength = 0; + } + } + } + + writer.Write('\n'); + writer.Write("!!NLEND "); + writer.Write(relations.ToString(CultureInfo.InvariantCulture)); + writer.Write('\n'); + } + + return buffer.ToArray(); + } + + private static int WriteSiteRecord(StreamWriter writer, NinpathsSite site) + { + var id = Encoding.Latin1.GetString(site.Id); + var count = site.SentTo.ToString(CultureInfo.InvariantCulture); + writer.Write(id); + writer.Write(' '); + writer.Write(count); + return id.Length + 1 + count.Length; + } + + private static int WriteColonNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write(':'); + writer.Write(text); + return 1 + text.Length; + } + + private static int WriteBangNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write('!'); + writer.Write(text); + return 1 + text.Length; + } + + private static int WriteCommaNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write(','); + writer.Write(text); + return 1 + text.Length; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportMail.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportMail.cs new file mode 100644 index 00000000..b511eb87 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportMail.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Builds one for a compact ninpaths dump (sendinpaths To/Subject/Auto-Submitted). + /// + internal static class NinpathsReportMail + { + /// + /// Creates a single message with every recipient in To. Returns + /// when there is no dump or no recipients. + /// + public static EmailMessage? TryCreate( + ReadOnlyMemory report, + string fqdn, + string fromMailbox, + IReadOnlyList recipients) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + ArgumentException.ThrowIfNullOrWhiteSpace(fromMailbox); + ArgumentNullException.ThrowIfNull(recipients); + if (report.IsEmpty || recipients.Count == 0) + { + return null; + } + + var to = new EmailAddress[recipients.Count]; + for (var i = 0; i < recipients.Count; i++) + { + to[i] = new EmailAddress(recipients[i]); + } + + return new EmailMessage + { + From = new EmailAddress(fromMailbox), + To = to, + Subject = NinpathsConstants.EmailSubjectPrefix + fqdn, + Body = report, + ContentType = "text/plain", + Charset = "us-ascii", + Headers = [new EmailHeader("Auto-Submitted", "auto-generated")], + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportPeriod.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportPeriod.cs new file mode 100644 index 00000000..f793ad78 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsReportPeriod.cs @@ -0,0 +1,66 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Derives INN dump timestamps from a completed Path-survey file and . + /// + /// + /// INN starttime is process start and endtime is time(NULL) at dump. + /// For a completed daily inpaths-yyyyMMdd.log the reporting period start is local + /// midnight of that date. Articles are aggregated as a batch at dump time, so + /// avgtime = (atimes / total) + starttime equals dump time (INN would add + /// time(0) - starttime once per article during a same-moment batch). + /// + internal static class NinpathsReportPeriod + { + /// + /// Resolves starttime, endtime, and avgtime Unix seconds for a dump. + /// + public static (long StartUnix, long EndUnix, long AverageUnix) Resolve( + string completedFilePath, + TimeProvider time) + { + ArgumentException.ThrowIfNullOrWhiteSpace(completedFilePath); + ArgumentNullException.ThrowIfNull(time); + + var now = time.GetUtcNow(); + var end = now.ToUnixTimeSeconds(); + if (!TryParseSurveyDate(completedFilePath, out var date)) + { + return (end, end, end); + } + + var unspecified = new DateTime(date.Year, date.Month, date.Day, 0, 0, 0, DateTimeKind.Unspecified); + var offset = time.LocalTimeZone.GetUtcOffset(unspecified); + var start = new DateTimeOffset(unspecified, offset).ToUnixTimeSeconds(); + return (start, end, end); + } + + /// Parses inpaths-yyyyMMdd.log from a file name or path. + internal static bool TryParseSurveyDate(string completedFilePath, out DateOnly date) + { + date = default; + var name = Path.GetFileName(completedFilePath.AsSpan()); + const string prefix = "inpaths-"; + const string suffix = ".log"; + if (name.Length != prefix.Length + 8 + suffix.Length) + { + return false; + } + + if (!name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + || !name.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return DateOnly.TryParseExact( + name.Slice(prefix.Length, 8), + "yyyyMMdd", + CultureInfo.InvariantCulture, + DateTimeStyles.None, + out date); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsStatistics.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsStatistics.cs new file mode 100644 index 00000000..739ca514 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsStatistics.cs @@ -0,0 +1,217 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Compact INN ninpaths aggregation: unique sites and pairwise Path relations. + /// + /// + /// Memory is O(unique sites + unique directed relations), not O(articles). + /// Raw Path observations are not retained. Hash-table layout matches INN + /// hosthash (FNV-style hash, chain prepend, dump numbering). + /// + public sealed class NinpathsStatistics + { + private readonly NinpathsSite?[] _buckets = new NinpathsSite?[NinpathsConstants.HashTableSize]; + private long _sites; + private long _total; + + /// Gets the number of Path observations that INN would count in total. + public long TotalArticles => _total; + + /// Gets the number of unique interned sites. + public long SiteCount => _sites; + + /// + /// Applies one Path value after INN last-element chopping (may be empty). + /// Does not increment ; the reader does that + /// for valid lines even when this method stops early on an overlong token. + /// + internal void AddChoppedPath(ReadOnlySpan chopped) + { + NinpathsSite? previous = null; + var remaining = chopped; + while (!remaining.IsEmpty) + { + var bang = remaining.IndexOf((byte)'!'); + ReadOnlySpan token; + if (bang < 0) + { + token = remaining; + remaining = default; + } + else + { + token = remaining[..bang]; + remaining = remaining[(bang + 1)..]; + while (!remaining.IsEmpty && remaining[0] == (byte)'!') + { + remaining = remaining[1..]; + } + } + + if (token.Length > NinpathsConstants.MaxHostChars) + { + return; + } + + var site = GetOrAdd(token); + site.SentTo++; + if (previous is not null && !ReferenceEquals(previous, site)) + { + IncrementRelation(previous, site); + } + + previous = site; + } + } + + /// Counts one valid Path line (INN ++total). + internal void AddValidArticle() => _total++; + + internal NinpathsSite?[] Buckets => _buckets; + + /// Looks up INN sentto for a site id, using case-sensitive strcmp semantics. + public bool TryGetSentTo(ReadOnlySpan siteId, out long sentTo) + { + var site = Find(siteId); + if (site is null) + { + sentTo = 0; + return false; + } + + sentTo = site.SentTo; + return true; + } + + /// Looks up the directed relation tally from to . + public bool TryGetRelation(ReadOnlySpan from, ReadOnlySpan to, out long tally) + { + tally = 0; + var source = Find(from); + var target = Find(to); + if (source is null || target is null) + { + return false; + } + + for (var rel = source.Relations; rel is not null; rel = rel.Next) + { + if (ReferenceEquals(rel.Target, target)) + { + tally = rel.Tally; + return true; + } + } + + return false; + } + + /// INN hash() over a NUL-free token (FNV-style). + internal static int Hash(ReadOnlySpan str) + { + ulong val = 0; + for (var i = 0; i < str.Length; i++) + { + val *= NinpathsConstants.HashPrime; + val ^= str[i]; + } + + return (int)(val & (NinpathsConstants.HashTableSize - 1)); + } + + internal NinpathsSite? Find(ReadOnlySpan token) + { + var index = Hash(token); + for (var site = _buckets[index]; site is not null; site = site.BucketNext) + { + if (token.SequenceEqual(site.Id)) + { + return site; + } + } + + return null; + } + + /// Latin-1 helper for tests that start from managed strings. + public bool TryGetSentTo(string siteId, out long sentTo) + { + ArgumentNullException.ThrowIfNull(siteId); + return TryGetSentTo(Encoding.Latin1.GetBytes(siteId), out sentTo); + } + + /// Latin-1 helper for tests that start from managed strings. + public bool TryGetRelation(string from, string to, out long tally) + { + ArgumentNullException.ThrowIfNull(from); + ArgumentNullException.ThrowIfNull(to); + return TryGetRelation(Encoding.Latin1.GetBytes(from), Encoding.Latin1.GetBytes(to), out tally); + } + + private NinpathsSite GetOrAdd(ReadOnlySpan token) + { + var index = Hash(token); + for (var site = _buckets[index]; site is not null; site = site.BucketNext) + { + if (token.SequenceEqual(site.Id)) + { + return site; + } + } + + var created = new NinpathsSite + { + Id = token.ToArray(), + BucketNext = _buckets[index], + }; + _buckets[index] = created; + _sites++; + return created; + } + + private static void IncrementRelation(NinpathsSite from, NinpathsSite to) + { + for (var rel = from.Relations; rel is not null; rel = rel.Next) + { + if (ReferenceEquals(rel.Target, to)) + { + rel.Tally++; + return; + } + } + + from.Relations = new NinpathsRelation + { + Target = to, + Tally = 1, + Next = from.Relations, + }; + } + } + + /// One interned Path hop (INN nrec). + internal sealed class NinpathsSite + { + public NinpathsSite? BucketNext; + + public NinpathsRelation? Relations; + + public byte[] Id { get; set; } = []; + + public long Number; + + public long SentTo; + } + + /// One directed relation (INN trec). + internal sealed class NinpathsRelation + { + public NinpathsRelation? Next; + + public NinpathsSite Target { get; set; } = null!; + + public long Tally; + } +} diff --git a/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsTop1000.cs b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsTop1000.cs new file mode 100644 index 00000000..b322fdd7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Ninpaths/NinpathsTop1000.cs @@ -0,0 +1,51 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Ninpaths +{ + /// + /// Resolves Nntpd:Top1000 recipients. Missing, null, empty, and + /// whitespace-only entries disable or skip; remaining mailboxes enable reporting. + /// + internal static class NinpathsTop1000 + { + /// Returns distinct trimmed mailboxes, ignoring whitespace-only entries. + public static string[] GetRecipients(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Top1000 is null || options.Top1000.Length == 0) + { + return []; + } + + List? list = null; + foreach (var raw in options.Top1000) + { + if (!EmailOptionsValidator.TryValidateMailbox(raw, out var mailbox)) + { + continue; + } + + list ??= new List(options.Top1000.Length); + var exists = false; + for (var i = 0; i < list.Count; i++) + { + if (string.Equals(list[i], mailbox, StringComparison.OrdinalIgnoreCase)) + { + exists = true; + break; + } + } + + if (!exists) + { + list.Add(mailbox); + } + } + + return list is null ? [] : [.. list]; + } + + /// True when at least one usable Top1000 mailbox is configured. + public static bool IsEnabled(NntpdOptions options) => GetRecipients(options).Length > 0; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/AccountByteConsumeResult.cs b/src/VectorNNTPD.ReaderService/NntpDb/AccountByteConsumeResult.cs new file mode 100644 index 00000000..6688488e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/AccountByteConsumeResult.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Outcome of a durable account_byte_limit consume or remaining-quota read. + public enum AccountByteConsumeStatus + { + /// The account row exists and the remaining value is valid. + Consumed, + + /// No nntpusers row matched the plaintext account name. + AccountNotFound, + } + + /// Durable byte-quota result. is never negative. + public readonly struct AccountByteConsumeResult + { + /// Initializes a clamped durable quota result. + public AccountByteConsumeResult(AccountByteConsumeStatus status, long remaining, long consumed) + { + ArgumentOutOfRangeException.ThrowIfNegative(remaining); + ArgumentOutOfRangeException.ThrowIfNegative(consumed); + Status = status; + Remaining = remaining; + Consumed = consumed; + } + + /// Gets whether the row was found or missing. + public AccountByteConsumeStatus Status { get; } + + /// Gets remaining bytes after the operation. Always >= 0. + public long Remaining { get; } + + /// Gets how many bytes were actually subtracted. Always >= 0. + public long Consumed { get; } + + /// Gets whether the row exists and remaining quota is known. + public bool Succeeded => Status == AccountByteConsumeStatus.Consumed; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/INntpArticlePolicySource.cs b/src/VectorNNTPD.ReaderService/NntpDb/INntpArticlePolicySource.cs new file mode 100644 index 00000000..49ce8b0b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/INntpArticlePolicySource.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// + /// Supplies the article-size and Path-tracker values captured for one NNTPD article operation. + /// + public interface INntpArticlePolicySource + { + /// Copies the published shared configuration. + /// Published maxartsize in bytes. + /// Published sitename. + /// when no snapshot has been published. + bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName); + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/INntpDbConnection.cs b/src/VectorNNTPD.ReaderService/NntpDb/INntpDbConnection.cs new file mode 100644 index 00000000..4f594482 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/INntpDbConnection.cs @@ -0,0 +1,91 @@ +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// + /// NNTPD query surface over one logical MySQL connection. + /// Dispose returns the physical connection to MySqlConnector's pool. + /// + public interface INntpDbConnection : INntpDbSession + { + /// + /// Executes and returns every row. + /// + /// Token used to cancel the query. + /// The complete result set. The caller must not retain this connection. + ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken); + + /// + /// Looks up one nntpusers row for reader authentication. + /// + /// Plaintext wire username. Bound as @account_name. + /// Token used to cancel the query. + /// The mapped account, or when no row exists. + ValueTask QueryUserAccountAsync( + string accountName, + CancellationToken cancellationToken); + + /// + /// Executes and returns every row. + /// + /// Token used to cancel the query. + /// Enabled moderator rows in moderator_id order. + ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken); + + /// + /// Loads one published PostFilter revision and the collection rows that + /// belong to that revision. + /// + /// Token used to cancel the queries. + /// + /// The mapped policy, or when + /// nntppostfiltercurrent.policy_id = 1 is missing. + /// + ValueTask QueryPostFilterPolicyAsync( + CancellationToken cancellationToken); + + /// + /// Loads the singleton published Transit catalogue. + /// + /// Token used to cancel the queries. + /// + /// The mapped catalogue, or when + /// nntptransitcurrent.policy_id = 1 is missing. + /// + ValueTask QueryTransitCatalogueAsync( + CancellationToken cancellationToken); + + /// + /// Inserts one PostFilter rejection evidence row. Not used on the accept path. + /// + ValueTask InsertPostFilterRejectionAsync( + PostFilter.PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken); + + /// + /// Atomically subtracts from + /// account_byte_limit, clamping at zero. Remaining is never negative. + /// + /// Plaintext wire username. Bound as @account_name. + /// Non-negative byte count to subtract. + /// Token used to cancel the transaction. + /// + /// Found status, bytes actually subtracted, and remaining after the update. + /// + ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken); + + /// + /// Reads remaining account_byte_limit without mutating. Does not use the + /// AUTHINFO user-record cache. + /// + /// Plaintext wire username. Bound as @account_name. + /// Token used to cancel the query. + ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/MySqlNntpDbConnection.cs b/src/VectorNNTPD.ReaderService/NntpDb/MySqlNntpDbConnection.cs new file mode 100644 index 00000000..156c7771 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/MySqlNntpDbConnection.cs @@ -0,0 +1,803 @@ +using System.Data; +using System.Globalization; +using MySqlConnector; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// NNTPD queries over a shared logical session. Dispose returns it to the provider pool. + internal sealed class MySqlNntpDbConnection : INntpDbConnection + { + private readonly INntpDbSession _session; + + /// Initializes a new instance wrapping an already-open provider connection. + /// Open connection. This instance owns its disposal. + public MySqlNntpDbConnection(MySqlConnection connection) + : this(new MySqlNntpDbSession(connection)) + { + } + + /// Initializes a new instance wrapping a shared session. + /// Open session. This instance owns its disposal. + public MySqlNntpDbConnection(INntpDbSession session) + { + ArgumentNullException.ThrowIfNull(session); + _session = session; + } + + /// + public ValueTask SelectOneAsync(CancellationToken cancellationToken) => + _session.SelectOneAsync(cancellationToken); + + /// + public MySqlCommand CreateCommand() => _session.CreateCommand(); + + /// + public ValueTask BeginTransactionAsync(CancellationToken cancellationToken) => + _session.BeginTransactionAsync(cancellationToken); + + /// + public ValueTask BeginTransactionAsync( + IsolationLevel isolationLevel, + CancellationToken cancellationToken) => + _session.BeginTransactionAsync(isolationLevel, cancellationToken); + + /// + public async ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken) + { + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpGroupQueries.SelectNewsgroups; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var rows = new List(); + var index = 0; + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rows.Add(ReadGroupRow(reader, index)); + index++; + } + + return rows; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not Newsgroups.NewsgroupCatalogueException) + { + throw new NntpDbUnavailableException("MySQL newsgroup catalogue query failed.", ex); + } + } + + /// + public async ValueTask QueryUserAccountAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpUserQueries.SelectUserByName; + command.Parameters.AddWithValue("@account_name", accountName); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + return MapUserRecord(reader, accountName); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntpusers lookup failed.", ex); + } + } + + /// + public async ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken) + { + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpModeratorQueries.SelectEnabledModerators; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var rows = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rows.Add(MapModeratorRow(reader)); + } + + return rows; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntpmoderators catalogue query failed.", ex); + } + } + + /// + public async ValueTask QueryPostFilterPolicyAsync( + CancellationToken cancellationToken) + { + try + { + await using var transaction = await _session + .BeginTransactionAsync(IsolationLevel.RepeatableRead, cancellationToken) + .ConfigureAwait(false); + try + { + var revision = await ReadCurrentRevisionAsync(transaction, cancellationToken) + .ConfigureAwait(false); + if (revision is null) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return null; + } + + PostFilterPolicyRecord record; + await using (var policyCommand = CreateRevisionCommand( + NntpPostFilterQueries.SelectPolicy, + revision.Value, + transaction)) + { + await using var policyReader = await policyCommand + .ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false); + if (!await policyReader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException( + "nntppostfiltercurrent.revision " + + revision.Value.ToString(CultureInfo.InvariantCulture) + + " has no nntppostfilterpolicy row."); + } + + record = MapPolicyScalars(policyReader); + } + + var deniedAccounts = new List(); + var allowlistedAccounts = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectAccounts, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindDeny, + NntpPostFilterQueries.ListKindAllow, + deniedAccounts, + allowlistedAccounts, + cancellationToken) + .ConfigureAwait(false); + + var deniedCidrs = new List(); + var allowlistedCidrs = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectCidrs, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindDeny, + NntpPostFilterQueries.ListKindAllow, + deniedCidrs, + allowlistedCidrs, + cancellationToken) + .ConfigureAwait(false); + + var rejectArtTypes = new List(); + var excludeArtTypes = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectArtTypes, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindReject, + NntpPostFilterQueries.ListKindSaExclude, + rejectArtTypes, + excludeArtTypes, + cancellationToken) + .ConfigureAwait(false); + + var hosts = await ReadHostsAsync(revision.Value, transaction, cancellationToken) + .ConfigureAwait(false); + record.Options.DeniedAccounts = [.. deniedAccounts]; + record.Options.AllowlistedAccounts = [.. allowlistedAccounts]; + record.Options.DeniedCidrs = [.. deniedCidrs]; + record.Options.AllowlistedCidrs = [.. allowlistedCidrs]; + record.Options.RejectArtTypes = [.. rejectArtTypes]; + record.Options.SpamAssassin.ExcludeArtTypes = [.. excludeArtTypes]; + record.Options.SpamAssassin.Hosts = [.. hosts]; + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return record; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not InvalidOperationException) + { + throw new NntpDbUnavailableException("MySQL nntppostfilterpolicy query failed.", ex); + } + } + + /// + public async ValueTask InsertPostFilterRejectionAsync( + PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(evidence); + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpPostFilterQueries.InsertRejection; + command.Parameters.AddWithValue("@rejected_utc", evidence.RejectedUtc.UtcDateTime); + command.Parameters.AddWithValue("@revision", evidence.PolicyRevision); + command.Parameters.AddWithValue( + "@account_name", + evidence.AccountName is null ? DBNull.Value : evidence.AccountName); + var sourceIp = command.Parameters.Add("@source_ip", MySqlDbType.VarBinary, 16); + sourceIp.Value = SourceAddressIdentity.ToNetworkBytes(evidence.SourceAddress); + command.Parameters.AddWithValue("@art_type", (uint)evidence.ArtType); + command.Parameters.AddWithValue( + "@message_id", + evidence.MessageId is null ? DBNull.Value : evidence.MessageId); + command.Parameters.AddWithValue("@article_size", evidence.ArticleSize); + command.Parameters.AddWithValue("@stage", evidence.Stage.ToString()); + command.Parameters.AddWithValue("@reason", evidence.Reason); + command.Parameters.AddWithValue( + "@sa_status", + evidence.SpamAssassinStatus is { } saStatus ? saStatus.ToString() : DBNull.Value); + command.Parameters.AddWithValue( + "@sa_score", + evidence.SpamAssassinScore is { } score ? score : DBNull.Value); + command.Parameters.AddWithValue( + "@sa_threshold", + evidence.SpamAssassinThreshold is { } threshold ? threshold : DBNull.Value); + var payload = evidence.ArticlePayload; + command.Parameters.AddWithValue( + "@article_payload", + payload is { Length: > 0 } ? payload.Value.ToArray() : DBNull.Value); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntppostfilterrejections insert failed.", ex); + } + } + + /// + public async ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + try + { + await using var transaction = await _session + .BeginTransactionAsync(cancellationToken) + .ConfigureAwait(false); + try + { + var locked = await ReadByteQuotaAsync( + NntpUserQueries.SelectByteQuotaForUpdate, + accountName, + transaction, + cancellationToken) + .ConfigureAwait(false); + if (locked is null) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0); + } + + var current = ClampNonNegative(locked.Value.Remaining); + var consumed = bytes > current ? current : bytes; + await using (var update = _session.CreateCommand()) + { + update.Transaction = transaction; + update.CommandText = NntpUserQueries.ConsumeAccountBytes; + update.Parameters.AddWithValue("@account_name", accountName); + update.Parameters.AddWithValue("@bytes", bytes); + await update.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var after = await ReadByteQuotaAsync( + NntpUserQueries.SelectAccountByteRemaining, + accountName, + transaction, + cancellationToken) + .ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + var remaining = after is { } row + ? ClampNonNegative(row.Remaining) + : 0L; + return new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, consumed); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL account byte-quota consume failed.", ex); + } + } + + /// + public async ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + var row = await ReadByteQuotaAsync( + NntpUserQueries.SelectAccountByteRemaining, + accountName, + transaction: null, + cancellationToken) + .ConfigureAwait(false); + if (row is null) + { + return new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0); + } + + return new AccountByteConsumeResult( + AccountByteConsumeStatus.Consumed, + ClampNonNegative(row.Value.Remaining), + 0); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL account byte-quota query failed.", ex); + } + } + + /// + public ValueTask DisposeAsync() => _session.DisposeAsync(); + + /// Maps one enabled nntpmoderators row. CHAR columns are trimmed. + internal static NntpModeratorRow MapModeratorRow(MySqlDataReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + var id = Convert.ToInt64(reader.GetValue(0), System.Globalization.CultureInfo.InvariantCulture); + var pattern = reader.IsDBNull(1) ? string.Empty : reader.GetString(1).Trim(); + var address = reader.IsDBNull(2) ? string.Empty : reader.GetString(2).Trim(); + var account = reader.IsDBNull(3) ? string.Empty : reader.GetString(3).Trim(); + return new NntpModeratorRow(id, pattern, address, account); + } + + /// + /// Maps one nntpusers row from . + /// Flag columns are true only for Y. NULL rate/byte limits map to 0. + /// Ordinals: 0 pass, 1 salt, 2 iterations, 3 stored key, 4 server key, 5 plain, + /// 6 scram256, 7 rate bps, 8 byte remaining, 9 session limit, 10 srcip limit, + /// 11 enabled, 12 customer, 13 account_art_type. + /// + internal static NntpUserRecord MapUserRecord(MySqlDataReader reader, string accountName) + { + ArgumentNullException.ThrowIfNull(reader); + var password = reader.IsDBNull(0) ? string.Empty : reader.GetString(0); + var scramSalt = ReadBinaryColumn(reader, 1); + var scramIterations = reader.IsDBNull(2) ? 0 : Convert.ToInt32(reader.GetValue(2)); + var scramStoredKey = ReadBinaryColumn(reader, 3); + var scramServerKey = ReadBinaryColumn(reader, 4); + var allowAuthPlain = IsYesFlag(reader, 5); + var allowAuthScram256 = IsYesFlag(reader, 6); + var rateLimitBps = reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7)); + var byteLimit = reader.IsDBNull(8) ? 0L : ConvertByteLimit(reader.GetValue(8)); + var sessionLimit = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)); + var srcIpLimit = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)); + var isEnabled = IsYesFlag(reader, 11); + var customerId = ReadCustomerId(reader, 12); + var allowedArtTypes = ReadAccountArtType(reader, 13); + return new NntpUserRecord( + accountName, + password, + allowAuthPlain, + allowAuthScram256, + scramSalt, + scramIterations, + scramStoredKey, + scramServerKey, + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + isEnabled, + customerId, + allowedArtTypes); + } + + private async ValueTask ReadByteQuotaAsync( + string commandText, + string accountName, + MySqlTransaction? transaction, + CancellationToken cancellationToken) + { + await using var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = commandText; + command.Parameters.AddWithValue("@account_name", accountName); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + var remaining = reader.IsDBNull(0) ? 0L : ConvertByteLimit(reader.GetValue(0)); + return new ByteQuotaRow(remaining); + } + + /// + /// Converts a MySQL integer account_byte_limit to a non-negative . + /// Unsigned values above saturate rather than wrap. + /// Negative signed values become 0 (exhausted / never allowed). + /// + internal static long ConvertByteLimit(object? value) => + value switch + { + null or DBNull => 0, + long signed64 => ClampNonNegative(signed64), + ulong unsigned64 => unsigned64 > long.MaxValue ? long.MaxValue : (long)unsigned64, + int signed32 => ClampNonNegative(signed32), + uint unsigned32 => unsigned32, + short signed16 => ClampNonNegative(signed16), + ushort unsigned16 => unsigned16, + sbyte signed8 => ClampNonNegative(signed8), + byte unsigned8 => unsigned8, + decimal dec when dec < 0 => 0, + decimal dec when dec > long.MaxValue => long.MaxValue, + decimal dec => (long)dec, + _ => ClampNonNegative(Convert.ToInt64(value, System.Globalization.CultureInfo.InvariantCulture)), + }; + + private static long ClampNonNegative(long value) => value < 0 ? 0 : value; + + private readonly record struct ByteQuotaRow(long Remaining); + + private static bool IsYesFlag(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return false; + } + + return string.Equals(Convert.ToString(reader.GetValue(ordinal)), "Y", StringComparison.OrdinalIgnoreCase); + } + + private static string ReadCustomerId(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return string.Empty; + } + + var value = reader.GetValue(ordinal); + return value switch + { + Guid guid => guid.ToString("D"), + string text => text, + _ => Convert.ToString(value) ?? string.Empty, + }; + } + + private static ArticleType ReadAccountArtType(MySqlDataReader reader, int ordinal) + { + if (ordinal >= reader.FieldCount) + { + throw new InvalidOperationException( + "nntpusers.account_art_type was not selected. Apply docs/schema/nntpusers-account-art-type.sql."); + } + + if (reader.IsDBNull(ordinal)) + { + throw new InvalidOperationException("nntpusers.account_art_type is NULL."); + } + + return (ArticleType)Convert.ToUInt32(reader.GetValue(ordinal), CultureInfo.InvariantCulture); + } + + private static ReadOnlyMemory ReadBinaryColumn(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return ReadOnlyMemory.Empty; + } + + var value = reader.GetValue(ordinal); + return value switch + { + byte[] bytes when bytes.Length == 0 => ReadOnlyMemory.Empty, + byte[] bytes => bytes, + ReadOnlyMemory memory => memory, + _ => ReadOnlyMemory.Empty, + }; + } + + /// Reads one nntpgroups row, preserving unsigned integer width. + internal static NntpGroupRow ReadGroupRow(MySqlDataReader reader, int index) + { + ArgumentNullException.ThrowIfNull(reader); + if (reader.IsDBNull(0)) + { + throw new Newsgroups.NewsgroupCatalogueException($"nntpgroups row {index} has a NULL group_name."); + } + + var name = reader.GetString(0); + var description = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); + var high = ConvertUnsignedCount(reader.GetValue(2), "count_high", name); + var low = ConvertUnsignedCount(reader.GetValue(3), "count_low", name); + var status = ConvertPostingStatus(reader.GetValue(4), name); + return new NntpGroupRow(name, description, high, low, status); + } + + /// + /// Converts a MySQL unsigned/signed integer column to without + /// narrowing or floating-point conversion. + /// + internal static ulong ConvertUnsignedCount(object? value, string column, string groupName) => + value switch + { + ulong unsigned64 => unsigned64, + uint unsigned32 => unsigned32, + ushort unsigned16 => unsigned16, + byte unsigned8 => unsigned8, + long signed64 when signed64 >= 0 => (ulong)signed64, + int signed32 when signed32 >= 0 => (ulong)signed32, + short signed16 when signed16 >= 0 => (ulong)signed16, + sbyte signed8 when signed8 >= 0 => (ulong)signed8, + _ => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' column {column} is not a non-negative integer."), + }; + + /// + /// Reads posting_status as a single supported octet. + /// MySQL ENUM/CHAR typically arrives as a one-character string from MySqlConnector; + /// byte/sbyte/char are also accepted. Values other than y/n/m/x/j + /// are rejected, including RFC 6048 =<newsgroup>. + /// + internal static byte ConvertPostingStatus(object? value, string groupName) + { + var status = value switch + { + byte b => b, + sbyte sb => (byte)sb, + char ch => (byte)ch, + string text when text.Length == 1 => (byte)text[0], + string text when text.Length > 0 && text[0] == '=' => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' uses unsupported RFC 6048 = posting_status."), + _ => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' has an unreadable posting_status."), + }; + + if (!Newsgroups.NewsgroupPostingStatusOctets.IsSupported(status)) + { + var reason = status == (byte)'=' + ? $"nntpgroups row '{groupName}' uses unsupported RFC 6048 = posting_status." + : $"nntpgroups row '{groupName}' has invalid posting_status 0x{status:X2}."; + throw new Newsgroups.NewsgroupCatalogueException(reason); + } + + return status; + } + + /// + /// Converts a MySQL SELECT 1 scalar, including the connector's result type. + /// + internal static int ConvertSelectOneScalar(object? result) => + MySqlNntpDbSession.ConvertSelectOneScalar(result); + + private static PostFilterPolicyRecord MapPolicyScalars(MySqlDataReader reader) + { + var revision = Convert.ToInt64(reader.GetValue(0), System.Globalization.CultureInfo.InvariantCulture); + var updated = reader.GetDateTime(1); + var updatedUtc = updated.Kind == DateTimeKind.Unspecified + ? new DateTimeOffset(DateTime.SpecifyKind(updated, DateTimeKind.Utc)) + : new DateTimeOffset(updated.ToUniversalTime()); + var gateText = reader.GetString(2).Trim(); + if (!Enum.TryParse(gateText, ignoreCase: true, out var gate) + || !Enum.IsDefined(gate)) + { + throw new InvalidOperationException($"PostFilter gate is not a defined value: '{gateText}'."); + } + + var options = new PostFilterOptions + { + Gate = gate, + Quota = new PostFilterQuotaOptions + { + LongWindow = TimeSpan.FromMilliseconds(ReadInt64(reader, 3)), + ShortWindow = TimeSpan.FromMilliseconds(ReadInt64(reader, 4)), + MaxMessagesLong = ReadInt64(reader, 5), + MaxBytesLong = ReadInt64(reader, 6), + MaxIdenticalLong = ReadInt64(reader, 7), + MaxMessagesShort = ReadInt64(reader, 8), + MaxBytesShort = ReadInt64(reader, 9), + MaxIdenticalShort = ReadInt64(reader, 10), + }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = IsYes(reader, 11), + OnFailure = ReadOnFailure(reader, 12), + MaxArticleSize = checked((int)ReadInt64(reader, 13)), + Port = checked((int)ReadInt64(reader, 14)), + ProtocolVersion = reader.IsDBNull(15) ? string.Empty : reader.GetString(15).Trim(), + MaxConnections = checked((int)ReadInt64(reader, 16)), + HostSelection = ReadHostSelection(reader, 17), + ConnectTimeout = TimeSpan.FromMilliseconds(ReadInt64(reader, 18)), + OperationTimeout = TimeSpan.FromMilliseconds(ReadInt64(reader, 19)), + }, + }; + + return new PostFilterPolicyRecord(revision, updatedUtc, options); + } + + private async Task ReadCurrentRevisionAsync( + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpPostFilterQueries.SelectCurrentRevision; + var scalar = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return scalar is null or DBNull + ? null + : Convert.ToInt64(scalar, CultureInfo.InvariantCulture); + } + + private MySqlCommand CreateRevisionCommand( + string commandText, + long revision, + MySqlTransaction transaction) + { + var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = commandText; + command.Parameters.AddWithValue("@revision", revision); + return command; + } + + private async Task ReadKeyedStringsAsync( + string commandText, + long revision, + MySqlTransaction transaction, + string firstKind, + string secondKind, + List first, + List second, + CancellationToken cancellationToken) + { + await using var command = CreateRevisionCommand(commandText, revision, transaction); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var kind = reader.IsDBNull(0) ? string.Empty : reader.GetString(0).Trim(); + var value = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); + if (string.Equals(kind, firstKind, StringComparison.OrdinalIgnoreCase)) + { + first.Add(value); + } + else if (string.Equals(kind, secondKind, StringComparison.OrdinalIgnoreCase)) + { + second.Add(value); + } + else + { + throw new InvalidOperationException($"PostFilter collection list_kind is unknown: '{kind}'."); + } + } + } + + private async Task> ReadHostsAsync( + long revision, + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = CreateRevisionCommand( + NntpPostFilterQueries.SelectHosts, + revision, + transaction); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var hosts = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + hosts.Add(reader.IsDBNull(0) ? string.Empty : reader.GetString(0)); + } + + return hosts; + } + + private static long ReadInt64(MySqlDataReader reader, int ordinal) => + Convert.ToInt64(reader.GetValue(ordinal), System.Globalization.CultureInfo.InvariantCulture); + + private static bool IsYes(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return false; + } + + var value = reader.GetValue(ordinal); + return value switch + { + string text => text.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase), + char ch => ch is 'Y' or 'y', + byte b => b is (byte)'Y' or (byte)'y', + _ => false, + }; + } + + private static PostFilterSpamOnFailure? ReadOnFailure(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return null; + } + + var text = reader.GetString(ordinal).Trim(); + if (text.Length == 0) + { + return null; + } + + if (!Enum.TryParse(text, ignoreCase: true, out var parsed) + || !Enum.IsDefined(parsed)) + { + throw new InvalidOperationException($"PostFilter SpamAssassin OnFailure is not a defined value: '{text}'."); + } + + return parsed; + } + + private static PostFilterSpamAssassinHostSelection ReadHostSelection(MySqlDataReader reader, int ordinal) + { + var text = reader.IsDBNull(ordinal) + ? PostFilterSpamAssassinHostSelection.RoundRobin.ToString() + : reader.GetString(ordinal).Trim(); + if (!Enum.TryParse(text, ignoreCase: true, out var parsed) + || !Enum.IsDefined(parsed)) + { + throw new InvalidOperationException($"PostFilter SpamAssassin HostSelection is not a defined value: '{text}'."); + } + + return parsed; + } + + /// + public async ValueTask QueryTransitCatalogueAsync(CancellationToken cancellationToken) + { + try + { + return await NntpTransitCatalogueLoader.LoadAsync(_session, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not InvalidOperationException) + { + throw new NntpDbUnavailableException("MySQL Transit catalogue query failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpArticlePolicyCapture.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpArticlePolicyCapture.cs new file mode 100644 index 00000000..9b3be219 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpArticlePolicyCapture.cs @@ -0,0 +1,48 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// + /// Captures one article-size ceiling and Path tracker for a single article operation. + /// + internal static class NntpArticlePolicyCapture + { + /// + /// Copies the published maxartsize when the session has a shared-configuration snapshot. + /// Otherwise uses . + /// The returned ceiling is the smaller of that value and the connection's captured Receive + /// MaxArticleBytes. A peer cannot exceed the published global ceiling. + /// A non-positive peer size, or a peer size above , adds no further limit. + /// + /// Session that may hold the catalogue and the identified peer. + /// Limit used when no snapshot is published on the session. + /// Article-size boundary for this operation. + /// UTF-8 tracker component, or when the static tracker remains in use. + internal static void Capture( + NntpSession session, + int fallbackMaxArticleBytes, + out int maxArticleBytes, + out byte[]? siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(session); + int globalOrFallback; + if (session.SharedConfiguration is { } catalogue + && catalogue.TryGetArticlePolicy(out var publishedMax, out var siteName)) + { + globalOrFallback = publishedMax; + siteNameUtf8 = Encoding.UTF8.GetBytes(siteName); + } + else + { + globalOrFallback = fallbackMaxArticleBytes; + siteNameUtf8 = null; + } + + maxArticleBytes = TransitReceivePolicy.EffectiveMaxArticleBytes( + globalOrFallback, + session.Authorization.TransitPeerPolicy); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpDbConnections.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpDbConnections.cs new file mode 100644 index 00000000..97d31f37 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpDbConnections.cs @@ -0,0 +1,29 @@ +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Opens an NNTPD query connection from the shared database service. + internal static class NntpDbConnections + { + /// + /// Opens a session and returns the NNTPD query surface. + /// A session that already implements is returned unchanged. + /// + /// Started database service. + /// Token used to cancel the open. + /// An open query connection. The caller disposes it. + public static async ValueTask OpenAsync( + NntpDbService service, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(service); + var session = await service.OpenAsync(cancellationToken).ConfigureAwait(false); + if (session is INntpDbConnection connection) + { + return connection; + } + + return new MySqlNntpDbConnection(session); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupQueries.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupQueries.cs new file mode 100644 index 00000000..d0eef9e6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupQueries.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Authoritative NntpDB queries used by the newsgroup catalogue. + public static class NntpGroupQueries + { + /// + /// Complete newsgroup catalogue source. Column set is exactly the fields + /// required by LIST ACTIVE / LIST NEWSGROUPS / GROUP metadata. + /// + public const string SelectNewsgroups = + """ + SELECT + group_name, + group_description, + count_high, + count_low, + posting_status + FROM nntpgroups + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupRow.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupRow.cs new file mode 100644 index 00000000..508fb00f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpGroupRow.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// + /// One raw nntpgroups row as read from MySQL. Validation and wire encoding + /// happen when a newsgroup snapshot is constructed. + /// + /// Original group_name text. + /// group_description, empty when the column is null. + /// Unsigned count_high water mark. + /// Unsigned count_low water mark. + /// Single status octet as stored (y/n/m/x/j). The RFC 6048 =<newsgroup> form is not represented. + public readonly record struct NntpGroupRow( + string GroupName, + string GroupDescription, + ulong CountHigh, + ulong CountLow, + byte PostingStatus); +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpModeratorQueries.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpModeratorQueries.cs new file mode 100644 index 00000000..5fe6980f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpModeratorQueries.cs @@ -0,0 +1,16 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Authoritative NntpDB query for the moderator authorization catalogue. + public static class NntpModeratorQueries + { + /// + /// Enabled nntpmoderators rows in first-match order (moderator_id ASC). + /// + public const string SelectEnabledModerators = + "SELECT " + + "moderator_id, group_pattern, moderator_address, account_name " + + "FROM nntpmoderators " + + "WHERE is_enabled = 'Y' " + + "ORDER BY moderator_id"; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpPostFilterQueries.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpPostFilterQueries.cs new file mode 100644 index 00000000..0b7e3733 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpPostFilterQueries.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Authoritative NntpDB queries for the cluster PostFilter policy. + /// + /// NNTPD only SELECTs one published revision. Schema DDL is + /// docs/schema/postfilter.sql. There is no application migration runner. + /// + public static class NntpPostFilterQueries + { + /// Published revision pointer. policy_id is always 1. + public const string SelectCurrentRevision = + "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"; + + /// Scalar policy for one revision. + public const string SelectPolicy = + """ + SELECT + revision, + updated_utc, + gate, + long_window_ms, + short_window_ms, + max_messages_long, + max_bytes_long, + max_identical_long, + max_messages_short, + max_bytes_short, + max_identical_short, + sa_enabled, + sa_on_failure, + sa_max_article_size, + sa_port, + sa_protocol_version, + sa_max_connections, + sa_host_selection, + sa_connect_timeout_ms, + sa_operation_timeout_ms + FROM nntppostfilterpolicy + WHERE revision = @revision + """; + + /// Deny/allow MD5 account identifiers for @revision. + public const string SelectAccounts = + "SELECT list_kind, account_name FROM nntppostfilteraccounts " + + "WHERE revision = @revision ORDER BY list_kind, account_name"; + + /// + /// Inserts one PostFilter rejection evidence row. Payload is the unstuffed + /// article when available. Never used on the accept path. + /// + public const string InsertRejection = + """ + INSERT INTO nntppostfilterrejections ( + rejected_utc, + revision, + account_name, + source_ip, + art_type, + message_id, + article_size, + stage, + reason, + sa_status, + sa_score, + sa_threshold, + article_payload + ) VALUES ( + @rejected_utc, + @revision, + @account_name, + @source_ip, + @art_type, + @message_id, + @article_size, + @stage, + @reason, + @sa_status, + @sa_score, + @sa_threshold, + @article_payload + ) + """; + + /// Deny/allow client CIDRs for @revision. + public const string SelectCidrs = + "SELECT list_kind, cidr FROM nntppostfiltercidrs " + + "WHERE revision = @revision ORDER BY list_kind, cidr"; + + /// ArtType lists for @revision. + public const string SelectArtTypes = + "SELECT list_kind, art_type FROM nntppostfilterarttypes " + + "WHERE revision = @revision ORDER BY list_kind, art_type"; + + /// SPAMD hosts for @revision in compiled order. + public const string SelectHosts = + "SELECT host FROM nntppostfiltersahosts WHERE revision = @revision ORDER BY host_order"; + + /// Account list kind stored for deny. + public const string ListKindDeny = "deny"; + + /// Account/CIDR list kind stored for allowlist. + public const string ListKindAllow = "allow"; + + /// ArtType list kind stored for Stage 3 reject. + public const string ListKindReject = "reject"; + + /// ArtType list kind stored for SpamAssassin exclusion. + public const string ListKindSaExclude = "sa_exclude"; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationLogMessages.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationLogMessages.cs new file mode 100644 index 00000000..4e612709 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Source-generated shared-configuration lifecycle log messages. + internal static partial class NntpSharedConfigurationLogMessages + { + [LoggerMessage( + EventId = 2260, + Level = LogLevel.Information, + Message = "Loading initial nntpsharedconfig")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2261, + Level = LogLevel.Information, + Message = "Published nntpsharedconfig maxArticleBytes={MaxArticleBytes} siteName={SiteName}")] + public static partial void SnapshotPublished(ILogger logger, int MaxArticleBytes, string SiteName); + + [LoggerMessage( + EventId = 2262, + Level = LogLevel.Error, + Message = "Initial nntpsharedconfig load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2263, + Level = LogLevel.Error, + Message = "nntpsharedconfig refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2264, + Level = LogLevel.Information, + Message = "nntpsharedconfig refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2265, + Level = LogLevel.Information, + Message = "nntpsharedconfig refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationService.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationService.cs new file mode 100644 index 00000000..766dd6f6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpSharedConfigurationService.cs @@ -0,0 +1,263 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// + /// Loads nntpsharedconfig during startup and refreshes it on this service's own loop. + /// + /// + /// The initial load fails startup. A later refresh failure keeps the last known-good snapshot. + /// One refresh runs at a time. This service does not query on an article path. + /// + internal sealed class NntpSharedConfigurationService : INntpSharedConfigurationCatalogue, INntpArticlePolicySource, IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. Owned by this service, not by a shared scheduler. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly NntpDbService _nntpDb; + private readonly IOptions _options; + private readonly IOptions _rabbitMq; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private PublishedSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + /// Started database service. + /// Live NNTPD options. ACME, DNS, and Cloudflare credential fields are replaced from each published snapshot. + /// Live RabbitMQ options. Username and password are replaced from each published snapshot. + /// Lifecycle logger. + public NntpSharedConfigurationService( + NntpDbService nntpDb, + IOptions options, + IOptions rabbitMq, + ILogger logger) + : this(nntpDb, options, logger, TimeProvider.System, RefreshInterval, rabbitMq) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + /// Started database service. + /// Live NNTPD options. ACME, DNS, and Cloudflare credential fields are replaced from each published snapshot. + /// Lifecycle logger. + /// Clock used by the refresh delay. + /// Delay between refreshes. + /// Live RabbitMQ options. Username and password are replaced from each published snapshot. + internal NntpSharedConfigurationService( + NntpDbService nntpDb, + IOptions options, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval, + IOptions? rabbitMq = null) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _options = options; + _rabbitMq = rabbitMq ?? Options.Create(new RabbitMqOptions()); + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "NntpSharedConfiguration"; + + /// + public Task? Execution => _execution; + + /// + public bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName) + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + maxArticleBytes = 0; + siteName = string.Empty; + return false; + } + + maxArticleBytes = snapshot.Value.MaxArticleBytes; + siteName = snapshot.Value.SiteName; + return true; + } + + /// + public NntpSharedConfiguration Current + { + get + { + var snapshot = Volatile.Read(ref _current); + return snapshot?.Value ?? throw new InvalidOperationException("nntpsharedconfig has not been published."); + } + } + + private sealed class PublishedSnapshot + { + internal PublishedSnapshot(NntpSharedConfiguration value) => Value = value; + + internal NntpSharedConfiguration Value { get; } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + NntpSharedConfigurationLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + NntpSharedConfigurationLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + NntpSharedConfigurationLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + NntpSharedConfigurationLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NntpSharedConfigurationLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadAsync(CancellationToken cancellationToken) + { + await using var session = await _nntpDb.OpenAsync(cancellationToken).ConfigureAwait(false); + return await NntpSharedConfigurationReader.ReadAsync(session, cancellationToken).ConfigureAwait(false); + } + + private void Publish(NntpSharedConfiguration snapshot) + { + var options = _options.Value; + if (options.ServerId is { } serverId && ServerIdRules.IsInRange(serverId)) + { + var prefix = string.IsNullOrWhiteSpace(options.HostApplicationPrefix) + ? NntpdOptions.ApplicationPrefix + : options.HostApplicationPrefix; + var fqdn = snapshot.RequireApplicationFqdn(prefix, serverId); + if (options.IsTlsListenerEnabled) + { + DnsZoneCoverage.RequireIdentitiesInDnsZone( + CertificateIdentities.ForFqdn(fqdn, options.IncludeNewsHostnameInCertificate), + snapshot.DnsSuffix); + } + } + + snapshot.CopyAcmeDnsTo(options); + snapshot.CopyRabbitMqCredentialsTo(_rabbitMq.Value); + Volatile.Write(ref _current, new PublishedSnapshot(snapshot)); + NntpSharedConfigurationLogMessages.SnapshotPublished(_logger, snapshot.MaxArticleBytes, snapshot.SiteName); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitCatalogueLoader.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitCatalogueLoader.cs new file mode 100644 index 00000000..07deed09 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitCatalogueLoader.cs @@ -0,0 +1,416 @@ +using System.Data; +using System.Globalization; +using MySqlConnector; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Reads one coherent Transit publication inside a repeatable-read transaction. + internal static class NntpTransitCatalogueLoader + { + /// + /// Loads nntptransitcurrent. Returns when the singleton row is missing. + /// + public static async ValueTask LoadAsync( + INntpDbSession session, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + await using var transaction = await session + .BeginTransactionAsync(IsolationLevel.RepeatableRead, cancellationToken) + .ConfigureAwait(false); + try + { + var header = await ReadHeaderAsync(session, transaction, cancellationToken).ConfigureAwait(false); + if (header is null) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return null; + } + + var receiveCount = await ReadCountAsync( + session, + transaction, + NntpTransitQueries.CountReceive, + "@receive_revision", + header.ReceiveRevision, + cancellationToken) + .ConfigureAwait(false); + var sendCount = await ReadCountAsync( + session, + transaction, + NntpTransitQueries.CountSend, + "@send_revision", + header.SendRevision, + cancellationToken) + .ConfigureAwait(false); + var peers = await ReadPeersAsync(session, transaction, header, cancellationToken).ConfigureAwait(false); + if (peers.Count != receiveCount || peers.Count != sendCount) + { + throw new InvalidOperationException( + "Transit receive revision " + + header.ReceiveRevision.ToString(CultureInfo.InvariantCulture) + + " and send revision " + + header.SendRevision.ToString(CultureInfo.InvariantCulture) + + " do not contain the same peer identifiers."); + } + + var allowFrom = await ReadStringsAsync( + session, + transaction, + NntpTransitQueries.SelectAllowFrom, + "@receive_revision", + header.ReceiveRevision, + "entry", + cancellationToken) + .ConfigureAwait(false); + var endpoints = await ReadEndpointsAsync(session, transaction, header.SendRevision, cancellationToken) + .ConfigureAwait(false); + var exclusions = await ReadStringsAsync( + session, + transaction, + NntpTransitQueries.SelectPathExclusions, + "@send_revision", + header.SendRevision, + "path_token", + cancellationToken) + .ConfigureAwait(false); + var records = new List(peers.Count); + foreach (var peer in peers) + { + if (!allowFrom.TryGetValue(peer.Identifier, out var allow)) + { + allow = []; + } + + if (!endpoints.TryGetValue(peer.Identifier, out var ends)) + { + ends = []; + } + + if (!exclusions.TryGetValue(peer.Identifier, out var excluded)) + { + excluded = []; + } + + records.Add(peer.WithChildren(allow, ends, excluded)); + } + + RejectUnknown(allowFrom, peers, "AllowFrom"); + RejectUnknown(endpoints, peers, "Send endpoint"); + RejectUnknown(exclusions, peers, "Path exclusion"); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return new TransitCatalogueRecord( + header.PublicationId, + header.ReceiveRevision, + header.SendRevision, + header.GlobalRevision, + header.WantTrash, + header.LogTrash, + records); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + private static async ValueTask ReadHeaderAsync( + INntpDbSession session, + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectCurrent; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + var header = new Header( + reader.GetInt64(0), + reader.GetInt64(1), + reader.GetInt64(2), + reader.GetInt64(3), + ReadYn(reader.GetString(4)), + ReadYn(reader.GetString(5))); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("nntptransitcurrent returned more than one publication."); + } + + return header; + } + + private static async ValueTask ReadCountAsync( + INntpDbSession session, + MySqlTransaction transaction, + string sql, + string parameterName, + long revision, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = sql; + command.Parameters.AddWithValue(parameterName, revision); + var scalar = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); + } + + private static async ValueTask> ReadPeersAsync( + INntpDbSession session, + MySqlTransaction transaction, + Header header, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectPeers; + command.Parameters.AddWithValue("@receive_revision", header.ReceiveRevision); + command.Parameters.AddWithValue("@send_revision", header.SendRevision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var peers = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + peers.Add(new PeerBuilder( + reader.GetString(0), + reader.GetString(1), + reader.GetInt32(2), + reader.GetString(3), + reader.GetString(4), + ReadYn(reader.GetString(5)), + reader.GetInt32(6), + reader.GetInt32(7), + reader.GetString(8), + reader.GetInt32(9), + reader.GetString(10), + reader.GetString(11), + reader.GetString(12), + ReadSendMaxArticleBytes(reader, 13), + reader.GetInt32(14), + reader.GetString(15), + reader.GetString(16))); + } + + return peers; + } + + private static async ValueTask>> ReadStringsAsync( + INntpDbSession session, + MySqlTransaction transaction, + string sql, + string parameterName, + long revision, + string valueColumn, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = sql; + command.Parameters.AddWithValue(parameterName, revision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var valueOrdinal = reader.GetOrdinal(valueColumn); + var map = new Dictionary>(StringComparer.Ordinal); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var identifier = reader.GetString(0); + if (!map.TryGetValue(identifier, out var list)) + { + list = []; + map.Add(identifier, list); + } + + list.Add(reader.GetString(valueOrdinal)); + } + + return map; + } + + private static async ValueTask>> ReadEndpointsAsync( + INntpDbSession session, + MySqlTransaction transaction, + long sendRevision, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectEndpoints; + command.Parameters.AddWithValue("@send_revision", sendRevision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var map = new Dictionary>(StringComparer.Ordinal); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var identifier = reader.GetString(0); + if (!map.TryGetValue(identifier, out var list)) + { + list = []; + map.Add(identifier, list); + } + + list.Add(new TransitCatalogueEndpointRecord(reader.GetString(1), reader.GetInt32(2))); + } + + return map; + } + + private static void RejectUnknown( + Dictionary children, + List peers, + string kind) + { + foreach (var identifier in children.Keys) + { + var found = false; + foreach (var peer in peers) + { + if (string.Equals(peer.Identifier, identifier, StringComparison.Ordinal)) + { + found = true; + break; + } + } + + if (!found) + { + throw new InvalidOperationException( + "Transit " + kind + " row '" + identifier + "' is not in the published peer set."); + } + } + } + + /// + /// Reads nntptransitsend.max_article_bytes. + /// SQL NULL is unlimited and stays . A stored integer stays that ceiling. + /// + /// The peer row. + /// The send max_article_bytes column. + /// when the column is SQL NULL; otherwise the stored byte ceiling. + internal static int? ReadSendMaxArticleBytes(IDataRecord record, int ordinal) + { + ArgumentNullException.ThrowIfNull(record); + return record.IsDBNull(ordinal) ? null : record.GetInt32(ordinal); + } + + private static bool ReadYn(string value) => + value switch + { + "Y" => true, + "N" => false, + _ => throw new InvalidOperationException("Transit catalogue flag is not Y or N."), + }; + + private sealed class Header( + long publicationId, + long receiveRevision, + long sendRevision, + long globalRevision, + bool wantTrash, + bool logTrash) + { + public long PublicationId { get; } = publicationId; + + public long ReceiveRevision { get; } = receiveRevision; + + public long SendRevision { get; } = sendRevision; + + public long GlobalRevision { get; } = globalRevision; + + public bool WantTrash { get; } = wantTrash; + + public bool LogTrash { get; } = logTrash; + } + + private sealed class PeerBuilder + { + private readonly string _identifier; + private readonly string _peerName; + private readonly int _maxInbound; + private readonly string _receiveUsername; + private readonly string _receivePassword; + private readonly bool _deferOnDuplicate; + private readonly int _receiveMaxArticleBytes; + private readonly int _receiveArticleTypes; + private readonly string _receivePatterns; + private readonly int _maxOutbound; + private readonly string _sslMode; + private readonly string _sendUsername; + private readonly string _sendPassword; + private readonly int? _sendMaxArticleBytes; + private readonly int _sendArticleTypes; + private readonly string _sendPatterns; + private readonly string _pathToken; + + public PeerBuilder( + string identifier, + string peerName, + int maxInbound, + string receiveUsername, + string receivePassword, + bool deferOnDuplicate, + int receiveMaxArticleBytes, + int receiveArticleTypes, + string receivePatterns, + int maxOutbound, + string sslMode, + string sendUsername, + string sendPassword, + int? sendMaxArticleBytes, + int sendArticleTypes, + string sendPatterns, + string pathToken) + { + _identifier = identifier; + _peerName = peerName; + _maxInbound = maxInbound; + _receiveUsername = receiveUsername; + _receivePassword = receivePassword; + _deferOnDuplicate = deferOnDuplicate; + _receiveMaxArticleBytes = receiveMaxArticleBytes; + _receiveArticleTypes = receiveArticleTypes; + _receivePatterns = receivePatterns; + _maxOutbound = maxOutbound; + _sslMode = sslMode; + _sendUsername = sendUsername; + _sendPassword = sendPassword; + _sendMaxArticleBytes = sendMaxArticleBytes; + _sendArticleTypes = sendArticleTypes; + _sendPatterns = sendPatterns; + _pathToken = pathToken; + } + + public string Identifier => _identifier; + + public TransitCataloguePeerRecord WithChildren( + IReadOnlyList allowFrom, + IReadOnlyList endpoints, + IReadOnlyList pathExclusions) => + new( + _identifier, + _peerName, + _maxInbound, + _receiveUsername, + _receivePassword, + _deferOnDuplicate, + _receiveMaxArticleBytes, + _receiveArticleTypes, + _receivePatterns, + allowFrom, + _maxOutbound, + _sslMode, + _sendUsername, + _sendPassword, + _sendMaxArticleBytes, + _sendArticleTypes, + _sendPatterns, + _pathToken, + endpoints, + pathExclusions); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitQueries.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitQueries.cs new file mode 100644 index 00000000..4442eecf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpTransitQueries.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// SQL for the published Transit catalogue. NNTPD only reads these tables. + internal static class NntpTransitQueries + { + /// The singleton current-publication row and its global flags. + public const string SelectCurrent = """ + SELECT p.publication_id, p.receive_revision, p.send_revision, p.global_revision, + g.want_trash, g.log_trash + FROM nntptransitcurrent c + INNER JOIN nntptransitpublication p ON p.publication_id = c.publication_id + INNER JOIN nntptransitglobalrevision g ON g.revision = p.global_revision + WHERE c.policy_id = 1 + """; + + /// Receive and send rows for the publication's two revisions, joined on identifier. + public const string SelectPeers = """ + SELECT r.identifier, peer.peer_name, + r.max_inbound, r.username, r.password, r.defer_on_duplicate, + r.max_article_bytes, r.article_types, r.patterns, + s.max_outbound, s.ssl_mode, s.username, s.password, + s.max_article_bytes, s.article_types, s.patterns, s.path_token + FROM nntptransitreceive r + INNER JOIN nntptransitsend s + ON s.identifier = r.identifier AND s.revision = @send_revision + INNER JOIN nntptransitpeer peer ON peer.identifier = r.identifier + WHERE r.revision = @receive_revision + ORDER BY r.identifier + """; + + /// Counts receive rows so a dropped join cannot hide a missing send peer. + public const string CountReceive = """ + SELECT COUNT(*) FROM nntptransitreceive WHERE revision = @receive_revision + """; + + /// Counts send rows so a dropped join cannot hide a missing receive peer. + public const string CountSend = """ + SELECT COUNT(*) FROM nntptransitsend WHERE revision = @send_revision + """; + + /// AllowFrom entries for the receive revision, in configuration order. + public const string SelectAllowFrom = """ + SELECT identifier, entry + FROM nntptransitallowfrom + WHERE revision = @receive_revision + ORDER BY identifier, ordinal + """; + + /// Send endpoints for the send revision, in configuration order. + public const string SelectEndpoints = """ + SELECT identifier, host, port + FROM nntptransitendpoint + WHERE revision = @send_revision + ORDER BY identifier, ordinal + """; + + /// Send Path exclusions for the send revision, in configuration order. + public const string SelectPathExclusions = """ + SELECT identifier, path_token + FROM nntptransitsendpathexclude + WHERE revision = @send_revision + ORDER BY identifier, ordinal + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/NntpDb/NntpUserQueries.cs b/src/VectorNNTPD.ReaderService/NntpDb/NntpUserQueries.cs new file mode 100644 index 00000000..9f6dc97b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/NntpDb/NntpUserQueries.cs @@ -0,0 +1,59 @@ +namespace VectorNNTPD.ReaderService.NntpDb +{ + /// Authoritative NntpDB query for reader account authentication. + public static class NntpUserQueries + { + /// + /// Looks up one nntpusers row. account_name is stored as MD5 hex of the + /// plaintext wire username. account_pass is decrypted in SQL with + /// AES_DECRYPT(..., UNHEX(SHA2(account_name, 256))) using the stored hash column. + /// Does not select account_type; every account has both remaining-byte and rate + /// policies. + /// + public const string SelectUserByName = + "SELECT " + + "CAST(AES_DECRYPT(account_pass, UNHEX(SHA2(account_name, 256))) AS CHAR) AS account_pass, " + + "scram_salt, scram_iterations, scram_stored_key, scram_server_key, " + + "allow_auth_plain, allow_auth_scram256, " + + "account_rate_limit, account_byte_limit, account_session_limit, account_srcip_limit, " + + "is_enabled, customer_id, account_art_type " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name)"; + + /// + /// Additive nntpusers extension from + /// docs/schema/nntpusers-account-art-type.sql. Does not drop the table. + /// Default 65535 is ArticleTypeCapabilities.All. + /// + public const string AddAccountArtTypeColumn = + "ALTER TABLE nntpusers " + + "ADD COLUMN account_art_type INT UNSIGNED NOT NULL DEFAULT 65535 " + + "COMMENT 'ArticleType flags; 65535=ArticleTypeCapabilities.All (unrestricted)'"; + + /// Locks one nntpusers row so a consume can read remaining atomically. + public const string SelectByteQuotaForUpdate = + "SELECT account_byte_limit " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name) " + + "FOR UPDATE"; + + /// + /// Subtracts consumed bytes from remaining quota, clamping at zero. + /// Uses CASE rather than unchecked subtraction so an unsigned column + /// cannot wrap before the floor is applied. + /// + public const string ConsumeAccountBytes = + "UPDATE nntpusers " + + "SET account_byte_limit = CASE " + + "WHEN account_byte_limit IS NULL THEN 0 " + + "WHEN account_byte_limit > @bytes THEN account_byte_limit - @bytes " + + "ELSE 0 END " + + "WHERE account_name = MD5(@account_name)"; + + /// Reads remaining quota without locking. + public const string SelectAccountByteRemaining = + "SELECT account_byte_limit " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name)"; + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilter.cs b/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilter.cs new file mode 100644 index 00000000..848d304d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Accepts every article without quota or SpamAssassin. Used when no filter is wired. + internal sealed class DisabledPostFilter : IPostFilter + { + /// Shared instance. + public static DisabledPostFilter Instance { get; } = new(); + + /// + public ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + _ = request; + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(PostFilterResult.AcceptedDisabled()); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..50cc33d5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// No-op evidence queue used when the writer is not registered. + public sealed class DisabledPostFilterRejectionEvidenceQueue : IPostFilterRejectionEvidenceQueue + { + /// Shared disabled instance. + public static DisabledPostFilterRejectionEvidenceQueue Instance { get; } = new(); + + /// + public int Capacity => 0; + + /// + public int Count => 0; + + /// + public long Dropped => 0; + + /// + public long Written => 0; + + /// + public long WriteFailures => 0; + + /// + public bool TryEnqueue(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + return true; + } + + /// + public void Complete() + { + } + + /// + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + ValueTask.FromResult(null); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/IPostFilter.cs b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilter.cs new file mode 100644 index 00000000..3fc6b4ae --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilter.cs @@ -0,0 +1,67 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// NNTPD POST-only accept filter. Does not own CreateQueued / TryAdmit / History. + public interface IPostFilter + { + /// + /// Evaluates one CanonicalV1 article. On Accept, + /// is set when a reservation was taken. + /// + ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default); + } + + /// Inputs already available at the POST insertion point. + public readonly struct PostFilterRequest + { + /// Initializes a request. Does not copy . + public PostFilterRequest( + ArticleRecord article, + string? accountName, + ConnectionClientIdentity clientIdentity, + NntpAccountPolicy? accountPolicy, + DateTimeOffset injectionUtc, + string[] newsgroups, + DateTimeOffset now, + string? serverFqdn = null) + { + Article = article; + AccountName = accountName; + ClientIdentity = clientIdentity; + AccountPolicy = accountPolicy; + InjectionUtc = injectionUtc; + Newsgroups = newsgroups; + Now = now; + ServerFqdn = string.IsNullOrWhiteSpace(serverFqdn) ? "localhost" : serverFqdn.Trim(); + } + + /// Gets the CanonicalV1 record (read-only). + public ArticleRecord Article { get; } + + /// Gets the authenticated account, when present. + public string? AccountName { get; } + + /// Gets the PROXY-aware client identity. + public ConnectionClientIdentity ClientIdentity { get; } + + /// Gets the AUTH-time account policy, when present. + public NntpAccountPolicy? AccountPolicy { get; } + + /// Gets injection time from StreamingPost. + public DateTimeOffset InjectionUtc { get; } + + /// Gets newsgroup names already parsed by StreamingPost. + public string[] Newsgroups { get; } + + /// Gets the evaluation clock. + public DateTimeOffset Now { get; } + + /// Gets the server FQDN used for synthetic SPAMD scan headers. + public string ServerFqdn { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicyRepository.cs b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicyRepository.cs new file mode 100644 index 00000000..9ca2361c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicyRepository.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Loads the cluster PostFilter policy from NntpDB. Must not be called from the POST hot path. + /// + public interface IPostFilterPolicyRepository + { + /// + /// Reads the singleton policy and its collection tables. + /// + /// Token used to cancel the load. + /// The complete policy document. + /// + /// MySQL is unavailable or the query failed. + /// + /// + /// The singleton row is missing or the row set cannot be mapped. + /// + ValueTask LoadAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicySource.cs b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicySource.cs new file mode 100644 index 00000000..1203334f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterPolicySource.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Publishes the current immutable PostFilter snapshot. + internal interface IPostFilterPolicySource + { + /// Gets the last-known-good snapshot. Throws when none has been published. + PostFilterPolicySnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..a39a9a84 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,37 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Bounded asynchronous queue for PostFilter rejection evidence. + /// Enqueue never changes the PostFilter or NNTP decision. + /// + public interface IPostFilterRejectionEvidenceQueue + { + /// Gets the configured capacity. + int Capacity { get; } + + /// Gets the approximate queued count. + int Count { get; } + + /// Gets dropped-enqueue count (queue full or completed). + long Dropped { get; } + + /// Gets successfully written rows. + long Written { get; } + + /// Gets failed database writes. + long WriteFailures { get; } + + /// + /// Attempts to enqueue evidence. Returns when the + /// queue is full or completed. The caller must still return the original + /// NNTP rejection. + /// + bool TryEnqueue(PostFilterRejectionEvidence evidence); + + /// Completes the writer so the consumer can drain and exit. + void Complete(); + + /// Dequeues the next item, or when completed and empty. + ValueTask DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterSpamAssassin.cs b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterSpamAssassin.cs new file mode 100644 index 00000000..678d8a6f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/IPostFilterSpamAssassin.cs @@ -0,0 +1,91 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// NNTPD-only SPAMD CHECK seam. Does not mutate . + internal interface IPostFilterSpamAssassin + { + /// Runs CHECK. Cancellation is rethrown and is not an on_failure event. + ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default); + } + + /// Peer and server identity used only to build the disposable SPAMD scan message. + internal readonly record struct SpamdScanContext( + IPAddress ClientAddress, + string ServerFqdn, + DateTimeOffset Now); + + /// SPAMD endpoints, pool, and timeouts captured with one PostFilter snapshot. + internal readonly record struct PostFilterSpamAssassinTarget( + IReadOnlyList Hosts, + int Port, + TimeSpan ConnectTimeout, + TimeSpan OperationTimeout, + string ProtocolVersion, + int MaxConnections, + PostFilterSpamAssassinHostSelection HostSelection); + + /// CHECK outcome classified for PostFilter policy. + public enum PostFilterSpamAssassinStatus + { + /// SPAMD reported ham. + Ham = 0, + + /// SPAMD reported spam. + Spam = 1, + + /// Connect, protocol, timeout, or malformed response. + Failed = 2, + } + + /// CHECK result. + internal readonly struct PostFilterSpamAssassinResult + { + /// Initializes a result. + public PostFilterSpamAssassinResult( + PostFilterSpamAssassinStatus status, + string detail, + decimal? score = null, + decimal? threshold = null) + { + Status = status; + Detail = detail; + Score = score; + Threshold = threshold; + } + + /// Gets the classified status. + public PostFilterSpamAssassinStatus Status { get; } + + /// Gets an internal detail. + public string Detail { get; } + + /// Gets the SPAMD score when the Spam header included one. + public decimal? Score { get; } + + /// Gets the SPAMD threshold when the Spam header included one. + public decimal? Threshold { get; } + + /// Ham. + public static PostFilterSpamAssassinResult Ham(decimal? score = null, decimal? threshold = null) => + new(PostFilterSpamAssassinStatus.Ham, "ham", score, threshold); + + /// Spam. + public static PostFilterSpamAssassinResult Spam( + string detail, + decimal? score = null, + decimal? threshold = null) => + new(PostFilterSpamAssassinStatus.Spam, detail, score, threshold); + + /// Scanner fault. + public static PostFilterSpamAssassinResult Failed(string detail) => + new(PostFilterSpamAssassinStatus.Failed, detail); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/MySqlPostFilterPolicyRepository.cs b/src/VectorNNTPD.ReaderService/PostFilter/MySqlPostFilterPolicyRepository.cs new file mode 100644 index 00000000..8244f31c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/MySqlPostFilterPolicyRepository.cs @@ -0,0 +1,53 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Loads the published PostFilter revision through the existing NntpDB connection pool. + public sealed class MySqlPostFilterPolicyRepository : IPostFilterPolicyRepository + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlPostFilterPolicyRepository( + NntpDbService nntpDb, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask LoadAsync(CancellationToken cancellationToken = default) + { + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var record = await connection.QueryPostFilterPolicyAsync(cancellationToken).ConfigureAwait(false); + return record ?? throw new InvalidOperationException( + "nntppostfiltercurrent is missing policy_id = 1. NNTPD will not invent a local PostFilter policy."); + } + catch (OperationCanceledException) + { + throw; + } + catch (InvalidOperationException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PostFilterLogMessages.PolicyRepositoryFailed(_logger, ex); + throw new NntpDbUnavailableException("MySQL nntppostfilterpolicy lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/NullPostFilterSpamAssassin.cs b/src/VectorNNTPD.ReaderService/PostFilter/NullPostFilterSpamAssassin.cs new file mode 100644 index 00000000..2978ed30 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/NullPostFilterSpamAssassin.cs @@ -0,0 +1,27 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// SpamAssassin seam used when CHECK is not configured. + internal sealed class NullPostFilterSpamAssassin : IPostFilterSpamAssassin + { + /// Shared instance. + public static NullPostFilterSpamAssassin Instance { get; } = new(); + + /// + public ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = article; + _ = accountName; + _ = target; + _ = scanContext; + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(PostFilterSpamAssassinResult.Failed("spamassassin not configured")); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterAccountIdentity.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterAccountIdentity.cs new file mode 100644 index 00000000..e2e2b4cf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterAccountIdentity.cs @@ -0,0 +1,64 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// PostFilter account-list identity: lowercase hexadecimal MD5 of the AUTH + /// username using the same trim-on-policy / raw-on-request contract as the + /// evaluator. Does not replace ArticleId or ArtHash. + /// + public static class PostFilterAccountIdentity + { + /// Stored identifier length (CHAR(32) lowercase hex). + public const int HexLength = 32; + + /// + /// MD5 of as UTF-8 bytes, lowercase hex. + /// Does not trim; request matching uses the session username as stored. + /// + public static string FromUsername(string username) + { + ArgumentNullException.ThrowIfNull(username); + var digest = MD5.HashData(Encoding.UTF8.GetBytes(username)); + return Convert.ToHexString(digest).ToLowerInvariant(); + } + + /// + /// Policy-list normalization: trim then hash, unless the trimmed value is + /// already a stored 32-character lowercase hex identifier. + /// + public static string FromPolicyEntry(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + var trimmed = value.Trim(); + return IsStoredIdentifier(trimmed) ? trimmed : FromUsername(trimmed); + } + + /// Returns whether is 32 lowercase hex characters. + public static bool IsStoredIdentifier(ReadOnlySpan value) + { + if (value.Length != HexLength) + { + return false; + } + + foreach (var ch in value) + { + if (ch is not (>= '0' and <= '9' or >= 'a' and <= 'f')) + { + return false; + } + } + + return true; + } + + /// + /// Hashes a request username when it is non-empty; otherwise returns + /// . Does not trim. + /// + public static string? TryFromRequest(string? accountName) => + accountName is { Length: > 0 } ? FromUsername(accountName) : null; + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterBodyHash.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterBodyHash.cs new file mode 100644 index 00000000..9936d2ce --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterBodyHash.cs @@ -0,0 +1,44 @@ +using System.Globalization; +using System.IO.Hashing; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// XXH3-64 of canonical body octets for identical-body quota. + /// + /// Does not use . Encoded ArtType bits do not participate. + /// + internal static class PostFilterBodyHash + { + /// ArtType bits excluded from identical-body quota. + public const ArticleType EncodedTypes = + ArticleType.YEncoded + | ArticleType.Binary + | ArticleType.UuEncode + | ArticleType.Base64 + | ArticleType.BinHex; + + /// + /// Returns lowercase hex XXH3-64 of the body after the first CRLF CRLF, + /// or when the article must not participate. + /// + public static string? TryCompute(in ArticleRecord article) + { + if ((article.ArtType & EncodedTypes) != 0) + { + return null; + } + + var data = article.ArtData.Span; + var separator = data.IndexOf("\r\n\r\n"u8); + if (separator < 0) + { + return null; + } + + var body = data[(separator + 4)..]; + var hash = XxHash3.HashToUInt64(body); + return hash.ToString("x16", CultureInfo.InvariantCulture); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterEvaluator.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterEvaluator.cs new file mode 100644 index 00000000..f633dd14 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterEvaluator.cs @@ -0,0 +1,257 @@ +using System.Globalization; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// POST-only PostFilter pipeline. Does not call CreateQueued or History. + internal sealed class PostFilterEvaluator : IPostFilter + { + private readonly IPostFilterPolicySource _policy; + private readonly IPostFilterQuotaStore _quota; + private readonly PostFilterReservationIdentity _identity; + private readonly IPostFilterSpamAssassin _spamAssassin; + private readonly ILogger _logger; + + /// Initializes the evaluator. + public PostFilterEvaluator( + IPostFilterPolicySource policy, + IPostFilterQuotaStore quota, + PostFilterReservationIdentity identity, + IPostFilterSpamAssassin spamAssassin, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(policy); + ArgumentNullException.ThrowIfNull(quota); + ArgumentNullException.ThrowIfNull(identity); + ArgumentNullException.ThrowIfNull(spamAssassin); + ArgumentNullException.ThrowIfNull(logger); + _policy = policy; + _quota = quota; + _identity = identity; + _spamAssassin = spamAssassin; + _logger = logger; + } + + /// + public async ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var snapshot = _policy.Current; + var account = request.AccountName ?? string.Empty; + var artId = FormatArtId(request.Article); + + if (snapshot.Gate == PostFilterGateState.Disabled) + { + PostFilterLogMessages.Accepted(_logger, PostFilterStage.Gate, account, artId, reserved: false); + return PostFilterResult.AcceptedDisabled(snapshot.Revision); + } + + if (snapshot.Gate == PostFilterGateState.Closed) + { + return Reject(snapshot, PostFilterStage.Gate, "closed", account, artId); + } + + if (IsDenied(snapshot, request)) + { + return Reject(snapshot, PostFilterStage.Deny, "denied", account, artId); + } + + if (snapshot.RejectArtTypes != ArticleType.None + && (request.Article.ArtType & snapshot.RejectArtTypes) != 0) + { + return Reject(snapshot, PostFilterStage.ArtType, "arttype", account, artId); + } + + var allowlisted = IsAllowlisted(snapshot, request); + if (string.IsNullOrWhiteSpace(request.AccountName)) + { + return Reject(snapshot, PostFilterStage.Quota, "unauthenticated", account, artId); + } + + var reservation = _identity.Next(); + var bodyHex = PostFilterBodyHash.TryCompute(request.Article); + var mpUnits = bodyHex is null ? 0 : 1; + PostFilterQuotaReserveStatus reserved; + try + { + reserved = await _quota.ReserveAsync( + request.AccountName, + reservation, + request.Now, + snapshot.Windows, + snapshot.Ceilings, + messages: 1, + bytes: request.Article.ArtSize, + mpUnits, + bodyHex, + cancellationToken, + snapshot.ReservationTtlMs) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await _quota.ReleaseAsync( + request.AccountName, + reservation, + request.Now, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + if (reserved != PostFilterQuotaReserveStatus.Accepted) + { + return Reject(snapshot, PostFilterStage.Quota, reserved.ToString(), account, artId, reserved); + } + + var lease = new PostFilterLease( + _quota, + request.AccountName, + reservation, + snapshot.Windows, + snapshot.Ceilings); + + if (!allowlisted && ShouldScan(snapshot, request.Article)) + { + PostFilterSpamAssassinResult scan; + try + { + scan = await _spamAssassin + .CheckAsync( + request.Article, + request.AccountName, + snapshot.SpamAssassinTarget, + new SpamdScanContext( + request.ClientIdentity.ClientAddress, + request.ServerFqdn, + request.Now), + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + throw; + } + + if (scan.Status == PostFilterSpamAssassinStatus.Spam) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + PostFilterLogMessages.Rejected(_logger, PostFilterStage.SpamAssassin, "spam", account, artId); + return PostFilterResult.RejectedSpamAssassin("spam", snapshot.Revision, scan); + } + + if (scan.Status == PostFilterSpamAssassinStatus.Failed) + { + var action = snapshot.SpamAssassinOnFailure + ?? throw new InvalidOperationException( + "SpamAssassin is enabled but OnFailure is not configured."); + PostFilterLogMessages.SpamAssassinFailed( + _logger, + action.ToString(), + scan.Detail, + account, + artId); + if (action == PostFilterSpamOnFailure.Reject) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + PostFilterLogMessages.Rejected( + _logger, + PostFilterStage.SpamAssassin, + "scanner-failure", + account, + artId); + return PostFilterResult.RejectedSpamAssassin( + "scanner-failure", + snapshot.Revision, + scan); + } + } + } + + PostFilterLogMessages.Accepted(_logger, PostFilterStage.Complete, account, artId, reserved: true); + return PostFilterResult.Accepted(lease, snapshot.Revision); + } + + private PostFilterResult Reject( + PostFilterPolicySnapshot snapshot, + PostFilterStage stage, + string reason, + string account, + string artId, + PostFilterQuotaReserveStatus? quotaStatus = null) + { + PostFilterLogMessages.Rejected(_logger, stage, reason, account, artId); + return PostFilterResult.Rejected(stage, reason, quotaStatus, snapshot.Revision); + } + + private async ValueTask ReleaseQuietAsync(PostFilterLease lease, DateTimeOffset now) + { + var status = await lease.ReleaseAsync(now, CancellationToken.None).ConfigureAwait(false); + PostFilterLogMessages.Released(_logger, status, lease.AccountName, lease.Reservation.Token); + } + + private static bool ShouldScan(PostFilterPolicySnapshot snapshot, in ArticleRecord article) + { + if (!snapshot.SpamAssassinEnabled) + { + return false; + } + + if (snapshot.SpamAssassinMaxArticleSize > 0 + && article.ArtSize >= snapshot.SpamAssassinMaxArticleSize) + { + return false; + } + + return (article.ArtType & snapshot.SpamAssassinExcludeArtTypes) == 0; + } + + private static bool IsDenied(PostFilterPolicySnapshot snapshot, in PostFilterRequest request) + { + var accountId = PostFilterAccountIdentity.TryFromRequest(request.AccountName); + if (accountId is not null && snapshot.DeniedAccounts.Contains(accountId)) + { + return true; + } + + var address = request.ClientIdentity.ClientAddress; + foreach (var network in snapshot.DeniedCidrs) + { + if (network.Contains(address)) + { + return true; + } + } + + return false; + } + + private static bool IsAllowlisted(PostFilterPolicySnapshot snapshot, in PostFilterRequest request) + { + var accountId = PostFilterAccountIdentity.TryFromRequest(request.AccountName); + if (accountId is not null && snapshot.AllowlistedAccounts.Contains(accountId)) + { + return true; + } + + var address = request.ClientIdentity.ClientAddress; + foreach (var network in snapshot.AllowlistedCidrs) + { + if (network.Contains(address)) + { + return true; + } + } + + return false; + } + + private static string FormatArtId(in ArticleRecord article) => + article.ArtHash.ToString("x16", CultureInfo.InvariantCulture); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterLogMessages.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterLogMessages.cs new file mode 100644 index 00000000..9b1ad1f2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterLogMessages.cs @@ -0,0 +1,94 @@ +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Source-generated PostFilter logs. Reasons stay off the NNTP wire. + internal static partial class PostFilterLogMessages + { + [LoggerMessage(EventId = 2800, Level = LogLevel.Information, Message = "PostFilter rejected stage={Stage} reason={Reason} account={Account} artId={ArtId}")] + public static partial void Rejected(ILogger logger, PostFilterStage stage, string reason, string account, string artId); + + [LoggerMessage(EventId = 2801, Level = LogLevel.Debug, Message = "PostFilter accepted stage={Stage} account={Account} artId={ArtId} reserved={Reserved}")] + public static partial void Accepted(ILogger logger, PostFilterStage stage, string account, string artId, bool reserved); + + [LoggerMessage(EventId = 2802, Level = LogLevel.Warning, Message = "PostFilter COMMIT NOOP account={Account} token={Token} artId={ArtId}")] + public static partial void CommitNoop(ILogger logger, string account, string token, string artId); + + [LoggerMessage(EventId = 2803, Level = LogLevel.Warning, Message = "PostFilter COMMIT unavailable after admit account={Account} token={Token} artId={ArtId}")] + public static partial void CommitUnavailable(ILogger logger, string account, string token, string artId); + + [LoggerMessage(EventId = 2804, Level = LogLevel.Debug, Message = "PostFilter RELEASE status={Status} account={Account} token={Token}")] + public static partial void Released(ILogger logger, PostFilterQuotaReleaseStatus status, string account, string token); + + [LoggerMessage(EventId = 2805, Level = LogLevel.Warning, Message = "PostFilter policy refresh failed; retaining last-known-good snapshot")] + public static partial void PolicyRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2806, Level = LogLevel.Warning, Message = "PostFilter SpamAssassin failure action={Action} detail={Detail} account={Account} artId={ArtId}")] + public static partial void SpamAssassinFailed(ILogger logger, string action, string detail, string account, string artId); + + [LoggerMessage( + EventId = 2808, + Level = LogLevel.Information, + Message = "PostFilter policy published revision={Revision} gate={Gate} spamAssassin={SpamAssassin} maxArticleSize={MaxArticleSize} excludeArtTypes={ExcludeArtTypes} hosts={Hosts} maxConnections={MaxConnections}")] + public static partial void PolicyPublished( + ILogger logger, + long revision, + string gate, + bool spamAssassin, + int maxArticleSize, + string excludeArtTypes, + int hosts, + int maxConnections); + + [LoggerMessage(EventId = 2809, Level = LogLevel.Error, Message = "PostFilter initial policy load failed")] + public static partial void PolicyInitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2810, Level = LogLevel.Information, Message = "PostFilter policy revision changed from={FromRevision} to={ToRevision}")] + public static partial void PolicyRevisionChanged(ILogger logger, long fromRevision, long toRevision); + + [LoggerMessage(EventId = 2811, Level = LogLevel.Error, Message = "PostFilter policy repository failed")] + public static partial void PolicyRepositoryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2840, + Level = LogLevel.Information, + Message = "PostFilter evidence writer started capacity={Capacity}")] + public static partial void EvidenceWriterStarted(ILogger logger, int capacity); + + [LoggerMessage( + EventId = 2841, + Level = LogLevel.Information, + Message = "PostFilter evidence writer stopped written={Written} dropped={Dropped} writeFailures={WriteFailures}")] + public static partial void EvidenceWriterStopped(ILogger logger, long written, long dropped, long writeFailures); + + [LoggerMessage( + EventId = 2842, + Level = LogLevel.Warning, + Message = "PostFilter evidence writer stopped with queued items remaining={Remaining}")] + public static partial void EvidenceWriterStoppedWithQueued(ILogger logger, int remaining); + + [LoggerMessage( + EventId = 2843, + Level = LogLevel.Error, + Message = "PostFilter evidence write failed stage={Stage} reason={Reason}")] + public static partial void EvidenceWriteFailed(ILogger logger, Exception exception, PostFilterStage stage, string reason); + + [LoggerMessage( + EventId = 2844, + Level = LogLevel.Warning, + Message = "PostFilter evidence queue full; rejection evidence dropped stage={Stage} reason={Reason}")] + public static partial void EvidenceQueueFull(ILogger logger, PostFilterStage stage, string reason); + + [LoggerMessage( + EventId = 2807, + Level = LogLevel.Information, + Message = "PostFilter SPAMD transport stopped connects={Connects} checks={Checks} reuses={Reuses} reconnects={Reconnects} evictions={Evictions}")] + public static partial void SpamAssassinTransportStopped( + ILogger logger, + long connects, + long checks, + long reuses, + long reconnects, + long evictions); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterMetrics.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterMetrics.cs new file mode 100644 index 00000000..5cc9f343 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterMetrics.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Process counters for PostFilter reservation outcomes. + public sealed class PostFilterMetrics + { + private long _commitNoop; + private long _commitUnavailable; + + /// Gets COMMIT NOOP after a successful TryAdmit. + public long CommitNoop => Volatile.Read(ref _commitNoop); + + /// Gets COMMIT Unavailable after a successful TryAdmit. + public long CommitUnavailable => Volatile.Read(ref _commitUnavailable); + + /// Records a COMMIT NOOP. The article remains queued. + public void RecordCommitNoop() => Interlocked.Increment(ref _commitNoop); + + /// Records COMMIT Unavailable after admit. Does not RELEASE. + public void RecordCommitUnavailable() => Interlocked.Increment(ref _commitUnavailable); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyCompiler.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyCompiler.cs new file mode 100644 index 00000000..9da03b1c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyCompiler.cs @@ -0,0 +1,222 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Compiles into an immutable snapshot. + internal static class PostFilterPolicyCompiler + { + /// Builds a snapshot or throws when the options are not usable. + public static PostFilterPolicySnapshot Compile(PostFilterOptions options, long revision = 0) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + ArgumentOutOfRangeException.ThrowIfNegative(revision); + if (!Enum.IsDefined(options.Gate)) + { + throw new InvalidOperationException($"PostFilter Gate is not a defined value: '{options.Gate}'."); + } + + if (options.Quota.LongWindow <= TimeSpan.Zero || options.Quota.ShortWindow <= TimeSpan.Zero) + { + throw new InvalidOperationException("PostFilter quota windows must be positive."); + } + + var windows = new PostFilterQuotaWindows( + (long)options.Quota.LongWindow.TotalMilliseconds, + (long)options.Quota.ShortWindow.TotalMilliseconds); + windows.Validate(); + + var sa = options.SpamAssassin; + if (sa.Enabled) + { + if (sa.OnFailure is null) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin OnFailure is required when SpamAssassin is enabled."); + } + + if (sa.Hosts.Length == 0) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin Hosts is required when SpamAssassin is enabled."); + } + + var uniqueHosts = new HashSet(StringComparer.Ordinal); + foreach (var host in sa.Hosts) + { + if (string.IsNullOrWhiteSpace(host)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin Hosts contains a malformed empty entry."); + } + + if (!uniqueHosts.Add(host.Trim())) + { + throw new InvalidOperationException( + $"PostFilter SpamAssassin Hosts contains a duplicate entry: '{host.Trim()}'."); + } + } + + if (sa.Port is < 1 or > 65535) + { + throw new InvalidOperationException("PostFilter SpamAssassin Port must be 1–65535."); + } + + if (sa.OperationTimeout < PostFilterSpamAssassinOptions.MinOperationTimeout + || sa.OperationTimeout > PostFilterSpamAssassinOptions.MaxOperationTimeout) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin OperationTimeout must be between 1 second and 2 minutes when SpamAssassin is enabled."); + } + + if (sa.ConnectTimeout <= TimeSpan.Zero || sa.ConnectTimeout > sa.OperationTimeout) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin ConnectTimeout must be positive and must not exceed OperationTimeout."); + } + + if (string.IsNullOrWhiteSpace(sa.ProtocolVersion)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin ProtocolVersion is required when SpamAssassin is enabled."); + } + + if (sa.MaxConnections < PostFilterSpamAssassinOptions.MinMaxConnections + || sa.MaxConnections > PostFilterSpamAssassinOptions.MaxMaxConnections) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin MaxConnections must be between 1 and 32 when SpamAssassin is enabled."); + } + + if (!Enum.IsDefined(sa.HostSelection)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin HostSelection is not a defined strategy."); + } + } + + if (sa.MaxArticleSize < 0) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin MaxArticleSize must be zero or positive."); + } + + var ceilings = new PostFilterQuotaCeilings( + options.Quota.MaxMessagesLong, + options.Quota.MaxBytesLong, + options.Quota.MaxIdenticalLong, + options.Quota.MaxMessagesShort, + options.Quota.MaxBytesShort, + options.Quota.MaxIdenticalShort); + ceilings.Validate(); + + return new PostFilterPolicySnapshot( + options.Gate, + ToSet(options.DeniedAccounts, "DeniedAccounts"), + ToNetworks(options.DeniedCidrs, "DeniedCidrs"), + ToSet(options.AllowlistedAccounts, "AllowlistedAccounts"), + ToNetworks(options.AllowlistedCidrs, "AllowlistedCidrs"), + ParseArtTypes(options.RejectArtTypes, "RejectArtTypes"), + windows, + ceilings, + sa.Enabled, + sa.OnFailure, + sa.MaxArticleSize, + ParseArtTypes(sa.ExcludeArtTypes, "SpamAssassin:ExcludeArtTypes"), + sa.Enabled ? sa.Hosts.Select(static host => host.Trim()).ToArray() : [], + sa.Port, + string.IsNullOrWhiteSpace(sa.ProtocolVersion) + ? PostFilterSpamAssassinOptions.DefaultProtocolVersion + : sa.ProtocolVersion.Trim(), + sa.MaxConnections, + sa.HostSelection, + sa.ConnectTimeout, + sa.OperationTimeout, + PostFilterQuotaDefaults.HoldMilliseconds(sa.Enabled, sa.OperationTimeout), + revision); + } + + private static HashSet ToSet(string[] values, string name) + { + var set = new HashSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + var identifier = PostFilterAccountIdentity.FromPolicyEntry(trimmed); + if (!set.Add(identifier)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + } + + return set; + } + + private static List ToNetworks(string[] values, string name) + { + var networks = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + if (!seen.Add(trimmed)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + + if (!IPNetwork.TryParse(trimmed, out var network)) + { + throw new InvalidOperationException($"PostFilter {name} contains an invalid CIDR: '{value}'."); + } + + networks.Add(network); + } + + return networks; + } + + private static ArticleType ParseArtTypes(string[] values, string name) + { + var mask = ArticleType.None; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + if (!seen.Add(trimmed)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + + if (!Enum.TryParse(trimmed, ignoreCase: true, out var flag) + || !Enum.IsDefined(flag) + || flag == ArticleType.None) + { + throw new InvalidOperationException($"PostFilter {name} contains an unknown ArtType: '{value}'."); + } + + mask |= flag; + } + + return mask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyRecord.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyRecord.cs new file mode 100644 index 00000000..bf737d0c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyRecord.cs @@ -0,0 +1,29 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Authoritative cluster PostFilter policy loaded from NntpDB, before compile. + /// + public sealed class PostFilterPolicyRecord + { + /// Initializes a loaded policy document. + public PostFilterPolicyRecord(long revision, DateTimeOffset updatedUtc, PostFilterOptions options) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(revision); + ArgumentNullException.ThrowIfNull(options); + Revision = revision; + UpdatedUtc = updatedUtc; + Options = options; + } + + /// Gets the published NntpDB revision (nntppostfiltercurrent.revision). + public long Revision { get; } + + /// Gets nntppostfilterpolicy.updated_utc. + public DateTimeOffset UpdatedUtc { get; } + + /// Gets the compile input. Not bound from Nntpd:PostFilter. + public PostFilterOptions Options { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyService.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyService.cs new file mode 100644 index 00000000..b4c3c182 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicyService.cs @@ -0,0 +1,215 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Loads PostFilter policy from NntpDB, publishes an immutable snapshot, + /// and refreshes it every 60 seconds. + /// + /// + /// Initial load failure prevents RUNNING. Refresh failure retains last-known-good. + /// A successful refresh of the same published revision does not replace the snapshot. + /// POST observes via a volatile snapshot read and never queries MySQL. + /// There is no appsettings fallback. + /// + internal sealed class PostFilterPolicyService : IPostFilterPolicySource, IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly IPostFilterPolicyRepository _repository; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private PostFilterPolicySnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a production policy service. + public PostFilterPolicyService( + IPostFilterPolicyRepository repository, + ILogger logger) + : this(repository, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a service with an explicit clock and interval (tests). + internal PostFilterPolicyService( + IPostFilterPolicyRepository repository, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(repository); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _repository = repository; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "PostFilterPolicy"; + + /// + public Task? Execution => _execution; + + /// + public PostFilterPolicySnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + throw new InvalidOperationException("PostFilter policy has not been published."); + } + + return snapshot; + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot, previousRevision: null); + _execution = RunAsync(_runCts.Token); + await Task.CompletedTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + PostFilterLogMessages.PolicyInitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is not null) + { + try + { + await _execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var previous = Volatile.Read(ref _current); + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + if (previous is not null && previous.Revision == snapshot.Revision) + { + return; + } + + Publish(snapshot, previous?.Revision); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PostFilterLogMessages.PolicyRefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + var record = await _repository.LoadAsync(cancellationToken).ConfigureAwait(false); + return PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + } + + private void Publish(PostFilterPolicySnapshot snapshot, long? previousRevision) + { + Volatile.Write(ref _current, snapshot); + PostFilterLogMessages.PolicyPublished( + _logger, + snapshot.Revision, + snapshot.Gate.ToString(), + snapshot.SpamAssassinEnabled, + snapshot.SpamAssassinMaxArticleSize, + snapshot.SpamAssassinExcludeArtTypes.ToString(), + snapshot.SpamAssassinHosts.Count, + snapshot.SpamAssassinMaxConnections); + if (previousRevision is { } prior && prior != snapshot.Revision) + { + PostFilterLogMessages.PolicyRevisionChanged(_logger, prior, snapshot.Revision); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicySnapshot.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicySnapshot.cs new file mode 100644 index 00000000..08ea0222 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterPolicySnapshot.cs @@ -0,0 +1,163 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Immutable PostFilter policy observed for one POST evaluation. + internal sealed class PostFilterPolicySnapshot + { + /// Initializes a compiled snapshot. + public PostFilterPolicySnapshot( + PostFilterGateState gate, + IReadOnlySet deniedAccounts, + IReadOnlyList deniedCidrs, + IReadOnlySet allowlistedAccounts, + IReadOnlyList allowlistedCidrs, + ArticleType rejectArtTypes, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + bool spamAssassinEnabled, + PostFilterSpamOnFailure? spamAssassinOnFailure, + int spamAssassinMaxArticleSize, + ArticleType spamAssassinExcludeArtTypes, + IReadOnlyList spamAssassinHosts, + int spamAssassinPort, + string spamAssassinProtocolVersion, + int spamAssassinMaxConnections, + PostFilterSpamAssassinHostSelection spamAssassinHostSelection, + TimeSpan spamAssassinConnectTimeout, + TimeSpan spamAssassinOperationTimeout, + long reservationTtlMs, + long revision = 0) + { + Gate = gate; + DeniedAccounts = deniedAccounts; + DeniedCidrs = deniedCidrs; + AllowlistedAccounts = allowlistedAccounts; + AllowlistedCidrs = allowlistedCidrs; + RejectArtTypes = rejectArtTypes; + Windows = windows; + Ceilings = ceilings; + SpamAssassinEnabled = spamAssassinEnabled; + SpamAssassinOnFailure = spamAssassinOnFailure; + SpamAssassinMaxArticleSize = spamAssassinMaxArticleSize; + SpamAssassinExcludeArtTypes = spamAssassinExcludeArtTypes; + SpamAssassinHosts = spamAssassinHosts; + SpamAssassinPort = spamAssassinPort; + SpamAssassinProtocolVersion = spamAssassinProtocolVersion; + SpamAssassinMaxConnections = spamAssassinMaxConnections; + SpamAssassinHostSelection = spamAssassinHostSelection; + SpamAssassinConnectTimeout = spamAssassinConnectTimeout; + SpamAssassinOperationTimeout = spamAssassinOperationTimeout; + ReservationTtlMs = reservationTtlMs; + Revision = revision; + } + + /// Gets the operational gate. + public PostFilterGateState Gate { get; } + + /// Gets denied account MD5 identifiers (ordinal lowercase hex). + public IReadOnlySet DeniedAccounts { get; } + + /// Gets denied client networks. + public IReadOnlyList DeniedCidrs { get; } + + /// Gets allowlisted account MD5 identifiers (ordinal lowercase hex). + public IReadOnlySet AllowlistedAccounts { get; } + + /// Gets allowlisted client networks. + public IReadOnlyList AllowlistedCidrs { get; } + + /// Gets the ArtType reject mask. disables the stage. + public ArticleType RejectArtTypes { get; } + + /// Gets quota windows. + public PostFilterQuotaWindows Windows { get; } + + /// Gets quota ceilings. + public PostFilterQuotaCeilings Ceilings { get; } + + /// Gets whether SpamAssassin CHECK is enabled. + public bool SpamAssassinEnabled { get; } + + /// Gets scanner-fault behaviour when SpamAssassin is enabled. + public PostFilterSpamOnFailure? SpamAssassinOnFailure { get; } + + /// + /// Gets the exclusive SA size gate. Articles with + /// ArtSize >= SpamAssassinMaxArticleSize skip CHECK. + /// 0 disables the size gate. + /// + public int SpamAssassinMaxArticleSize { get; } + + /// Gets ArtType bits excluded from CHECK. + public ArticleType SpamAssassinExcludeArtTypes { get; } + + /// Gets compiled SPAMD hosts for this evaluation. + public IReadOnlyList SpamAssassinHosts { get; } + + /// Gets the compiled SPAMD port. + public int SpamAssassinPort { get; } + + /// Gets the compiled SPAMC protocol version. + public string SpamAssassinProtocolVersion { get; } + + /// Gets the compiled persistent-connection pool size. + public int SpamAssassinMaxConnections { get; } + + /// Gets the compiled host-selection strategy. + public PostFilterSpamAssassinHostSelection SpamAssassinHostSelection { get; } + + /// Gets the compiled SPAMD connect timeout. + public TimeSpan SpamAssassinConnectTimeout { get; } + + /// Gets the compiled SPAMD CHECK wall-clock budget. + public TimeSpan SpamAssassinOperationTimeout { get; } + + /// + /// Gets the reservation hold compiled for this snapshot. + /// Floor is 10s; when SpamAssassin is enabled this covers + /// plus hold skew. + /// + public long ReservationTtlMs { get; } + + /// Gets the NntpDB policy revision compiled into this snapshot. + public long Revision { get; } + + /// Gets the CHECK target captured with this snapshot. + public PostFilterSpamAssassinTarget SpamAssassinTarget => + new( + SpamAssassinHosts, + SpamAssassinPort, + SpamAssassinConnectTimeout, + SpamAssassinOperationTimeout, + SpamAssassinProtocolVersion, + SpamAssassinMaxConnections, + SpamAssassinHostSelection); + + /// Disabled gate, no lists, no type policy, no SA, all quota dimensions off. + public static PostFilterPolicySnapshot Disabled { get; } = new( + PostFilterGateState.Disabled, + new HashSet(StringComparer.Ordinal), + [], + new HashSet(StringComparer.Ordinal), + [], + ArticleType.None, + new PostFilterQuotaWindows(86_400_000, 600_000), + new PostFilterQuotaCeilings(0, 0, 0, 0, 0, 0), + spamAssassinEnabled: false, + spamAssassinOnFailure: null, + spamAssassinMaxArticleSize: 0, + spamAssassinExcludeArtTypes: ArticleType.None, + spamAssassinHosts: [], + spamAssassinPort: 783, + spamAssassinProtocolVersion: PostFilterSpamAssassinOptions.DefaultProtocolVersion, + spamAssassinMaxConnections: PostFilterSpamAssassinOptions.DefaultMaxConnections, + spamAssassinHostSelection: PostFilterSpamAssassinHostSelection.RoundRobin, + spamAssassinConnectTimeout: TimeSpan.FromSeconds(5), + spamAssassinOperationTimeout: TimeSpan.FromSeconds(30), + reservationTtlMs: (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidence.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidence.cs new file mode 100644 index 00000000..eff2d63c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidence.cs @@ -0,0 +1,144 @@ +using System.Net; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// One PostFilter rejection queued for asynchronous NntpDB persistence. + /// The payload is the exact unstuffed article available at the decision, or + /// when those bytes were not available. + /// + public sealed class PostFilterRejectionEvidence + { + /// Creates an evidence snapshot. Does not copy . + public PostFilterRejectionEvidence( + DateTimeOffset rejectedUtc, + long policyRevision, + string? accountName, + IPAddress sourceAddress, + ArticleType artType, + string? messageId, + int articleSize, + PostFilterStage stage, + string reason, + PostFilterSpamAssassinStatus? spamAssassinStatus, + decimal? spamAssassinScore, + decimal? spamAssassinThreshold, + ReadOnlyMemory? articlePayload) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + ArgumentNullException.ThrowIfNull(sourceAddress); + ArgumentOutOfRangeException.ThrowIfNegative(articleSize); + RejectedUtc = rejectedUtc; + PolicyRevision = policyRevision; + AccountName = PostFilterAccountIdentity.TryFromRequest(accountName); + SourceAddress = sourceAddress; + ArtType = artType; + MessageId = string.IsNullOrWhiteSpace(messageId) ? null : messageId.Trim(); + ArticleSize = articleSize; + Stage = stage; + Reason = reason; + SpamAssassinStatus = spamAssassinStatus; + SpamAssassinScore = spamAssassinScore; + SpamAssassinThreshold = spamAssassinThreshold; + ArticlePayload = articlePayload; + } + + /// Gets the rejection timestamp. + public DateTimeOffset RejectedUtc { get; } + + /// Gets the published policy revision that produced the decision. + public long PolicyRevision { get; } + + /// Gets the MD5 account identifier, or when unauthenticated. + public string? AccountName { get; } + + /// Gets the client source address. + public IPAddress SourceAddress { get; } + + /// Gets the classifier flags from the article, not a policy ENUM. + public ArticleType ArtType { get; } + + /// Gets the Message-ID when available. + public string? MessageId { get; } + + /// Gets the article size in bytes (0 when unknown). + public int ArticleSize { get; } + + /// Gets the PostFilter stage. + public PostFilterStage Stage { get; } + + /// Gets the internal PostFilter reason. + public string Reason { get; } + + /// Gets the SA status for SA-stage rejections. + public PostFilterSpamAssassinStatus? SpamAssassinStatus { get; } + + /// Gets the SA score when present. + public decimal? SpamAssassinScore { get; } + + /// Gets the SA threshold when present. + public decimal? SpamAssassinThreshold { get; } + + /// + /// Gets the complete unstuffed article, or when the + /// complete article was not available at the rejection point. + /// + public ReadOnlyMemory? ArticlePayload { get; } + + /// Builds evidence from a POST evaluation that already has an . + public static PostFilterRejectionEvidence FromEvaluation( + in PostFilterRequest request, + in PostFilterResult result, + DateTimeOffset rejectedUtc) + { + var article = request.Article; + ReadOnlyMemory? payload = article.ArtSize > 0 ? article.ArtData : null; + return new( + rejectedUtc, + result.PolicyRevision, + request.AccountName, + request.ClientIdentity.ClientAddress, + article.ArtType, + EncodingMessageId(article), + article.ArtSize, + result.Stage, + result.Reason, + result.SpamAssassinStatus, + result.SpamAssassinScore, + result.SpamAssassinThreshold, + payload); + } + + /// Builds evidence for a post-accept admission failure while the article is still held. + public static PostFilterRejectionEvidence FromAdmissionFailure( + in PostFilterRequest request, + in PostFilterResult accepted, + string reason, + DateTimeOffset rejectedUtc) + { + var article = request.Article; + ReadOnlyMemory? payload = article.ArtSize > 0 ? article.ArtData : null; + return new( + rejectedUtc, + accepted.PolicyRevision, + request.AccountName, + request.ClientIdentity.ClientAddress, + article.ArtType, + EncodingMessageId(article), + article.ArtSize, + PostFilterStage.Admission, + reason, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + payload); + } + + private static string? EncodingMessageId(in ArticleRecord article) + { + var messageId = article.MessageId; + return messageId.IsEmpty ? null : System.Text.Encoding.ASCII.GetString(messageId); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..a19c0d05 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,87 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Bounded rejection-evidence queue. Full enqueues are dropped and counted; + /// they never invert a 441/240 decision. + /// + public sealed class PostFilterRejectionEvidenceQueue : IPostFilterRejectionEvidenceQueue + { + /// Default bound. Evidence rows can include full article bytes. + public const int DefaultCapacity = 128; + + private readonly Channel _channel; + private int _count; + private long _dropped; + private long _written; + private long _writeFailures; + + /// Initializes a bounded queue. + public PostFilterRejectionEvidenceQueue(int capacity = DefaultCapacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + Capacity = capacity; + _channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public int Capacity { get; } + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public long Dropped => Volatile.Read(ref _dropped); + + /// + public long Written => Volatile.Read(ref _written); + + /// + public long WriteFailures => Volatile.Read(ref _writeFailures); + + /// + public bool TryEnqueue(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + if (!_channel.Writer.TryWrite(evidence)) + { + Interlocked.Increment(ref _dropped); + return false; + } + + Interlocked.Increment(ref _count); + return true; + } + + /// + public void Complete() => _channel.Writer.TryComplete(); + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var evidence)) + { + Interlocked.Decrement(ref _count); + return evidence; + } + } + + return null; + } + + /// Records a successful database insert. + internal void RecordWritten() => Interlocked.Increment(ref _written); + + /// Records a failed database insert. + internal void RecordWriteFailure() => Interlocked.Increment(ref _writeFailures); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceService.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceService.cs new file mode 100644 index 00000000..b94facb4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterRejectionEvidenceService.cs @@ -0,0 +1,122 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Background writer for PostFilter rejection evidence. Completes independently + /// of the POST decision. Write failures are counted and logged; they never + /// change 441/240 semantics. + /// + public sealed class PostFilterRejectionEvidenceService : IApplicationService + { + private readonly NntpDbService _nntpDb; + private readonly PostFilterRejectionEvidenceQueue _queue; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes the writer. + public PostFilterRejectionEvidenceService( + NntpDbService nntpDb, + PostFilterRejectionEvidenceQueue queue, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _queue = queue; + _logger = logger; + } + + /// + public string Name => "PostFilterRejectionEvidence"; + + /// + public Task? Execution => _execution; + + /// Gets the bounded queue consumed by this service. + public PostFilterRejectionEvidenceQueue Queue => _queue; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Complete(); + await _runCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + PostFilterLogMessages.EvidenceWriterStoppedWithQueued(_logger, _queue.Count); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + PostFilterLogMessages.EvidenceWriterStarted(_logger, _queue.Capacity); + while (true) + { + PostFilterRejectionEvidence? evidence; + try + { + evidence = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (evidence is null) + { + break; + } + + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, CancellationToken.None) + .ConfigureAwait(false); + await connection.InsertPostFilterRejectionAsync(evidence, CancellationToken.None) + .ConfigureAwait(false); + _queue.RecordWritten(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + _queue.RecordWriteFailure(); + PostFilterLogMessages.EvidenceWriteFailed(_logger, ex, evidence.Stage, evidence.Reason); + } + + if (cancellationToken.IsCancellationRequested && _queue.Count == 0) + { + break; + } + } + + PostFilterLogMessages.EvidenceWriterStopped(_logger, _queue.Written, _queue.Dropped, _queue.WriteFailures); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterReservationIdentity.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterReservationIdentity.cs new file mode 100644 index 00000000..9b3c60ed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterReservationIdentity.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Process-lifetime reservation identity: node + incarnation + monotonic generation. + internal sealed class PostFilterReservationIdentity + { + private readonly string _nodeId; + private readonly string _incarnation; + private long _generation; + + /// Initializes identity for this process. + public PostFilterReservationIdentity(string nodeId, string? incarnation = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + _nodeId = nodeId; + _incarnation = string.IsNullOrWhiteSpace(incarnation) + ? Guid.NewGuid().ToString("N") + : incarnation; + } + + /// Gets the node identity. + public string NodeId => _nodeId; + + /// Gets the process incarnation. + public string Incarnation => _incarnation; + + /// Allocates the next per-POST generation. + public PostFilterReservationId Next() => + new(_nodeId, _incarnation, Interlocked.Increment(ref _generation)); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/PostFilterResults.cs b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterResults.cs new file mode 100644 index 00000000..e666428f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/PostFilterResults.cs @@ -0,0 +1,245 @@ +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Client-visible PostFilter decision. Moderate is not implemented. + public enum PostFilterDecision + { + /// Continue to CreateQueued. + Accept = 0, + + /// Do not queue. Client receives 441. + Reject = 1, + } + + /// Stage that produced the decision. + public enum PostFilterStage + { + /// No remaining stage ran (disabled gate). + Gate = 0, + + /// Account or client-IP deny. + Deny = 1, + + /// ArtType policy mask. + ArtType = 2, + + /// Redis accept-quota reservation. + Quota = 3, + + /// SpamAssassin CHECK. + SpamAssassin = 4, + + /// Accepted after all enabled stages. + Complete = 5, + + /// Queue admission after an Accept decision (TryAdmit). + Admission = 6, + } + + /// Lease held after a successful RESERVE. POST owns commit/release. + public sealed class PostFilterLease + { + private readonly IPostFilterQuotaStore _store; + + /// Initializes a live reservation lease. + internal PostFilterLease( + IPostFilterQuotaStore store, + string accountName, + PostFilterReservationId reservation, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + _store = store; + AccountName = accountName; + Reservation = reservation; + Windows = windows; + Ceilings = ceilings; + } + + /// Gets the authenticated account key. + public string AccountName { get; } + + /// Gets the reservation token for logs. + public string Token => Reservation.Token; + + /// Gets the reservation identity. + internal PostFilterReservationId Reservation { get; } + + /// Gets the windows used at reserve time. + internal PostFilterQuotaWindows Windows { get; } + + /// Gets the ceilings used at reserve time. + internal PostFilterQuotaCeilings Ceilings { get; } + + /// COMMITs reserved units into the current buckets. + internal ValueTask CommitAsync( + DateTimeOffset now, + CancellationToken cancellationToken = default) => + _store.CommitAsync(AccountName, Reservation, now, Windows, Ceilings, cancellationToken); + + /// RELEASEs the reservation. Never decrements committed usage. + internal ValueTask ReleaseAsync( + DateTimeOffset now, + CancellationToken cancellationToken = default) => + _store.ReleaseAsync(AccountName, Reservation, now, cancellationToken); + } + + /// PostFilter evaluation outcome. + public readonly struct PostFilterResult + { + /// Initializes a result. + public PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease) + : this(decision, stage, reason, lease, quotaStatus: null) + { + } + + /// Initializes a result with a quota reserve status. + internal PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease, + PostFilterQuotaReserveStatus? quotaStatus) + : this( + decision, + stage, + reason, + lease, + quotaStatus, + policyRevision: 0, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null) + { + } + + /// Initializes a result with policy revision and optional SA metadata. + internal PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease, + PostFilterQuotaReserveStatus? quotaStatus, + long policyRevision, + PostFilterSpamAssassinStatus? spamAssassinStatus, + decimal? spamAssassinScore, + decimal? spamAssassinThreshold) + { + Decision = decision; + Stage = stage; + Reason = reason; + Lease = lease; + QuotaStatus = quotaStatus; + PolicyRevision = policyRevision; + SpamAssassinStatus = spamAssassinStatus; + SpamAssassinScore = spamAssassinScore; + SpamAssassinThreshold = spamAssassinThreshold; + } + + /// Gets Accept or Reject. + public PostFilterDecision Decision { get; } + + /// Gets the decisive stage. + public PostFilterStage Stage { get; } + + /// Gets an internal reason (never sent on the wire). + public string Reason { get; } + + /// Gets the reservation lease when Accept reserved quota. + public PostFilterLease? Lease { get; } + + /// Gets the RESERVE status when the quota stage ran. + internal PostFilterQuotaReserveStatus? QuotaStatus { get; } + + /// Gets the published policy revision observed for this decision. + public long PolicyRevision { get; } + + /// Gets the SA status when the SpamAssassin stage produced the decision. + internal PostFilterSpamAssassinStatus? SpamAssassinStatus { get; } + + /// Gets the SA score when the Spam header included one. + public decimal? SpamAssassinScore { get; } + + /// Gets the SA threshold when the Spam header included one. + public decimal? SpamAssassinThreshold { get; } + + /// Accept with no reservation (disabled gate). + public static PostFilterResult AcceptedDisabled(long policyRevision = 0) => + new( + PostFilterDecision.Accept, + PostFilterStage.Gate, + "disabled", + lease: null, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Accept after enabled stages. + public static PostFilterResult Accepted(PostFilterLease? lease, long policyRevision = 0) => + new( + PostFilterDecision.Accept, + PostFilterStage.Complete, + "accepted", + lease, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject. + public static PostFilterResult Rejected(PostFilterStage stage, string reason, long policyRevision = 0) => + new( + PostFilterDecision.Reject, + stage, + reason, + lease: null, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject with a quota reserve status. + internal static PostFilterResult Rejected( + PostFilterStage stage, + string reason, + PostFilterQuotaReserveStatus? quotaStatus, + long policyRevision = 0) => + new( + PostFilterDecision.Reject, + stage, + reason, + lease: null, + quotaStatus, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject after SpamAssassin CHECK. + internal static PostFilterResult RejectedSpamAssassin( + string reason, + long policyRevision, + PostFilterSpamAssassinResult scan) => + new( + PostFilterDecision.Reject, + PostFilterStage.SpamAssassin, + reason, + lease: null, + quotaStatus: null, + policyRevision, + scan.Status, + scan.Score, + scan.Threshold); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/IPostFilterQuotaStore.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/IPostFilterQuotaStore.cs new file mode 100644 index 00000000..d80a474d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/IPostFilterQuotaStore.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Cluster-wide PostFilter accept-quota reservations. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Reserve, commit, and release each use one atomic operation. + /// + internal interface IPostFilterQuotaStore + { + /// + /// Atomically evaluates enabled ceilings and creates one reservation, or writes nothing. + /// reservationTtlMs of 0 uses . + /// When SpamAssassin CHECK can run, callers pass + /// . + /// + ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0); + + /// + /// Transfers units from the live reservation into the current buckets. + /// The caller does not supply units. + /// + ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default); + + /// Deletes a matching live reservation. Never decrements committed usage. + ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs new file mode 100644 index 00000000..ef382684 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs @@ -0,0 +1,124 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Process-local cluster fake for accept-quota. Two stores sharing one engine + /// behave as two NNTP nodes. Ordinary unit tests must not require live Redis. + /// + internal sealed class InMemoryPostFilterQuotaStore : IPostFilterQuotaStore + { + private readonly PostFilterQuotaEngine _engine; + + /// Initializes a store with a dedicated engine. + public InMemoryPostFilterQuotaStore() + : this(new PostFilterQuotaEngine()) + { + } + + /// Initializes a store that shares with other fakes. + public InMemoryPostFilterQuotaStore(PostFilterQuotaEngine engine) + { + ArgumentNullException.ThrowIfNull(engine); + _engine = engine; + } + + /// When set, operations fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Gets the shared algorithm (tests). + internal PostFilterQuotaEngine Engine => _engine; + + /// + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReserveStatus.Unavailable); + } + + var ttl = reservationTtlMs > 0 + ? reservationTtlMs + : (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds; + var code = _engine.Reserve( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds(), + ttl, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex ?? string.Empty); + return ValueTask.FromResult((PostFilterQuotaReserveStatus)code); + } + + /// + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Unavailable); + } + + var code = _engine.Commit( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds(), + windows, + ceilings); + return ValueTask.FromResult((PostFilterQuotaCommitStatus)code); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReleaseStatus.Unavailable); + } + + var code = _engine.Release( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds()); + return ValueTask.FromResult((PostFilterQuotaReleaseStatus)code); + } + + /// In-memory quota hash identity matching the Redis key bytes. + internal static string QuotaKey(string accountName) => + Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateQuota(accountName)); + + /// In-memory multipost hash identity matching the Redis key bytes. + internal static string MultipostKey(string accountName) => + Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateMultipost(accountName)); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaCeilings.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaCeilings.cs new file mode 100644 index 00000000..a3d30e2e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaCeilings.cs @@ -0,0 +1,43 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Per-window ceilings. 0 disables that dimension. + /// + /// Message ceiling for L. 0 is disabled. + /// Byte ceiling for L. 0 is disabled. + /// Identical-body ceiling for L. 0 is disabled. + /// Message ceiling for S. 0 is disabled. + /// Byte ceiling for S. 0 is disabled. + /// Identical-body ceiling for S. 0 is disabled. + internal readonly record struct PostFilterQuotaCeilings( + long MaxMessagesLong, + long MaxBytesLong, + long MaxIdenticalLong, + long MaxMessagesShort, + long MaxBytesShort, + long MaxIdenticalShort) + { + /// Throws when any ceiling is negative. + public void Validate() + { + ArgumentOutOfRangeException.ThrowIfNegative(MaxMessagesLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxBytesLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxIdenticalLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxMessagesShort); + ArgumentOutOfRangeException.ThrowIfNegative(MaxBytesShort); + ArgumentOutOfRangeException.ThrowIfNegative(MaxIdenticalShort); + } + + /// Returns the message ceiling for . + public long MaxMessages(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxMessagesLong : MaxMessagesShort; + + /// Returns the byte ceiling for . + public long MaxBytes(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxBytesLong : MaxBytesShort; + + /// Returns the identical-body ceiling for . + public long MaxIdentical(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxIdenticalLong : MaxIdenticalShort; + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaDefaults.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaDefaults.cs new file mode 100644 index 00000000..83563e90 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaDefaults.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// Fixed crash-recovery timings for PostFilter accept-quota reservations. + /// + /// Reservation TTL is not a quota window and is not a v1 operator knob. + /// The floor () is the crash-recovery hold when + /// SpamAssassin CHECK is not in the POST path. When CHECK is enabled, the + /// effective hold is derived so a live reservation outlives the configured + /// SPAMD OperationTimeout plus . + /// Idle key TTL is garbage collection only; bucket fields define window expiry. + /// + internal static class PostFilterQuotaDefaults + { + /// Live-reservation crash-recovery floor written by RESERVE when CHECK is off. + public static readonly TimeSpan ReservationTtl = TimeSpan.FromSeconds(10); + + /// + /// Extra hold after the configured SpamAssassin operation budget so + /// CreateQueued / TryAdmit / COMMIT still see a live reservation. + /// + public static readonly TimeSpan ReservationHoldSkew = TimeSpan.FromSeconds(1); + + /// + /// Extra milliseconds added to max(window) + reservation hold when refreshing + /// idle Redis key expiry. Does not change bucket accounting. + /// + public const long IdleTtlSkewMs = 1_000; + + /// + /// Effective reservation hold in milliseconds for one POST. + /// Floor is . When SpamAssassin is enabled, + /// hold is at least plus + /// . + /// + public static long HoldMilliseconds(bool spamAssassinEnabled, TimeSpan operationTimeout) + { + var floor = (long)ReservationTtl.TotalMilliseconds; + if (!spamAssassinEnabled) + { + return floor; + } + + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(operationTimeout, TimeSpan.Zero); + var needed = (long)operationTimeout.TotalMilliseconds + + (long)ReservationHoldSkew.TotalMilliseconds; + return Math.Max(floor, needed); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaEngine.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaEngine.cs new file mode 100644 index 00000000..91f21cb6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaEngine.cs @@ -0,0 +1,547 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Atomic accept-quota algorithm. Redis Lua is a faithful port of these operations. + /// + internal sealed class PostFilterQuotaEngine + { + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// Atomically evaluates ceilings and creates one reservation, or writes nothing. + public long Reserve( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs, + long reservationTtlMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + ValidateReserve(quotaKey, multipostKey, nowMs, reservationTtlMs, windows, ceilings, messages, bytes, mpUnits, ref bodyHex); + + lock (_gate) + { + var quota = GetOrCreateHash(quotaKey); + var multipost = GetOrCreateHash(multipostKey); + PruneQuota(quota, windows, nowMs); + PruneMultipost(multipost, windows, nowMs); + + var field = reservation.Field; + if (quota.TryGetValue(field, out var existing) + && PostFilterQuotaKeys.TryParseReservation( + existing, out var exp, out var storedGen, out var sm, out var sb, out var smp, out var sbody) + && exp > nowMs) + { + if (storedGen == reservation.Generation + && sm == messages + && sb == bytes + && smp == mpUnits + && string.Equals(sbody, bodyHex, StringComparison.Ordinal)) + { + return PostFilterQuotaReserveCodes.Accept; + } + + return PostFilterQuotaReserveCodes.Conflict; + } + + var deny = FirstDenial(quota, multipost, windows, ceilings, nowMs, messages, bytes, mpUnits, bodyHex); + if (deny != PostFilterQuotaReserveCodes.Accept) + { + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return deny; + } + + quota[field] = PostFilterQuotaKeys.PackReservation( + nowMs + reservationTtlMs, + reservation.Generation, + messages, + bytes, + mpUnits, + bodyHex); + return PostFilterQuotaReserveCodes.Accept; + } + } + + /// Transfers a live reservation into the current buckets, or no-ops. + public long Commit( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + windows.Validate(); + ceilings.Validate(); + + lock (_gate) + { + var quota = GetOrCreateHash(quotaKey); + var multipost = GetOrCreateHash(multipostKey); + PruneQuota(quota, windows, nowMs); + PruneMultipost(multipost, windows, nowMs); + + var field = reservation.Field; + if (!quota.TryGetValue(field, out var raw) + || !PostFilterQuotaKeys.TryParseReservation( + raw, out var exp, out var storedGen, out var messages, out var bytes, out var mpUnits, out var bodyHex)) + { + if (quota.ContainsKey(field)) + { + quota.Remove(field); + } + + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Noop; + } + + if (storedGen != reservation.Generation) + { + return PostFilterQuotaCommitCodes.Noop; + } + + if (exp <= nowMs) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Noop; + } + + IncrementCurrent(quota, multipost, windows, ceilings, nowMs, messages, bytes, mpUnits, bodyHex); + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Committed; + } + } + + /// Deletes a matching live reservation. Never decrements committed usage. + public long Release( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + + lock (_gate) + { + if (!_hashes.TryGetValue(quotaKey, out var quota)) + { + return PostFilterQuotaReleaseCodes.Noop; + } + + var field = reservation.Field; + if (!quota.TryGetValue(field, out var raw) + || !PostFilterQuotaKeys.TryParseReservation( + raw, out var exp, out var storedGen, out _, out _, out _, out _)) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Noop; + } + + if (storedGen != reservation.Generation) + { + return PostFilterQuotaReleaseCodes.Noop; + } + + if (exp <= nowMs) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Noop; + } + + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Released; + } + } + + /// Returns committed messages in the current bucket, or zero. + internal long CommittedMessages(string quotaKey, char window, long bucketId) => + ReadLong(quotaKey, PostFilterQuotaKeys.CommittedMessagesField(window, bucketId)); + + /// Returns committed bytes in the current bucket, or zero. + internal long CommittedBytes(string quotaKey, char window, long bucketId) => + ReadLong(quotaKey, PostFilterQuotaKeys.CommittedBytesField(window, bucketId)); + + /// Returns committed identical count, or zero. + internal long CommittedIdentical(string multipostKey, string bodyHex, char window, long bucketId) => + ReadLong(multipostKey, PostFilterQuotaKeys.MultipostField(bodyHex, window, bucketId)); + + /// Returns whether a reservation field is present (including expired until pruned). + internal bool HasReservationField(string quotaKey, PostFilterReservationId reservation) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) && quota.ContainsKey(reservation.Field); + } + } + + /// Returns whether a live (unexpired) reservation exists. + internal bool HasLiveReservation(string quotaKey, PostFilterReservationId reservation, long nowMs) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) + && quota.TryGetValue(reservation.Field, out var raw) + && PostFilterQuotaKeys.TryParseReservation(raw, out var exp, out var gen, out _, out _, out _, out _) + && gen == reservation.Generation + && exp > nowMs; + } + } + + /// Returns the number of fields on the quota hash. + internal int QuotaFieldCount(string quotaKey) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) ? quota.Count : 0; + } + } + + /// Returns the number of fields on the multipost hash. + internal int MultipostFieldCount(string multipostKey) + { + lock (_gate) + { + return _hashes.TryGetValue(multipostKey, out var multipost) ? multipost.Count : 0; + } + } + + /// Returns reservation field count (prefix r:). + internal int ReservationFieldCount(string quotaKey) + { + lock (_gate) + { + if (!_hashes.TryGetValue(quotaKey, out var quota)) + { + return 0; + } + + var count = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var field in quota.Keys) + { + if (field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + count++; + } + } + + return count; + } + } + + /// Writes a reservation field (tests: expiry / generation mismatch). + internal void SeedReservation( + string quotaKey, + PostFilterReservationId reservation, + long expiryMs, + long storedGeneration, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + lock (_gate) + { + GetOrCreateHash(quotaKey)[reservation.Field] = PostFilterQuotaKeys.PackReservation( + expiryMs, storedGeneration, messages, bytes, mpUnits, bodyHex); + } + } + + /// Writes a committed counter field (tests). + internal void SeedCommitted(string key, string field, long value) + { + lock (_gate) + { + GetOrCreateHash(key)[field] = value.ToString(CultureInfo.InvariantCulture); + } + } + + private long ReadLong(string key, string field) + { + lock (_gate) + { + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var raw) + || !long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value)) + { + return 0; + } + + return value; + } + } + + private static long FirstDenial( + Dictionary quota, + Dictionary multipost, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long nowMs, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + SumLive(quota, nowMs, bodyHex, mpUnits, out var resMsg, out var resBytes, out var resIdentical); + var bucketL = windows.BucketId(PostFilterQuotaKeys.LongWindow, nowMs); + var bucketS = windows.BucketId(PostFilterQuotaKeys.ShortWindow, nowMs); + var cml = ReadField(quota, PostFilterQuotaKeys.CommittedMessagesField(PostFilterQuotaKeys.LongWindow, bucketL)); + var cbl = ReadField(quota, PostFilterQuotaKeys.CommittedBytesField(PostFilterQuotaKeys.LongWindow, bucketL)); + var cms = ReadField(quota, PostFilterQuotaKeys.CommittedMessagesField(PostFilterQuotaKeys.ShortWindow, bucketS)); + var cbs = ReadField(quota, PostFilterQuotaKeys.CommittedBytesField(PostFilterQuotaKeys.ShortWindow, bucketS)); + var cil = mpUnits == 1 + ? ReadField(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, PostFilterQuotaKeys.LongWindow, bucketL)) + : 0; + var cis = mpUnits == 1 + ? ReadField(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, PostFilterQuotaKeys.ShortWindow, bucketS)) + : 0; + + if (ceilings.MaxMessagesLong > 0 && cml + resMsg + messages > ceilings.MaxMessagesLong) + { + return PostFilterQuotaReserveCodes.DeniedMessagesLong; + } + + if (ceilings.MaxBytesLong > 0 && cbl + resBytes + bytes > ceilings.MaxBytesLong) + { + return PostFilterQuotaReserveCodes.DeniedBytesLong; + } + + if (mpUnits == 1 + && ceilings.MaxIdenticalLong > 0 + && cil + resIdentical + mpUnits > ceilings.MaxIdenticalLong) + { + return PostFilterQuotaReserveCodes.DeniedIdenticalLong; + } + + if (ceilings.MaxMessagesShort > 0 && cms + resMsg + messages > ceilings.MaxMessagesShort) + { + return PostFilterQuotaReserveCodes.DeniedMessagesShort; + } + + if (ceilings.MaxBytesShort > 0 && cbs + resBytes + bytes > ceilings.MaxBytesShort) + { + return PostFilterQuotaReserveCodes.DeniedBytesShort; + } + + if (mpUnits == 1 + && ceilings.MaxIdenticalShort > 0 + && cis + resIdentical + mpUnits > ceilings.MaxIdenticalShort) + { + return PostFilterQuotaReserveCodes.DeniedIdenticalShort; + } + + return PostFilterQuotaReserveCodes.Accept; + } + + private static void SumLive( + Dictionary quota, + long nowMs, + string bodyHex, + int requestMpUnits, + out long messages, + out long bytes, + out long identical) + { + messages = 0; + bytes = 0; + identical = 0; + foreach (var (field, value) in quota) + { + if (!field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + continue; + } + + if (!PostFilterQuotaKeys.TryParseReservation(value, out var exp, out _, out var m, out var b, out var mp, out var body) + || exp <= nowMs) + { + continue; + } + + messages += m; + bytes += b; + if (requestMpUnits == 1 && mp == 1 && string.Equals(body, bodyHex, StringComparison.Ordinal)) + { + identical += mp; + } + } + } + + private static void IncrementCurrent( + Dictionary quota, + Dictionary multipost, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long nowMs, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + foreach (var window in new[] { PostFilterQuotaKeys.LongWindow, PostFilterQuotaKeys.ShortWindow }) + { + var bucket = windows.BucketId(window, nowMs); + if (ceilings.MaxMessages(window) > 0) + { + Add(quota, PostFilterQuotaKeys.CommittedMessagesField(window, bucket), messages); + } + + if (ceilings.MaxBytes(window) > 0) + { + Add(quota, PostFilterQuotaKeys.CommittedBytesField(window, bucket), bytes); + } + + if (mpUnits == 1 && bodyHex.Length > 0 && ceilings.MaxIdentical(window) > 0) + { + Add(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, window, bucket), mpUnits); + } + } + } + + private static void Add(Dictionary hash, string field, long delta) + { + var current = ReadField(hash, field); + hash[field] = (current + delta).ToString(CultureInfo.InvariantCulture); + } + + private static long ReadField(Dictionary hash, string field) => + hash.TryGetValue(field, out var raw) + && long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) + ? value + : 0; + + private static void PruneQuota( + Dictionary quota, + PostFilterQuotaWindows windows, + long nowMs) + { + List? dead = null; + foreach (var (field, value) in quota) + { + if (field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + if (!PostFilterQuotaKeys.TryParseReservation(value, out var exp, out _, out _, out _, out _, out _) + || exp <= nowMs) + { + (dead ??= []).Add(field); + } + + continue; + } + + if (PostFilterQuotaKeys.TryParseCommittedField(field, out _, out var window, out var bucket) + && windows.BucketEnded(window, bucket, nowMs)) + { + (dead ??= []).Add(field); + } + } + + DeleteDead(quota, dead); + } + + private static void PruneMultipost( + Dictionary multipost, + PostFilterQuotaWindows windows, + long nowMs) + { + List? dead = null; + // codeql[cs/linq/missed-where] TryParse out-params; collect-then-remove without LINQ allocation. + foreach (var field in multipost.Keys) + { + if (PostFilterQuotaKeys.TryParseMultipostField(field, out _, out var window, out var bucket) + && windows.BucketEnded(window, bucket, nowMs)) + { + (dead ??= []).Add(field); + } + } + + DeleteDead(multipost, dead); + } + + private static void DeleteDead(Dictionary hash, List? dead) + { + if (dead is null) + { + return; + } + + foreach (var field in dead) + { + hash.Remove(field); + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private static void ValidateReserve( + string quotaKey, + string multipostKey, + long nowMs, + long reservationTtlMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + ref string bodyHex) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + ArgumentOutOfRangeException.ThrowIfNegative(nowMs); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(reservationTtlMs, 0); + windows.Validate(); + ceilings.Validate(); + ArgumentOutOfRangeException.ThrowIfNegative(messages); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + if (mpUnits is not (0 or 1)) + { + throw new ArgumentOutOfRangeException(nameof(mpUnits), mpUnits, "mpUnits must be 0 or 1."); + } + + bodyHex ??= string.Empty; + if (mpUnits == 0) + { + bodyHex = string.Empty; + return; + } + + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaKeys.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaKeys.cs new file mode 100644 index 00000000..8a667593 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaKeys.cs @@ -0,0 +1,211 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Redis key and field material for PostFilter accept-quota state. + /// + /// + /// Keys: nntpd:pf:q:{sha256hex(account)} and nntpd:pf:m:{sha256hex(account)}. + /// Account hashing matches nntpd:sess: / nntpd:account: + /// (UTF-8 SHA-256, lowercase hex). Reservation value: + /// expiryMs|generation|messages|bytes|mpUnits|bodyHex. + /// + internal static class PostFilterQuotaKeys + { + /// ASCII namespace for committed + reservation hashes. + public static ReadOnlySpan QuotaNamespacePrefix => "nntpd:pf:q:"u8; + + /// ASCII namespace for committed identical-body hashes. + public static ReadOnlySpan MultipostNamespacePrefix => "nntpd:pf:m:"u8; + + /// Prefix of live reservation fields. + public const string ReservationFieldPrefix = "r:"; + + /// Separates packed reservation fields. + public const char ValueSeparator = '|'; + + /// Long / sustained window identifier. + public const char LongWindow = 'L'; + + /// Short / burst window identifier. + public const char ShortWindow = 'S'; + + /// Builds the quota HASH key for . + public static byte[] CreateQuota(string accountName) => Create(QuotaNamespacePrefix, accountName); + + /// Builds the multipost HASH key for . + public static byte[] CreateMultipost(string accountName) => Create(MultipostNamespacePrefix, accountName); + + /// Builds c:m:{W}:{bucketId}. + public static string CommittedMessagesField(char window, long bucketId) => + string.Concat("c:m:", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + + /// Builds c:b:{W}:{bucketId}. + public static string CommittedBytesField(char window, long bucketId) => + string.Concat("c:b:", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + + /// Builds {bodyHex}:{W}:{bucketId}. + public static string MultipostField(string bodyHex, char window, long bucketId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + return string.Concat(bodyHex, ":", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + } + + /// Packs a reservation value. + public static string PackReservation( + long expiryMs, + long generation, + long messages, + long bytes, + int mpUnits, + string bodyHex) => + string.Concat( + expiryMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + messages.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + bytes.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + mpUnits.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + bodyHex); + + /// Parses a reservation value. Returns when malformed. + public static bool TryParseReservation( + string value, + out long expiryMs, + out long generation, + out long messages, + out long bytes, + out int mpUnits, + out string bodyHex) + { + expiryMs = 0; + generation = 0; + messages = 0; + bytes = 0; + mpUnits = 0; + bodyHex = string.Empty; + if (string.IsNullOrEmpty(value)) + { + return false; + } + + // Lua split_res takes the first five '|' fields; the remainder is bodyHex + // and may itself contain '|'. Require exactly those five separators. + var fields = new string[5]; + var start = 0; + for (var i = 0; i < 5; i++) + { + var sep = value.IndexOf(ValueSeparator, start); + if (sep < 0) + { + return false; + } + + fields[i] = value[start..sep]; + start = sep + 1; + } + + if (!long.TryParse(fields[0], NumberStyles.None, CultureInfo.InvariantCulture, out expiryMs) + || !long.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out generation) + || !long.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out messages) + || !long.TryParse(fields[3], NumberStyles.None, CultureInfo.InvariantCulture, out bytes) + || !int.TryParse(fields[4], NumberStyles.None, CultureInfo.InvariantCulture, out mpUnits)) + { + return false; + } + + bodyHex = value[start..]; + return true; + } + + /// Parses c:m|b:{W}:{bucket}. Returns when not a committed field. + public static bool TryParseCommittedField(string field, out char kind, out char window, out long bucketId) + { + kind = default; + window = default; + bucketId = 0; + if (field.Length < 7 || field[0] != 'c' || field[1] != ':') + { + return false; + } + + if (field[2] is not ('m' or 'b') || field[3] != ':') + { + return false; + } + + kind = field[2]; + if (field[4] is not (LongWindow or ShortWindow) || field[5] != ':') + { + return false; + } + + window = field[4]; + return long.TryParse(field.AsSpan(6), NumberStyles.None, CultureInfo.InvariantCulture, out bucketId); + } + + /// Parses {bodyHex}:{W}:{bucket} from the right. + public static bool TryParseMultipostField(string field, out string bodyHex, out char window, out long bucketId) + { + bodyHex = string.Empty; + window = default; + bucketId = 0; + var last = field.LastIndexOf(':'); + if (last <= 2 || last >= field.Length - 1) + { + return false; + } + + var mid = field.LastIndexOf(':', last - 1); + if (mid <= 0 || last != mid + 2) + { + return false; + } + + var w = field[mid + 1]; + if (w is not (LongWindow or ShortWindow)) + { + return false; + } + + if (!long.TryParse(field.AsSpan(last + 1), NumberStyles.None, CultureInfo.InvariantCulture, out bucketId)) + { + return false; + } + + window = w; + bodyHex = field[..mid]; + return bodyHex.Length > 0; + } + + private static byte[] Create(ReadOnlySpan prefix, string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaResults.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaResults.cs new file mode 100644 index 00000000..8ead9b96 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaResults.cs @@ -0,0 +1,107 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// Integer codes returned by the RESERVE EVAL (and the matching engine). + internal static class PostFilterQuotaReserveCodes + { + /// Accepted, including an idempotent live-token hit. + public const long Accept = 0; + + /// First failing enabled dimension: messages L. + public const long DeniedMessagesLong = 1; + + /// Bytes L. + public const long DeniedBytesLong = 2; + + /// Identical L. + public const long DeniedIdenticalLong = 3; + + /// Messages S. + public const long DeniedMessagesShort = 4; + + /// Bytes S. + public const long DeniedBytesShort = 5; + + /// Identical S. + public const long DeniedIdenticalShort = 6; + + /// Live token exists with different units or body hash. No write. + public const long Conflict = 7; + } + + /// Mapped RESERVE outcome including store-level Redis unavailability. + internal enum PostFilterQuotaReserveStatus + { + /// Reservation is live and counts toward the ceiling. + Accepted = 0, + + /// Messages L would exceed. + DeniedMessagesLong = 1, + + /// Bytes L would exceed. + DeniedBytesLong = 2, + + /// Identical L would exceed. + DeniedIdenticalLong = 3, + + /// Messages S would exceed. + DeniedMessagesShort = 4, + + /// Bytes S would exceed. + DeniedBytesShort = 5, + + /// Identical S would exceed. + DeniedIdenticalShort = 6, + + /// Same token, different payload. State unchanged. + Conflict = 7, + + /// Redis unavailable. No reservation. Caller fail-closes. + Unavailable = 8, + } + + /// Mapped COMMIT outcome. + internal enum PostFilterQuotaCommitStatus + { + /// No live matching reservation. Committed counters unchanged. + Noop = 0, + + /// Units transferred into the current buckets and the reservation deleted. + Committed = 1, + + /// Redis unavailable. Reservation, if any, is unchanged. + Unavailable = 2, + } + + /// Mapped RELEASE outcome. + internal enum PostFilterQuotaReleaseStatus + { + /// Nothing deleted. + Noop = 0, + + /// Matching live reservation deleted. + Released = 1, + + /// Redis unavailable. Reservation, if any, is unchanged. + Unavailable = 2, + } + + /// Integer codes returned by the COMMIT EVAL. + internal static class PostFilterQuotaCommitCodes + { + /// No transfer. + public const long Noop = 0; + + /// Transferred. + public const long Committed = 1; + } + + /// Integer codes returned by the RELEASE EVAL. + internal static class PostFilterQuotaReleaseCodes + { + /// No delete. + public const long Noop = 0; + + /// Deleted. + public const long Released = 1; + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaScripts.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaScripts.cs new file mode 100644 index 00000000..83bd709a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaScripts.cs @@ -0,0 +1,454 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Redis Lua scripts for atomic PostFilter accept-quota reserve / commit / release. + /// Semantics match . + /// + internal static class PostFilterQuotaScripts + { + /// + /// Evaluates all enabled ceilings and creates one reservation, or writes nothing. + /// + /// + /// KEYS[1] quota hash, KEYS[2] multipost hash. + /// ARGV: nowMs, reservationTtlMs, reservationToken, generation, messages, bytes, + /// mpUnits, bodyHex, longMs, shortMs, maxML, maxBL, maxIL, maxMS, maxBS, maxIS, idleMs. + /// Returns 0 accept, 1–6 first deny, 7 conflict. + /// + public const string Reserve = + """ + -- nntpd-pf-reserve + local qKey = KEYS[1] + local mKey = KEYS[2] + local now = tonumber(ARGV[1]) + local ttl = tonumber(ARGV[2]) + local resId = ARGV[3] + local gen = ARGV[4] + local messages = tonumber(ARGV[5]) + local bytes = tonumber(ARGV[6]) + local mpUnits = tonumber(ARGV[7]) + local bodyHex = ARGV[8] + local longMs = tonumber(ARGV[9]) + local shortMs = tonumber(ARGV[10]) + local maxML = tonumber(ARGV[11]) + local maxBL = tonumber(ARGV[12]) + local maxIL = tonumber(ARGV[13]) + local maxMS = tonumber(ARGV[14]) + local maxBS = tonumber(ARGV[15]) + local maxIS = tonumber(ARGV[16]) + local idleMs = tonumber(ARGV[17]) + local rField = 'r:' .. resId + + local function window_ms(w) + if w == 'L' then return longMs end + return shortMs + end + + local function bucket_id(w) + return math.floor(now / window_ms(w)) + end + + local function bucket_ended(w, b) + return ((b + 1) * window_ms(w)) <= now + end + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local function parse_c_field(f) + if string.sub(f, 1, 2) ~= 'c:' then return nil end + local kind = string.sub(f, 3, 3) + if (kind ~= 'm' and kind ~= 'b') or string.sub(f, 4, 4) ~= ':' then return nil end + local w = string.sub(f, 5, 5) + if (w ~= 'L' and w ~= 'S') or string.sub(f, 6, 6) ~= ':' then return nil end + local b = tonumber(string.sub(f, 7)) + if b == nil then return nil end + return kind, w, b + end + + local function parse_m_field(f) + local last = string.len(f) + local colon2 = nil + for i = last, 1, -1 do + if string.sub(f, i, i) == ':' then + if colon2 == nil then + colon2 = i + else + local w = string.sub(f, i + 1, colon2 - 1) + local b = tonumber(string.sub(f, colon2 + 1)) + local body = string.sub(f, 1, i - 1) + if (w == 'L' or w == 'S') and b ~= nil and string.len(body) > 0 then + return body, w, b + end + return nil + end + end + end + return nil + end + + local function hdel_list(key, dead) + if #dead == 0 then return end + redis.call('HDEL', key, unpack(dead)) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + local function prune_q() + local entries = redis.call('HGETALL', qKey) + local dead = {} + for i = 1, #entries, 2 do + local f = entries[i] + if string.sub(f, 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec == nil or rec.exp <= now then + dead[#dead + 1] = f + end + else + local _, w, b = parse_c_field(f) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = f + end + end + end + hdel_list(qKey, dead) + end + + local function prune_m() + local entries = redis.call('HGETALL', mKey) + local dead = {} + for i = 1, #entries, 2 do + local body, w, b = parse_m_field(entries[i]) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = entries[i] + end + end + hdel_list(mKey, dead) + end + + local function touch() + if idleMs ~= nil and idleMs > 0 then + if redis.call('EXISTS', qKey) == 1 then + redis.call('PEXPIRE', qKey, idleMs) + end + if redis.call('EXISTS', mKey) == 1 then + redis.call('PEXPIRE', mKey, idleMs) + end + end + end + + if mpUnits == 0 then + bodyHex = '' + end + + prune_q() + prune_m() + + local existing = redis.call('HGET', qKey, rField) + if existing then + local rec = split_res(existing) + if rec and rec.exp > now and rec.gen == gen and rec.messages == messages and rec.bytes == bytes and rec.mp == mpUnits and rec.body == bodyHex then + return 0 + end + return 7 + end + + local bL = bucket_id('L') + local bS = bucket_id('S') + local function hnum(key, field) + local v = redis.call('HGET', key, field) + if not v then return 0 end + return tonumber(v) or 0 + end + + local cml = hnum(qKey, 'c:m:L:' .. bL) + local cbl = hnum(qKey, 'c:b:L:' .. bL) + local cms = hnum(qKey, 'c:m:S:' .. bS) + local cbs = hnum(qKey, 'c:b:S:' .. bS) + local cil = 0 + local cis = 0 + if mpUnits == 1 and bodyHex ~= '' then + cil = hnum(mKey, bodyHex .. ':L:' .. bL) + cis = hnum(mKey, bodyHex .. ':S:' .. bS) + end + + local resMsg, resBytes, resIdent = 0, 0, 0 + local entries = redis.call('HGETALL', qKey) + for i = 1, #entries, 2 do + if string.sub(entries[i], 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec and rec.exp > now then + resMsg = resMsg + rec.messages + resBytes = resBytes + rec.bytes + if mpUnits == 1 and rec.mp == 1 and rec.body == bodyHex then + resIdent = resIdent + rec.mp + end + end + end + end + + if maxML > 0 and (cml + resMsg + messages) > maxML then return 1 end + if maxBL > 0 and (cbl + resBytes + bytes) > maxBL then return 2 end + if mpUnits == 1 and maxIL > 0 and (cil + resIdent + mpUnits) > maxIL then return 3 end + if maxMS > 0 and (cms + resMsg + messages) > maxMS then return 4 end + if maxBS > 0 and (cbs + resBytes + bytes) > maxBS then return 5 end + if mpUnits == 1 and maxIS > 0 and (cis + resIdent + mpUnits) > maxIS then return 6 end + + redis.call('HSET', qKey, rField, tostring(now + ttl) .. '|' .. gen .. '|' .. tostring(messages) .. '|' .. tostring(bytes) .. '|' .. tostring(mpUnits) .. '|' .. bodyHex) + touch() + return 0 + """; + + /// + /// Transfers reservation units into the current buckets, or no-ops. + /// + /// + /// KEYS[1] quota, KEYS[2] multipost. + /// ARGV: nowMs, reservationToken, generation, longMs, shortMs, + /// maxML, maxBL, maxIL, maxMS, maxBS, maxIS, idleMs. + /// Units are read only from the reservation. Returns 0 no-op, 1 committed. + /// + public const string Commit = + """ + -- nntpd-pf-commit + local qKey = KEYS[1] + local mKey = KEYS[2] + local now = tonumber(ARGV[1]) + local resId = ARGV[2] + local gen = ARGV[3] + local longMs = tonumber(ARGV[4]) + local shortMs = tonumber(ARGV[5]) + local maxML = tonumber(ARGV[6]) + local maxBL = tonumber(ARGV[7]) + local maxIL = tonumber(ARGV[8]) + local maxMS = tonumber(ARGV[9]) + local maxBS = tonumber(ARGV[10]) + local maxIS = tonumber(ARGV[11]) + local idleMs = tonumber(ARGV[12]) + local rField = 'r:' .. resId + + local function window_ms(w) + if w == 'L' then return longMs end + return shortMs + end + + local function bucket_id(w) + return math.floor(now / window_ms(w)) + end + + local function bucket_ended(w, b) + return ((b + 1) * window_ms(w)) <= now + end + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local function parse_c_field(f) + if string.sub(f, 1, 2) ~= 'c:' then return nil end + local kind = string.sub(f, 3, 3) + if (kind ~= 'm' and kind ~= 'b') or string.sub(f, 4, 4) ~= ':' then return nil end + local w = string.sub(f, 5, 5) + if (w ~= 'L' and w ~= 'S') or string.sub(f, 6, 6) ~= ':' then return nil end + local b = tonumber(string.sub(f, 7)) + if b == nil then return nil end + return kind, w, b + end + + local function parse_m_field(f) + local last = string.len(f) + local colon2 = nil + for i = last, 1, -1 do + if string.sub(f, i, i) == ':' then + if colon2 == nil then + colon2 = i + else + local w = string.sub(f, i + 1, colon2 - 1) + local b = tonumber(string.sub(f, colon2 + 1)) + local body = string.sub(f, 1, i - 1) + if (w == 'L' or w == 'S') and b ~= nil and string.len(body) > 0 then + return body, w, b + end + return nil + end + end + end + return nil + end + + local function hdel_list(key, dead) + if #dead == 0 then return end + redis.call('HDEL', key, unpack(dead)) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + local function prune_q() + local entries = redis.call('HGETALL', qKey) + local dead = {} + for i = 1, #entries, 2 do + local f = entries[i] + if string.sub(f, 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec == nil or rec.exp <= now then + dead[#dead + 1] = f + end + else + local _, w, b = parse_c_field(f) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = f + end + end + end + hdel_list(qKey, dead) + end + + local function prune_m() + local entries = redis.call('HGETALL', mKey) + local dead = {} + for i = 1, #entries, 2 do + local body, w, b = parse_m_field(entries[i]) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = entries[i] + end + end + hdel_list(mKey, dead) + end + + prune_q() + prune_m() + + local existing = redis.call('HGET', qKey, rField) + local rec = split_res(existing) + if rec == nil then + if existing then + redis.call('HDEL', qKey, rField) + end + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + if rec.gen ~= gen then + return 0 + end + if rec.exp <= now then + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + + local bL = bucket_id('L') + local bS = bucket_id('S') + if maxML > 0 then redis.call('HINCRBY', qKey, 'c:m:L:' .. bL, rec.messages) end + if maxBL > 0 then redis.call('HINCRBY', qKey, 'c:b:L:' .. bL, rec.bytes) end + if maxMS > 0 then redis.call('HINCRBY', qKey, 'c:m:S:' .. bS, rec.messages) end + if maxBS > 0 then redis.call('HINCRBY', qKey, 'c:b:S:' .. bS, rec.bytes) end + if rec.mp == 1 and rec.body ~= '' then + if maxIL > 0 then redis.call('HINCRBY', mKey, rec.body .. ':L:' .. bL, rec.mp) end + if maxIS > 0 then redis.call('HINCRBY', mKey, rec.body .. ':S:' .. bS, rec.mp) end + end + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + if idleMs ~= nil and idleMs > 0 then + if redis.call('EXISTS', qKey) == 1 then redis.call('PEXPIRE', qKey, idleMs) end + if redis.call('EXISTS', mKey) == 1 then redis.call('PEXPIRE', mKey, idleMs) end + end + return 1 + """; + + /// + /// Deletes a matching live reservation. Never decrements committed usage. + /// + /// + /// KEYS[1] quota, KEYS[2] multipost (unused except for KEYS contract). + /// ARGV: nowMs, reservationToken, generation. + /// Returns 0 no-op, 1 released. + /// + public const string Release = + """ + -- nntpd-pf-release + local qKey = KEYS[1] + local now = tonumber(ARGV[1]) + local resId = ARGV[2] + local gen = ARGV[3] + local rField = 'r:' .. resId + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local existing = redis.call('HGET', qKey, rField) + local rec = split_res(existing) + if rec == nil then + if existing then + redis.call('HDEL', qKey, rField) + end + if redis.call('EXISTS', qKey) == 1 and redis.call('HLEN', qKey) == 0 then + redis.call('DEL', qKey) + end + return 0 + end + if rec.gen ~= gen then + return 0 + end + if rec.exp <= now then + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaWindows.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaWindows.cs new file mode 100644 index 00000000..4fe055a9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterQuotaWindows.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// Fixed-epoch window lengths in milliseconds. + /// Sustained (L) window. Must be greater than zero. + /// Burst (S) window. Must be greater than zero. + internal readonly record struct PostFilterQuotaWindows(long LongWindowMs, long ShortWindowMs) + { + /// Returns the current bucket id for at . + public long BucketId(char window, long nowMs) + { + var ms = WindowMs(window); + return nowMs / ms; + } + + /// Returns whether bucket of has ended. + public bool BucketEnded(char window, long bucketId, long nowMs) => + (bucketId + 1) * WindowMs(window) <= nowMs; + + /// Returns the millisecond length of . + public long WindowMs(char window) => window == PostFilterQuotaKeys.LongWindow + ? LongWindowMs + : ShortWindowMs; + + /// Throws when either window length is not positive. + public void Validate() + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(LongWindowMs, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(ShortWindowMs, 0); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterReservationId.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterReservationId.cs new file mode 100644 index 00000000..39d1aa7c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/PostFilterReservationId.cs @@ -0,0 +1,41 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// One in-flight POST reservation: {nodeId}:{incarnation}:{generation}. + /// + /// + /// Not session occupancy. nodeId and incarnation follow + /// DistributedSessionStateTracker ownership. generation is unique + /// per reservation so one process can hold many concurrent POSTs. + /// + internal readonly record struct PostFilterReservationId + { + /// Initializes a reservation identity. + public PostFilterReservationId(string nodeId, string incarnation, long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentException.ThrowIfNullOrWhiteSpace(incarnation); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(generation, 0); + NodeId = nodeId; + Incarnation = incarnation; + Generation = generation; + } + + /// Gets the NNTPD node identity. + public string NodeId { get; } + + /// Gets the process-lifetime incarnation. + public string Incarnation { get; } + + /// Gets the per-reservation generation. + public long Generation { get; } + + /// Gets {nodeId}:{incarnation}:{generation}. + public string Token => string.Concat(NodeId, ":", Incarnation, ":", Generation.ToString(CultureInfo.InvariantCulture)); + + /// Gets the quota-hash field r:{token}. + public string Field => string.Concat(PostFilterQuotaKeys.ReservationFieldPrefix, Token); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/Quota/RedisPostFilterQuotaStore.cs b/src/VectorNNTPD.ReaderService/PostFilter/Quota/RedisPostFilterQuotaStore.cs new file mode 100644 index 00000000..4f4c6be5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/Quota/RedisPostFilterQuotaStore.cs @@ -0,0 +1,212 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.PostFilter.Quota +{ + /// + /// Redis-backed accept-quota using one atomic Lua EVAL per reserve / commit / release + /// on the shared connection. + /// + internal sealed class RedisPostFilterQuotaStore : IPostFilterQuotaStore + { + private readonly IRedisService _redis; + + /// Initializes a Redis-backed quota store. + public RedisPostFilterQuotaStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + windows.Validate(); + ceilings.Validate(); + ArgumentOutOfRangeException.ThrowIfNegative(messages); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + if (mpUnits is not (0 or 1)) + { + throw new ArgumentOutOfRangeException(nameof(mpUnits), mpUnits, "mpUnits must be 0 or 1."); + } + + var ttl = ResolveReservationTtlMs(reservationTtlMs); + var hash = mpUnits == 1 ? bodyHex : string.Empty; + if (mpUnits == 1) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + hash = bodyHex; + } + + return ExecuteAsync( + PostFilterQuotaScripts.Reserve, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(ttl), + Utf8(reservation.Token), + Utf8(reservation.Generation), + Utf8(messages), + Utf8(bytes), + Utf8(mpUnits), + Utf8(hash ?? string.Empty), + Utf8(windows.LongWindowMs), + Utf8(windows.ShortWindowMs), + Utf8(ceilings.MaxMessagesLong), + Utf8(ceilings.MaxBytesLong), + Utf8(ceilings.MaxIdenticalLong), + Utf8(ceilings.MaxMessagesShort), + Utf8(ceilings.MaxBytesShort), + Utf8(ceilings.MaxIdenticalShort), + Utf8(IdleTtlMs(windows, ttl)), + ], + static result => result is >= 0 and <= 7 + ? (PostFilterQuotaReserveStatus)result + : PostFilterQuotaReserveStatus.Unavailable, + PostFilterQuotaReserveStatus.Unavailable, + cancellationToken); + } + + /// + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + windows.Validate(); + ceilings.Validate(); + return ExecuteAsync( + PostFilterQuotaScripts.Commit, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(reservation.Token), + Utf8(reservation.Generation), + Utf8(windows.LongWindowMs), + Utf8(windows.ShortWindowMs), + Utf8(ceilings.MaxMessagesLong), + Utf8(ceilings.MaxBytesLong), + Utf8(ceilings.MaxIdenticalLong), + Utf8(ceilings.MaxMessagesShort), + Utf8(ceilings.MaxBytesShort), + Utf8(ceilings.MaxIdenticalShort), + Utf8(IdleTtlMs(windows, (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds)), + ], + static result => result == PostFilterQuotaCommitCodes.Committed + ? PostFilterQuotaCommitStatus.Committed + : PostFilterQuotaCommitStatus.Noop, + PostFilterQuotaCommitStatus.Unavailable, + cancellationToken); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) => + ExecuteAsync( + PostFilterQuotaScripts.Release, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(reservation.Token), + Utf8(reservation.Generation), + ], + static result => result == PostFilterQuotaReleaseCodes.Released + ? PostFilterQuotaReleaseStatus.Released + : PostFilterQuotaReleaseStatus.Noop, + PostFilterQuotaReleaseStatus.Unavailable, + cancellationToken); + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + long result; + try + { + // Observe EVAL even if the caller cancelled. WaitAsync on the Redis + // task would otherwise abandon a completed Lua write and leak a + // reservation until TTL. After the result is known, propagate + // cancel so EvaluateAsync RELEASE can delete the row. + result = await database.ScriptEvaluateAsync( + script, + [PostFilterQuotaKeys.CreateQuota(accountName), PostFilterQuotaKeys.CreateMultipost(accountName)], + values, + CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + cancellationToken.ThrowIfCancellationRequested(); + return map(result); + } + + private static long ResolveReservationTtlMs(long reservationTtlMs) => + reservationTtlMs > 0 + ? reservationTtlMs + : (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds; + + private static long IdleTtlMs(PostFilterQuotaWindows windows, long reservationTtlMs) => + Math.Max(windows.LongWindowMs, windows.ShortWindowMs) + + reservationTtlMs + + PostFilterQuotaDefaults.IdleTtlSkewMs; + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + + private static byte[] Utf8(long value) => + Encoding.UTF8.GetBytes(value.ToString(CultureInfo.InvariantCulture)); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/SpamdCheckClient.cs b/src/VectorNNTPD.ReaderService/PostFilter/SpamdCheckClient.cs new file mode 100644 index 00000000..98dca870 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/SpamdCheckClient.cs @@ -0,0 +1,639 @@ +using System.Globalization; +using System.Net.Sockets; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Persistent SPAMD CHECK client. Builds a disposable scan message and never + /// mutates . + /// + /// + /// + /// SPAMD PROTOCOL has no request identifiers and says each side shuts down after + /// writing. This client therefore does not pipeline or multiplex CHECKs on one + /// socket. Each pooled connection serializes requests. Concurrent CHECKs use + /// distinct connections up to the snapshot pool size. + /// + /// + /// After a complete CHECK response the socket is returned to the pool when it is + /// still connected and has no leftover bytes. A broken connection is evicted + /// before another request can use it. Hosts, port, pool size, and timeouts come + /// from the PostFilter snapshot captured for this POST. + /// + /// + internal sealed class SpamdCheckClient : IPostFilterSpamAssassin, IAsyncDisposable + { + private readonly object _gate = new(); + private readonly Dictionary<(string Host, int Port), Queue> _idle = new(); + private readonly List _waiters = []; + private readonly ILogger? _logger; + private int _live; + private int _roundRobin; + private bool _disposed; + + /// Initializes the client with process-wide transport counters. + public SpamdCheckClient() + : this(new SpamdTransportMetrics(), logger: null) + { + } + + /// Initializes the client with injected counters and optional shutdown logging. + public SpamdCheckClient(SpamdTransportMetrics metrics, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(metrics); + Metrics = metrics; + _logger = logger; + } + + /// Gets process-wide transport counters. + public SpamdTransportMetrics Metrics { get; } + + /// + public async ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_disposed) + { + return PostFilterSpamAssassinResult.Failed("disposed"); + } + + if (target.Hosts.Count == 0) + { + return PostFilterSpamAssassinResult.Failed("no-hosts"); + } + + byte[] scan; + try + { + scan = SpamdScanArticleBuilder.Build(article, scanContext); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + } + + var start = SelectStartIndex(target); + Exception? lastConnectFailure = null; + for (var attempt = 0; attempt < target.Hosts.Count; attempt++) + { + var host = target.Hosts[(start + attempt) % target.Hosts.Count]; + if (string.IsNullOrWhiteSpace(host)) + { + lastConnectFailure = new InvalidOperationException("empty-host"); + continue; + } + + host = host.Trim(); + SpamdConnection? leased = null; + var reused = false; + try + { + (leased, reused) = await RentAsync(host, target, cancellationToken).ConfigureAwait(false); + var started = TimeProvider.System.GetTimestamp(); + try + { + var result = await CheckOnConnectionAsync( + leased, + scan, + accountName, + target, + cancellationToken) + .ConfigureAwait(false); + if (reused && IsStaleIdleFailure(result)) + { + throw new IOException(result.Detail); + } + + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + Return(leased, reusable: result.Status != PostFilterSpamAssassinStatus.Failed); + return result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Evict(leased); + throw; + } + catch (OperationCanceledException) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (reused && IsConnectionFailure(ex)) + { + Evict(leased); + (leased, reused) = await RentAsync(host, target, cancellationToken).ConfigureAwait(false); + started = TimeProvider.System.GetTimestamp(); + try + { + var retry = await CheckOnConnectionAsync( + leased, + scan, + accountName, + target, + cancellationToken) + .ConfigureAwait(false); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + Return(leased, reusable: retry.Status != PostFilterSpamAssassinStatus.Failed); + return retry; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Evict(leased); + throw; + } + catch (OperationCanceledException) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception retryEx) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return ClassifyAfterWrite(retryEx); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return ClassifyAfterWrite(ex); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (leased is null) + { + lastConnectFailure = new TimeoutException("connect-timeout"); + if (attempt + 1 < target.Hosts.Count) + { + continue; + } + + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (leased is null && IsConnectionFailure(ex) && attempt + 1 < target.Hosts.Count) + { + lastConnectFailure = ex; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (leased is null) + { + return ClassifyConnect(ex); + } + finally + { + _ = reused; + } + } + + return PostFilterSpamAssassinResult.Failed(lastConnectFailure?.GetType().Name ?? "no-hosts"); + } + + /// + public async ValueTask DisposeAsync() + { + List idle; + List waiters; + lock (_gate) + { + if (_disposed) + { + return; + } + + _disposed = true; + idle = []; + foreach (var queue in _idle.Values) + { + while (queue.Count > 0) + { + idle.Add(queue.Dequeue()); + } + } + + _idle.Clear(); + _live = 0; + waiters = [.. _waiters]; + _waiters.Clear(); + } + + foreach (var waiter in waiters) + { + waiter.TrySetCanceled(); + } + + foreach (var connection in idle) + { + connection.Dispose(); + } + + if (_logger is not null) + { + PostFilterLogMessages.SpamAssassinTransportStopped( + _logger, + Metrics.ConnectionsEstablished, + Metrics.CheckRequests, + Metrics.ConnectionReuses, + Metrics.Reconnects, + Metrics.Evictions); + } + + await ValueTask.CompletedTask.ConfigureAwait(false); + } + + private int SelectStartIndex(in PostFilterSpamAssassinTarget target) + { + if (target.Hosts.Count == 0) + { + return 0; + } + + return SpamdHostSelector.NextStart(target.HostSelection, target.Hosts.Count, ref _roundRobin); + } + + private async Task<(SpamdConnection Connection, bool Reused)> RentAsync( + string host, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + Task wait; + lock (_gate) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (TryDequeueIdle(host, target.Port, out var idle)) + { + Metrics.RecordReuse(); + return (idle, true); + } + + if (_live < target.MaxConnections) + { + _live++; + wait = Task.CompletedTask; + } + else + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add(tcs); + wait = tcs.Task; + } + } + + if (wait.IsCompleted) + { + try + { + var connection = await ConnectAsync(host, target, cancellationToken).ConfigureAwait(false); + return (connection, false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + lock (_gate) + { + _live--; + PulseWaiter(); + } + + throw; + } + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private bool TryDequeueIdle(string host, int port, out SpamdConnection connection) + { + var key = (host, port); + if (_idle.TryGetValue(key, out var queue)) + { + while (queue.Count > 0) + { + var candidate = queue.Dequeue(); + if (candidate.IsReusable) + { + connection = candidate; + return true; + } + + // codeql[cs/dispose-not-called-on-throw] Candidate is pool-owned from Dequeue; Dispose here is best-effort eviction. Dequeue cannot create a new undisposed resource, and ConnectAsync already disposes TcpClient on connect failure. + candidate.Dispose(); + _live--; + Metrics.RecordEviction(); + Metrics.RecordReconnect(); + } + } + + connection = null!; + return false; + } + + private async Task ConnectAsync( + string host, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + var client = new TcpClient(); + try + { + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(target.ConnectTimeout); + await client.ConnectAsync(host, target.Port, connectCts.Token).ConfigureAwait(false); + Metrics.RecordConnect(); + return new SpamdConnection(client, host, target.Port); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + client.Dispose(); + throw; + } + } + + private void Return(SpamdConnection connection, bool reusable) + { + lock (_gate) + { + if (_disposed || !reusable || !connection.IsReusable) + { + connection.Dispose(); + if (!_disposed) + { + _live--; + Metrics.RecordEviction(); + PulseWaiter(); + } + + return; + } + + var key = (connection.Host, connection.Port); + if (!_idle.TryGetValue(key, out var queue)) + { + queue = new Queue(); + _idle[key] = queue; + } + + queue.Enqueue(connection); + PulseWaiter(); + } + } + + private void Evict(SpamdConnection connection) + { + connection.Dispose(); + lock (_gate) + { + if (!_disposed) + { + _live--; + Metrics.RecordEviction(); + Metrics.RecordReconnect(); + PulseWaiter(); + } + } + } + + private void PulseWaiter() + { + if (_waiters.Count == 0) + { + return; + } + + var waiter = _waiters[0]; + _waiters.RemoveAt(0); + waiter.TrySetResult(); + } + + private static async Task CheckOnConnectionAsync( + SpamdConnection connection, + ReadOnlyMemory scan, + string? accountName, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(target.OperationTimeout); + await WriteCheckAsync(connection.Stream, scan, accountName, target.ProtocolVersion, timeoutCts.Token) + .ConfigureAwait(false); + return await ReadCheckAsync(connection.Stream, timeoutCts.Token).ConfigureAwait(false); + } + + private static async Task WriteCheckAsync( + NetworkStream stream, + ReadOnlyMemory scan, + string? accountName, + string protocolVersion, + CancellationToken cancellationToken) + { + var version = string.IsNullOrWhiteSpace(protocolVersion) + ? PostFilterSpamAssassinOptions.DefaultProtocolVersion + : protocolVersion.Trim(); + var header = new StringBuilder(64); + header.Append("CHECK SPAMC/"); + header.Append(version); + header.Append("\r\n"); + header.Append("Content-length: "); + header.Append(scan.Length.ToString(CultureInfo.InvariantCulture)); + header.Append("\r\n"); + if (!string.IsNullOrWhiteSpace(accountName)) + { + header.Append("User: "); + header.Append(accountName); + header.Append("\r\n"); + } + + header.Append("\r\n"); + var headerBytes = Encoding.ASCII.GetBytes(header.ToString()); + await stream.WriteAsync(headerBytes, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(scan, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadCheckAsync( + NetworkStream stream, + CancellationToken cancellationToken) + { + var buffer = new byte[2048]; + var total = 0; + while (total < buffer.Length) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + break; + } + + total += read; + if (buffer.AsSpan(0, total).IndexOf("\r\n\r\n"u8) >= 0) + { + break; + } + } + + if (total == 0) + { + return PostFilterSpamAssassinResult.Failed("closed"); + } + + var terminator = buffer.AsSpan(0, total).IndexOf("\r\n\r\n"u8); + if (terminator < 0) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + if (terminator + 4 != total) + { + return PostFilterSpamAssassinResult.Failed("leftover"); + } + + var text = Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + if (!text.StartsWith("SPAMD/", StringComparison.Ordinal)) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + var firstLineEnd = text.IndexOf("\r\n", StringComparison.Ordinal); + if (firstLineEnd < 0) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + var statusLine = text[..firstLineEnd]; + var parts = statusLine.Split(' ', 3, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2 || parts[1] != "0") + { + return PostFilterSpamAssassinResult.Failed(statusLine); + } + + var spamIndex = text.IndexOf("Spam:", StringComparison.OrdinalIgnoreCase); + if (spamIndex < 0) + { + return PostFilterSpamAssassinResult.Failed("missing-spam-header"); + } + + var lineEnd = text.IndexOf("\r\n", spamIndex, StringComparison.Ordinal); + var spamLine = lineEnd < 0 ? text[spamIndex..] : text[spamIndex..lineEnd]; + TryParseSpamScores(spamLine, out var score, out var threshold); + if (spamLine.Contains("True", StringComparison.OrdinalIgnoreCase)) + { + return PostFilterSpamAssassinResult.Spam(spamLine, score, threshold); + } + + if (spamLine.Contains("False", StringComparison.OrdinalIgnoreCase)) + { + return PostFilterSpamAssassinResult.Ham(score, threshold); + } + + return PostFilterSpamAssassinResult.Failed(spamLine); + } + + private static void TryParseSpamScores(string spamLine, out decimal? score, out decimal? threshold) + { + score = null; + threshold = null; + var separator = spamLine.IndexOf(';'); + if (separator < 0) + { + return; + } + + var scores = spamLine[(separator + 1)..].Split('/', 2, StringSplitOptions.TrimEntries); + if (scores.Length == 2 + && decimal.TryParse(scores[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsedScore) + && decimal.TryParse(scores[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsedThreshold)) + { + score = parsedScore; + threshold = parsedThreshold; + } + } + + private static PostFilterSpamAssassinResult ClassifyConnect(Exception ex) => + ex is OperationCanceledException + ? PostFilterSpamAssassinResult.Failed("timeout") + : PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + + private static PostFilterSpamAssassinResult ClassifyAfterWrite(Exception ex) => + ex is IOException or SocketException + ? PostFilterSpamAssassinResult.Failed("connection") + : PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + + private static bool IsConnectionFailure(Exception ex) => + ex is SocketException or IOException or TimeoutException or ObjectDisposedException; + + private static bool IsStaleIdleFailure(in PostFilterSpamAssassinResult result) => + result.Status == PostFilterSpamAssassinStatus.Failed + && result.Detail is "closed" or "connection"; + + private sealed class SpamdConnection : IDisposable + { + private readonly TcpClient _client; + + public SpamdConnection(TcpClient client, string host, int port) + { + _client = client; + Host = host; + Port = port; + Stream = client.GetStream(); + } + + public string Host { get; } + + public int Port { get; } + + public NetworkStream Stream { get; } + + public bool IsReusable + { + get + { + try + { + var socket = _client.Client; + if (!_client.Connected || socket is null || !socket.Connected) + { + return false; + } + + if (socket.Poll(0, SelectMode.SelectRead) && socket.Available == 0) + { + return false; + } + + return !Stream.DataAvailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return false; + } + } + } + + public void Dispose() => _client.Dispose(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/SpamdHostSelector.cs b/src/VectorNNTPD.ReaderService/PostFilter/SpamdHostSelector.cs new file mode 100644 index 00000000..c1a5353a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/SpamdHostSelector.cs @@ -0,0 +1,33 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Deterministic SPAMD host index selection. Does not allocate. + internal static class SpamdHostSelector + { + /// Returns the first host index to try for this CHECK. + public static int NextStart( + PostFilterSpamAssassinHostSelection selection, + int hostCount, + ref int roundRobin) + { + if (hostCount <= 0) + { + return 0; + } + + if (selection == PostFilterSpamAssassinHostSelection.Failover) + { + return 0; + } + + var next = Interlocked.Increment(ref roundRobin) - 1; + if (next < 0) + { + next = 0; + } + + return next % hostCount; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/SpamdScanArticleBuilder.cs b/src/VectorNNTPD.ReaderService/PostFilter/SpamdScanArticleBuilder.cs new file mode 100644 index 00000000..b0bd905e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/SpamdScanArticleBuilder.cs @@ -0,0 +1,267 @@ +using System.Buffers; +using System.Globalization; +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// + /// Builds a disposable email-like SPAMD CHECK payload from a canonical article. + /// + /// + /// Does not mutate or any other article field. + /// The returned buffer is independent of the article and is only the bytes sent to SPAMD. + /// + internal static class SpamdScanArticleBuilder + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + private static readonly byte[] HeaderSeparator = "\r\n\r\n"u8.ToArray(); + + /// Constructs the SPAMD scan representation. + /// Canonical article. Read-only. + /// Peer address, server FQDN, and evaluation clock. + /// Independent scan bytes. Never aliases . + public static byte[] Build(in ArticleRecord article, in SpamdScanContext context) + { + ArgumentNullException.ThrowIfNull(context.ClientAddress); + var fqdn = SanitizeToken(context.ServerFqdn); + var artData = article.ArtData.Span; + SplitArticle(artData, out var headerBlock, out var body); + ScanOriginalHeaders( + headerBlock, + out var hasDate, + out var messageId, + out var newsgroups, + out var preservedHeaders); + + if (messageId.Length == 0 && article.Fields.MessageId.IsPresent) + { + messageId = article.MessageId.ToArray(); + } + + if (newsgroups.Length == 0 && article.Fields.Newsgroups.IsPresent) + { + newsgroups = article.Newsgroups.ToArray(); + } + + var writer = new ArrayBufferWriter(256 + artData.Length); + WriteReceived(writer, context.ClientAddress, fqdn, messageId, context.Now); + WriteAscii(writer, "To: usenet@"); + WriteAscii(writer, fqdn); + writer.Write(Crlf); + if (newsgroups.Length > 0) + { + WriteAscii(writer, "X-Usenet-Newsgroups: "); + writer.Write(newsgroups); + writer.Write(Crlf); + } + + if (!hasDate) + { + WriteAscii(writer, "Date: "); + WriteAscii(writer, FormatDate(context.Now)); + writer.Write(Crlf); + } + + writer.Write(preservedHeaders); + writer.Write(Crlf); + writer.Write(body); + return writer.WrittenSpan.ToArray(); + } + + private static void WriteReceived( + ArrayBufferWriter writer, + IPAddress client, + string fqdn, + ReadOnlySpan messageId, + DateTimeOffset now) + { + WriteAscii(writer, "Received: from ["); + WriteAscii(writer, client.ToString()); + WriteAscii(writer, "] by "); + WriteAscii(writer, fqdn); + WriteAscii(writer, " with NNTP"); + if (!messageId.IsEmpty) + { + WriteAscii(writer, " id "); + writer.Write(messageId); + } + + WriteAscii(writer, "; "); + WriteAscii(writer, FormatDate(now)); + writer.Write(Crlf); + } + + private static void SplitArticle( + ReadOnlySpan artData, + out ReadOnlySpan headers, + out ReadOnlySpan body) + { + var separator = artData.IndexOf(HeaderSeparator); + if (separator < 0) + { + headers = artData; + body = ReadOnlySpan.Empty; + return; + } + + headers = artData[..separator]; + body = artData[(separator + HeaderSeparator.Length)..]; + } + + private static void ScanOriginalHeaders( + ReadOnlySpan headerBlock, + out bool hasDate, + out byte[] messageId, + out byte[] newsgroups, + out byte[] preservedHeaders) + { + hasDate = false; + messageId = []; + newsgroups = []; + if (headerBlock.IsEmpty) + { + preservedHeaders = []; + return; + } + + var preserved = new ArrayBufferWriter(headerBlock.Length); + var offset = 0; + while (offset < headerBlock.Length) + { + var blockStart = offset; + if (!TryReadLine(headerBlock, ref offset, out var line)) + { + break; + } + + while (offset < headerBlock.Length + && headerBlock[offset] is (byte)' ' or (byte)'\t') + { + if (!TryReadLine(headerBlock, ref offset, out _)) + { + break; + } + } + + var block = headerBlock[blockStart..offset]; + var colon = line.IndexOf((byte)':'); + var name = colon < 0 ? line : line[..colon]; + if (IsStripped(name)) + { + continue; + } + + if (AsciiEqualsIgnoreCase(name, "Date"u8)) + { + hasDate = true; + } + else if (AsciiEqualsIgnoreCase(name, "Message-ID"u8) && messageId.Length == 0) + { + messageId = ExtractValue(line, colon); + } + else if (AsciiEqualsIgnoreCase(name, "Newsgroups"u8) && newsgroups.Length == 0) + { + newsgroups = ExtractValue(line, colon); + } + + preserved.Write(block); + if (block.Length < 2 + || block[^2] != (byte)'\r' + || block[^1] != (byte)'\n') + { + preserved.Write(Crlf); + } + } + + preservedHeaders = preserved.WrittenSpan.ToArray(); + } + + private static bool TryReadLine(ReadOnlySpan headers, ref int offset, out ReadOnlySpan line) + { + var remaining = headers[offset..]; + var eol = remaining.IndexOf("\r\n"u8); + if (eol < 0) + { + line = remaining; + offset = headers.Length; + return !line.IsEmpty; + } + + line = remaining[..eol]; + offset += eol + 2; + return true; + } + + private static byte[] ExtractValue(ReadOnlySpan line, int colon) + { + if (colon < 0 || colon + 1 >= line.Length) + { + return []; + } + + var value = line[(colon + 1)..]; + while (!value.IsEmpty && value[0] is (byte)' ' or (byte)'\t') + { + value = value[1..]; + } + + return value.ToArray(); + } + + private static bool IsStripped(ReadOnlySpan name) => + AsciiEqualsIgnoreCase(name, "Path"u8) + || AsciiEqualsIgnoreCase(name, "Xref"u8) + || AsciiEqualsIgnoreCase(name, "Injection-Info"u8) + || AsciiEqualsIgnoreCase(name, "X-Trace"u8) + || AsciiEqualsIgnoreCase(name, "X-Complaints-To"u8) + || AsciiEqualsIgnoreCase(name, "NNTP-Posting-Host"u8); + + private static bool AsciiEqualsIgnoreCase(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (ToLower(left[i]) != ToLower(right[i])) + { + return false; + } + } + + return true; + } + + private static byte ToLower(byte value) => + value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; + + private static string SanitizeToken(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "localhost"; + } + + var trimmed = value.Trim(); + foreach (var ch in trimmed) + { + if (ch is '\r' or '\n' or '\0') + { + return "localhost"; + } + } + + return trimmed; + } + + private static string FormatDate(DateTimeOffset now) => + now.UtcDateTime.ToString("ddd, dd MMM yyyy HH:mm:ss +0000", CultureInfo.InvariantCulture); + + private static void WriteAscii(ArrayBufferWriter writer, string value) => + writer.Write(Encoding.ASCII.GetBytes(value)); + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/SpamdTransportMetrics.cs b/src/VectorNNTPD.ReaderService/PostFilter/SpamdTransportMetrics.cs new file mode 100644 index 00000000..7d8b27bc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/SpamdTransportMetrics.cs @@ -0,0 +1,56 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Process-wide SPAMD transport counters. Not per-POST logging. + internal sealed class SpamdTransportMetrics + { + private long _connectionsEstablished; + private long _checkRequests; + private long _connectionReuses; + private long _reconnects; + private long _evictions; + private long _totalCheckTicks; + + /// Gets TCP connections opened to SPAMD. + public long ConnectionsEstablished => Volatile.Read(ref _connectionsEstablished); + + /// Gets CHECK operations attempted after a connection was obtained. + public long CheckRequests => Volatile.Read(ref _checkRequests); + + /// Gets CHECKs that reused an idle persistent connection. + public long ConnectionReuses => Volatile.Read(ref _connectionReuses); + + /// Gets replacements opened after a broken or half-closed connection. + public long Reconnects => Volatile.Read(ref _reconnects); + + /// Gets connections removed from the pool before another request could use them. + public long Evictions => Volatile.Read(ref _evictions); + + /// Gets total CHECK duration in . + public long TotalCheckTicks => Volatile.Read(ref _totalCheckTicks); + + /// Gets average CHECK latency, or when none completed. + public TimeSpan AverageCheckLatency => + CheckRequests == 0 + ? TimeSpan.Zero + : TimeSpan.FromTicks(TotalCheckTicks / CheckRequests); + + /// Records a newly established TCP connection. + public void RecordConnect() => Interlocked.Increment(ref _connectionsEstablished); + + /// Records a CHECK that reused an idle connection. + public void RecordReuse() => Interlocked.Increment(ref _connectionReuses); + + /// Records a reconnect after eviction or unexpected close. + public void RecordReconnect() => Interlocked.Increment(ref _reconnects); + + /// Records removal of a broken connection. + public void RecordEviction() => Interlocked.Increment(ref _evictions); + + /// Records one CHECK duration. + public void RecordCheck(TimeSpan elapsed) + { + Interlocked.Increment(ref _checkRequests); + Interlocked.Add(ref _totalCheckTicks, elapsed.Ticks); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/PostFilter/StaticPostFilterPolicySource.cs b/src/VectorNNTPD.ReaderService/PostFilter/StaticPostFilterPolicySource.cs new file mode 100644 index 00000000..403ff3b9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/PostFilter/StaticPostFilterPolicySource.cs @@ -0,0 +1,16 @@ +namespace VectorNNTPD.ReaderService.PostFilter +{ + /// Fixed snapshot for tests and the disabled-gate default. + internal sealed class StaticPostFilterPolicySource : IPostFilterPolicySource + { + /// Initializes a source that always returns . + public StaticPostFilterPolicySource(PostFilterPolicySnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Current = snapshot; + } + + /// + public PostFilterPolicySnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Program.cs b/src/VectorNNTPD.ReaderService/Program.cs new file mode 100644 index 00000000..9fb7287f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Program.cs @@ -0,0 +1,80 @@ +using Serilog; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService +{ + /// Process entry point for the NNTP reader-server host. + internal static class Program + { + /// + /// Builds and runs the reader Generic Host, or runs startup smoke when that environment variable is set. + /// + /// Command-line arguments passed to the host builder. + /// Zero on clean shutdown or successful smoke; one on unexpected termination. + private static async Task Main(string[] args) + { + NntpdLoggingExtensions.UseAutoFlushConsoleOutput(); + Log.Logger = NntpdLoggingExtensions.CreateBootstrapLogger(); + + try + { + Log.Information("{Application} host starting", ApplicationJsonConfiguration.EntryAssemblyName); + + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + Args = args, + ContentRootPath = AppContext.BaseDirectory, + }); + builder.Environment.ContentRootPath = AppContext.BaseDirectory; + ApplicationJsonConfiguration.UseEntryAssemblyJsonFiles( + builder.Configuration, + builder.Environment.EnvironmentName); + ApplicationJsonConfiguration.AddSharedRabbitMqJsonFile( + builder.Configuration, + builder.Environment.EnvironmentName, + NntpServerJsonFiles.SharedRabbitMqFileName); + + var startupSmoke = ReaderStartupSmoke.IsRequested( + Environment.GetEnvironmentVariable(ReaderStartupSmoke.EnvironmentVariable)); + if (startupSmoke) + { + ReaderStartupSmoke.UseIsolatedConfiguration(builder.Configuration); + } + + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpReaderHosting(); + + var host = builder.Build(); + NntpCommandLoggers.Configure(host.Services.GetRequiredService()); + NntpdLoggingExtensions.WriteLoggingInitialized( + host.Services, + builder.Environment.EnvironmentName, + builder.Environment.ContentRootPath); + if (startupSmoke) + { + return ReaderStartupSmoke.Complete(host, Console.Out); + } + + await host.RunAsync().ConfigureAwait(false); + + Log.Information("{Application} host stopped cleanly", ApplicationJsonConfiguration.EntryAssemblyName); + return 0; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + Log.Fatal(ex, "{Application} host terminated unexpectedly", ApplicationJsonConfiguration.EntryAssemblyName); + return 1; + } + finally + { + await Log.CloseAndFlushAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs new file mode 100644 index 00000000..94265b87 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// NNTPD policy for sequential Backfill Scheduler attempts. + /// + /// + /// + /// BackFiller v1 outcomes classify a single work item. The scheduler publishes at most + /// one ArticleWork request at a time and never fans out concurrently. + /// + /// + /// Attempt policy: + /// + /// + /// + /// completes the logical lookup immediately. + /// + /// + /// + /// + /// is a definitive, lookup-terminal + /// miss (provider 430). BackFiller ACKs the delivery. NNTPD must not publish + /// another request to a different backbone, requeue, or invent a replacement work item. + /// + /// + /// + /// + /// is definitive for the selected + /// backbone. BackFiller ACKs the delivery. NNTPD may publish a NEW request to another + /// eligible backbone (unusable article on one provider is not proof of absence elsewhere). + /// + /// + /// + /// + /// is also attempt-terminal for the + /// selected backbone (protocol rejection from that destination); another eligible + /// backbone may still be attempted. + /// + /// + /// + /// + /// + /// In-flight attempts cannot be cancelled on the broker. A per-attempt wait that + /// expires without a definitive wire outcome ends the logical lookup without starting + /// another backbone (avoids duplicate upstream retrieval). + /// + /// + internal static class ArticleWorkAggregatePolicy + { + /// Returns whether this outcome completes the whole logical lookup successfully. + internal static bool IsLookupSuccess(ArticleWorkOutcome outcome) => + outcome == ArticleWorkOutcome.Success; + + /// + /// Returns whether this wire outcome ends the current backbone attempt + /// (success or definitive absence/rejection). + /// + internal static bool IsAttemptTerminal(ArticleWorkOutcome outcome) => + outcome is ArticleWorkOutcome.Success + or ArticleWorkOutcome.ArticleNotFound + or ArticleWorkOutcome.InvalidArticle + or ArticleWorkOutcome.InvalidRequest; + + /// + /// Returns whether the scheduler may try another eligible backbone after this outcome. + /// + /// + /// is intentionally omitted: a definitive + /// provider 430 ends the entire logical lookup. + /// + internal static bool ShouldTryNextBackbone(ArticleWorkOutcome outcome) => + outcome is ArticleWorkOutcome.InvalidArticle + or ArticleWorkOutcome.InvalidRequest; + + /// Legacy name: only Success completed the old parallel aggregate. + internal static bool IsAggregateTerminal(ArticleWorkOutcome outcome) => + IsLookupSuccess(outcome); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs new file mode 100644 index 00000000..53d802e9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs @@ -0,0 +1,93 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// In-flight article-work lookup shared by every publication of one requestId. + internal sealed class ArticleWorkLookupOperation + { + private readonly TaskCompletionSource _completion = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Initializes a new lookup operation. + internal ArticleWorkLookupOperation(Guid requestId, string messageId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + RequestId = requestId; + MessageId = messageId; + } + + /// Gets the application request identity shared by every publication. + internal Guid RequestId { get; } + + /// Gets the Message-ID being retrieved. + internal string MessageId { get; } + + /// Gets the task that completes when the lookup reaches a terminal classification. + internal Task Completion => _completion.Task; + + /// Returns whether the operation has already completed. + internal bool IsCompleted => _completion.Task.IsCompleted; + + /// Returns whether a response belongs to this operation's application identity. + internal bool Matches(ArticleWorkResponse response) + { + ArgumentNullException.ThrowIfNull(response); + if (response.RequestId.HasValue && response.RequestId.Value != RequestId) + { + return false; + } + + if (response.MessageId is not null + && !string.Equals(response.MessageId, MessageId, StringComparison.Ordinal)) + { + return false; + } + + return true; + } + + /// + /// Applies a validated response immediately. Attempt-terminal outcomes + /// (, NotFound, InvalidArticle, InvalidRequest) + /// complete this attempt's wait so the scheduler can decide whether to continue. + /// + internal void OnResponse(string exchange, ArticleWorkResponse response) + { + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + ArgumentNullException.ThrowIfNull(response); + + if (!ArticleWorkAggregatePolicy.IsAttemptTerminal(response.Outcome)) + { + return; + } + + TryComplete(new ArticleWorkRpcResult( + response.Outcome, + RequestId, + MessageId, + response.Backbone, + response.Fqdn, + response.VatpPort, + response.ArticleId, + response.Error, + exchange)); + } + + /// Completes the operation if it is still pending. + internal bool TryComplete(ArticleWorkRpcResult result) => _completion.TrySetResult(result); + + /// Cancels the operation during shutdown. + internal bool TryCancel() => _completion.TrySetCanceled(); + + /// Returns the completed result when available. + internal bool TryGetResult(out ArticleWorkRpcResult result) + { + if (_completion.Task is { IsCompletedSuccessfully: true } task) + { + result = task.Result; + return true; + } + + result = default!; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs new file mode 100644 index 00000000..58659560 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Canonical BackFiller v1 article-work response outcomes that NNTPD classifies. + public enum ArticleWorkOutcome + { + /// The source located the article and supplied a cache URI. + Success = 0, + + /// The source did not have the article. + ArticleNotFound = 1, + + /// The source rejected the article as invalid. + InvalidArticle = 2, + + /// The source rejected the request as protocol-invalid. + InvalidRequest = 3, + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRequest.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRequest.cs new file mode 100644 index 00000000..0cc074a5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRequest.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Application JSON fields for a BackFiller v1 article-work request. + /// + /// AMQP CorrelationId and ReplyTo are transport metadata and are never + /// serialized into this payload. + /// + /// Application protocol version. Current value is 1. + /// Application work identity retained across storage and provider publications. + /// Canonical NNTP Message-ID as supplied on the command line. + /// Destination backbone identity for the publication target. + internal sealed record ArticleWorkRequest( + int Version, + Guid RequestId, + string MessageId, + string Backbone); +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkResponse.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkResponse.cs new file mode 100644 index 00000000..928eee5b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkResponse.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Parsed BackFiller v1 article-work response JSON. + /// Application protocol version. + /// Application request identity when the responder could establish it. + /// Request Message-ID when the responder could establish it. + /// Request backbone when the responder could establish it. + /// Terminal response outcome. + /// Success-only BackFiller FQDN. Absent for non-success outcomes. + /// Success-only TLS VATP listen port. Absent for non-success outcomes. + /// Success-only CanonicalV1 ArtId. Absent for non-success outcomes. + /// Terminal-failure detail. Absent for success. + internal sealed record ArticleWorkResponse( + int Version, + Guid? RequestId, + string? MessageId, + string? Backbone, + ArticleWorkOutcome Outcome, + string? Fqdn, + int? VatpPort, + VectorNNTPD.Common.Articles.ArticleId? ArticleId, + string? Error); +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs new file mode 100644 index 00000000..311282e1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs @@ -0,0 +1,56 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// AMQP transport constants for article-work RPC publications. + internal static class ArticleWorkRpcAmqp + { + /// AMQP header/property name carrying the logical lookup UUID. + internal const string RequestIdPropertyName = "RequestId"; + + /// + /// Documented BackFiller response AMQP Expiration (milliseconds). + /// NNTPD work requests omit Expiration so RabbitMQ may buffer ready work while + /// consumers are busy. A one-second request TTL silently discarded accepted lookups and + /// conflicted with the five-second . + /// Leftover replies on exclusive ReplyTo still use this value on the BackFiller publish path. + /// NNTPD does not delete or purge queues. + /// + internal const string ResponseExpirationMilliseconds = "1000"; + + /// + /// Obsolete alias retained only for transitional call sites that assert response TTL. + /// Prefer . Not applied to request publishes. + /// + internal const string ExpirationMilliseconds = ResponseExpirationMilliseconds; + + /// Creates the AMQP header table that carries . + internal static Dictionary CreateRequestIdHeaders(string requestId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(requestId); + return new Dictionary(StringComparer.Ordinal) + { + [RequestIdPropertyName] = requestId, + }; + } + + /// Reads the logical lookup UUID from AMQP headers, if present. + internal static string? ReadRequestId(IDictionary? headers) + { + if (headers is null + || !headers.TryGetValue(RequestIdPropertyName, out var value) + || value is null) + { + return null; + } + + return value switch + { + string text => text, + byte[] bytes => Encoding.UTF8.GetString(bytes), + ReadOnlyMemory memory => Encoding.UTF8.GetString(memory.Span), + _ => null, + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs new file mode 100644 index 00000000..07b6dbfb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs @@ -0,0 +1,304 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// Sequential Backfill Scheduler: one ArticleWork publish per attempt, weighted by + /// active consumer count, excluding already-attempted backbones. + /// + /// + /// + /// Never fans out concurrently. Definitive ArticleNotFound (provider 430) + /// from BackFiller is ACKed there and ends the logical lookup — NNTPD does not publish + /// to another backbone. InvalidArticle / InvalidRequest remain + /// attempt-terminal for the selected backbone and may advance to another eligible + /// backbone. In-flight attempts cannot be cancelled on the broker — an attempt wait + /// that expires without a definitive wire outcome ends the logical lookup without + /// starting another backbone. + /// + /// + internal sealed class ArticleWorkRpcClient : IArticleWorkRpcClient + { + private readonly IArticleWorkRpcPublisher _publisher; + private readonly ArticleWorkRpcResponseRouter _router; + private readonly IBackfillConsumerAvailability _availability; + private readonly IBackboneSelector _selector; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly Func _currentGeneration; + + /// Initializes a new orchestrator. + internal ArticleWorkRpcClient( + IArticleWorkRpcPublisher publisher, + ArticleWorkRpcResponseRouter router, + IBackfillConsumerAvailability availability, + IBackboneSelector selector, + TimeProvider timeProvider, + ILogger logger, + Func? currentGeneration = null) + { + ArgumentNullException.ThrowIfNull(publisher); + ArgumentNullException.ThrowIfNull(router); + ArgumentNullException.ThrowIfNull(availability); + ArgumentNullException.ThrowIfNull(selector); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(logger); + _publisher = publisher; + _router = router; + _availability = availability; + _selector = selector; + _timeProvider = timeProvider; + _logger = logger; + _currentGeneration = currentGeneration ?? (() => 0); + } + + /// + public async Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var started = _timeProvider.GetUtcNow(); + var decoded = DecodeMessageId(messageId); + var requestId = Guid.NewGuid(); + var attempted = new HashSet(StringComparer.OrdinalIgnoreCase); + var deadline = started + ArticleWorkRpcTiming.LookupDeadline; + + using var deadlineCts = new CancellationTokenSource(ArticleWorkRpcTiming.LookupDeadline, _timeProvider); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, deadlineCts.Token); + var lookupToken = linked.Token; + + try + { + var attempt = 0; + while (true) + { + lookupToken.ThrowIfCancellationRequested(); + var now = _timeProvider.GetUtcNow(); + if (now >= deadline) + { + var timedOut = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article lookup deadline elapsed"); + LogCompleted(timedOut, started); + return timedOut; + } + + var selected = SelectNext(attempted); + if (selected is null) + { + var exhausted = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + attempted.Count == 0 + ? "No BackFiller ArticleWork consumers are active" + : "All eligible BackFiller backbones returned not-found"); + LogCompleted(exhausted, started); + return exhausted; + } + + attempted.Add(selected.Value.Backbone); + attempt++; + ArticleWorkRpcLogMessages.SchedulerAttempt( + _logger, + requestId, + decoded, + selected.Value.Backbone, + selected.Value.ConsumerCount, + attempt); + + var operation = new ArticleWorkLookupOperation(requestId, decoded); + try + { + await PublishAsync(operation, selected.Value, lookupToken).ConfigureAwait(false); + lookupToken.ThrowIfCancellationRequested(); + + await WaitUntilAsync(operation, deadline, lookupToken).ConfigureAwait(false); + + if (!operation.TryGetResult(out var attemptResult)) + { + // No definitive wire outcome before the remaining lookup deadline. + // Do not start another backbone: in-flight work cannot be cancelled. + var uncertain = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article-work attempt timed out without a definitive response"); + LogCompleted(uncertain, started); + return uncertain; + } + + if (ArticleWorkAggregatePolicy.IsLookupSuccess(attemptResult.Outcome)) + { + LogCompleted(attemptResult, started); + return attemptResult; + } + + if (!ArticleWorkAggregatePolicy.ShouldTryNextBackbone(attemptResult.Outcome)) + { + // Includes definitive ArticleNotFound (430): lookup ends here. + LogCompleted(attemptResult, started); + return attemptResult; + } + + // InvalidArticle / InvalidRequest for this backbone — try another eligible. + } + finally + { + _router.UnregisterAll(operation); + } + } + } + catch (OperationCanceledException) when (deadlineCts.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + var expired = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article lookup deadline elapsed"); + LogCompleted(expired, started); + return expired; + } + } + + private BackfillEligibleBackbone? SelectNext(HashSet attempted) + { + var eligible = _availability.GetEligibleBackbones(); + if (eligible.Count == 0) + { + return null; + } + + List? filtered = null; + for (var i = 0; i < eligible.Count; i++) + { + var candidate = eligible[i]; + if (candidate.ConsumerCount < 1 || attempted.Contains(candidate.Backbone)) + { + continue; + } + + filtered ??= new List(eligible.Count); + filtered.Add(candidate); + } + + if (filtered is null || filtered.Count == 0) + { + return null; + } + + return _selector.Select(filtered); + } + + private async Task PublishAsync( + ArticleWorkLookupOperation operation, + BackfillEligibleBackbone destination, + CancellationToken cancellationToken) + { + var correlationId = Guid.NewGuid().ToString("D"); + var request = new ArticleWorkRequest( + ArticleWorkWireProtocol.CurrentVersion, + operation.RequestId, + operation.MessageId, + destination.Backbone); + var body = ArticleWorkWireProtocol.SerializeRequestV1(request); + var generation = _currentGeneration(); + if (!_router.TryRegister(correlationId, operation, destination.Exchange, generation)) + { + return; + } + + try + { + await _publisher + .PublishAsync( + destination.Exchange, + destination.RoutingKey, + operation.RequestId, + correlationId, + body, + cancellationToken) + .ConfigureAwait(false); + + ArticleWorkRpcLogMessages.Published( + _logger, + operation.RequestId, + correlationId, + operation.MessageId, + destination.Exchange, + generation); + } + catch (OperationCanceledException) + { + _router.Unregister(correlationId); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _router.Unregister(correlationId); + ArticleWorkRpcLogMessages.PublishFailed( + _logger, + ex, + operation.RequestId, + correlationId, + operation.MessageId, + destination.Exchange, + generation); + throw; + } + } + + private async Task WaitUntilAsync( + ArticleWorkLookupOperation operation, + DateTimeOffset deadline, + CancellationToken cancellationToken) + { + var remaining = deadline - _timeProvider.GetUtcNow(); + if (remaining <= TimeSpan.Zero || operation.IsCompleted) + { + return; + } + + using var delayCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var delay = Task.Delay(remaining, _timeProvider, delayCts.Token); + var completed = await Task.WhenAny(operation.Completion, delay).ConfigureAwait(false); + if (completed == operation.Completion) + { + await delayCts.CancelAsync().ConfigureAwait(false); + _ = await operation.Completion.ConfigureAwait(false); + } + else + { + await delay.ConfigureAwait(false); + } + } + + private void LogCompleted(ArticleWorkRpcResult result, DateTimeOffset started) + { + var elapsedMs = (long)Math.Max(0, (_timeProvider.GetUtcNow() - started).TotalMilliseconds); + ArticleWorkRpcLogMessages.Completed( + _logger, + result.RequestId, + result.MessageId, + result.Outcome.ToString(), + result.SourceExchange, + elapsedMs); + } + + /// + /// Converts command-line Message-ID bytes to the JSON string at the RPC application boundary. + /// NNTP Message-IDs are ASCII; this is not a protocol-data-plane default representation. + /// + internal static string DecodeMessageId(ReadOnlyMemory messageId) + { + if (messageId.IsEmpty || !NntpMessageIdValidation.IsValidMessageId(messageId.Span)) + { + throw new ArgumentException("Article-work RPC requires a well-formed Message-ID.", nameof(messageId)); + } + + return Encoding.ASCII.GetString(messageId.Span); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs new file mode 100644 index 00000000..99beec75 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs @@ -0,0 +1,111 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Source-generated article-work RPC log messages. Never includes payloads or credentials. + internal static partial class ArticleWorkRpcLogMessages + { + [LoggerMessage( + EventId = 2830, + Level = LogLevel.Information, + Message = "Article-work RPC ready (replyTo={ReplyTo}, generation={Generation})")] + public static partial void Ready(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2831, + Level = LogLevel.Information, + Message = "Article-work RPC session replaced (replyTo={ReplyTo}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2832, + Level = LogLevel.Error, + Message = "Article-work RPC is not ready")] + public static partial void NotReady(ILogger logger); + + [LoggerMessage( + EventId = 2833, + Level = LogLevel.Debug, + Message = "Article-work request published (requestId={RequestId}, correlationId={CorrelationId}, messageId={MessageId}, exchange={Exchange}, generation={Generation})")] + public static partial void Published( + ILogger logger, + Guid RequestId, + string CorrelationId, + string MessageId, + string Exchange, + long Generation); + + [LoggerMessage( + EventId = 2834, + Level = LogLevel.Warning, + Message = "Article-work publication failed (requestId={RequestId}, correlationId={CorrelationId}, messageId={MessageId}, exchange={Exchange}, generation={Generation})")] + public static partial void PublishFailed( + ILogger logger, + Exception exception, + Guid RequestId, + string CorrelationId, + string MessageId, + string Exchange, + long Generation); + + [LoggerMessage( + EventId = 2835, + Level = LogLevel.Information, + Message = "Article-work RPC completed (requestId={RequestId}, messageId={MessageId}, outcome={Outcome}, exchange={Exchange}, elapsedMs={ElapsedMs})")] + public static partial void Completed( + ILogger logger, + Guid RequestId, + string MessageId, + string Outcome, + string? Exchange, + long ElapsedMs); + + [LoggerMessage( + EventId = 2836, + Level = LogLevel.Debug, + Message = "Article-work RPC response ignored (correlationId={CorrelationId}, reason={Reason})")] + public static partial void ResponseIgnored(ILogger logger, string CorrelationId, string Reason); + + [LoggerMessage( + EventId = 2837, + Level = LogLevel.Warning, + Message = "Article-work RPC consumer callback failed")] + public static partial void ConsumerCallbackFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2838, + Level = LogLevel.Error, + Message = "Article-work RPC session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2839, + Level = LogLevel.Warning, + Message = "ARTICLE message-id RPC lookup failed (messageId={MessageId})")] + public static partial void ArticleLookupFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2840, + Level = LogLevel.Warning, + Message = "Article-work Management API availability refresh failed; retaining last successful snapshot")] + public static partial void ManagementAvailabilityRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2841, + Level = LogLevel.Debug, + Message = "Article-work scheduler attempt (requestId={RequestId}, messageId={MessageId}, backbone={Backbone}, consumers={Consumers}, attempt={Attempt})")] + public static partial void SchedulerAttempt( + ILogger logger, + Guid RequestId, + string MessageId, + string Backbone, + int Consumers, + int Attempt); + + [LoggerMessage( + EventId = 2842, + Level = LogLevel.Debug, + Message = "Article-work availability snapshot refreshed (eligibleCount={EligibleCount})")] + public static partial void AvailabilitySnapshotRefreshed(ILogger logger, int EligibleCount); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs new file mode 100644 index 00000000..0a04f313 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs @@ -0,0 +1,176 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// Routes consumed RPC responses to outstanding publications by AMQP + /// CorrelationId and logical RequestId. + /// + /// + /// This type owns NNTPD in-process correlation state only. It does not delete, + /// purge, or cancel RabbitMQ messages. Unknown and stale correlations are a + /// normal cheap ignore path after the logical lookup has completed. + /// + internal sealed class ArticleWorkRpcResponseRouter + { + private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); + private readonly ILogger _logger; + + /// Initializes a new router. + internal ArticleWorkRpcResponseRouter(ILogger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// Gets the number of outstanding correlation registrations. + internal int OutstandingCount => _pending.Count; + + /// + /// Registers a publication so a later response can be correlated. + /// Completed lookups are not registered. + /// + internal bool TryRegister( + string correlationId, + ArticleWorkLookupOperation operation, + string exchange, + long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + ArgumentNullException.ThrowIfNull(operation); + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + + if (operation.IsCompleted) + { + return false; + } + + if (!_pending.TryAdd(correlationId, new PendingPublication(operation, exchange, generation))) + { + throw new InvalidOperationException($"Duplicate article-work correlation '{correlationId}'."); + } + + if (operation.IsCompleted) + { + _pending.TryRemove(correlationId, out _); + return false; + } + + return true; + } + + /// Registers a publication so a later response can be correlated. + internal void Register( + string correlationId, + ArticleWorkLookupOperation operation, + string exchange, + long generation) + { + _ = TryRegister(correlationId, operation, exchange, generation); + } + + /// Removes one publication registration. Safe when already removed. + internal void Unregister(string correlationId) + { + if (string.IsNullOrWhiteSpace(correlationId)) + { + return; + } + + _pending.TryRemove(correlationId, out _); + } + + /// Removes every registration owned by . + internal void UnregisterAll(ArticleWorkLookupOperation operation) + { + ArgumentNullException.ThrowIfNull(operation); + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _pending) + { + if (ReferenceEquals(pair.Value.Operation, operation)) + { + _pending.TryRemove(pair.Key, out _); + } + } + } + + /// Cancels every outstanding operation during shutdown. + internal void CancelAll() + { + foreach (var pair in _pending) + { + pair.Value.Operation.TryCancel(); + _pending.TryRemove(pair.Key, out _); + } + } + + /// + /// Dispatches one consumed body. Unknown, stale, malformed, generation-mismatched, + /// already-completed, or identity-mismatched responses are ignored. + /// + internal void Dispatch( + string? correlationId, + ReadOnlyMemory body, + string? amqpRequestId, + long deliveryGeneration) + { + if (string.IsNullOrWhiteSpace(correlationId)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, "(missing)", "missing-correlation"); + return; + } + + if (!_pending.TryGetValue(correlationId, out var pending)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "unknown-correlation"); + return; + } + + if (pending.Operation.IsCompleted) + { + UnregisterAll(pending.Operation); + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "lookup-already-complete"); + return; + } + + if (pending.Generation != deliveryGeneration) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "generation-mismatch"); + return; + } + + if (string.IsNullOrWhiteSpace(amqpRequestId) + || !Guid.TryParse(amqpRequestId, out var headerRequestId) + || headerRequestId != pending.Operation.RequestId) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "request-id-mismatch"); + return; + } + + if (!ArticleWorkWireProtocol.TryParseResponseV1(body.Span, out var response, out var reason) + || response is null) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, reason); + return; + } + + if (!pending.Operation.Matches(response)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "identity-mismatch"); + return; + } + + pending.Operation.OnResponse(pending.Exchange, response); + if (pending.Operation.IsCompleted) + { + UnregisterAll(pending.Operation); + } + } + + private readonly record struct PendingPublication( + ArticleWorkLookupOperation Operation, + string Exchange, + long Generation); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs new file mode 100644 index 00000000..2b103b10 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Structured article-work RPC result returned to NNTP command callers. + /// + /// This type does not contain article bytes. Callers that need the article dial + /// and , then OPEN with . + /// + /// Classified RPC outcome. + /// Application request identity used for the lookup. + /// Message-ID that was queried. + /// Winning source backbone when a wire response completed the operation. + /// Success-only BackFiller FQDN from the winning response. + /// Success-only TLS VATP listen port from the winning response. + /// Success-only CanonicalV1 ArtId from the winning response. + /// Failure detail when the classified outcome is not success. + /// Exchange that produced the winning response, when applicable. + public sealed record ArticleWorkRpcResult( + ArticleWorkOutcome Outcome, + Guid RequestId, + string MessageId, + string? Backbone, + string? Fqdn, + int? VatpPort, + VectorNNTPD.Common.Articles.ArticleId? ArticleId, + string? Error, + string? SourceExchange) + { + /// Builds an overall not-found result after the lookup deadline elapses. + internal static ArticleWorkRpcResult NotFound(Guid requestId, string messageId, string error) => + new( + ArticleWorkOutcome.ArticleNotFound, + requestId, + messageId, + Backbone: null, + Fqdn: null, + VatpPort: null, + ArticleId: null, + error, + SourceExchange: null); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs new file mode 100644 index 00000000..e56a3556 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs @@ -0,0 +1,395 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// NNTPD-owned article-work RPC session: shared reply consumer, generation-scoped channels, + /// and sequential Backfill Scheduler orchestration. + /// + /// + /// remains the sole TCP connection owner. This service + /// opens caller-owned channels, declares an exclusive auto-delete reply queue, and + /// rebuilds that session after . + /// Outstanding application operations survive generation replacement in process memory; + /// a stale channel cannot publish into or mutate the replacement session. + /// BackFiller availability is discovered via the RabbitMQ Management HTTP API + /// (eventually consistent ~5s snapshot), not via AMQP passive queue declares. + /// + internal sealed class ArticleWorkRpcService : IApplicationService, IArticleWorkRpcClient, IArticleWorkRpcPublisher + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly ArticleWorkRpcResponseRouter _router; + private readonly BackfillConsumerAvailabilityService _availability; + private readonly ArticleWorkRpcClient _client; + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly object _sessionGate = new(); + private readonly Guid _instanceId = Guid.NewGuid(); + private CancellationTokenSource? _shutdownCts; + private RpcSession? _session; + private int _started; + + /// Initializes a new article-work RPC service. + public ArticleWorkRpcService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IRabbitMqManagementQueueInventory managementInventory, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(managementInventory); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _router = new ArticleWorkRpcResponseRouter(logger); + _availability = new BackfillConsumerAvailabilityService(managementInventory, logger, _timeProvider); + _client = new ArticleWorkRpcClient( + this, + _router, + _availability, + new WeightedBackboneSelector(), + _timeProvider, + logger, + () => _rabbitMq.ConnectionGeneration); + } + + /// + public string Name => "RabbitMQArticleWorkRpc"; + + /// + public Task? Execution => null; + + /// + public string ReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo + ?? throw new InvalidOperationException("Article-work RPC reply queue is not ready."); + } + } + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + await _availability.RefreshNowAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + /// + public async Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref _started) == 0 || _shutdownCts is null) + { + throw new InvalidOperationException("Article-work RPC is not started."); + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token); + return await _client.LookupByMessageIdAsync(messageId, linked.Token).ConfigureAwait(false); + } + + /// + public async Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + ArgumentNullException.ThrowIfNull(routingKey); + if (requestId == Guid.Empty) + { + throw new ArgumentException("Article-work RPC requires a non-empty RequestId.", nameof(requestId)); + } + + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + + RpcSession session; + lock (_sessionGate) + { + session = _session ?? throw new InvalidOperationException("Article-work RPC session is not ready."); + } + + if (!_rabbitMq.TryGetCurrent(out var handle) + || !handle.IsCurrent + || handle.Generation != session.Generation) + { + throw new InvalidOperationException( + $"Article-work RPC publication failed: connection generation {session.Generation} is not current."); + } + + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + lock (_sessionGate) + { + if (!ReferenceEquals(_session, session)) + { + throw new InvalidOperationException( + "Article-work RPC publication failed: session was replaced."); + } + } + + await session.PublishChannel + .PublishAsync( + exchange, + routingKey, + correlationId, + requestId.ToString("D"), + session.ReplyTo, + ArticleWorkWireProtocol.JsonContentType, + expiration: string.Empty, + body, + cancellationToken) + .ConfigureAwait(false); + } + finally + { + _publishGate.Release(); + } + } + + internal int OutstandingCorrelations => _router.OutstandingCount; + + internal string? CurrentReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo; + } + } + } + + internal long? CurrentSessionGeneration + { + get + { + lock (_sessionGate) + { + return _session?.Generation; + } + } + } + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + ArticleWorkRpcLogMessages.NotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for article-work RPC."); + } + + var generation = handle.Generation; + IRabbitMqRpcChannel? publishChannel = null; + IRabbitMqRpcChannel? consumeChannel = null; + try + { + publishChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + consumeChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + var replyTo = CreateReplyQueueName(); + await consumeChannel.QueueDeclareAsync( + replyTo, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + _ = await consumeChannel + .ConsumeAsync(replyTo, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new RpcSession(generation, replyTo, publishChannel, consumeChannel); + RpcSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + ArticleWorkRpcLogMessages.SessionReplaced(_logger, replyTo, generation); + } + else + { + ArticleWorkRpcLogMessages.Ready(_logger, replyTo, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + ArticleWorkRpcLogMessages.SessionAttachFailed(_logger, ex, generation); + if (publishChannel is not null) + { + await publishChannel.DisposeAsync().ConfigureAwait(false); + } + + if (consumeChannel is not null) + { + await consumeChannel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqRpcDelivery delivery) + { + try + { + _router.Dispatch( + delivery.CorrelationId, + delivery.Body, + delivery.RequestId, + delivery.Generation); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.ConsumerCallbackFailed(_logger, ex); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + _router.CancelAll(); + + RpcSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + await _availability.DisposeAsync().ConfigureAwait(false); + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + } + + private string CreateReplyQueueName() + { + var serverId = _nntpdOptions.Value.ServerId ?? 0; + return RabbitMqTopologyNames.Normalize($"nntpd.{serverId:00}.{_instanceId:N}.rpc"); + } + + private sealed class RpcSession : IAsyncDisposable + { + internal RpcSession( + long generation, + string replyTo, + IRabbitMqRpcChannel publishChannel, + IRabbitMqRpcChannel consumeChannel) + { + Generation = generation; + ReplyTo = replyTo; + PublishChannel = publishChannel; + ConsumeChannel = consumeChannel; + } + + internal long Generation { get; } + + internal string ReplyTo { get; } + + internal IRabbitMqRpcChannel PublishChannel { get; } + + internal IRabbitMqRpcChannel ConsumeChannel { get; } + + public async ValueTask DisposeAsync() + { + await PublishChannel.DisposeAsync().ConfigureAwait(false); + await ConsumeChannel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs new file mode 100644 index 00000000..bd30ab32 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Fixed article-work RPC deadlines. These are not configuration keys. + internal static class ArticleWorkRpcTiming + { + /// + /// Maximum total Backfill Scheduler lookup duration measured from request start. + /// Also the per-attempt wait budget (remaining time until this deadline). + /// + internal static readonly TimeSpan LookupDeadline = TimeSpan.FromSeconds(5); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs new file mode 100644 index 00000000..8c79c5fb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs @@ -0,0 +1,628 @@ +using System.Buffers; +using System.Text.Encodings.Web; +using System.Text.Json; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// Canonical compact System.Text.Json contract for BackFiller article-work v1 request and response bodies. + /// + /// + /// Property names are version, requestId, messageId, backbone, + /// and on responses outcome, fqdn, vatpPort, articleId, and error. + /// A response that includes uri is rejected. AMQP CorrelationId, + /// ReplyTo, and Expiration are never JSON fields. JSON requestId must + /// match the AMQP RequestId property. Serialization writes compact UTF-8 with no indentation. + /// + internal static partial class ArticleWorkWireProtocol + { + /// Current application protocol version. + internal const int CurrentVersion = 1; + + /// AMQP ContentType required by the BackFiller RPC contract. + internal const string JsonContentType = "application/json"; + + /// Serializes a version-1 request into compact UTF-8 JSON. + /// Application fields to write. Must be a concrete valid v1 request. + /// UTF-8 bytes containing only the four canonical request properties. + internal static byte[] SerializeRequestV1(ArticleWorkRequest request) + { + ArgumentNullException.ThrowIfNull(request); + if (request.Version != CurrentVersion) + { + throw new InvalidOperationException($"Unsupported article-work request version '{request.Version}'."); + } + + if (request.RequestId == Guid.Empty) + { + throw new InvalidOperationException("Canonical request serialization requires a concrete non-empty requestId."); + } + + if (string.IsNullOrWhiteSpace(request.MessageId)) + { + throw new InvalidOperationException("Canonical request serialization requires a canonical non-empty messageId."); + } + + if (string.IsNullOrWhiteSpace(request.Backbone)) + { + throw new InvalidOperationException("Canonical request serialization requires a non-empty backbone."); + } + + var writer = new ArrayBufferWriter(); + using var jsonWriter = new Utf8JsonWriter(writer, new JsonWriterOptions + { + Indented = false, + SkipValidation = false, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }); + + jsonWriter.WriteStartObject(); + jsonWriter.WriteNumber("version", request.Version); + jsonWriter.WriteString("requestId", request.RequestId); + jsonWriter.WriteString("messageId", request.MessageId); + jsonWriter.WriteString("backbone", request.Backbone); + jsonWriter.WriteEndObject(); + jsonWriter.Flush(); + return writer.WrittenSpan.ToArray(); + } + + /// Attempts to parse and validate a version-1 response payload. + /// UTF-8 JSON body. + /// Parsed response when validation succeeds. + /// Rejection reason when parsing fails. + /// when the payload is a valid v1 response. + internal static bool TryParseResponseV1( + ReadOnlySpan payload, + out ArticleWorkResponse? response, + out string reason) + { + response = null; + reason = "Response payload is empty."; + if (payload.IsEmpty) + { + return false; + } + + JsonDocument document; + try + { + document = JsonDocument.Parse(payload.ToArray()); + } + catch (JsonException) + { + reason = "Response payload was not valid JSON."; + return false; + } + + using (document) + { + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + reason = "Response payload must be a JSON object."; + return false; + } + + if (!root.TryGetProperty("version", out var versionElement) + || versionElement.ValueKind != JsonValueKind.Number + || !versionElement.TryGetInt32(out var version)) + { + reason = "Response payload is missing required integer property 'version'."; + return false; + } + + if (version != CurrentVersion) + { + reason = $"Unsupported response version '{version}'."; + return false; + } + + if (!TryReadRequiredString(root, "outcome", out var outcomeText) || outcomeText is null) + { + reason = "Response payload is missing required 'outcome'."; + return false; + } + + if (!TryParseOutcome(outcomeText, out var outcome)) + { + reason = $"Unsupported response outcome '{outcomeText}'."; + return false; + } + + if (!TryReadOptionalGuid(root, "requestId", out var requestId, out reason)) + { + return false; + } + + if (!TryReadMessageId(payload, out var messageId, out reason)) + { + return false; + } + + if (!TryReadOptionalString(root, "backbone", out var backbone, out reason)) + { + return false; + } + + if (root.TryGetProperty("uri", out _)) + { + reason = "Response payload must not include 'uri'."; + return false; + } + + var hasFqdn = root.TryGetProperty("fqdn", out var fqdnElement); + var hasVatpPort = root.TryGetProperty("vatpPort", out var vatpPortElement); + var hasError = root.TryGetProperty("error", out var errorElement); + var hasArticleId = root.TryGetProperty("articleId", out var articleIdElement); + string? fqdn = null; + int? vatpPort = null; + string? error = null; + ArticleId? articleId = null; + if (hasFqdn && !TryReadStringValue(fqdnElement, "fqdn", out fqdn, out reason)) + { + return false; + } + + if (hasVatpPort) + { + if (vatpPortElement.ValueKind != JsonValueKind.Number || !vatpPortElement.TryGetInt32(out var parsedPort)) + { + reason = "Response payload property 'vatpPort' must be an integer."; + return false; + } + + vatpPort = parsedPort; + } + + if (hasError && !TryReadStringValue(errorElement, "error", out error, out reason)) + { + return false; + } + + if (hasArticleId) + { + if (!TryReadStringValue(articleIdElement, "articleId", out var articleIdText, out reason)) + { + return false; + } + + if (articleIdText is not null) + { + if (!ArticleId.TryParseLowerHex(articleIdText, out var parsedArticleId)) + { + reason = "Response payload property 'articleId' is not a 64-character lowercase hexadecimal ArticleId."; + return false; + } + + articleId = parsedArticleId; + } + } + + if (!TryValidateOutcomeContract( + outcome, + requestId, + messageId, + backbone, + hasFqdn, + fqdn, + hasVatpPort, + vatpPort, + hasArticleId, + articleId, + hasError, + error, + out reason)) + { + return false; + } + + response = new ArticleWorkResponse( + version, + requestId, + messageId, + backbone, + outcome, + fqdn, + vatpPort, + articleId, + error); + reason = string.Empty; + return true; + } + } + + private static bool TryValidateOutcomeContract( + ArticleWorkOutcome outcome, + Guid? requestId, + string? messageId, + string? backbone, + bool hasFqdn, + string? fqdn, + bool hasVatpPort, + int? vatpPort, + bool hasArticleId, + ArticleId? articleId, + bool hasError, + string? error, + out string reason) + { + reason = string.Empty; + if (outcome == ArticleWorkOutcome.Success) + { + if (!requestId.HasValue || requestId.Value == Guid.Empty) + { + reason = "Success response payload requires a concrete non-empty 'requestId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(messageId)) + { + reason = "Success response payload requires a canonical non-empty 'messageId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(backbone)) + { + reason = "Success response payload requires a non-empty 'backbone'."; + return false; + } + + if (!hasFqdn || !VatpEndpointFields.IsCanonicalFqdn(fqdn)) + { + reason = "Success response payload requires a lowercase dotted DNS 'fqdn'."; + return false; + } + + if (!hasVatpPort || vatpPort is not int port || !VatpEndpointFields.IsCanonicalPort(port)) + { + reason = "Success response payload requires a 'vatpPort' in the range 1–65535."; + return false; + } + + if (!hasArticleId || articleId is null) + { + reason = "Success response payload requires a concrete 'articleId'."; + return false; + } + + if (hasError) + { + reason = "Success response payload must not include 'error'."; + return false; + } + + return true; + } + + if (outcome is ArticleWorkOutcome.ArticleNotFound or ArticleWorkOutcome.InvalidArticle) + { + if (!requestId.HasValue || requestId.Value == Guid.Empty) + { + reason = "Terminal non-invalid-request response payload requires a concrete non-empty 'requestId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(messageId)) + { + reason = "Terminal non-invalid-request response payload requires a canonical non-empty 'messageId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(backbone)) + { + reason = "Terminal non-invalid-request response payload requires a non-empty 'backbone'."; + return false; + } + + if (hasFqdn || hasVatpPort) + { + reason = "Terminal failure response payload must not include 'fqdn' or 'vatpPort'."; + return false; + } + + if (hasArticleId) + { + reason = "Terminal failure response payload must not include 'articleId'."; + return false; + } + + if (!hasError || string.IsNullOrWhiteSpace(error)) + { + reason = "Terminal failure response payload requires a non-empty 'error'."; + return false; + } + + return true; + } + + if (requestId.HasValue && requestId.Value == Guid.Empty) + { + reason = "InvalidRequest payload must not use Guid.Empty sentinel request identity."; + return false; + } + + if (messageId is not null && string.IsNullOrWhiteSpace(messageId)) + { + reason = "InvalidRequest payload messageId, when provided, must be canonical and non-empty."; + return false; + } + + if (backbone is not null && string.IsNullOrWhiteSpace(backbone)) + { + reason = "InvalidRequest payload backbone, when provided, must be non-empty."; + return false; + } + + if (hasFqdn || hasVatpPort) + { + reason = "InvalidRequest payload must not include 'fqdn' or 'vatpPort'."; + return false; + } + + if (hasArticleId) + { + reason = "InvalidRequest payload must not include 'articleId'."; + return false; + } + + if (!hasError || string.IsNullOrWhiteSpace(error)) + { + reason = "InvalidRequest payload requires a non-empty 'error'."; + return false; + } + + return true; + } + + private static bool TryParseOutcome(string outcome, out ArticleWorkOutcome parsed) + { + if (string.Equals(outcome, nameof(ArticleWorkOutcome.Success), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.Success; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.ArticleNotFound), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.ArticleNotFound; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.InvalidArticle), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.InvalidArticle; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.InvalidRequest), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.InvalidRequest; + return true; + } + + parsed = default; + return false; + } + + private static bool TryReadRequiredString(JsonElement root, string propertyName, out string? value) + { + value = null; + return root.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.String + && (value = property.GetString()) is not null; + } + + private static bool TryReadOptionalGuid( + JsonElement root, + string propertyName, + out Guid? value, + out string reason) + { + value = null; + reason = string.Empty; + if (!root.TryGetProperty(propertyName, out var property)) + { + reason = $"Response payload is missing required '{propertyName}'."; + return false; + } + + if (property.ValueKind == JsonValueKind.Null) + { + return true; + } + + if (property.ValueKind != JsonValueKind.String) + { + reason = $"Response payload property '{propertyName}' must be a JSON string or null."; + return false; + } + + var text = property.GetString(); + if (string.IsNullOrWhiteSpace(text) || !Guid.TryParse(text, out var parsed) || parsed == Guid.Empty) + { + reason = $"Response payload property '{propertyName}' is not a concrete GUID."; + return false; + } + + value = parsed; + return true; + } + + /// + /// Reads the top-level messageId property from the original UTF-8 payload and validates those bytes. + /// + /// Response payload. Already known to be a JSON object. + /// ASCII text of a valid Message-ID, or null when the property is JSON null. + /// Rejection text when the result is . + /// when the property is missing, not a string, or not a valid Message-ID. + private static bool TryReadMessageId(ReadOnlySpan json, out string? messageId, out string reason) + { + messageId = null; + reason = string.Empty; + Span buffer = stackalloc byte[NntpMessageIdValidation.MaxMessageIdLength]; + switch (TryCopyTopLevelJsonString(json, "messageId"u8, buffer, out var length)) + { + case TopLevelJsonString.Missing: + reason = "Response payload is missing required 'messageId'."; + return false; + case TopLevelJsonString.Null: + return true; + case TopLevelJsonString.NotString: + reason = "Response payload property 'messageId' must be a JSON string or null."; + return false; + case TopLevelJsonString.TooLong: + reason = "Response payload property 'messageId' must be a canonical Message-ID."; + return false; + case TopLevelJsonString.Copied: + var token = buffer[..length]; + if (length == 0 || !NntpMessageIdValidation.IsValidMessageId(token)) + { + reason = "Response payload property 'messageId' must be a canonical Message-ID."; + return false; + } + + messageId = System.Text.Encoding.ASCII.GetString(token); + return true; + default: + reason = "Response payload is missing required 'messageId'."; + return false; + } + } + + private enum TopLevelJsonString + { + Missing, + Null, + NotString, + TooLong, + Copied, + } + + /// Copies one top-level JSON string property into without allocating. + private static TopLevelJsonString TryCopyTopLevelJsonString( + ReadOnlySpan json, + ReadOnlySpan propertyName, + Span destination, + out int written) + { + written = 0; + var reader = new Utf8JsonReader(json); + if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject) + { + return TopLevelJsonString.Missing; + } + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject && reader.CurrentDepth == 0) + { + return TopLevelJsonString.Missing; + } + + if (reader.TokenType != JsonTokenType.PropertyName || reader.CurrentDepth != 1) + { + continue; + } + + var match = reader.ValueTextEquals(propertyName); + if (!reader.Read()) + { + return TopLevelJsonString.Missing; + } + + if (!match) + { + if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + reader.Skip(); + } + + continue; + } + + if (reader.TokenType == JsonTokenType.Null) + { + return TopLevelJsonString.Null; + } + + if (reader.TokenType != JsonTokenType.String) + { + return TopLevelJsonString.NotString; + } + + if (!reader.ValueIsEscaped) + { + var value = reader.ValueSpan; + if (value.Length > destination.Length) + { + return TopLevelJsonString.TooLong; + } + + value.CopyTo(destination); + written = value.Length; + return TopLevelJsonString.Copied; + } + + try + { + written = reader.CopyString(destination); + return TopLevelJsonString.Copied; + } + catch (ArgumentException) + { + written = 0; + return TopLevelJsonString.TooLong; + } + } + + return TopLevelJsonString.Missing; + } + + private static bool TryReadOptionalString( + JsonElement root, + string propertyName, + out string? value, + out string reason) + { + value = null; + reason = string.Empty; + if (!root.TryGetProperty(propertyName, out var property)) + { + reason = $"Response payload is missing required '{propertyName}'."; + return false; + } + + return TryReadStringValue(property, propertyName, out value, out reason); + } + + private static bool TryReadStringValue( + JsonElement property, + string propertyName, + out string? value, + out string reason) + { + value = null; + reason = string.Empty; + switch (property.ValueKind) + { + case JsonValueKind.Null: + return true; + case JsonValueKind.String: + value = property.GetString(); + if (value is null) + { + reason = $"Response payload property '{propertyName}' string value was null."; + return false; + } + + return true; + default: + reason = $"Response payload property '{propertyName}' must be a JSON string or null."; + return false; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs new file mode 100644 index 00000000..1b6f80ef --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs @@ -0,0 +1,207 @@ +using System.Collections.Immutable; +using VectorNNTPD.ReaderService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// Cached BackFiller ArticleWork consumer counts from the RabbitMQ Management HTTP API. + /// + /// + /// + /// Discovery plane: Management API queue inventory refreshed on a fixed ~5s interval. + /// AMQP remains the ArticleWork data plane. Availability is intentionally eventually + /// consistent and may be approximately 5 seconds stale. + /// + /// + /// A provider queue is eligible only when the Management API reports + /// consumers > 0 for that queue name. Missing queues and zero-consumer queues + /// are unavailable and are omitted from the snapshot. Absent/zero-consumer states are + /// normal and do not produce exceptions. + /// + /// + /// A failed Management API refresh retains the last successfully retrieved snapshot + /// (last-known-good). There is no additional long-term stale expiry beyond the normal + /// ~5s refresh cadence. + /// + /// + internal sealed class BackfillConsumerAvailabilityService : IBackfillConsumerAvailability, IAsyncDisposable + { + /// Minimum interval between Management API refresh attempts. + internal static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5); + + private readonly IRabbitMqManagementQueueInventory _inventory; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly object _gate = new(); + private readonly SemaphoreSlim _refreshGate = new(1, 1); + private ImmutableArray _snapshot = []; + private DateTimeOffset _nextRefresh = DateTimeOffset.MinValue; + private int _disposed; + + /// Initializes a new availability cache. + internal BackfillConsumerAvailabilityService( + IRabbitMqManagementQueueInventory inventory, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(inventory); + ArgumentNullException.ThrowIfNull(logger); + _inventory = inventory; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public IReadOnlyList GetEligibleBackbones() + { + RefreshIfDueAsync(force: false, CancellationToken.None).GetAwaiter().GetResult(); + return _snapshot; + } + + /// Forces a refresh regardless of interval (startup / tests). + internal Task RefreshNowAsync(CancellationToken cancellationToken = default) => + RefreshIfDueAsync(force: true, cancellationToken); + + private async Task RefreshIfDueAsync(bool force, CancellationToken cancellationToken) + { + if (Volatile.Read(ref _disposed) != 0) + { + return; + } + + var now = _timeProvider.GetUtcNow(); + if (!force) + { + lock (_gate) + { + if (now < _nextRefresh) + { + return; + } + } + } + + await _refreshGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (Volatile.Read(ref _disposed) != 0) + { + return; + } + + now = _timeProvider.GetUtcNow(); + if (!force) + { + lock (_gate) + { + if (now < _nextRefresh) + { + return; + } + } + } + + await RefreshCoreAsync(cancellationToken).ConfigureAwait(false); + lock (_gate) + { + _nextRefresh = _timeProvider.GetUtcNow() + RefreshInterval; + } + } + finally + { + _refreshGate.Release(); + } + } + + private async Task RefreshCoreAsync(CancellationToken cancellationToken) + { + try + { + var queues = await _inventory.ListQueuesAsync(cancellationToken).ConfigureAwait(false); + var consumersByName = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < queues.Count; i++) + { + var queue = queues[i]; + if (string.IsNullOrWhiteSpace(queue.Name)) + { + continue; + } + + // Last duplicate name wins; Management API should not duplicate names in one vhost. + consumersByName[queue.Name] = queue.Consumers; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + if (!consumersByName.TryGetValue(definition.QueueName, out var consumers) + || consumers <= 0) + { + continue; + } + + builder.Add(new BackfillEligibleBackbone( + definition.Provider, + definition.ExchangeName, + definition.RoutingKey, + definition.QueueName, + consumers)); + } + + var next = builder.ToImmutable(); + lock (_gate) + { + _snapshot = next; + } + + ArticleWorkRpcLogMessages.AvailabilitySnapshotRefreshed(_logger, next.Length); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // Retain last-known-good snapshot. Do not treat Management outage as zero consumers. + ArticleWorkRpcLogMessages.ManagementAvailabilityRefreshFailed(_logger, ex); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _refreshGate.Dispose(); + await Task.CompletedTask.ConfigureAwait(false); + } + } + + /// Test double / static snapshot availability. + internal sealed class StaticBackfillConsumerAvailability : IBackfillConsumerAvailability + { + private volatile IReadOnlyList _eligible; + + /// Creates availability from an initial snapshot. + internal StaticBackfillConsumerAvailability(IReadOnlyList eligible) + { + ArgumentNullException.ThrowIfNull(eligible); + _eligible = eligible; + } + + /// Replaces the snapshot (tests). + internal void SetEligible(IReadOnlyList eligible) + { + ArgumentNullException.ThrowIfNull(eligible); + _eligible = eligible; + } + + /// + public IReadOnlyList GetEligibleBackbones() => _eligible; + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs new file mode 100644 index 00000000..bf6c23dd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs @@ -0,0 +1,119 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// One eligible BackFiller backbone with a positive active consumer count. + /// Canonical provider identifier (JSON backbone casing). + /// Normalized exchange name. + /// Normalized routing key. + /// Normalized queue name used for passive consumer inspection. + /// Broker-reported consumer count (must be > 0). + internal readonly record struct BackfillEligibleBackbone( + string Backbone, + string Exchange, + string RoutingKey, + string QueueName, + int ConsumerCount); + + /// + /// Snapshot of BackFiller ArticleWork consumer availability for scheduler selection. + /// + internal interface IBackfillConsumerAvailability + { + /// + /// Returns provider backbones with a positive active consumer count. + /// Zero-consumer and unknown queues are omitted. + /// + IReadOnlyList GetEligibleBackbones(); + } + + /// + /// Selects one eligible backbone weighted by active consumer count. + /// + internal interface IBackboneSelector + { + /// + /// Chooses one candidate. Returns when + /// is empty. + /// + BackfillEligibleBackbone? Select(IReadOnlyList candidates); + } + + /// + /// Weighted random backbone selection. Isolates randomness for deterministic tests. + /// + internal sealed class WeightedBackboneSelector : IBackboneSelector + { + private readonly Func _nextInt; + + /// Creates a selector using . + public WeightedBackboneSelector() + : this(static exclusiveMax => Random.Shared.Next(exclusiveMax)) + { + } + + /// + /// Creates a selector with an injectable RNG. receives + /// an exclusive upper bound and must return a value in [0, exclusiveMax). + /// + internal WeightedBackboneSelector(Func nextInt) + { + ArgumentNullException.ThrowIfNull(nextInt); + _nextInt = nextInt; + } + + /// + public BackfillEligibleBackbone? Select(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + if (candidates.Count == 0) + { + return null; + } + + long total = 0; + for (var i = 0; i < candidates.Count; i++) + { + var weight = candidates[i].ConsumerCount; + if (weight < 1) + { + continue; + } + + total += weight; + } + + if (total < 1) + { + return null; + } + + if (total > int.MaxValue) + { + total = int.MaxValue; + } + + var roll = _nextInt((int)total); + if (roll < 0 || roll >= total) + { + throw new InvalidOperationException("Backbone selector RNG returned a value outside [0, total)."); + } + + long cursor = 0; + for (var i = 0; i < candidates.Count; i++) + { + var candidate = candidates[i]; + if (candidate.ConsumerCount < 1) + { + continue; + } + + cursor += candidate.ConsumerCount; + if (roll < cursor) + { + return candidate; + } + } + + return candidates[^1]; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs new file mode 100644 index 00000000..6978d1ee --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs @@ -0,0 +1,25 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// + /// Reusable article-work RPC entry point for ARTICLE, HEAD, BODY, and STAT message-id lookups. + /// + /// + /// This boundary converts command-line Message-ID bytes into the JSON application payload. + /// It does not retrieve article bytes or write NNTP success responses. + /// + public interface IArticleWorkRpcClient + { + /// + /// Runs the sequential Backfill Scheduler: selects one eligible backbone at a time + /// (weighted by active consumer count), publishes one ArticleWork request per attempt, + /// and returns on Success, exhausted not-found, cancellation, or the 5-second lookup + /// deadline. The scheduler never exceeds that 5-second budget from its start time. + /// + /// Command-line Message-ID bytes, including angle brackets. + /// Caller or session cancellation. + /// The structured RPC result. Never includes article content. + Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs new file mode 100644 index 00000000..81194722 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.RabbitMq.ArticleWork +{ + /// Publishes one article-work RPC request onto the current RabbitMQ generation. + internal interface IArticleWorkRpcPublisher + { + /// Gets the NNTPD-owned reply queue name used as AMQP ReplyTo. + string ReplyTo { get; } + + /// + /// Publishes a compact v1 JSON body with AMQP RPC metadata: + /// RequestId, CorrelationId, ReplyTo, and + /// ContentType=application/json. Work requests omit AMQP Expiration + /// so the broker may buffer ready messages until a BackFiller consumer delivers them. + /// + /// Destination fanout exchange. + /// Routing key matching the declared binding. + /// Logical lookup UUID. Also written to JSON requestId. + /// Fresh AMQP correlation identity for this publication only. + /// Canonical compact request JSON. + /// Token used to cancel the publish. + Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/BackfillArticleRetrievalTopology.cs b/src/VectorNNTPD.ReaderService/RabbitMq/BackfillArticleRetrievalTopology.cs new file mode 100644 index 00000000..d633f625 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/BackfillArticleRetrievalTopology.cs @@ -0,0 +1,149 @@ +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// Immutable BackFiller-compatible topology for one article-retrieval provider. + /// + /// Canonical provider identifier. Casing is preserved. + /// Exchange that receives article-retrieval work for the provider. + /// RabbitMQ exchange type. BackFiller uses fanout. + /// Whether the exchange survives broker restart. + /// Whether the exchange is auto-deleted when unused. + /// Queue bound to the provider exchange. + /// Whether the queue survives broker restart. + /// Whether the queue is exclusive to a single connection. + /// Whether the queue is auto-deleted when unused. + /// Routing key used when binding the queue to the exchange. + /// + /// Queue arguments applied during declaration. Durable provider work queues require + /// x-queue-type=quorum. + /// + internal sealed record BackfillArticleRetrievalTopologyDefinition( + string Provider, + string ExchangeName, + string ExchangeType, + bool ExchangeDurable, + bool ExchangeAutoDelete, + string QueueName, + bool QueueDurable, + bool QueueExclusive, + bool QueueAutoDelete, + string RoutingKey, + IReadOnlyDictionary QueueArguments) + { + /// Projects this BackFiller definition onto the shared declare shape. + internal RabbitMqArticleRetrievalEndpoint ToEndpoint() => + new( + ExchangeName, + ExchangeType, + ExchangeDurable, + ExchangeAutoDelete, + QueueName, + QueueDurable, + QueueExclusive, + QueueAutoDelete, + RoutingKey, + QueueArguments); + } + + /// + /// Fixed article-retrieval topology shared with VectorNNTP.BackFiller. + /// + /// + /// + /// Provider identifiers are application constants, not configuration. The twelve names + /// match BackFiller's backbone set. Entity names are backfiller.{provider} + /// after normalization, for the exchange, queue, + /// and routing key. + /// + /// + /// Every queue name matches the durable quorum fanout shape BackFiller declares when a + /// backbone becomes usable (x-queue-type=quorum). NNTPD does not declare or consume + /// these queues; it publishes sequential ArticleWork RPC requests onto selected provider + /// queues that already exist and have active consumers. + /// + /// + internal static class BackfillArticleRetrievalTopology + { + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName; + + /// Required queue type for durable BackFiller ArticleWork provider queues. + internal const string QuorumQueueType = RabbitMqArticleRetrievalEndpoints.QuorumQueueType; + + /// + /// Canonical BackFiller provider identifiers. Casing is significant and is not normalized. + /// + internal static readonly string[] Providers = + [ + "Abavia", + "Altopia", + "BaseIP", + "Eweka", + "Elbracht", + "Giganews", + "GTT", + "Highwinds", + "ItsHosted", + "Novia", + "UExpress", + "UsenetNode1", + ]; + + /// Complete article-retrieval topology in provider-list order. + internal static IReadOnlyList Definitions { get; } = + BuildDefinitions(Providers); + + /// + /// Builds topology definitions for the supplied provider identifiers. + /// + /// Provider names whose exchanges and queues must exist. + /// One definition per non-blank unique provider, preserving first-seen order. + internal static IReadOnlyList BuildDefinitions( + IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + + var uniqueProviders = new HashSet(StringComparer.OrdinalIgnoreCase); + var definitions = new List(); + + foreach (var provider in providers) + { + if (string.IsNullOrWhiteSpace(provider)) + { + continue; + } + + var canonicalProvider = provider.Trim(); + if (!uniqueProviders.Add(canonicalProvider)) + { + continue; + } + + var endpoint = RabbitMqArticleRetrievalEndpoints.CreateFanoutQuorumBinding( + BuildProviderEntityName(canonicalProvider)); + definitions.Add(new BackfillArticleRetrievalTopologyDefinition( + Provider: canonicalProvider, + ExchangeName: endpoint.ExchangeName, + ExchangeType: endpoint.ExchangeType, + ExchangeDurable: endpoint.ExchangeDurable, + ExchangeAutoDelete: endpoint.ExchangeAutoDelete, + QueueName: endpoint.QueueName, + QueueDurable: endpoint.QueueDurable, + QueueExclusive: endpoint.QueueExclusive, + QueueAutoDelete: endpoint.QueueAutoDelete, + RoutingKey: endpoint.RoutingKey, + QueueArguments: endpoint.QueueArguments)); + } + + return definitions; + } + + /// + /// Builds the NNTPD BackFiller exchange, queue, and routing-key name for a provider. + /// + /// Provider identifier. Casing and surrounding whitespace are normalized. + /// The normalized entity name in the form backfiller.{provider}. + internal static string BuildProviderEntityName(string provider) => + RabbitMqTopologyNames.Compose("backfiller", provider); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/CacheBroadcastTopology.cs b/src/VectorNNTPD.ReaderService/RabbitMq/CacheBroadcastTopology.cs new file mode 100644 index 00000000..e73b6fa1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/CacheBroadcastTopology.cs @@ -0,0 +1,61 @@ +using VectorNNTPD.Common.Messaging.Cache; +using RabbitMQ.Client; + +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// NNTPD-owned cache.requests fleet article-lookup fanout exchange. + /// + /// + /// There is no shared cache.requests work queue. Each StorageServer binds an + /// ephemeral cache.<storage-fqdn> queue. Per-NNTPD reply queues for lookup + /// responses are owned by StorageArticleLookupService. + /// + internal static class CacheRequestsTopology + { + /// Fanout exchange that carries "who has article X?" lookups. + internal const string ExchangeName = CacheFleetTopology.RequestsExchangeName; + + /// Exchange type. + internal const string ExchangeTypeName = ExchangeType.Fanout; + + /// Whether the exchange survives broker restart. + internal const bool ExchangeDurable = true; + + /// Whether the exchange auto-deletes when unused. + internal const bool ExchangeAutoDelete = false; + + /// Per-message AMQP expiration for lookup requests/responses. + internal const string ExpirationMilliseconds = CacheFleetTopology.LookupExpirationMilliseconds; + } + + /// + /// NNTPD-owned cache.broadcast fanout exchange declaration constants. + /// + /// + /// Per-instance ephemeral consume queues are declared by + /// , not by + /// . + /// + internal static class CacheBroadcastTopology + { + /// Fanout exchange that carries StorageServer advertisements. + internal const string ExchangeName = CacheFleetTopology.BroadcastExchangeName; + + /// Exchange type. + internal const string ExchangeTypeName = ExchangeType.Fanout; + + /// Whether the exchange survives broker restart. + internal const bool ExchangeDurable = true; + + /// Whether the exchange auto-deletes when unused. + internal const bool ExchangeAutoDelete = false; + + /// Per-message AMQP expiration for advertisements. + internal const string ExpirationMilliseconds = CacheFleetTopology.AdvertisementExpirationMilliseconds; + + /// Builds cache.{nntpdFqdn} after normalization. + internal static string BuildInstanceQueueName(string nntpdFqdn) => + CacheFleetTopology.BuildNntpdBroadcastQueueName(nntpdFqdn); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs b/src/VectorNNTPD.ReaderService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs new file mode 100644 index 00000000..8af1b6b8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs @@ -0,0 +1,20 @@ +using VectorNNTPD.Common.Messaging.RabbitMq; +namespace VectorNNTPD.ReaderService.RabbitMq.Management +{ + /// + /// Reads the RabbitMQ Management HTTP API queue inventory for one virtual host. + /// + /// + /// Discovery plane only. Does not publish or consume ArticleWork AMQP messages. + /// + internal interface IRabbitMqManagementQueueInventory + { + /// + /// Lists all queues in the configured virtual host via + /// GET /api/queues/{vhost} (vhost URL-encoded). + /// + /// Token used to cancel the HTTP request. + /// Queue name and consumer count rows. Never . + Task> ListQueuesAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementHttpClient.cs b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementHttpClient.cs new file mode 100644 index 00000000..504ce6d4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementHttpClient.cs @@ -0,0 +1,148 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.RabbitMq.Management +{ + /// + /// HTTP client for RabbitMQ Management API queue inventory + /// (GET /api/queues/{vhost}). + /// + /// + /// Authenticates with HTTP Basic using / + /// . Never logs credentials or the + /// Authorization header. Uses source-generated JSON deserialization. + /// + internal sealed class RabbitMqManagementHttpClient : IRabbitMqManagementQueueInventory + { + /// Named client for Management API calls. + internal const string HttpClientName = "RabbitMqManagement"; + + private readonly HttpClient _httpClient; + private readonly IOptions _options; + + /// Initializes a new Management API client. + public RabbitMqManagementHttpClient( + HttpClient httpClient, + IOptions options) + { + ArgumentNullException.ThrowIfNull(httpClient); + ArgumentNullException.ThrowIfNull(options); + _httpClient = httpClient; + _options = options; + } + + /// + public async Task> ListQueuesAsync( + CancellationToken cancellationToken) + { + var rabbitMq = _options.Value; + var management = rabbitMq.Management + ?? throw new InvalidOperationException("RabbitMQ:Management is required."); + var baseUrl = management.BaseUrl?.Trim(); + if (string.IsNullOrWhiteSpace(baseUrl)) + { + throw new InvalidOperationException("RabbitMQ:Management:BaseUrl is required."); + } + + var username = rabbitMq.Username?.Trim(); + var password = rabbitMq.Password; + if (string.IsNullOrWhiteSpace(username) || password is null) + { + throw new InvalidOperationException( + "RabbitMQ:Username and RabbitMQ:Password are required for Management API access."); + } + + var virtualHost = string.IsNullOrWhiteSpace(rabbitMq.VirtualHost) + ? "/" + : rabbitMq.VirtualHost.Trim(); + var path = $"api/queues/{Uri.EscapeDataString(virtualHost)}"; + + var timeoutSeconds = management.RequestTimeoutSeconds is > 0 + ? management.RequestTimeoutSeconds.Value + : 5; + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds)); + + using var request = new HttpRequestMessage(HttpMethod.Get, path); + request.Headers.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}"))); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + HttpResponseMessage response; + try + { + response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"RabbitMQ Management API request timed out after {timeoutSeconds} seconds."); + } + + using (response) + { + if (!response.IsSuccessStatusCode) + { + // Do not include response bodies that may echo credentials or tokens. + throw new HttpRequestException( + $"RabbitMQ Management API returned {(int)response.StatusCode} ({response.StatusCode}) for queue inventory."); + } + + RabbitMqManagementQueueDto[]? payload; + try + { + payload = await response.Content + .ReadFromJsonAsync( + RabbitMqManagementJsonContext.Default.RabbitMqManagementQueueDtoArray, + timeoutCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"RabbitMQ Management API response read timed out after {timeoutSeconds} seconds."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + "RabbitMQ Management API returned a malformed queue inventory payload.", + ex); + } + + if (payload is null) + { + throw new InvalidOperationException( + "RabbitMQ Management API returned an empty queue inventory payload."); + } + + var results = new List(payload.Length); + for (var i = 0; i < payload.Length; i++) + { + var row = payload[i]; + if (row is null || string.IsNullOrWhiteSpace(row.Name)) + { + continue; + } + + var consumers = row.Consumers; + if (consumers < 0) + { + consumers = 0; + } + + results.Add(new RabbitMqManagementQueueInfo(row.Name.Trim(), consumers)); + } + + return results; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementJsonContext.cs b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementJsonContext.cs new file mode 100644 index 00000000..026e2296 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementJsonContext.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace VectorNNTPD.ReaderService.RabbitMq.Management +{ + /// + /// Source-generated JSON metadata for RabbitMQ Management API queue inventory responses. + /// + [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] + [JsonSerializable(typeof(RabbitMqManagementQueueDto[]))] + [JsonSerializable(typeof(List))] + internal sealed partial class RabbitMqManagementJsonContext : JsonSerializerContext; +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementQueueDto.cs b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementQueueDto.cs new file mode 100644 index 00000000..00b02812 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/Management/RabbitMqManagementQueueDto.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace VectorNNTPD.ReaderService.RabbitMq.Management +{ + /// + /// Minimal RabbitMQ Management HTTP API queue inventory entry + /// (GET /api/queues/{vhost}). + /// + internal sealed class RabbitMqManagementQueueDto + { + /// Queue name as reported by the Management API (name). + [JsonPropertyName("name")] + public string? Name { get; set; } + + /// Broker-reported AMQP consumer count (consumers). + [JsonPropertyName("consumers")] + public int Consumers { get; set; } + } + + /// Normalized queue inventory row used by availability discovery. + /// Queue name. + /// AMQP consumer count. + internal readonly record struct RabbitMqManagementQueueInfo(string Name, int Consumers); +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/OverviewDbTopology.cs b/src/VectorNNTPD.ReaderService/RabbitMq/OverviewDbTopology.cs new file mode 100644 index 00000000..6c394042 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/OverviewDbTopology.cs @@ -0,0 +1,57 @@ +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// Broker topology for the one-way OverviewDB ingest handoff. + /// + /// + /// Publishes through the default exchange with routing key + /// . No dedicated exchange, bind, reply queue, or RPC + /// topology is declared. + /// + internal static class OverviewDbTopology + { + /// + /// Durable quorum queue consumed independently by OverviewDB. + /// + /// + /// Declared with x-queue-type=quorum. Per-message AMQP expiration is + /// supported on the current broker; there is no queue-wide x-message-ttl. + /// + internal const string QueueName = "overviewdb.queue"; + + /// Default-exchange routing key; equal to . + internal const string RoutingKey = QueueName; + + /// AMQP default exchange (empty name). + internal const string DefaultExchange = ""; + + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName; + + /// Required queue type for the OverviewDB ingest queue. + internal const string QuorumQueueType = RabbitMqArticleRetrievalEndpoints.QuorumQueueType; + + /// Queue arguments applied during declaration. + internal static IReadOnlyDictionary QueueArguments { get; } = + new Dictionary(StringComparer.Ordinal) + { + [QueueTypeArgumentName] = QuorumQueueType, + }; + + /// Per-message AMQP expiration in milliseconds (expiration shortstr). + /// + /// The broker interprets this as milliseconds. RabbitMQ discards expired + /// messages when they reach the head of the queue; Ready counts can still + /// include expired messages queued behind a non-expired head. This is not a + /// queue-wide x-message-ttl. Value restored from the OverviewDB handoff + /// design: 2000. + /// + internal const string ExpirationMilliseconds = "2000"; + + /// + /// AMQP mandatory flag. Unroutable publications are returned to the publisher + /// instead of being confirmed and dropped. + /// + internal const bool Mandatory = true; + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs new file mode 100644 index 00000000..bc74b312 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs @@ -0,0 +1,86 @@ +using RabbitMQ.Client; + +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// One declare-only article-retrieval exchange, durable quorum queue, and binding. + /// + /// Exchange that receives article-retrieval requests. + /// RabbitMQ exchange type. Article-retrieval uses fanout. + /// Whether the exchange survives broker restart. + /// Whether the exchange is auto-deleted when unused. + /// Queue bound to the exchange. + /// Whether the queue survives broker restart. + /// Whether the queue is exclusive to a single connection. + /// Whether the queue is auto-deleted when unused. + /// Routing key used when binding the queue to the exchange. + /// + /// Queue arguments applied during declaration. Durable article-retrieval work queues + /// require x-queue-type=quorum. + /// + internal sealed record RabbitMqArticleRetrievalEndpoint( + string ExchangeName, + string ExchangeType, + bool ExchangeDurable, + bool ExchangeAutoDelete, + string QueueName, + bool QueueDurable, + bool QueueExclusive, + bool QueueAutoDelete, + string RoutingKey, + IReadOnlyDictionary QueueArguments); + + /// + /// Shared factory for NNTPD/BackFiller article-retrieval broker entities. + /// + /// + /// Used by BackFiller provider endpoints. Storage fleet lookup uses + /// (fanout exchange only) and is not declared through + /// this factory. + /// + internal static class RabbitMqArticleRetrievalEndpoints + { + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = "x-queue-type"; + + /// Required queue type for every durable article-retrieval work queue. + internal const string QuorumQueueType = "quorum"; + + /// + /// Builds a durable fanout exchange, durable quorum queue, and binding that share + /// the normalized as the exchange, queue, and routing key. + /// + internal static RabbitMqArticleRetrievalEndpoint CreateFanoutQuorumBinding(string entityName) + { + var name = RabbitMqTopologyNames.Normalize(entityName); + return new RabbitMqArticleRetrievalEndpoint( + ExchangeName: name, + ExchangeType: ExchangeType.Fanout, + ExchangeDurable: true, + ExchangeAutoDelete: false, + QueueName: name, + QueueDurable: true, + QueueExclusive: false, + QueueAutoDelete: false, + RoutingKey: name, + QueueArguments: new Dictionary(StringComparer.Ordinal) + { + [QueueTypeArgumentName] = QuorumQueueType, + }); + } + } + + /// + /// NNTPD-owned BackFiller-compatible article-retrieval endpoints declared at topology start. + /// + /// + /// Empty: per-backbone backfiller.* topology is declared by BackFiller. + /// Storage fleet lookup declares only the cache.requests fanout exchange (no shared queue). + /// + internal static class ArticleRetrievalTopology + { + /// NNTPD-owned BackFiller-style article-retrieval endpoints declared at startup. + internal static IReadOnlyList Required { get; } = + Array.Empty(); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyLogMessages.cs b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyLogMessages.cs new file mode 100644 index 00000000..4db50f70 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyLogMessages.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// Source-generated RabbitMQ topology log messages. Never includes credentials. + internal static partial class RabbitMqTopologyLogMessages + { + [LoggerMessage( + EventId = 2820, + Level = LogLevel.Information, + Message = "Establishing NNTPD RabbitMQ topology (fanoutExchanges={ExchangeCount})")] + public static partial void Establishing(ILogger logger, int ExchangeCount); + + [LoggerMessage( + EventId = 2821, + Level = LogLevel.Information, + Message = "NNTPD RabbitMQ topology established (fanoutExchanges={ExchangeCount}, generation={Generation})")] + public static partial void Established(ILogger logger, int ExchangeCount, long Generation); + + [LoggerMessage( + EventId = 2822, + Level = LogLevel.Error, + Message = "NNTPD RabbitMQ topology declaration failed (endpoint={Endpoint}, exchange={Exchange}, queue={Queue})")] + public static partial void DeclarationFailed( + ILogger logger, + Exception exception, + string Endpoint, + string Exchange, + string Queue); + + [LoggerMessage( + EventId = 2823, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for topology declaration")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2824, + Level = LogLevel.Information, + Message = "Declared OverviewDB handoff queue {Queue} (generation={Generation})")] + public static partial void OverviewQueueDeclared(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2825, + Level = LogLevel.Information, + Message = "Declared cache broadcast exchange {Exchange} (generation={Generation})")] + public static partial void CacheBroadcastDeclared(ILogger logger, string Exchange, long Generation); + + [LoggerMessage( + EventId = 2826, + Level = LogLevel.Information, + Message = "Declared cache requests fanout exchange {Exchange} (generation={Generation})")] + public static partial void CacheRequestsDeclared(ILogger logger, string Exchange, long Generation); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyNames.cs b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyNames.cs new file mode 100644 index 00000000..74e9bce7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyNames.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// Deterministic, culture-independent names for RabbitMQ topology entities. + /// + /// + /// Every exchange name, queue name, and routing key is trimmed and converted with + /// . Callers must not assume their input is already + /// normalized. + /// + internal static class RabbitMqTopologyNames + { + /// + /// Normalizes a topology entity name: trim surrounding whitespace, then invariant lower-case. + /// + /// Exchange, queue, routing-key, or other topology entity name. + /// The trimmed invariant-lowercase name. + /// Thrown when is null or whitespace. + internal static string Normalize(string entityName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityName); + return entityName.Trim().ToLowerInvariant(); + } + + /// + /// Builds a two-part entity name {prefix}.{name} after normalizing both parts. + /// + /// Namespace such as backfiller. + /// Unqualified identifier such as a provider name. + /// The composed normalized entity name. + internal static string Compose(string prefix, string name) => + Normalize($"{Normalize(prefix)}.{Normalize(name)}"); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyService.cs b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyService.cs new file mode 100644 index 00000000..5b38f3ed --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/RabbitMqTopologyService.cs @@ -0,0 +1,171 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// Declares NNTPD-owned RabbitMQ topology: cache fleet exchanges and OverviewDB. + /// + /// + /// + /// Declares the durable cache.requests fanout exchange (no shared work queue), + /// the durable cache.broadcast fanout exchange, and the OverviewDB ingest queue + /// overviewdb.queue. Per-backbone backfiller.* provider topology is declared + /// by VectorNNTP.BackFiller. Per-NNTPD broadcast queues and StorageServer request queues + /// are ephemeral and owned by their respective consumers. Article-work and storage-lookup + /// reply queues are exclusive auto-delete non-durable classic queues owned by their RPC + /// services. + /// + /// + /// Declaration uses RabbitMQ's normal idempotent declare/bind operations. Existing + /// entities are never deleted, purged, or mutated. An incompatible existing entity + /// fails startup. + /// + /// + public sealed class RabbitMqTopologyService : IApplicationService + { + private readonly IRabbitMqService _rabbitMq; + private readonly ILogger _logger; + private int _started; + + /// Initializes a new instance of the class. + public RabbitMqTopologyService( + IRabbitMqService rabbitMq, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _logger = logger; + } + + /// + public string Name => "RabbitMQTopology"; + + /// + public Task? Execution => null; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + await DeclareRequiredTopologyAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + Interlocked.Exchange(ref _started, 0); + throw; + } + catch (OperationCanceledException) + { + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + /// Declares cache.requests, cache.broadcast, and overviewdb.queue. + /// + internal async Task DeclareRequiredTopologyAsync(CancellationToken cancellationToken) + { + RabbitMqTopologyLogMessages.Establishing(_logger, ExchangeCount: 2); + + if (!_rabbitMq.TryGetCurrent(out var handle)) + { + RabbitMqTopologyLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for topology declaration."); + } + + IRabbitMqTopologyChannel? channel = null; + try + { + channel = await handle.Connection + .CreateTopologyChannelAsync(cancellationToken) + .ConfigureAwait(false); + + await DeclareFanoutExchangeAsync( + channel, + CacheRequestsTopology.ExchangeName, + CacheRequestsTopology.ExchangeDurable, + CacheRequestsTopology.ExchangeAutoDelete, + cancellationToken) + .ConfigureAwait(false); + + await DeclareFanoutExchangeAsync( + channel, + CacheBroadcastTopology.ExchangeName, + CacheBroadcastTopology.ExchangeDurable, + CacheBroadcastTopology.ExchangeAutoDelete, + cancellationToken) + .ConfigureAwait(false); + + await DeclareOverviewDbQueueAsync(channel, cancellationToken).ConfigureAwait(false); + + RabbitMqTopologyLogMessages.Established(_logger, ExchangeCount: 2, handle.Generation); + RabbitMqTopologyLogMessages.CacheRequestsDeclared( + _logger, + CacheRequestsTopology.ExchangeName, + handle.Generation); + RabbitMqTopologyLogMessages.CacheBroadcastDeclared( + _logger, + CacheBroadcastTopology.ExchangeName, + handle.Generation); + RabbitMqTopologyLogMessages.OverviewQueueDeclared(_logger, OverviewDbTopology.QueueName, handle.Generation); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + RabbitMqTopologyLogMessages.DeclarationFailed( + _logger, + ex, + CacheFleetTopology.RequestsExchangeName, + CacheFleetTopology.RequestsExchangeName, + OverviewDbTopology.QueueName); + throw; + } + finally + { + if (channel is not null) + { + await channel.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static Task DeclareFanoutExchangeAsync( + IRabbitMqTopologyChannel channel, + string exchange, + bool durable, + bool autoDelete, + CancellationToken cancellationToken) => + channel.ExchangeDeclareAsync( + exchange, + CacheFleetTopology.FanoutExchangeType, + durable, + autoDelete, + arguments: null, + cancellationToken); + + private static Task DeclareOverviewDbQueueAsync( + IRabbitMqTopologyChannel channel, + CancellationToken cancellationToken) => + channel.QueueDeclareAsync( + OverviewDbTopology.QueueName, + durable: true, + exclusive: false, + autoDelete: false, + OverviewDbTopology.QueueArguments, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/RabbitMq/StorageArticleRetrievalTopology.cs b/src/VectorNNTPD.ReaderService/RabbitMq/StorageArticleRetrievalTopology.cs new file mode 100644 index 00000000..e386f16a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/RabbitMq/StorageArticleRetrievalTopology.cs @@ -0,0 +1,26 @@ +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.ReaderService.RabbitMq +{ + /// + /// Constants for the StorageServer fleet article-presence lookup path. + /// + /// + /// cache.requests is a fanout exchange only. There is no shared quorum work queue + /// and no BackFiller provider semantics. See . + /// + internal static class StorageArticleRetrievalTopology + { + /// Fanout exchange name for fleet article-presence lookups. + internal const string ExchangeName = CacheFleetTopology.RequestsExchangeName; + + /// Alias for used by older ArticleWork test fixtures. + internal const string EntityName = ExchangeName; + + /// + /// Historical JSON backbone label used only in ArticleWork tests that assert + /// Storage is not a BackFiller provider. Not a BackFiller backbone identifier. + /// + internal const string Backbone = "Storage"; + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/IRedisConnection.cs b/src/VectorNNTPD.ReaderService/Redis/IRedisConnection.cs new file mode 100644 index 00000000..53924a76 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/IRedisConnection.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// Long-lived Redis connection owned by . + public interface IRedisConnection : IAsyncDisposable + { + /// Gets the shared logical database for this connection. + IRedisDatabase GetDatabase(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/IRedisConnectionFactory.cs b/src/VectorNNTPD.ReaderService/Redis/IRedisConnectionFactory.cs new file mode 100644 index 00000000..69d1230e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/IRedisConnectionFactory.cs @@ -0,0 +1,11 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Redis +{ + /// Creates the process-wide Redis connection from configuration. + public interface IRedisConnectionFactory + { + /// Establishes one long-lived Redis connection for the configured topology. + Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/IRedisDatabase.cs b/src/VectorNNTPD.ReaderService/Redis/IRedisDatabase.cs new file mode 100644 index 00000000..560c8ce4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/IRedisDatabase.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// Generic Redis key/value operations used by application consumers. + public interface IRedisDatabase + { + /// Sends PING and returns the observed round-trip time. + Task PingAsync(CancellationToken cancellationToken = default); + + /// Returns whether exists. + ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Returns the value of , or when missing. + ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Sets to with a Redis-native TTL. + ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default); + + /// + /// Evaluates a Lua script atomically and returns the integer result. + /// + ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/IRedisService.cs b/src/VectorNNTPD.ReaderService/Redis/IRedisService.cs new file mode 100644 index 00000000..6337416d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/IRedisService.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// + /// Application Redis infrastructure: one shared connection and generic key operations. + /// + /// + /// Does not contain HistoryDB or CHECK policy. Consumers compose keys and TTLs themselves. + /// + public interface IRedisService + { + /// Gets the shared Redis database after a successful start. + IRedisDatabase Database { get; } + + /// + /// Gets whether Redis is in the cooldown window after a failure. CHECK uses + /// rather than this property so a recovery probe can proceed. + /// + bool IsUnavailable { get; } + + /// + /// Attempts to start a Redis operation. Returns during cooldown + /// or when a recovery probe is already in flight. + /// + /// + /// Set when this caller is the single recovery probe after cooldown. + /// Only that probe may mark the circuit healthy. + /// + bool TryBeginOperation(out bool isRecoveryProbe); + + /// Completes an operation started by . + void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null); + + /// + /// Releases a grant without treating the attempt as a Redis failure. + /// Used for caller/session cancellation. + /// + void AbandonOperation(bool isRecoveryProbe); + + /// Returns whether exists. + ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Returns the value of , or when missing. + ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Sets to with a Redis-native TTL. + ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/RedisAvailability.cs b/src/VectorNNTPD.ReaderService/Redis/RedisAvailability.cs new file mode 100644 index 00000000..555478e0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/RedisAvailability.cs @@ -0,0 +1,111 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// + /// Short-lived Redis unavailable / recovery-probe state shared by EXISTS, SET, and EVAL. + /// + /// + /// Healthy operations proceed without a lock. After a failure, callers see + /// return until the cooldown expires. + /// Exactly one caller may become the recovery probe. Only that probe may return + /// the circuit to healthy. Caller cancellation abandons the grant and does not + /// open or close the cooldown. CHECK never takes a global lock. + /// + internal sealed class RedisAvailability + { + internal static readonly TimeSpan DefaultCooldown = TimeSpan.FromSeconds(1); + + private readonly TimeProvider _timeProvider; + private readonly long _cooldownTicks; + private long _unavailableUntilTicks; + private int _probeInFlight; + private int _openLogged; + + /// Initializes a new instance of the class. + public RedisAvailability(TimeProvider timeProvider, TimeSpan cooldown) + { + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(cooldown, TimeSpan.Zero); + _timeProvider = timeProvider; + _cooldownTicks = cooldown.Ticks; + } + + /// Gets whether a cooldown is active (a recovery probe is not counted as available). + public bool IsUnavailable + { + get + { + var until = Volatile.Read(ref _unavailableUntilTicks); + return until != 0 && _timeProvider.GetUtcNow().UtcTicks < until; + } + } + + /// + /// Attempts to start a Redis call. is set only for the + /// single caller allowed to probe after cooldown. + /// + public bool TryBegin(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + var until = Volatile.Read(ref _unavailableUntilTicks); + if (until == 0) + { + return true; + } + + if (_timeProvider.GetUtcNow().UtcTicks < until) + { + return false; + } + + if (Interlocked.CompareExchange(ref _probeInFlight, 1, 0) != 0) + { + return false; + } + + isRecoveryProbe = true; + return true; + } + + /// + /// Records a finished Redis call. Only a recovery probe may clear cooldown on success. + /// + public void Complete( + bool isRecoveryProbe, + bool succeeded, + out bool becameUnavailable, + out bool recovered) + { + becameUnavailable = false; + recovered = false; + if (succeeded) + { + if (isRecoveryProbe) + { + var previous = Interlocked.Exchange(ref _unavailableUntilTicks, 0); + Interlocked.Exchange(ref _openLogged, 0); + Interlocked.Exchange(ref _probeInFlight, 0); + recovered = previous != 0; + } + + return; + } + + var until = _timeProvider.GetUtcNow().UtcTicks + _cooldownTicks; + Interlocked.Exchange(ref _unavailableUntilTicks, until); + becameUnavailable = Interlocked.Exchange(ref _openLogged, 1) == 0; + if (isRecoveryProbe) + { + Interlocked.Exchange(ref _probeInFlight, 0); + } + } + + /// Releases a recovery-probe grant without changing cooldown state. + public void Abandon(bool isRecoveryProbe) + { + if (isRecoveryProbe) + { + Interlocked.Exchange(ref _probeInFlight, 0); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/RedisLogMessages.cs b/src/VectorNNTPD.ReaderService/Redis/RedisLogMessages.cs new file mode 100644 index 00000000..39c18ef2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/RedisLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// Source-generated Redis infrastructure log messages. + internal static partial class RedisLogMessages + { + [LoggerMessage( + EventId = 2100, + Level = LogLevel.Information, + Message = "Connecting to Redis ({HostCount} endpoint(s), port {Port})")] + public static partial void Connecting(ILogger logger, int HostCount, int Port); + + [LoggerMessage( + EventId = 2101, + Level = LogLevel.Information, + Message = "Redis connection established (PING {ElapsedMs} ms)")] + public static partial void Connected(ILogger logger, long ElapsedMs); + + [LoggerMessage( + EventId = 2102, + Level = LogLevel.Error, + Message = "Redis connection failed during startup")] + public static partial void StartupFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2103, + Level = LogLevel.Information, + Message = "Redis connection disposed")] + public static partial void Disposed(ILogger logger); + + [LoggerMessage( + EventId = 2104, + Level = LogLevel.Error, + Message = "Redis operation failed ({Operation})")] + public static partial void OperationFailed(ILogger logger, Exception exception, string Operation); + + [LoggerMessage( + EventId = 2105, + Level = LogLevel.Warning, + Message = "Redis unavailable; CHECK and writes will return failure until a recovery probe")] + public static partial void BecameUnavailable(ILogger logger, Exception? exception); + + [LoggerMessage( + EventId = 2106, + Level = LogLevel.Information, + Message = "Redis recovered")] + public static partial void Recovered(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/RedisService.cs b/src/VectorNNTPD.ReaderService/Redis/RedisService.cs new file mode 100644 index 00000000..7ae99e85 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/RedisService.cs @@ -0,0 +1,159 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Redis +{ + /// + /// Process-wide Redis infrastructure: one long-lived connection and generic key operations. + /// + public sealed class RedisService : IRedisService, IApplicationService, IAsyncDisposable + { + private readonly IRedisConnectionFactory _connectionFactory; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly RedisAvailability _availability; + private IRedisConnection? _connection; + private IRedisDatabase? _database; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public RedisService( + IRedisConnectionFactory connectionFactory, + IOptions options, + ILogger logger) + : this(connectionFactory, options, logger, TimeProvider.System, RedisAvailability.DefaultCooldown) + { + } + + /// Initializes a new instance with an explicit circuit cooldown (tests). + internal RedisService( + IRedisConnectionFactory connectionFactory, + IOptions options, + ILogger logger, + TimeProvider timeProvider, + TimeSpan circuitCooldown) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + _connectionFactory = connectionFactory; + _options = options; + _logger = logger; + _availability = new RedisAvailability(timeProvider, circuitCooldown); + } + + /// + public string Name => "Redis"; + + /// + public Task? Execution => null; + + /// Gets the number of times a connection was established (tests). + internal int ConnectionCount { get; private set; } + + /// + public IRedisDatabase Database => + _database ?? throw new InvalidOperationException("Redis has not been started."); + + /// + public bool IsUnavailable => _availability.IsUnavailable; + + /// + public bool TryBeginOperation(out bool isRecoveryProbe) => _availability.TryBegin(out isRecoveryProbe); + + /// + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + _availability.Complete(isRecoveryProbe, succeeded, out var becameUnavailable, out var recovered); + if (becameUnavailable) + { + RedisLogMessages.BecameUnavailable(_logger, exception); + } + + if (recovered) + { + RedisLogMessages.Recovered(_logger); + } + } + + /// + public void AbandonOperation(bool isRecoveryProbe) => _availability.Abandon(isRecoveryProbe); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + var options = _options.Value; + RedisLogMessages.Connecting(_logger, options.Host.Length, options.Port); + try + { + _connection = await _connectionFactory.ConnectAsync(options, cancellationToken).ConfigureAwait(false); + _database = _connection.GetDatabase(); + var ping = await _database.PingAsync(cancellationToken).ConfigureAwait(false); + ConnectionCount++; + RedisLogMessages.Connected(_logger, (long)ping.TotalMilliseconds); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + RedisLogMessages.StartupFailed(_logger, ex); + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + _connection = null; + _database = null; + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await DisposeAsync().ConfigureAwait(false); + } + + /// + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.KeyExistsAsync(key, cancellationToken); + + /// + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.GetAsync(key, cancellationToken); + + /// + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + Database.SetAsync(key, value, expiry, cancellationToken); + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + RedisLogMessages.Disposed(_logger); + } + + _connection = null; + _database = null; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/RedisUnavailableException.cs b/src/VectorNNTPD.ReaderService/Redis/RedisUnavailableException.cs new file mode 100644 index 00000000..f1487af7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/RedisUnavailableException.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.ReaderService.Redis +{ + /// + /// Redis infrastructure failure (unavailable, timeout, or protocol error). + /// Distinct from a successful reply that the key does not exist. + /// + public sealed class RedisUnavailableException : Exception + { + /// Initializes a new instance of the class. + public RedisUnavailableException(string message, Exception? innerException = null) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnection.cs b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnection.cs new file mode 100644 index 00000000..f2a2a4f5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnection.cs @@ -0,0 +1,37 @@ +using StackExchange.Redis; + +namespace VectorNNTPD.ReaderService.Redis +{ + /// Owns one for the process. + internal sealed class StackExchangeRedisConnection : IRedisConnection + { + private readonly ConnectionMultiplexer _multiplexer; + private readonly StackExchangeRedisDatabase _database; + private int _disposed; + + /// Initializes a new instance of the class. + public StackExchangeRedisConnection(ConnectionMultiplexer multiplexer) + { + ArgumentNullException.ThrowIfNull(multiplexer); + _multiplexer = multiplexer; + _database = new StackExchangeRedisDatabase(multiplexer.GetDatabase()); + } + + /// Gets the shared multiplexer (tests). + internal ConnectionMultiplexer Multiplexer => _multiplexer; + + /// + public IRedisDatabase GetDatabase() => _database; + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _multiplexer.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnectionFactory.cs b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnectionFactory.cs new file mode 100644 index 00000000..3f84043a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisConnectionFactory.cs @@ -0,0 +1,47 @@ +using StackExchange.Redis; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Redis +{ + /// Creates one from . + public sealed class StackExchangeRedisConnectionFactory : IRedisConnectionFactory + { + /// + public async Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + + var configuration = new ConfigurationOptions + { + AbortOnConnectFail = true, + ConnectRetry = 2, + ConnectTimeout = 5000, + }; + + foreach (var host in options.Host) + { + configuration.EndPoints.Add(host.Trim(), options.Port); + } + + ConnectionMultiplexer multiplexer; + try + { + multiplexer = await ConnectionMultiplexer.ConnectAsync(configuration).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is RedisConnectionException or RedisException or TimeoutException) + { + throw new RedisUnavailableException("Redis ConnectAsync failed.", ex); + } + + if (cancellationToken.IsCancellationRequested) + { + await multiplexer.DisposeAsync().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } + + return new StackExchangeRedisConnection(multiplexer); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisDatabase.cs b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisDatabase.cs new file mode 100644 index 00000000..6aa0397c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Redis/StackExchangeRedisDatabase.cs @@ -0,0 +1,242 @@ +using System.Runtime.InteropServices; +using StackExchange.Redis; + +namespace VectorNNTPD.ReaderService.Redis +{ + /// Adapts to . + internal sealed class StackExchangeRedisDatabase : IRedisDatabase + { + private readonly IDatabase _database; + + /// Initializes a new instance of the class. + public StackExchangeRedisDatabase(IDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + _database = database; + } + + /// + public async Task PingAsync(CancellationToken cancellationToken = default) => + await Await(_database.PingAsync(), cancellationToken).ConfigureAwait(false); + + /// + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + try + { + var pending = _database.KeyExistsAsync(ToRedisKey(key)); + return AwaitExists(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis EXISTS failed.", ex)); + } + } + + /// + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + try + { + var pending = _database.StringGetAsync(ToRedisKey(key)); + return AwaitGet(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis GET failed.", ex)); + } + } + + /// + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) + { + try + { + var pending = _database.StringSetAsync(ToRedisKey(key), ToRedisValue(value), expiry); + return AwaitSet(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis SET failed.", ex)); + } + } + + /// + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(script); + ArgumentNullException.ThrowIfNull(keys); + ArgumentNullException.ThrowIfNull(values); + try + { + var redisKeys = new RedisKey[keys.Length]; + for (var i = 0; i < keys.Length; i++) + { + redisKeys[i] = ToRedisKey(keys[i]); + } + + var redisValues = new RedisValue[values.Length]; + for (var i = 0; i < values.Length; i++) + { + redisValues[i] = ToRedisValue(values[i]); + } + + var pending = _database.ScriptEvaluateAsync(script, redisKeys, redisValues); + return AwaitEval(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis EVAL failed.", ex)); + } + } + + private static RedisKey ToRedisKey(ReadOnlyMemory key) + { + // RedisKey is byte[]-backed and has no ReadOnlyMemory implicit. + // HistoryRedisKeys.Create returns a dedicated array; reuse it. + if (MemoryMarshal.TryGetArray(key, out var segment) + && segment.Array is { } array + && segment.Offset == 0 + && segment.Count == array.Length) + { + return (RedisKey)array; + } + + return (RedisKey)key.ToArray(); + } + + private static RedisValue ToRedisValue(ReadOnlyMemory value) => + value.IsEmpty ? RedisValue.EmptyString : (RedisValue)value; + + private static ValueTask AwaitExists(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(pending.Result); + } + + return AwaitExistsSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitExistsSlow(Task pending, CancellationToken cancellationToken) + { + try + { + return await Await(pending, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis EXISTS failed.", ex); + } + } + + private static ValueTask AwaitGet(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(ToBytes(pending.Result)); + } + + return AwaitGetSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitGetSlow(Task pending, CancellationToken cancellationToken) + { + try + { + var value = await Await(pending, cancellationToken).ConfigureAwait(false); + return ToBytes(value); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis GET failed.", ex); + } + } + + private static byte[]? ToBytes(RedisValue value) => value.IsNull ? null : (byte[]?)value; + + private static ValueTask AwaitEval(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(ToInt64(pending.Result)); + } + + return AwaitEvalSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitEvalSlow(Task pending, CancellationToken cancellationToken) + { + try + { + var result = await Await(pending, cancellationToken).ConfigureAwait(false); + return ToInt64(result); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis EVAL failed.", ex); + } + } + + private static long ToInt64(RedisResult result) + { + if (result.IsNull) + { + return 0; + } + + try + { + return (long)result; + } + catch (InvalidCastException) + { + return 0; + } + } + + private static ValueTask AwaitSet(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return ValueTask.CompletedTask; + } + + return AwaitSetSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitSetSlow(Task pending, CancellationToken cancellationToken) + { + try + { + _ = await Await(pending, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis SET failed.", ex); + } + } + + private static Task Await(Task pending, CancellationToken cancellationToken) => + cancellationToken.CanBeCanceled ? pending.WaitAsync(cancellationToken) : pending; + + private static bool IsInfrastructureFailure(Exception exception) => + exception is RedisException or RedisTimeoutException or TimeoutException or IOException; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs new file mode 100644 index 00000000..80702937 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// + /// Default authentication provider that rejects all credentials. + /// + /// + /// Register a real in DI when an account backend exists. + /// Cleartext AUTHINFO policy is controlled by Nntpd:AllowCleartextAuth, not this type. + /// + public sealed class DenyAllNntpAuthenticationProvider : INntpAuthenticationProvider + { + /// Shared singleton instance. + public static DenyAllNntpAuthenticationProvider Instance { get; } = new(); + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/INntpAuthenticationProvider.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/INntpAuthenticationProvider.cs new file mode 100644 index 00000000..348343a6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/INntpAuthenticationProvider.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// + /// Verifies NNTP AUTHINFO USER/PASS credentials and returns identity plus authorization. + /// + /// + /// Implementations must not log passwords. The session layer never stores the password after + /// this call returns. Whether AUTHINFO USER/PASS is allowed without TLS is a server policy + /// (Nntpd:AllowCleartextAuth), not a provider concern. + /// + public interface INntpAuthenticationProvider + { + /// Verifies username/password and returns authentication + authorization outcome. + ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default); + + /// + /// Verifies username/password with the effective client address for diagnostics and admission. + /// Default implementation ignores . + /// + ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) => + AuthenticateAsync(username, password, cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/ITransitPeerAuthenticator.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/ITransitPeerAuthenticator.cs new file mode 100644 index 00000000..cae21790 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/ITransitPeerAuthenticator.cs @@ -0,0 +1,22 @@ +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// Validates AUTHINFO USER/PASS against an identified Transit peer policy. + public interface ITransitPeerAuthenticator + { + /// + /// Authenticates / against + /// . On success, gains + /// and keeps its peer identity. + /// Reader and posting privileges are not granted. A blank username or password on the + /// peer disables this gate (); it is not a mismatch. + /// + NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs new file mode 100644 index 00000000..bca09d75 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs @@ -0,0 +1,80 @@ +using System.Security.Cryptography; +using System.Text; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// + /// AUTHINFO USER/PASS provider that recognizes only the configured newsmaster account. + /// + /// + /// Success grants posting plus so the + /// newsmaster can submit a well-formed Control: cancel article. It does not grant + /// transit or streaming privileges and does not accept any other Control verb. Passwords are + /// compared in constant time and are never logged. + /// When newsmaster credentials are not configured, returns + /// . + /// + public sealed class NewsmasterNntpAuthenticationProvider : INntpAuthenticationProvider + { + private readonly string _username; + private readonly byte[] _passwordUtf8; + + /// + /// Privileges granted to the configured newsmaster: authenticated reader + poster with + /// cancel-control permission only. + /// + public static NntpAuthorization Privileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false, + controlCancelPermitted: true); + + private NewsmasterNntpAuthenticationProvider(string username, string password) + { + _username = username; + _passwordUtf8 = Encoding.UTF8.GetBytes(password); + } + + /// + /// Returns a newsmaster provider when both credentials are configured; otherwise deny-all. + /// + public static INntpAuthenticationProvider Create(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.NewsmasterUser) + || string.IsNullOrEmpty(options.NewsmasterPassword)) + { + return DenyAllNntpAuthenticationProvider.Instance; + } + + return new NewsmasterNntpAuthenticationProvider( + options.NewsmasterUser.Trim(), + options.NewsmasterPassword); + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + + var provided = Encoding.UTF8.GetBytes(password); + var passwordMatch = provided.Length == _passwordUtf8.Length + && CryptographicOperations.FixedTimeEquals(provided, _passwordUtf8); + var userMatch = string.Equals(username, _username, StringComparison.Ordinal); + if (userMatch && passwordMatch) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, Privileges)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/NntpAuthenticationResult.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/NntpAuthenticationResult.cs new file mode 100644 index 00000000..a4de8401 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/NntpAuthenticationResult.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.ReaderService.Authentication; + +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// Classifies a non-success authentication outcome for wire mapping. + public enum NntpAuthenticationFailureKind + { + /// Success, or unused. + None, + + /// Invalid credentials, disabled account, or mechanism denied. Wire 481. + InvalidCredentials, + + /// Backend/store failure. Wire 503. Must not be presented as a bad password. + TransientFailure, + + /// Session admission rejected. Wire 481 Too many sessions. + TooManySessions, + + /// Source-IP admission rejected. Wire 481 Too many source addresses. + TooManySourceAddresses, + } + + /// Result of a credential verification attempt. + public sealed class NntpAuthenticationResult + { + /// Failed authentication (no identity, no authorization). + public static NntpAuthenticationResult Failed { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.InvalidCredentials, + policy: null); + + /// Backend failure. Distinct from invalid credentials. + public static NntpAuthenticationResult TransientFailure { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TransientFailure, + policy: null); + + /// Admission rejected: too many concurrent sessions. + public static NntpAuthenticationResult TooManySessions { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TooManySessions, + policy: null); + + /// Admission rejected: too many distinct source addresses. + public static NntpAuthenticationResult TooManySourceAddresses { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TooManySourceAddresses, + policy: null); + + private NntpAuthenticationResult( + bool succeeded, + string? username, + NntpAuthorization? authorization, + NntpAuthenticationFailureKind failure, + NntpAccountPolicy? policy) + { + Succeeded = succeeded; + Username = username; + Authorization = authorization; + Failure = failure; + Policy = policy; + } + + /// Gets a value indicating whether credentials were accepted. + public bool Succeeded { get; } + + /// Gets the authenticated identity username when is . + public string? Username { get; } + + /// + /// Gets authorization granted by the provider on success. + /// Authentication success does not imply any particular privilege flags. + /// + public NntpAuthorization? Authorization { get; } + + /// Gets the failure classification when is . + public NntpAuthenticationFailureKind Failure { get; } + + /// Gets account policy after a successful MySQL authentication. + public NntpAccountPolicy? Policy { get; } + + /// Creates a successful result with explicit authorization (flags are provider-defined). + public static NntpAuthenticationResult Success( + string username, + NntpAuthorization authorization, + NntpAccountPolicy? policy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(authorization); + return new(succeeded: true, username: username, authorization: authorization, NntpAuthenticationFailureKind.None, policy); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Authentication/TransitPeerAuthenticator.cs b/src/VectorNNTPD.ReaderService/Session/Authentication/TransitPeerAuthenticator.cs new file mode 100644 index 00000000..42551a59 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Authentication/TransitPeerAuthenticator.cs @@ -0,0 +1,31 @@ +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.Authentication +{ + /// Ordinal Transit peer USER/PASS check. No MySQL fall-through. + public sealed class TransitPeerAuthenticator : ITransitPeerAuthenticator + { + /// Shared production instance. + public static TransitPeerAuthenticator Instance { get; } = new(); + + /// + public NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(currentAuthorization); + if (policy is not null && policy.CredentialsMatch(username, password)) + { + return NntpAuthenticationResult.Success( + username, + currentAuthorization.With(authorizedTransit: true, streamingPermitted: true)); + } + + return NntpAuthenticationResult.Failed; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CheckPipeline.cs b/src/VectorNNTPD.ReaderService/Session/CheckPipeline.cs new file mode 100644 index 00000000..74c4de37 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CheckPipeline.cs @@ -0,0 +1,538 @@ +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Per-session bounded CHECK execution window: overlap Redis lookups, emit responses in send order. + /// + /// + /// Only CHECK uses this window. Non-CHECK commands drain it first and then run on the serial + /// dispatcher. A slot is occupied from admission until + /// + /// accepts the response (or the slot is cancelled on shutdown). Slots own a copy of the + /// Message-ID so STREAM/READER parser scratch can be reused. Every CHECK response passes through + /// the emit gate; there is no immediate path that bypasses it. + /// + internal sealed class CheckPipeline + { + /// + /// Outstanding CHECK executions per session. Architectural constant; not configurable. + /// + public const int Depth = 16; + + private readonly NntpSession _session; + private readonly NntpResponseWriter _response; + private readonly ILogger _logger; + private readonly Slot?[] _slots; + private readonly object _gate = new(); + // codeql[cs/missed-using-statement] Owned by CheckPipeline until ShutdownAsync; a using at this declaration would release it when the constructor returns. + private readonly SemaphoreSlim _emitGate = new(1, 1); + // codeql[cs/missed-using-statement] Owned by CheckPipeline until ShutdownAsync; a using at this declaration would cancel in-flight work when the constructor returns. + private readonly CancellationTokenSource _lifetimeCts = new(); + private int _head; + private int _count; + private int _peakOccupied; + private int _shutDown; + private int _inFlightCompletions; + private TaskCompletionSource _progress = NewProgress(); + + /// Initializes a new instance of the class. + public CheckPipeline(NntpSession session, NntpResponseWriter response) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + _session = session; + _response = response; + _logger = NntpCommandLoggers.For(typeof(Check)); + _slots = new Slot?[Depth]; + } + + /// Gets the number of occupied slots (lookup, waiting for prefix, or awaiting TX accept). + public int Occupied + { + get + { + lock (_gate) + { + return _count; + } + } + } + + /// Gets whether the window cannot accept another CHECK without waiting. + public bool IsFull => Occupied >= Depth; + + /// Gets the peak occupied slot count observed on this session. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Gets the number of CompleteWhen tasks still running (at most ). + internal int InFlightCompletions => Volatile.Read(ref _inFlightCompletions); + + /// + /// Test-only: awaited after response composition and before + /// . + /// The slot remains occupied. Used to model blocked TX without filling the writer Channel. + /// + internal Func? BeforeEnqueueProbe { get; set; } + + /// + /// Waits until a slot is free. Does not parse or drop commands; the caller must not read RX + /// while this is pending so the input Pipe can apply backpressure. + /// + public async ValueTask WaitForCapacityAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfShutDown(); + if (_count < Depth) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Starts one authorized, syntactically valid CHECK. The Message-ID is copied into the slot + /// so parser scratch can be reused immediately. + /// + public async ValueTask SubmitAsync( + NntpCommand command, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ThrowIfShutDown(); + var probe = _session.FeedProbe; + probe?.RecordCheck(); + _session.RecordPeerCheck(); + _session.SetActivityState(FeedSessionState.WaitingHistory); + var owned = command.ArgumentMemory(line).ToArray(); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var lookup = Check.LookupAsync(_session, owned, cancellationToken); + var emit = false; + lock (_gate) + { + ThrowIfShutDown(); + if (_count >= Depth) + { + throw new InvalidOperationException("CHECK pipeline accepted a command without a free slot."); + } + + var slot = new Slot + { + MessageId = owned, + StartedTimestamp = started, + Index = (_head + _count) % Depth, + }; + _slots[slot.Index] = slot; + _count++; + _session.BeginCommandWork(); + if (_count > _peakOccupied) + { + _peakOccupied = _count; + } + + if (lookup.IsCompletedSuccessfully) + { + slot.Completed = true; + slot.Result = lookup.Result; + probe?.RecordHistory(lookup.Result, System.Diagnostics.Stopwatch.GetTimestamp() - started); + emit = HeadIsReadyNoLock(); + } + else + { + _ = CompleteWhenAsync(slot, lookup, cancellationToken); + } + } + + if (emit) + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Waits until every slot has been accepted by the TX writer (or cancelled). Required before + /// any non-CHECK command. + /// + public async ValueTask DrainAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Stops emission, cancels in-flight enqueue waits, and waits until every slot is released. + /// Must run before the session disposes the writer. + /// + public async ValueTask ShutdownAsync() + { + if (Interlocked.Exchange(ref _shutDown, 1) == 1) + { + return; + } + + try + { + await _lifetimeCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + lock (_gate) + { + for (var i = 0; i < Depth; i++) + { + if (_slots[i] is { } slot) + { + slot.Cancelled = true; + } + } + + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + try + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.ConfigureAwait(false); + } + } + finally + { + _lifetimeCts.Dispose(); + } + } + + private async Task CompleteWhenAsync( + Slot slot, + ValueTask lookup, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _inFlightCompletions); + try + { + HistoryLookupResult result; + var historyStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + result = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) + { + result = HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + result = HistoryLookupResult.Unavailable; + } + + _session.FeedProbe?.RecordHistory( + result, + System.Diagnostics.Stopwatch.GetTimestamp() - historyStart); + + var emit = false; + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + if (Volatile.Read(ref _shutDown) == 1) + { + slot.Completed = true; + slot.Cancelled = true; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot.Completed = true; + slot.Result = result; + emit = HeadIsReadyNoLock(); + SignalProgressNoLock(); + } + + if (emit) + { + try + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + finally + { + Interlocked.Decrement(ref _inFlightCompletions); + } + } + + private async ValueTask EmitReadySerializedAsync(CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetimeCts.Token); + var token = linked.Token; + await _emitGate.WaitAsync(token).ConfigureAwait(false); + try + { + var emitted = 0; + while (true) + { + Slot? slot; + lock (_gate) + { + if (Volatile.Read(ref _shutDown) == 1) + { + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot = TryBeginEmitHeadNoLock(); + } + + if (slot is null) + { + if (emitted > 0 + && _response.HasCoalescedUnflushed + && Volatile.Read(ref _shutDown) == 0 + && !token.IsCancellationRequested) + { + await _response.FlushCoalescedAsync(token).ConfigureAwait(false); + } + + return; + } + + try + { + await EmitOccupiedAsync(slot, token).ConfigureAwait(false); + lock (_gate) + { + ReleaseHeadIfNoLock(slot); + SignalProgressNoLock(); + } + + emitted++; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + lock (_gate) + { + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + throw; + } + } + } + finally + { + _emitGate.Release(); + } + } + + private async ValueTask EmitOccupiedAsync(Slot slot, CancellationToken cancellationToken) + { + var deferOnDuplicate = Check.DeferOnDuplicate(_session); + var owned = Check.Compose(slot.Result, slot.MessageId, deferOnDuplicate); + if (BeforeEnqueueProbe is { } probe) + { + await probe().ConfigureAwait(false); + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + } + + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + await _response.EnqueueLineAsync(owned, cancellationToken).ConfigureAwait(false); + if (_logger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + _logger, + _session, + "CHECK", + System.Diagnostics.Stopwatch.GetElapsedTime(slot.StartedTimestamp), + statusLine: NntpCommandStatusText.FormatCheck(slot.Result, slot.MessageId, deferOnDuplicate)); + } + } + + private bool HeadIsReadyNoLock() + { + if (_count == 0) + { + return false; + } + + var slot = _slots[_head]; + return slot is { Completed: true, Cancelled: false, Emitting: false }; + } + + private Slot? TryBeginEmitHeadNoLock() + { + if (!HeadIsReadyNoLock()) + { + return null; + } + + var slot = _slots[_head]!; + slot.Emitting = true; + return slot; + } + + private void ReleaseHeadIfNoLock(Slot slot) + { + if (_count == 0 || !ReferenceEquals(_slots[_head], slot)) + { + return; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + slot.Emitting = false; + } + + private void CancelAndRelease(Slot slot) + { + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.Completed = true; + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + } + + private void ReleaseCancelledPrefixNoLock() + { + while (_count > 0) + { + var slot = _slots[_head]; + if (slot is null || slot.Emitting || !slot.Cancelled) + { + break; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + } + } + + private bool IsPresentNoLock(Slot slot) => + (uint)slot.Index < Depth && ReferenceEquals(_slots[slot.Index], slot); + + private void ThrowIfShutDown() + { + if (Volatile.Read(ref _shutDown) == 1) + { + throw new OperationCanceledException("CHECK pipeline has shut down."); + } + } + + private void SignalProgressNoLock() + { + var previous = _progress; + _progress = NewProgress(); + previous.TrySetResult(); + } + + private static TaskCompletionSource NewProgress() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class Slot + { + public required byte[] MessageId { get; init; } + + public required int Index { get; init; } + + public long StartedTimestamp { get; init; } + + public HistoryLookupResult Result { get; set; } + + public bool Completed { get; set; } + + public bool Cancelled { get; set; } + + public bool Emitting { get; set; } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/CommandLogMessages.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/CommandLogMessages.cs new file mode 100644 index 00000000..4733906a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/CommandLogMessages.cs @@ -0,0 +1,153 @@ +using System.Net.Sockets; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Source-generated command RX/TX and handler log messages. + internal static partial class CommandLogMessages + { + [LoggerMessage( + EventId = 1600, + Level = LogLevel.Debug, + Message = "[{Client}] RX: {Command}")] + public static partial void CommandRx(ILogger logger, string Client, string Command); + + [LoggerMessage( + EventId = 1601, + Level = LogLevel.Debug, + Message = "[{Client}] RX rejected: verb={Verb} qualifier={Qualifier} status={Status} [{Detail}]")] + public static partial void CommandRejected( + ILogger logger, + string Client, + NntpVerb Verb, + NntpVerb Qualifier, + NntpParseStatus Status, + string Detail); + + [LoggerMessage( + EventId = 1602, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} executed in {ElapsedSeconds:F3}s")] + public static partial void CommandTx(ILogger logger, string Client, string Command, double ElapsedSeconds); + + [LoggerMessage( + EventId = 1603, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} executed in {ElapsedSeconds:F3}s [{Detail}]")] + public static partial void CommandTxWithDetail( + ILogger logger, + string Client, + string Command, + double ElapsedSeconds, + string Detail); + + [LoggerMessage( + EventId = 1604, + Level = LogLevel.Error, + Message = "[{Client}] {Command} failed")] + public static partial void CommandFailed(ILogger logger, Exception exception, string Client, string Command); + + [LoggerMessage( + EventId = 1605, + Level = LogLevel.Warning, + Message = "[{Client}] TAKETHIS temporary failure; closing connection")] + public static partial void TakeThisTemporaryFailure(ILogger logger, string Client); + + [LoggerMessage( + EventId = 1606, + Level = LogLevel.Error, + Message = "[{Client}] STARTTLS handshake failed")] + public static partial void StartTlsHandshakeFailed(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1607, + Level = LogLevel.Warning, + Message = "[{Client}] COMPRESS DEFLATE refused before activation")] + public static partial void CompressRefusedBeforeActivation(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1608, + Level = LogLevel.Error, + Message = "[{Client}] COMPRESS DEFLATE activation failed after 206")] + public static partial void CompressActivationFailed(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1609, + Level = LogLevel.Debug, + Message = "[{Client}] Peer disconnected during QUIT termination")] + public static partial void QuitPeerDisconnected(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1610, + Level = LogLevel.Error, + Message = "[{Client}] QUIT uncaught during termination: {Type} socket={Socket}")] + public static partial void QuitUncaught( + ILogger logger, + Exception exception, + string Client, + string? Type, + SocketError? Socket); + + [LoggerMessage( + EventId = 1611, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST started peer={PeerId} peerName={PeerName}")] + public static partial void SpeedTestStarted(ILogger logger, string Client, string PeerId, string PeerName); + + [LoggerMessage( + EventId = 1612, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST completed peer={PeerId} peerName={PeerName} bytes={Bytes} durationMs={DurationMs:F3}")] + public static partial void SpeedTestCompleted( + ILogger logger, + string Client, + string PeerId, + string PeerName, + long Bytes, + double DurationMs); + + [LoggerMessage( + EventId = 1613, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST rejected peer={PeerId} peerName={PeerName} reason={Reason}")] + public static partial void SpeedTestRejected( + ILogger logger, + string Client, + string PeerId, + string PeerName, + string Reason); + + [LoggerMessage( + EventId = 1614, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST cancelled peer={PeerId} peerName={PeerName} reason={Reason}")] + public static partial void SpeedTestCancelled( + ILogger logger, + string Client, + string PeerId, + string PeerName, + string Reason); + + [LoggerMessage( + EventId = 1615, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} [{StatusLine}] executed in {ElapsedSeconds:F3}s")] + public static partial void CommandTxWithStatus( + ILogger logger, + string Client, + string Command, + string StatusLine, + double ElapsedSeconds); + + [LoggerMessage( + EventId = 1616, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} [{StatusLine}] executed in {ElapsedSeconds:F3}s [{Detail}]")] + public static partial void CommandTxWithStatusAndDetail( + ILogger logger, + string Client, + string Command, + string StatusLine, + double ElapsedSeconds, + string Detail); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs new file mode 100644 index 00000000..33e70429 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs @@ -0,0 +1,127 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Built-in NNTP command inventory, access flags, and display names. + /// + /// See docs/commands.md for the implementation checklist. Dispatch is enum/switch, not a dictionary. + /// + public static class DefaultNntpCommandCatalog + { + /// + /// Expected registry keys for the complete command inventory (stable order for tests). + /// + public static IReadOnlyList InventoryKeys { get; } = + [ + "ARTICLE", + "AUTHINFO PASS", + "AUTHINFO SASL", + "AUTHINFO USER", + "BODY", + "CAPABILITIES", + "CHECK", + "COMPRESS", + "DATE", + "GROUP", + "HDR", + "HEAD", + "HELP", + "IHAVE", + "LAST", + "LIST", + "LIST ACTIVE", + "LIST COUNTS", + "LIST HEADERS", + "LIST MOTD", + "LIST NEWSGROUPS", + "LIST OVERVIEW.FMT", + "LISTGROUP", + "MODE READER", + "MODE STREAM", + "NEXT", + "OVER", + "POST", + "QUIT", + "SPEEDTEST", + "STARTTLS", + "STAT", + "TAKETHIS", + ]; + + /// Registered keys including the internal BENCHIT facility. + public static IReadOnlyList GetRegisteredKeys() + { + var keys = new List(InventoryKeys.Count + 1); + keys.AddRange(InventoryKeys); + keys.Add("BENCHIT"); + keys.Sort(StringComparer.Ordinal); + return keys; + } + + /// Returns access flags for a syntactically valid built-in command. + public static NntpCommandAccess GetAccess(NntpVerb verb, NntpVerb qualifier) + { + var readerAccess = NntpCommandAccess.RequiresAuthentication | NntpCommandAccess.RequiresReader; + return verb switch + { + NntpVerb.Capabilities or NntpVerb.Help or NntpVerb.Date or NntpVerb.Quit + or NntpVerb.StartTls or NntpVerb.Compress or NntpVerb.BenchIt + or NntpVerb.AuthInfo => NntpCommandAccess.Public, + NntpVerb.Mode when qualifier == NntpVerb.Reader => NntpCommandAccess.Public, + NntpVerb.Mode when qualifier == NntpVerb.Stream => NntpCommandAccess.RequiresStreaming, + NntpVerb.Check or NntpVerb.TakeThis or NntpVerb.Ihave or NntpVerb.SpeedTest + => NntpCommandAccess.RequiresTransit, + NntpVerb.Post => NntpCommandAccess.RequiresAuthentication + | NntpCommandAccess.RequiresReader, + NntpVerb.List or NntpVerb.ListGroup or NntpVerb.Group + or NntpVerb.Article or NntpVerb.Head or NntpVerb.Body + or NntpVerb.Stat or NntpVerb.Last or NntpVerb.Next or NntpVerb.Over + or NntpVerb.Hdr => readerAccess, + _ => NntpCommandAccess.Public, + }; + } + + /// Returns the static uppercase display name used by TX completion logs. + public static string DisplayName(NntpVerb verb, NntpVerb qualifier) + { + return (verb, qualifier) switch + { + (NntpVerb.Mode, NntpVerb.Reader) => "MODE READER", + (NntpVerb.Mode, NntpVerb.Stream) => "MODE STREAM", + (NntpVerb.AuthInfo, NntpVerb.User) => "AUTHINFO USER", + (NntpVerb.AuthInfo, NntpVerb.Pass) => "AUTHINFO PASS", + (NntpVerb.AuthInfo, NntpVerb.Sasl) => "AUTHINFO SASL", + (NntpVerb.List, NntpVerb.Active) => "LIST ACTIVE", + (NntpVerb.List, NntpVerb.Counts) => "LIST COUNTS", + (NntpVerb.List, NntpVerb.Headers) => "LIST HEADERS", + (NntpVerb.List, NntpVerb.Motd) => "LIST MOTD", + (NntpVerb.List, NntpVerb.Newsgroups) => "LIST NEWSGROUPS", + (NntpVerb.List, NntpVerb.OverviewFmt) => "LIST OVERVIEW.FMT", + (NntpVerb.Article, _) => "ARTICLE", + (NntpVerb.AuthInfo, _) => "AUTHINFO", + (NntpVerb.Body, _) => "BODY", + (NntpVerb.Capabilities, _) => "CAPABILITIES", + (NntpVerb.Check, _) => "CHECK", + (NntpVerb.Compress, _) => "COMPRESS", + (NntpVerb.Date, _) => "DATE", + (NntpVerb.Group, _) => "GROUP", + (NntpVerb.Hdr, _) => "HDR", + (NntpVerb.Head, _) => "HEAD", + (NntpVerb.Help, _) => "HELP", + (NntpVerb.Ihave, _) => "IHAVE", + (NntpVerb.Last, _) => "LAST", + (NntpVerb.List, _) => "LIST", + (NntpVerb.ListGroup, _) => "LISTGROUP", + (NntpVerb.Mode, _) => "MODE", + (NntpVerb.Next, _) => "NEXT", + (NntpVerb.Over, _) => "OVER", + (NntpVerb.Post, _) => "POST", + (NntpVerb.Quit, _) => "QUIT", + (NntpVerb.SpeedTest, _) => "SPEEDTEST", + (NntpVerb.StartTls, _) => "STARTTLS", + (NntpVerb.Stat, _) => "STAT", + (NntpVerb.TakeThis, _) => "TAKETHIS", + (NntpVerb.BenchIt, _) => "BENCHIT", + _ => "INVALID", + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DirectTxPipeLease.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DirectTxPipeLease.cs new file mode 100644 index 00000000..973b03fd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/DirectTxPipeLease.cs @@ -0,0 +1,45 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// DIAGNOSTIC-ONLY exclusive lease of the existing TX . + /// + /// + /// SPEEDTEST holds this while writing the synthetic payload so the ResponseWriter pump + /// cannot produce to the same pipe. Dispose returns ownership to the Channel pump. + /// Does not construct a pipe, socket, or transport. + /// + internal sealed class DirectTxPipeLease : IAsyncDisposable + { + private readonly NntpResponseWriter _writer; + private int _disposed; + + internal DirectTxPipeLease(NntpResponseWriter writer) + { + ArgumentNullException.ThrowIfNull(writer); + _writer = writer; + } + + /// Gets the existing session TX (not a new pipe). + public PipeWriter PipeWriter => _writer.UnderlyingOutput; + + /// Gets direct-path FlushAsync calls recorded on this writer. + public long FlushCount => _writer.DirectPipeFlushCount; + + /// Copies to the existing TX pipe and flushes. + public ValueTask WriteAndFlushAsync(ReadOnlyMemory payload, CancellationToken cancellationToken = default) => + _writer.WriteDirectToOutputAsync(payload, cancellationToken); + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writer.ReleaseDirectTxPipe(); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/IHaveLogMessages.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/IHaveLogMessages.cs new file mode 100644 index 00000000..594c781c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/IHaveLogMessages.cs @@ -0,0 +1,72 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Source-generated IHAVE receive/queue diagnostics. Does not log bodies or headers. + internal static partial class IHaveLogMessages + { + [LoggerMessage( + EventId = 1700, + Level = LogLevel.Information, + Message = "[{Client}] IHAVE {MessageId} size={Size} pipeReads={PipeReads} receiveMs={ReceiveMs:F1} queued={Queued}")] + public static partial void Received( + ILogger logger, + string Client, + string MessageId, + int Size, + int PipeReads, + double ReceiveMs, + bool Queued); + + [LoggerMessage( + EventId = 1710, + Level = LogLevel.Warning, + Message = "IHAVE deferred: TransitQueueMemoryLimit exhausted (queued {QueuedBytes}/{MemoryLimit} bytes, {Count} articles)")] + public static partial void QueueBudgetExhausted( + ILogger logger, + long QueuedBytes, + long MemoryLimit, + int Count); + } + + /// Rate-limits IHAVE queue-budget exhaustion warnings. + internal static class IHaveAdmissionLog + { + internal static TimeSpan WarningInterval { get; set; } = TimeSpan.FromSeconds(5); + + private static long _nextWarningTicks; + + /// Emits at most one budget-exhausted warning per . + internal static void BudgetExhausted(ILogger logger, IArticleIngestionQueue queue) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(queue); + + var now = DateTime.UtcNow.Ticks; + var next = Volatile.Read(ref _nextWarningTicks); + if (now < next) + { + return; + } + + var until = now + WarningInterval.Ticks; + if (Interlocked.CompareExchange(ref _nextWarningTicks, until, next) != next) + { + return; + } + + IHaveLogMessages.QueueBudgetExhausted( + logger, + queue.QueuedBytes, + queue.MemoryLimitBytes, + queue.Count); + } + + /// Resets rate-limit state for tests. + internal static void ResetForTests() + { + Volatile.Write(ref _nextWarningTicks, 0); + WarningInterval = TimeSpan.FromSeconds(5); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs new file mode 100644 index 00000000..5a151da9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Chunk budget for the shared article TX data plane (Phase 1). + /// + /// + /// Architecture: default 64 KiB; allowed production range 64–256 KiB + /// (docs/NNTP-DATA-PLANE-IMPLEMENTATION.md). Kept as an internal seam for Phase 6 + /// measurement rather than a full configuration surface. + /// + public static class NntpArticleTxChunkBudget + { + /// Default owned TX chunk size (64 KiB). + public const int DefaultBytes = 64 * 1024; + + /// Minimum production chunk size (64 KiB). + public const int MinBytes = 64 * 1024; + + /// Maximum production chunk size (256 KiB). + public const int MaxBytes = 256 * 1024; + + /// Returns when is in the production range. + public static bool IsProductionRange(int chunkBytes) => + chunkBytes is >= MinBytes and <= MaxBytes; + + /// Throws if is outside the production range. + public static void ThrowIfNotProductionRange(int chunkBytes) + { + if (!IsProductionRange(chunkBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(chunkBytes), + chunkBytes, + $"Article TX chunk budget must be between {MinBytes} and {MaxBytes} bytes."); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxFraming.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxFraming.cs new file mode 100644 index 00000000..53a15b0c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpArticleTxFraming.cs @@ -0,0 +1,91 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Wire framing kind for shared article TX. Mode-independent: callers choose framing; the TX + /// primitive does not branch on MODE READER vs MODE STREAM and does not perform article lookup. + /// + /// + /// Customer* names mean RFC 3977 ARTICLE/BODY response shapes for externally supplied + /// bytes — not an NNTPD customer article store. + /// + public enum NntpArticleTxFrameKind + { + /// ARTICLE wire shape: 220 n|0 <message-id>\r\n + full article + terminator. + CustomerArticle = 0, + + /// Peer TAKETHIS-style feed: TAKETHIS <message-id>\r\n + article + terminator. + PeerTakeThis = 1, + + /// BODY wire shape: 222 n|0 <message-id>\r\n + body only + terminator. + CustomerBody = 2, + + /// HEAD wire shape: 221 n|0 <message-id>\r\n + headers only + terminator. + CustomerHead = 3, + } + + /// Framing parameters for shared article TX via . + public readonly struct NntpArticleTxFraming + { + /// Initializes framing. + /// Framing kind. + /// Message-id (including angle brackets when applicable). + /// + /// Article number for the status line, or 0 when the article was selected by message-id + /// (RFC 3977 §6.2.1 / §6.2.3). + /// + public NntpArticleTxFraming(NntpArticleTxFrameKind kind, string messageId, long articleNumber = 0) + { + ArgumentException.ThrowIfNullOrEmpty(messageId); + ArgumentOutOfRangeException.ThrowIfNegative(articleNumber); + Kind = kind; + MessageId = messageId; + ArticleNumber = articleNumber; + } + + /// Gets the framing kind. + public NntpArticleTxFrameKind Kind { get; } + + /// Gets the message-id (including angle brackets when applicable). + public string MessageId { get; } + + /// Gets the article number used in the status line (0 for message-id selection). + public long ArticleNumber { get; } + + /// Creates ARTICLE-style framing (220 n mid). + public static NntpArticleTxFraming CustomerArticle(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerArticle, messageId, articleNumber); + + /// Creates BODY-style framing (222 n mid). + public static NntpArticleTxFraming CustomerBody(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerBody, messageId, articleNumber); + + /// Creates HEAD-style framing (221 n mid). + public static NntpArticleTxFraming CustomerHead(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerHead, messageId, articleNumber); + + /// Creates TAKETHIS-style framing (TAKETHIS mid). + public static NntpArticleTxFraming PeerTakeThis(string messageId) => + new(NntpArticleTxFrameKind.PeerTakeThis, messageId); + + /// Builds the ASCII status/command header line including trailing CRLF. + public byte[] BuildHeaderBytes() + { + var number = ArticleNumber.ToString(CultureInfo.InvariantCulture); + return Kind switch + { + NntpArticleTxFrameKind.CustomerArticle => + Encoding.ASCII.GetBytes("220 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.CustomerHead => + Encoding.ASCII.GetBytes("221 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.CustomerBody => + Encoding.ASCII.GetBytes("222 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.PeerTakeThis => + Encoding.ASCII.GetBytes("TAKETHIS " + MessageId + "\r\n"), + _ => throw new ArgumentOutOfRangeException(nameof(Kind), Kind, "Unknown article TX framing kind."), + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpAscii.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpAscii.cs new file mode 100644 index 00000000..e100d832 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpAscii.cs @@ -0,0 +1,89 @@ +using System.Runtime.CompilerServices; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// ASCII helpers for NNTP command parsing. RFC 3977 §3.1 SP/TAB. No culture. + internal static class NntpAscii + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsWhitespace(byte b) => b == (byte)' ' || b == (byte)'\t'; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte FoldUpper(byte b) + { + if (b >= (byte)'a' && b <= (byte)'z') + { + return (byte)(b - 32); + } + + return b; + } + + public static bool EqualsFolded(ReadOnlySpan value, ReadOnlySpan upperAscii) + { + if (value.Length != upperAscii.Length) + { + return false; + } + + for (int i = 0; i < value.Length; i++) + { + if (FoldUpper(value[i]) != upperAscii[i]) + { + return false; + } + } + + return true; + } + + public static int SkipWhitespace(ReadOnlySpan line, int index) + { + while (index < line.Length && IsWhitespace(line[index])) + { + index++; + } + + return index; + } + + public static int SkipToken(ReadOnlySpan line, int index) + { + while (index < line.Length && !IsWhitespace(line[index])) + { + index++; + } + + return index; + } + + public static int TrimEndWhitespace(ReadOnlySpan line, int start, int end) + { + while (end > start && IsWhitespace(line[end - 1])) + { + end--; + } + + return end; + } + + public static int CountTokens(ReadOnlySpan line, int start, int end) + { + int count = 0; + int i = start; + while (i < end) + { + if (IsWhitespace(line[i])) + { + i++; + continue; + } + + count++; + i = SkipToken(line, i); + } + + return count; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommand.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommand.cs new file mode 100644 index 00000000..e9524560 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommand.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Immutable parse result: enums and indexes only. Safe to retain across await. + /// Argument bytes live in the current command buffer, not in this struct. + /// + public readonly struct NntpCommand + { + /// Initializes a new instance of the struct. + public NntpCommand( + NntpVerb verb, + NntpVerb qualifier, + int argumentStart, + int argumentLength, + int tokenCount, + NntpParseStatus status) + { + Verb = verb; + Qualifier = qualifier; + ArgumentStart = argumentStart; + ArgumentLength = argumentLength; + TokenCount = tokenCount; + Status = status; + } + + /// Gets the classified verb. + public NntpVerb Verb { get; } + + /// Gets the qualifier (MODE/AUTHINFO/LIST), or . + public NntpVerb Qualifier { get; } + + /// Gets the argument start index in the current command buffer. + public int ArgumentStart { get; } + + /// Gets the argument length in the current command buffer. + public int ArgumentLength { get; } + + /// Gets the number of tokens in the argument region. + public int TokenCount { get; } + + /// Gets the syntactic status. Only may reach a handler. + public NntpParseStatus Status { get; } + + /// Gets whether the command may be dispatched to a normal handler. + public bool IsValid => Status == NntpParseStatus.Ok; + + /// + /// A command line that exceeded . + /// Not a parsed prefix. Handlers must not run. + /// + public static NntpCommand OverlongLine { get; } = new( + NntpVerb.None, + NntpVerb.None, + argumentStart: 0, + argumentLength: 0, + tokenCount: 0, + NntpParseStatus.CommandLineTooLong); + + /// Returns the argument bytes from a still-valid command buffer. + public ReadOnlySpan ArgumentSpan(ReadOnlySpan line) + { + if (ArgumentLength <= 0 + || ArgumentStart < 0 + || ArgumentStart + ArgumentLength > line.Length) + { + return ReadOnlySpan.Empty; + } + + return line.Slice(ArgumentStart, ArgumentLength); + } + + /// Returns the argument bytes from a still-valid command buffer. + public ReadOnlyMemory ArgumentMemory(ReadOnlyMemory line) + { + if (ArgumentLength <= 0 + || ArgumentStart < 0 + || ArgumentStart + ArgumentLength > line.Length) + { + return ReadOnlyMemory.Empty; + } + + return line.Slice(ArgumentStart, ArgumentLength); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandAccess.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandAccess.cs new file mode 100644 index 00000000..9f383069 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandAccess.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Authorization requirements evaluated by the command dispatcher before invoking a handler. + [Flags] + public enum NntpCommandAccess + { + /// Command is allowed without authentication (public/pre-auth). + Public = 0, + + /// Peer must be authenticated (480 if not). + RequiresAuthentication = 1, + + /// Peer must have reader authorization (502 if authenticated but not authorized). + RequiresReader = 2, + + /// Peer must have transit authorization (502 if not). + RequiresTransit = 4, + + /// Peer must have posting permission (in addition to reader where applicable). + RequiresPosting = 8, + + /// Session must be in reader mode. + RequiresReaderMode = 16, + + /// Session must be in stream mode. + RequiresStreamMode = 32, + + /// Peer must have streaming permission to enter stream/transit mode. + RequiresStreaming = 64, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandContext.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandContext.cs new file mode 100644 index 00000000..10d8f073 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandContext.cs @@ -0,0 +1,77 @@ +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Per-command execution context supplied to handlers after syntactic validation. + public sealed class NntpCommandContext + { + /// Initializes a new instance of the class. + public NntpCommandContext( + NntpSession session, + NntpCommand command, + ReadOnlyMemory line, + NntpResponseWriter response, + NntpMultilineReadResult? preReadArticle = null) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + if (!command.IsValid) + { + throw new ArgumentException("Handlers receive only syntactically valid commands.", nameof(command)); + } + + Session = session; + Command = command; + Line = line; + Response = response; + PreReadArticle = preReadArticle; + } + + /// Gets the owning session. + public NntpSession Session { get; } + + /// Gets the validated command (indexes into ). + public NntpCommand Command { get; } + + /// + /// Gets the current command-line buffer. Valid until the next command is parsed. + /// Pipelined CHECK copies the Message-ID before the next parse; do not retain this + /// memory across a later command. + /// + public ReadOnlyMemory Line { get; } + + /// Gets the response writer for this command. + public NntpResponseWriter Response { get; } + + /// + /// Semantic first/status line for TX Debug logging. Set only when Debug is enabled; + /// first assignment wins. Never derived from wire bytes. + /// + internal string? StatusLine { get; set; } + + /// + /// Gets an article already consumed by the continuous RX scanner, when the session + /// pre-read a TAKETHIS body so the handler must not read the Pipe again. + /// + public NntpMultilineReadResult? PreReadArticle { get; } + + /// + /// Optional TX completion detail set by the handler after successful work + /// (for example negotiated TLS parameters after STARTTLS). + /// + public string? CompletionDetail { get; set; } + + /// Gets the underlying transport connection. + public INntpConnection Connection => Session.Connection; + + /// Gets the argument bytes from the current command buffer. + public ReadOnlySpan ArgumentSpan => Command.ArgumentSpan(Line.Span); + + /// + /// Gets the argument bytes as memory. Safe across await only while this command's + /// parser scratch remains the current line; CHECK pipeline slots copy the Message-ID first. + /// + public ReadOnlyMemory ArgumentMemory => Command.ArgumentMemory(Line); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandDispatcher.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandDispatcher.cs new file mode 100644 index 00000000..68851583 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandDispatcher.cs @@ -0,0 +1,426 @@ +using System.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Rejects invalid syntax, applies session/authorization gates, then switch-dispatches the handler. + /// + /// + /// Validation order: parse (caller) → syntax reject → authentication → authorization → mode → handler. + /// Command RX/TX completion records are owned by the session (RX) and command modules (TX), not this type. + /// + public sealed class NntpCommandDispatcher + { + private readonly ILogger _unknownCommandLogger; + + /// Initializes a new instance of the class. + public NntpCommandDispatcher(ILoggerFactory? loggerFactory = null) + { + NntpCommandLoggers.Configure(loggerFactory); + _unknownCommandLogger = NntpCommandLoggers.For(typeof(NntpCommandExecution)); + } + + /// Dispatches one already-parsed command. Invalid syntax never reaches a handler. + public async ValueTask DispatchAsync( + NntpSession session, + NntpCommand command, + ReadOnlyMemory line, + NntpResponseWriter response, + CancellationToken cancellationToken, + NntpMultilineReadResult? preReadArticle = null) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + + if (!command.IsValid) + { + await RejectInvalidAsync(session, command, response, preReadArticle, cancellationToken) + .ConfigureAwait(false); + return; + } + + var access = DefaultNntpCommandCatalog.GetAccess(command.Verb, command.Qualifier); + var display = DefaultNntpCommandCatalog.DisplayName(command.Verb, command.Qualifier); + var logger = LoggerFor(command); + var authz = session.Authorization; + var gateStarted = Stopwatch.GetTimestamp(); + + // Role isolation: CAPABILITIES is authoritative; opposite-role commands are unavailable. + if (session.ServerRole == NntpServerRole.Reader + && (access.HasFlag(NntpCommandAccess.RequiresTransit) + || access.HasFlag(NntpCommandAccess.RequiresStreaming))) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + "command not available on this server", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (session.ServerRole == NntpServerRole.Transit + && access.HasFlag(NntpCommandAccess.RequiresReader)) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + "command not available on this server", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresAuthentication) && !authz.IsAuthenticated) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.AuthenticationRequired, + NntpResponseStatus.AuthenticationRequired, + "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresReader) && !authz.AuthorizedReader) + { + var wire = authz.IsAuthenticated + ? NntpResponses.PermissionDenied + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.PermissionDenied + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "permission denied" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresTransit) && !authz.AuthorizedTransit) + { + var wire = authz.IsAuthenticated + ? NntpResponses.PermissionDenied + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.PermissionDenied + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "permission denied" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresPosting) && !authz.PostingPermitted) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.PostingNotPermitted, + NntpResponseStatus.PostingNotPermitted, + "posting not permitted", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresStreaming) && !authz.StreamingPermitted) + { + var wire = authz.IsAuthenticated + ? NntpResponses.StreamingNotPermitted + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.StreamingNotPermitted + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "streaming not permitted" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresReaderMode) && session.Mode != NntpSessionMode.Reader) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.NotInReaderMode, + NntpResponseStatus.NotInReaderMode, + "not in reader mode", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresStreamMode) && session.Mode != NntpSessionMode.Stream) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.NotInStreamMode, + NntpResponseStatus.NotInStreamMode, + "not in stream mode", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var context = new NntpCommandContext(session, command, line, response, preReadArticle); + try + { + await InvokeHandlerAsync(context, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + if (session.Connection.IsCompleted || session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + try + { + await response + .WriteLineAsync(NntpResponses.CommandFailed, cancellationToken) + .ConfigureAwait(false); + } + catch (InvalidOperationException) + { + // Output writer completed between the check and the write. + } + } + } + + private async ValueTask RejectInvalidAsync( + NntpSession session, + NntpCommand command, + NntpResponseWriter response, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + if (command.Verb == NntpVerb.TakeThis && preReadArticle is null) + { + try + { + NntpArticlePolicyCapture.Capture( + session, + session.ArticleIngestion.MaxArticleBytes, + out var maxArticleBytes, + out _); + _ = await NntpMultilineDataReader + .ReadArticleAsync(session.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + } + + var started = Stopwatch.GetTimestamp(); + var display = command.Status is NntpParseStatus.Empty + or NntpParseStatus.UnknownVerb + or NntpParseStatus.CommandLineTooLong + ? "INVALID" + : DefaultNntpCommandCatalog.DisplayName(command.Verb, command.Qualifier); + var (wire, detail, statusLine) = Rejection(command); + await response.WriteLineAsync(wire, cancellationToken).ConfigureAwait(false); + session.LogCommandRejected(command, detail); + if (_unknownCommandLogger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + _unknownCommandLogger, + session, + display, + Stopwatch.GetElapsedTime(started), + detail, + statusLine); + } + } + + private static async ValueTask WriteGatedResponseAsync( + NntpResponseWriter response, + ILogger logger, + NntpSession session, + string display, + ReadOnlyMemory wire, + string statusLine, + string detail, + long startedTimestamp, + CancellationToken cancellationToken) + { + await response.WriteLineAsync(wire, cancellationToken).ConfigureAwait(false); + if (logger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + logger, + session, + display, + Stopwatch.GetElapsedTime(startedTimestamp), + detail, + statusLine); + } + } + + private static (ReadOnlyMemory Wire, string Detail, string StatusLine) Rejection(NntpCommand command) + { + return command.Status switch + { + NntpParseStatus.CommandLineTooLong => + (NntpResponses.SyntaxError, "syntax error", NntpResponseStatus.SyntaxError), + NntpParseStatus.UnknownVerb => + (NntpResponses.UnknownCommand, "unknown command", NntpResponseStatus.UnknownCommand), + NntpParseStatus.UnknownQualifier => + (NntpResponses.UnknownCommandVariant, "unknown variant", NntpResponseStatus.UnknownCommandVariant), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.User => + (NntpResponses.AuthinfoUserRequiresUsername, "syntax error", NntpResponseStatus.AuthinfoUserRequiresUsername), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.Pass => + (NntpResponses.AuthinfoPassRequiresPassword, "syntax error", NntpResponseStatus.AuthinfoPassRequiresPassword), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.Sasl => + (NntpResponses.SaslMechanismRequired, "syntax error", NntpResponseStatus.SaslMechanismRequired), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressRequiresAlgorithm, "syntax error", NntpResponseStatus.CompressRequiresAlgorithm), + NntpParseStatus.InvalidArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressAlgorithmSyntaxInvalid, "syntax error", NntpResponseStatus.CompressAlgorithmSyntaxInvalid), + NntpParseStatus.ExtraArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressRequiresAlgorithm, "syntax error", NntpResponseStatus.CompressRequiresAlgorithm), + _ => (NntpResponses.SyntaxError, "syntax error", NntpResponseStatus.SyntaxError), + }; + } + + private static ILogger LoggerFor(NntpCommand command) + { + var type = command.Verb switch + { + NntpVerb.Check => typeof(Check), + NntpVerb.TakeThis => typeof(TakeThis), + NntpVerb.Mode => typeof(Mode), + NntpVerb.AuthInfo => typeof(AuthInfo), + NntpVerb.Quit => typeof(Quit), + NntpVerb.Help => typeof(Help), + NntpVerb.Date => typeof(Date), + NntpVerb.Capabilities => typeof(Capabilities), + NntpVerb.StartTls => typeof(StartTls), + NntpVerb.Compress => typeof(Compress), + NntpVerb.List => typeof(List), + NntpVerb.ListGroup => typeof(ListGroup), + NntpVerb.Group => typeof(Group), + NntpVerb.Article => typeof(Article), + NntpVerb.Head => typeof(Article), + NntpVerb.Body => typeof(Article), + NntpVerb.Stat => typeof(Article), + NntpVerb.Last => typeof(Last), + NntpVerb.Next => typeof(Next), + NntpVerb.Over => typeof(Over), + NntpVerb.Hdr => typeof(Hdr), + NntpVerb.Post => typeof(Post), + NntpVerb.Ihave => typeof(IHave), + NntpVerb.BenchIt => typeof(BenchIt), + NntpVerb.SpeedTest => typeof(Commands.SpeedTest), + _ => typeof(NntpCommandExecution), + }; + + return NntpCommandLoggers.For(type); + } + + private static ValueTask InvokeHandlerAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + return (context.Command.Verb, context.Command.Qualifier) switch + { + (NntpVerb.Capabilities, _) => Capabilities.HandleAsync(context, cancellationToken), + (NntpVerb.Mode, NntpVerb.Reader) => Mode.HandleReaderAsync(context, cancellationToken), + (NntpVerb.Mode, NntpVerb.Stream) => Mode.HandleStreamAsync(context, cancellationToken), + (NntpVerb.Help, _) => Help.HandleAsync(context, cancellationToken), + (NntpVerb.Date, _) => Date.HandleAsync(context, cancellationToken), + (NntpVerb.Quit, _) => Quit.HandleAsync(context, cancellationToken), + (NntpVerb.StartTls, _) => StartTls.HandleAsync(context, context.Session.CertificateProvider, cancellationToken), + (NntpVerb.Compress, _) => Compress.HandleAsync(context, cancellationToken), + (NntpVerb.BenchIt, _) => BenchIt.HandleAsync(context, cancellationToken), + (NntpVerb.SpeedTest, _) => Commands.SpeedTest.HandleAsync(context, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.User) => AuthInfo.HandleUserAsync( + context, context.Session.AuthenticationProvider, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.Pass) => AuthInfo.HandlePassAsync( + context, context.Session.AuthenticationProvider, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.Sasl) => AuthInfo.HandleSaslAsync(context, cancellationToken), + (NntpVerb.List, _) => List.HandleAsync(context, cancellationToken), + (NntpVerb.Group, _) => Group.HandleAsync(context, cancellationToken), + (NntpVerb.ListGroup, _) => ListGroup.HandleAsync(context, cancellationToken), + (NntpVerb.Article, _) => Article.HandleArticleAsync(context, cancellationToken), + (NntpVerb.Head, _) => Article.HandleHeadAsync(context, cancellationToken), + (NntpVerb.Body, _) => Article.HandleBodyAsync(context, cancellationToken), + (NntpVerb.Stat, _) => Article.HandleStatAsync(context, cancellationToken), + (NntpVerb.Last, _) => Last.HandleAsync(context, cancellationToken), + (NntpVerb.Next, _) => Next.HandleAsync(context, cancellationToken), + (NntpVerb.Over, _) => Over.HandleAsync(context, cancellationToken), + (NntpVerb.Hdr, _) => Hdr.HandleAsync(context, cancellationToken), + (NntpVerb.Post, _) => Post.HandleAsync(context, cancellationToken), + (NntpVerb.Ihave, _) => IHave.HandleAsync(context, cancellationToken), + (NntpVerb.Check, _) => Check.HandleAsync(context, cancellationToken), + (NntpVerb.TakeThis, _) => TakeThis.HandleAsync(context, cancellationToken), + _ => NntpCommandNotImplemented.HandleAsync( + context, + NntpCommandLoggers.For(typeof(NntpCommandExecution)), + cancellationToken), + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandExecution.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandExecution.cs new file mode 100644 index 00000000..54e2ac0f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandExecution.cs @@ -0,0 +1,133 @@ +using System.Diagnostics; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Times command-handler execution and emits exactly one TX completion record owned by the command module. + /// + /// + /// Timing covers handler work only (not socket receive wait). Completion is always written from + /// finally so cancellation and failures still produce one TX line. + /// + internal static class NntpCommandExecution + { + /// + /// Runs under a monotonic stopwatch and logs + /// [client] TX: COMMAND [status] executed in 0.000s exactly once when + /// the handler produced an NNTP status line; otherwise the historical no-status + /// completion format is retained. + /// + public static async ValueTask RunAsync( + ILogger logger, + NntpCommandContext context, + string command, + Func execute, + CancellationToken cancellationToken, + string? successDetail = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(context); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + ArgumentNullException.ThrowIfNull(execute); + + var started = Stopwatch.GetTimestamp(); + string? detail = successDetail; + var completedBefore = context.Connection.IsCompleted; + try + { + await execute(context, cancellationToken).ConfigureAwait(false); + if (context.CompletionDetail is not null) + { + // Handler-owned detail (e.g. QUIT peer disconnect, STARTTLS cipher, COMPRESS). + detail = context.CompletionDetail; + } + else if (!completedBefore && context.Connection.IsCompleted) + { + detail = "failed"; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + detail = "cancelled"; + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + detail = "failed"; + CommandLogMessages.CommandFailed( + logger, + ex, + NntpCommandLogFormat.Client(context.Session), + command); + throw; + } + finally + { + WriteCompletion( + logger, + context.Session, + command, + Stopwatch.GetElapsedTime(started), + detail, + context.StatusLine); + } + } + + /// + /// Emits a TX completion record for gate/reject paths that never enter a command handler body. + /// + /// + /// Uses the command module's logger category (not ). + /// + public static void WriteCompletion( + ILogger logger, + NntpSession session, + string command, + TimeSpan elapsed, + string? detail = null, + string? statusLine = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(session); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + + if (!logger.IsEnabled(LogLevel.Debug) + || NntpCommandLogFormat.SuppressHotPathCommandLog(command)) + { + return; + } + + var client = NntpCommandLogFormat.Client(session); + var seconds = elapsed.TotalSeconds; + if (statusLine is not null) + { + if (detail is null) + { + CommandLogMessages.CommandTxWithStatus(logger, client, command, statusLine, seconds); + } + else + { + CommandLogMessages.CommandTxWithStatusAndDetail( + logger, + client, + command, + statusLine, + seconds, + detail); + } + + return; + } + + if (detail is null) + { + CommandLogMessages.CommandTx(logger, client, command, seconds); + } + else + { + CommandLogMessages.CommandTxWithDetail(logger, client, command, seconds, detail); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLineReader.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLineReader.cs new file mode 100644 index 00000000..87e9c2e4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLineReader.cs @@ -0,0 +1,113 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Reads CRLF-delimited NNTP command lines from a . + public static class NntpCommandLineReader + { + /// Maximum command-line content before CRLF. Longer lines are rejected, not parsed. + public const int MaxCommandLineBytes = 2048; + + /// + /// result when the line before CRLF exceeds the destination + /// scratch. The line has been consumed. -1 remains EOF. + /// + public const int OverlongLine = -2; + + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + + /// + /// Reads the next command line (without trailing CRLF). Returns on EOF. + /// + public static async ValueTask ReadLineAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + if (TryReadLine(ref buffer, out var line)) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return line; + } + + reader.AdvanceTo(buffer.Start, buffer.End); + if (result.IsCompleted) + { + return null; + } + } + } + + private static bool TryReadLine(ref ReadOnlySequence buffer, out string? line) + { + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence lineBytes, Crlf)) + { + line = null; + return false; + } + + line = Encoding.ASCII.GetString(lineBytes); + buffer = buffer.Slice(reader.Position); + return true; + } + + /// + /// Reads the next command line into (without CRLF). + /// Returns the number of bytes written, or -1 on EOF. + /// + public static async ValueTask ReadLineBytesAsync( + PipeReader reader, + byte[] scratch, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(scratch); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + if (TryCopyLine(ref buffer, scratch, out var length)) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return length; + } + + reader.AdvanceTo(buffer.Start, buffer.End); + if (result.IsCompleted) + { + return -1; + } + } + } + + private static bool TryCopyLine(ref ReadOnlySequence buffer, byte[] scratch, out int length) + { + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence lineBytes, Crlf)) + { + length = 0; + return false; + } + + buffer = buffer.Slice(reader.Position); + if (lineBytes.Length > scratch.Length) + { + length = OverlongLine; + return true; + } + + length = checked((int)lineBytes.Length); + lineBytes.CopyTo(scratch); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLogFormat.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLogFormat.cs new file mode 100644 index 00000000..b8f92fff --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLogFormat.cs @@ -0,0 +1,155 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Formats effective client identity and redacts secrets for NNTP command RX/TX logs. + internal static class NntpCommandLogFormat + { + /// Formats src-ip:src-port from the session's effective client identity. + public static string Client(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + return string.Create( + CultureInfo.InvariantCulture, + $"{session.ClientAddress}:{session.ClientPort}"); + } + + /// + /// Returns whether per-command RX logging should be skipped for . + /// + /// + /// Previously suppressed because console Information + /// could not keep up. File logging is now Debug/Verbose; TAKETHIS diagnostics are restored. + /// + public static bool SuppressHotPathCommand(NntpVerb verb) + { + _ = verb; + return false; + } + + /// + /// Logging-boundary representation of a command. AUTHINFO secrets are redacted. + /// + /// + /// / + /// cannot accept + /// of bytes, and byte[] is not a useful operational + /// log representation. This ASCII conversion is the explicit logging string boundary. + /// Callers must check before + /// invoking this method. + /// + public static string RedactRxCommand(NntpCommand command, ReadOnlySpan line) + { + if (command.Verb == NntpVerb.AuthInfo && command.Qualifier == NntpVerb.Pass) + { + return "AUTHINFO PASS "; + } + + if (command.Verb == NntpVerb.AuthInfo && command.Qualifier == NntpVerb.Sasl) + { + return "AUTHINFO SASL "; + } + + return System.Text.Encoding.ASCII.GetString(line); + } + + /// + /// Returns a log-safe RX command line. Passwords and SASL credential material are redacted. + /// + public static string RedactRxLine(string rawLine) + { + ArgumentNullException.ThrowIfNull(rawLine); + if (rawLine.Length == 0) + { + return rawLine; + } + + // AUTHINFO PASS → AUTHINFO PASS + if (StartsWithCommand(rawLine, "AUTHINFO PASS")) + { + return "AUTHINFO PASS "; + } + + // AUTHINFO SASL may carry an initial response with credentials. + if (StartsWithCommand(rawLine, "AUTHINFO SASL")) + { + return "AUTHINFO SASL "; + } + + return rawLine; + } + + /// Extracts a display command name from a raw line when resolution is unavailable. + public static string DisplayNameFromRawLine(string rawLine) + { + ArgumentNullException.ThrowIfNull(rawLine); + if (rawLine.Length == 0) + { + return "INVALID"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO PASS")) + { + return "AUTHINFO PASS"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO SASL")) + { + return "AUTHINFO SASL"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO USER")) + { + return "AUTHINFO USER"; + } + + if (StartsWithCommand(rawLine, "MODE READER")) + { + return "MODE READER"; + } + + if (StartsWithCommand(rawLine, "MODE STREAM")) + { + return "MODE STREAM"; + } + + if (StartsWithCommand(rawLine, "COMPRESS")) + { + return "COMPRESS"; + } + + var span = rawLine.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space <= 0 ? span.ToString().ToUpperInvariant() : span[..space].ToString().ToUpperInvariant(); + } + + /// + /// Returns whether per-command RX/TX INFO traffic logging should be suppressed for this + /// command name or raw command line. + /// + /// + /// Previously suppressed TAKETHIS so feed volume did not flood the console. + /// Console is now Information-only; TAKETHIS Debug/Trace is restored for the file sink. + /// + public static bool SuppressHotPathCommandLog(string commandOrRawLine) + { + ArgumentNullException.ThrowIfNull(commandOrRawLine); + return false; + } + + private static bool StartsWithCommand(string line, string command) + { + if (line.Length < command.Length) + { + return false; + } + + if (!line.AsSpan(0, command.Length).Equals(command, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return line.Length == command.Length || char.IsWhiteSpace(line[command.Length]); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLoggers.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLoggers.cs new file mode 100644 index 00000000..e561d183 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandLoggers.cs @@ -0,0 +1,33 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Resolves per-command instances from the host . + /// + /// + /// Command modules own completion logging; categories use the command type's full name + /// (for example VectorNNTPD.ReaderService.Session.Commands.StartTls). + /// + internal static class NntpCommandLoggers + { + private static ILoggerFactory _factory = NullLoggerFactory.Instance; + private static readonly ConcurrentDictionary Cache = new(StringComparer.Ordinal); + + /// Configures the factory used for subsequent calls. + public static void Configure(ILoggerFactory? loggerFactory) + { + _factory = loggerFactory ?? NullLoggerFactory.Instance; + Cache.Clear(); + } + + /// Returns an whose category is 's full name. + public static ILogger For(Type commandType) + { + ArgumentNullException.ThrowIfNull(commandType); + var name = commandType.FullName ?? commandType.Name; + return Cache.GetOrAdd(name, static (category, factory) => factory.CreateLogger(category), _factory); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandNotImplemented.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandNotImplemented.cs new file mode 100644 index 00000000..6a1342b9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandNotImplemented.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Shared placeholder response for commands not yet implemented. + internal static class NntpCommandNotImplemented + { + /// Returns 500 Command not implemented (RFC 3977 for unsupported optional commands). + public static ValueTask HandleAsync( + NntpCommandContext context, + ILogger logger, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + logger, + NntpResponses.CommandNotImplemented, + NntpResponseStatus.CommandNotImplemented, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandParser.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandParser.cs new file mode 100644 index 00000000..60bd13e2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandParser.cs @@ -0,0 +1,341 @@ +using VectorNNTPD.Common.Articles.Validation; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Authoritative byte-oriented NNTP command parser and syntactic validation boundary. + /// + /// + /// Only may reach a command handler. Invalid commands are + /// rejected at dispatch. Argument bytes remain indexes into the caller's buffer. + /// Keywords are case-insensitive; SP and TAB are separators (RFC 3977 §3.1). + /// + public static class NntpCommandParser + { + /// Parses one command line (CRLF already removed). Never throws for ordinary malformed input. + public static NntpCommand Parse(ReadOnlySpan line) + { + var verb = NntpVerbClassifier.Classify(line, out _, out int verbEnd); + if (verb == NntpVerb.None) + { + return new NntpCommand(NntpVerb.None, NntpVerb.None, 0, 0, 0, NntpParseStatus.Empty); + } + + if (verb == NntpVerb.Unknown) + { + return Finish(NntpVerb.Unknown, NntpVerb.None, line, verbEnd, NntpParseStatus.UnknownVerb); + } + + int index = NntpAscii.SkipWhitespace(line, verbEnd); + var qualifier = NntpVerb.None; + + if (NeedsQualifier(verb)) + { + if (index >= line.Length) + { + return new NntpCommand(verb, NntpVerb.None, 0, 0, 0, NntpParseStatus.UnknownQualifier); + } + + int qEnd = NntpAscii.SkipToken(line, index); + qualifier = ClassifyQualifier(verb, line.Slice(index, qEnd - index)); + if (qualifier is NntpVerb.Unknown or NntpVerb.None) + { + return Finish(verb, NntpVerb.Unknown, line, qEnd, NntpParseStatus.UnknownQualifier); + } + + index = NntpAscii.SkipWhitespace(line, qEnd); + } + else if (verb == NntpVerb.List && index < line.Length) + { + int qEnd = NntpAscii.SkipToken(line, index); + var listQualifier = ClassifyListQualifier(line.Slice(index, qEnd - index)); + if (listQualifier == NntpVerb.Unknown) + { + return Finish(verb, NntpVerb.Unknown, line, qEnd, NntpParseStatus.UnknownQualifier); + } + + qualifier = listQualifier; + index = NntpAscii.SkipWhitespace(line, qEnd); + } + + int argEnd = NntpAscii.TrimEndWhitespace(line, index, line.Length); + int tokenCount = NntpAscii.CountTokens(line, index, argEnd); + int argLength = argEnd - index; + var status = Validate(verb, qualifier, line, index, argLength, tokenCount); + return new NntpCommand(verb, qualifier, index, argLength, tokenCount, status); + } + + private static NntpCommand Finish( + NntpVerb verb, + NntpVerb qualifier, + ReadOnlySpan line, + int after, + NntpParseStatus status) + { + int index = NntpAscii.SkipWhitespace(line, after); + int argEnd = NntpAscii.TrimEndWhitespace(line, index, line.Length); + return new NntpCommand( + verb, + qualifier, + index, + argEnd - index, + NntpAscii.CountTokens(line, index, argEnd), + status); + } + + private static bool NeedsQualifier(NntpVerb verb) => verb is NntpVerb.Mode or NntpVerb.AuthInfo; + + private static NntpParseStatus Validate( + NntpVerb verb, + NntpVerb qualifier, + ReadOnlySpan line, + int argumentStart, + int argumentLength, + int tokenCount) + { + var argument = argumentLength <= 0 + ? ReadOnlySpan.Empty + : line.Slice(argumentStart, argumentLength); + + switch (verb) + { + case NntpVerb.Check: + case NntpVerb.TakeThis: + case NntpVerb.Ihave: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpMessageIdValidation.IsValidMessageId(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.SpeedTest: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return IsSpeedTestPeerSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.Quit: + case NntpVerb.Help: + case NntpVerb.Post: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.Compress: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return IsCompressAlgorithmSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.AuthInfo when qualifier is NntpVerb.User or NntpVerb.Pass: + return argument.IsEmpty ? NntpParseStatus.MissingArgument : NntpParseStatus.Ok; + + case NntpVerb.AuthInfo when qualifier is NntpVerb.Sasl: + return argument.IsEmpty ? NntpParseStatus.MissingArgument : NntpParseStatus.Ok; + + case NntpVerb.Article: + case NntpVerb.Head: + case NntpVerb.Body: + case NntpVerb.Stat: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpMessageIdValidation.IsValidMessageId(argument) || NntpArticleNumber.IsSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.Group: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + return tokenCount == 1 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.ListGroup: + return tokenCount <= 2 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.None: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.Active or NntpVerb.Counts or NntpVerb.Newsgroups: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpWildmat.TryValidate(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.List when qualifier is NntpVerb.Motd or NntpVerb.OverviewFmt: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.Headers: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpAscii.EqualsFolded(argument, "MSGID"u8) + || NntpAscii.EqualsFolded(argument, "RANGE"u8) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + default: + return NntpParseStatus.Ok; + } + } + + /// + /// SPEEDTEST identifier: one NNTP token matching the Transit identifier grammar + /// (1–256 visible ASCII octets, no whitespace or controls). Exact ordinal match + /// is applied later against the snapshot. Not a quoted string or multi-token name. + /// + internal static bool IsSpeedTestPeerSyntax(ReadOnlySpan peer) => + TransitPeersOptionsValidator.IsPeerIdentifier(peer); + + /// RFC 8054 §5.3 algorithm syntax (case-sensitive). + internal static bool IsCompressAlgorithmSyntax(ReadOnlySpan algorithm) + { + if (algorithm.Length is < 1 or > 20) + { + return false; + } + + for (int i = 0; i < algorithm.Length; i++) + { + byte b = algorithm[i]; + if ((b >= (byte)'A' && b <= (byte)'Z') + || (b >= (byte)'0' && b <= (byte)'9') + || b == (byte)'-' + || b == (byte)'_') + { + continue; + } + + return false; + } + + return true; + } + + private static NntpVerb ClassifyQualifier(NntpVerb verb, ReadOnlySpan token) + { + if (verb == NntpVerb.Mode) + { + if (NntpAscii.EqualsFolded(token, "STREAM"u8)) + { + return NntpVerb.Stream; + } + + if (NntpAscii.EqualsFolded(token, "READER"u8)) + { + return NntpVerb.Reader; + } + + return NntpVerb.Unknown; + } + + if (verb == NntpVerb.AuthInfo) + { + if (NntpAscii.EqualsFolded(token, "USER"u8)) + { + return NntpVerb.User; + } + + if (NntpAscii.EqualsFolded(token, "PASS"u8)) + { + return NntpVerb.Pass; + } + + if (NntpAscii.EqualsFolded(token, "SASL"u8)) + { + return NntpVerb.Sasl; + } + + return NntpVerb.Unknown; + } + + return NntpVerb.None; + } + + private static NntpVerb ClassifyListQualifier(ReadOnlySpan token) + { + if (NntpAscii.EqualsFolded(token, "ACTIVE"u8)) + { + return NntpVerb.Active; + } + + if (NntpAscii.EqualsFolded(token, "COUNTS"u8)) + { + return NntpVerb.Counts; + } + + if (NntpAscii.EqualsFolded(token, "HEADERS"u8)) + { + return NntpVerb.Headers; + } + + if (NntpAscii.EqualsFolded(token, "MOTD"u8)) + { + return NntpVerb.Motd; + } + + if (NntpAscii.EqualsFolded(token, "NEWSGROUPS"u8)) + { + return NntpVerb.Newsgroups; + } + + if (NntpAscii.EqualsFolded(token, "OVERVIEW.FMT"u8)) + { + return NntpVerb.OverviewFmt; + } + + return NntpVerb.Unknown; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandReply.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandReply.cs new file mode 100644 index 00000000..2e21ed8f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandReply.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Writes existing wire bytes and, only when Debug command logging is enabled, records + /// the independently authored semantic status line on the command context. + /// + internal static class NntpCommandReply + { + /// + /// Records when Debug is enabled and no status is set yet. + /// Immortal strings are stored by reference; nothing is allocated when Debug is off. + /// + internal static void TryNote(NntpCommandContext context, ILogger logger, string statusLine) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(statusLine); + if (context.StatusLine is not null || !logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + context.StatusLine = statusLine; + } + + /// Writes and notes if Debug is on. + internal static ValueTask WriteAsync( + NntpCommandContext context, + ILogger logger, + ReadOnlyMemory wire, + string statusLine, + CancellationToken cancellationToken = default) + { + TryNote(context, logger, statusLine); + return context.Response.WriteLineAsync(wire, cancellationToken); + } + + /// + /// Enqueues immediately and notes if Debug is on. + /// + internal static ValueTask EnqueueImmediateAsync( + NntpCommandContext context, + ILogger logger, + ReadOnlyMemory wire, + string statusLine, + CancellationToken cancellationToken = default) + { + TryNote(context, logger, statusLine); + return context.Response.EnqueueLineImmediateAsync(wire, cancellationToken); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandStatusText.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandStatusText.cs new file mode 100644 index 00000000..9589bc61 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpCommandStatusText.cs @@ -0,0 +1,53 @@ +using System.Text; +using VectorNNTPD.ReaderService.History; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Debug-only semantic TX status lines for dynamic CHECK/TAKETHIS/DATE replies. + /// + internal static class NntpCommandStatusText + { + /// + /// Formats the CHECK status line from the lookup result and Message-ID octets. + /// Call only after . + /// + /// History lookup that selected the CHECK reply. It is not rewritten. + /// Message-ID octets already accepted for the command. + /// + /// Connection DeferOnDuplicate. When , a seen identifier is logged as 431. + /// + internal static string FormatCheck( + HistoryLookupResult result, + ReadOnlySpan messageId, + bool deferOnDuplicate = false) + { + var id = Encoding.ASCII.GetString(messageId); + return result switch + { + HistoryLookupResult.Seen when deferOnDuplicate => "431 " + id, + HistoryLookupResult.Seen => "438 " + id, + HistoryLookupResult.Unavailable => "431 " + id, + _ => "238 " + id + " send article to be transferred", + }; + } + + /// + /// Formats TAKETHIS 239 message-id. Call only after . + /// + internal static string FormatTakeThisAccepted(ReadOnlySpan messageId) => + "239 " + Encoding.ASCII.GetString(messageId); + + /// + /// Formats TAKETHIS 439 message-id. Call only after . + /// + internal static string FormatTakeThisRejected(ReadOnlySpan messageId) => + "439 " + Encoding.ASCII.GetString(messageId); + + /// + /// Formats 111 YYYYMMDDhhmmss from the same stamp characters used for the wire line. + /// Call only after . + /// + internal static string FormatDate(ReadOnlySpan stamp) => string.Concat("111 ".AsSpan(), stamp); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpParseStatus.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpParseStatus.cs new file mode 100644 index 00000000..3e92d161 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpParseStatus.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// Syntactic outcome of . + public enum NntpParseStatus : byte + { + /// Command is syntactically valid and may be dispatched to a handler. + Ok = 0, + + /// Empty or whitespace-only line. + Empty, + + /// Verb is not a supported command. + UnknownVerb, + + /// Verb is known but the qualifier/variant is missing or unsupported. + UnknownQualifier, + + /// A required argument is missing. + MissingArgument, + + /// An argument is present but syntactically invalid. + InvalidArgument, + + /// The command has more tokens than its grammar allows. + ExtraArgument, + + /// + /// The CRLF-terminated line is longer than . + /// The prefix is not parsed. + /// + CommandLineTooLong, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpPeerDisconnect.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpPeerDisconnect.cs new file mode 100644 index 00000000..8d9cd11f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpPeerDisconnect.cs @@ -0,0 +1,59 @@ +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Classifies transport failures that indicate the peer has already gone away. + /// + /// + /// Used by QUIT and by the session command-loop catch. Classification stays + /// reset/EOF-shaped so unrelated I/O failures are not treated as normal disconnects. + /// Authenticated-session release is guaranteed by session finalization, not by this classifier. + /// + internal static class NntpPeerDisconnect + { + /// + /// Returns whether indicates the peer disconnected or the + /// connection became terminal in a way expected during QUIT response/close. + /// + public static bool IsPeerDisconnect(Exception exception, INntpConnection connection) + { + ArgumentNullException.ThrowIfNull(exception); + ArgumentNullException.ThrowIfNull(connection); + + for (Exception? current = exception; current is not null; current = current.InnerException) + { + switch (current) + { + case SocketException socket when IsResetLike(socket.SocketErrorCode): + return true; + case IOException { InnerException: SocketException nested } when IsResetLike(nested.SocketErrorCode): + return true; + case IOException { Message: not null } io when + io.Message.Contains("zero bytes", StringComparison.OrdinalIgnoreCase): + return true; + case ObjectDisposedException when connection.IsCompleted + || connection.ConnectionClosed.IsCancellationRequested: + return true; + case InvalidOperationException { Message: not null } ioe when + ioe.Message.Contains("writer was completed", StringComparison.OrdinalIgnoreCase) + || ioe.Message.Contains("pipe completed", StringComparison.OrdinalIgnoreCase): + // Output PipeWriter completed (peer gone / CompleteAsync raced with QUIT flush). + return true; + } + } + + return false; + } + + private static bool IsResetLike(SocketError error) => + error is SocketError.ConnectionReset + or SocketError.ConnectionAborted + or SocketError.Shutdown + or SocketError.NotConnected + or SocketError.NetworkReset + // EPIPE (32) — not always present as SocketError.BrokenPipe on Windows TFMs. + or (SocketError)32; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseCompose.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseCompose.cs new file mode 100644 index 00000000..bb925d69 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseCompose.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Builds one owned wire buffer from static prefixes and dynamic protocol bytes. + /// + /// + /// The returned array is the caller's until it is passed to + /// + /// or . + /// The TX pump only reads it. Session scratch and Pipe memory must not be enqueued; + /// those spans are copied into this buffer first. + /// + internal static class NntpResponseCompose + { + /// Allocates prefix + dynamic + suffix as one owned array. + internal static byte[] Concat( + ReadOnlySpan prefix, + ReadOnlySpan dynamic, + ReadOnlySpan suffix) + { + var result = new byte[checked(prefix.Length + dynamic.Length + suffix.Length)]; + prefix.CopyTo(result); + dynamic.CopyTo(result.AsSpan(prefix.Length)); + suffix.CopyTo(result.AsSpan(prefix.Length + dynamic.Length)); + return result; + } + + /// Allocates the concatenation of as one owned array. + internal static byte[] Concatenate(ReadOnlySpan> parts) + { + var total = 0; + for (var i = 0; i < parts.Length; i++) + { + total = checked(total + parts[i].Length); + } + + var result = new byte[total]; + var offset = 0; + for (var i = 0; i < parts.Length; i++) + { + var part = parts[i].Span; + part.CopyTo(result.AsSpan(offset)); + offset += part.Length; + } + + return result; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseStatus.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseStatus.cs new file mode 100644 index 00000000..ce380d09 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseStatus.cs @@ -0,0 +1,91 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Immortal semantic first/status lines matching wire text + /// (without CRLF). Authored independently of the byte buffers; not decoded from them. + /// + internal static class NntpResponseStatus + { + internal const string HelpTextFollows = "100 Help text follows"; + internal const string CapabilityListFollows = "101 Capability list:"; + internal const string ListOfNewsgroupsFollows = "215 list of newsgroups follows"; + internal const string ListOverviewFmtFollows = "215 Order of fields in overview database."; + internal const string ListHeadersFollows = "215 headers and metadata items supported:"; + internal const string NoSuchNewsgroup = "411 No such newsgroup"; + internal const string NoNewsgroupSelected = "412 No newsgroup selected"; + internal const string CurrentArticleNumberInvalid = "420 Current article number is invalid"; + internal const string NoArticleWithNumber = "423 No article with that number"; + internal const string NoArticleWithMessageId = "430 No article with that message-id"; + internal const string GroupSelected = "211"; + internal const string ReaderModePostingPermitted = "200 Reader mode, posting permitted"; + internal const string ReaderModePostingProhibited = "201 Reader mode, posting prohibited"; + internal const string StreamingPermitted = "203 Streaming permitted"; + internal const string ConnectionClosing = "205 Connection closing"; + internal const string CompressionActive = "206 Compression active"; + internal const string IhaveTransferredOk = "235 Article transferred OK"; + internal const string ArticleReceivedOk = "240 Article received OK"; + internal const string IhaveSendArticle = "335 Send article to be transferred"; + internal const string PostSendArticle = "340 Input article; end with ."; + internal const string PostingProhibited = "440 Posting not permitted"; + internal const string PostingFailed = "441 Posting failed"; + internal const string AuthenticationAccepted = "281 Authentication accepted"; + internal const string PasswordRequired = "381 Password required"; + internal const string ContinueWithTls = "382 Continue with TLS negotiation"; + internal const string ServiceTemporarilyUnavailable = "400 Service temporarily unavailable"; + internal const string SpeedTestBusy = "400 SPEEDTEST BUSY"; + internal const string CommandFailed = "403 Command failed"; + internal const string UnableToActivateCompression = "403 Unable to activate compression"; + internal const string IhaveNotWanted = "435 Article not wanted"; + internal const string IhaveTryLater = "436 Transfer not possible; try again later"; + internal const string IhaveTransferFailed = "436 Transfer failed; try again later"; + internal const string IhaveRejected = "437 Transfer rejected; do not retry"; + internal const string AuthenticationRequired = "480 Authentication required"; + internal const string AuthenticationFailed = "481 Authentication failed"; + internal const string AuthenticationCancelled = "481 Authentication cancelled"; + internal const string TooManySessions = "481 Too many sessions"; + internal const string TooManySourceAddresses = "481 Too many source addresses"; + internal const string AuthenticationOutOfSequence = "482 Authentication commands issued out of sequence"; + internal const string PrivacyRequired = "483 Encryption or stronger authentication required"; + internal const string UnknownCommand = "500 Unknown command"; + internal const string CommandNotImplemented = "500 Command not implemented"; + internal const string SyntaxError = "501 Syntax error"; + internal const string UnknownCommandVariant = "501 Unknown command variant"; + internal const string AuthinfoUserRequiresUsername = "501 AUTHINFO USER requires a username"; + internal const string AuthinfoPassRequiresPassword = "501 AUTHINFO PASS requires a password"; + internal const string CompressRequiresAlgorithm = "501 COMPRESS requires a single algorithm argument"; + internal const string CompressAlgorithmSyntaxInvalid = "501 Syntactically incorrect compression algorithm"; + internal const string AuthinfoSaslNotImplemented = "501 AUTHINFO SASL not implemented"; + internal const string SaslMechanismRequired = "501 SASL mechanism required"; + internal const string SaslInvalidBase64 = "504 Base64 encoding error"; + internal const string SaslLoginUsernameChallenge = "383 VXNlcm5hbWU6"; + internal const string SaslLoginPasswordChallenge = "383 UGFzc3dvcmQ6"; + internal const string SaslEmptyChallenge = "383 ="; + internal const string SaslChallenge = "383 SASL challenge"; + internal const string SaslSuccessWithData = "283 Authentication accepted"; + internal const string SaslProtocolError = "482 SASL protocol error"; + internal const string SaslMechanismNotSupported = "503 Mechanism not supported"; + internal const string SaslContinuationNotSupported = "503 SASL continuation not supported"; + internal const string NoSaslExchangeInProgress = "503 No SASL exchange in progress"; + internal const string TemporaryAuthenticationFailure = "503 Temporary authentication failure"; + internal const string SaslScramUnavailable = "503 SCRAM not available"; + internal const string SaslCramUnavailable = "503 CRAM-MD5 not available"; + internal const string AlreadyAuthenticated = "502 Already authenticated"; + internal const string CommandUnavailableAfterAuthentication = "502 Command unavailable after authentication"; + internal const string CommandUnavailableAfterCompression = "502 Command unavailable after compression"; + internal const string DeflateAlreadyActive = "502 DEFLATE compression already active"; + internal const string CompressionAlreadyActive = "502 Compression already active"; + internal const string TlsAlreadyActive = "502 TLS already active"; + internal const string TlsProviderUnavailable = "502 TLS provider unavailable"; + internal const string PostingNotPermitted = "502 Posting not permitted"; + internal const string PermissionDenied = "502 Permission denied"; + internal const string CommandNotAvailable = "502 Command not available on this server"; + internal const string StreamingNotPermitted = "502 Streaming not permitted"; + internal const string NotInReaderMode = "502 Not in reader mode"; + internal const string NotInStreamMode = "502 Not in stream mode"; + internal const string SpeedTestUnknownPeer = "502 UNKNOWN SPEEDTEST PEER"; + internal const string SpeedTestPeerMismatch = "502 SPEEDTEST PEER MISMATCH"; + internal const string CompressionAlgorithmNotSupported = "503 Compression algorithm not supported"; + internal const string SpeedTestNotSupported = "503 SPEEDTEST NOT SUPPORTED"; + internal const string ListDataItemNotStored = "503 Data item not stored"; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseWriter.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseWriter.cs new file mode 100644 index 00000000..a8935d17 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponseWriter.cs @@ -0,0 +1,853 @@ +using System.IO.Pipelines; +using System.Text; +using System.Threading.Channels; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Writes NNTP single-line and multi-line responses to a through a + /// single ordered outbound pump. + /// + /// + /// + /// All writes are serialized onto one pump so response bytes remain ordered. Ordinary + /// awaits pump flush completion + /// (existing command semantics). + /// only waits until the line is + /// accepted by the ordered queue, + /// so TAKETHIS can continue reading the next pipelined article without waiting for network + /// delivery of the prior 239/439. + /// + /// + /// The pump coalesces fire-and-forget lines into batches of at most + /// before one . + /// Logical response bytes, CRLF framing, and channel order are unchanged; only write + /// granularity changes. Awaiting writes (status lines, article chunks, STARTTLS/COMPRESS/QUIT) + /// flush any held batch first. + /// + /// + /// Large article bodies use the WriteArticleAsync APIs: externally supplied destuffed bytes → + /// restuff → bounded owned chunks → the same ordered Channel (no second TX pipeline; + /// mode-independent). The writer does not own article storage or lookup. + /// + /// + public sealed class NntpResponseWriter : IAsyncDisposable + { + /// + /// Maximum fire-and-forget responses held into one . + /// + /// + /// Chosen from the measured N=8 reverse-path cell in + /// .artifacts/STREAM-TCP-WIRE-MECHANISM-AUDIT.md (~200 × 264-byte TCP segments/s + /// versus N=1 ~1450 × 33-byte segments/s). Not a configuration setting. + /// + internal const int CoalesceResponseBatchSize = 8; + + private static readonly byte[] DotCrlf = ".\r\n"u8.ToArray(); + + private readonly PipeWriter _output; + private readonly Channel _channel; + private readonly Task _pump; + // codeql[cs/missed-using-statement] Owned by NntpResponseWriter until Dispose; a using at this declaration would cancel the pump when the constructor returns. + private readonly CancellationTokenSource _pumpCts = new(); + private int _disposed; + private long _channelEnqueueCount; + private long _pipeFlushCount; + private long _ownedPayloadBytesEnqueued; + private int _coalescedUnflushed; + private int _directExclusive; + private long _directPipeFlushCount; + private IAccountByteSink _byteSink = NullAccountByteSink.Instance; + + private readonly struct WriteRequest + { + public WriteRequest( + ReadOnlyMemory payload, + TaskCompletionSource? completed, + bool flushAfter = false) + { + Payload = payload; + Completed = completed; + FlushAfter = flushAfter; + } + + /// + /// Payload bytes valid until the pump finishes writing this item. Immortal static + /// responses may share the same backing array across requests; the pump only reads. + /// + public ReadOnlyMemory Payload { get; } + + public TaskCompletionSource? Completed { get; } + + /// + /// When , the pump flushes after this fire-and-forget line + /// instead of holding it in a TAKETHIS coalesce batch. + /// + public bool FlushAfter { get; } + } + + /// Initializes a new instance of the class. + public NntpResponseWriter(PipeWriter output) + : this(output, channelCapacity: 4096) + { + } + + /// Test/harness constructor with an explicit bounded Channel capacity. + internal NntpResponseWriter(PipeWriter output, int channelCapacity) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentOutOfRangeException.ThrowIfLessThan(channelCapacity, 1); + _output = output; + var channelOptions = new BoundedChannelOptions(channelCapacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait, + AllowSynchronousContinuations = true, + }; + ChannelAllowsSynchronousContinuations = channelOptions.AllowSynchronousContinuations; + _channel = Channel.CreateBounded(channelOptions); + _pump = Task.Run(() => PumpAsync(_pumpCts.Token), CancellationToken.None); + } + + /// + /// Gets the TX response Channel AllowSynchronousContinuations value applied at construction. + /// + internal bool ChannelAllowsSynchronousContinuations { get; } + + /// + /// Installs the account byte sink. Counting starts after the next successful + /// PipeWriter.Advance, never at Channel enqueue. + /// + public void SetByteSink(IAccountByteSink sink) + { + ArgumentNullException.ThrowIfNull(sink); + Volatile.Write(ref _byteSink, sink); + } + + /// Gets the number of Channel items accepted (diagnostics / tests). + internal long ChannelEnqueueCount => Volatile.Read(ref _channelEnqueueCount); + + /// Gets the existing TX this writer pumps into. Not a second pipe. + internal PipeWriter UnderlyingOutput => _output; + + /// Gets whether SPEEDTEST currently holds exclusive TX PipeWriter ownership. + internal bool DirectPipeExclusive => Volatile.Read(ref _directExclusive) == 1; + + /// Gets calls made on the SPEEDTEST direct-pipe lease. + internal long DirectPipeFlushCount => Volatile.Read(ref _directPipeFlushCount); + + /// Gets the number of calls from the TX pump. + internal long PipeFlushCount => Volatile.Read(ref _pipeFlushCount); + + /// Gets total owned payload bytes accepted into the Channel. + internal long OwnedPayloadBytesEnqueued => Volatile.Read(ref _ownedPayloadBytesEnqueued); + + /// + /// Gets whether any fire-and-forget lines have been accepted and not yet flushed. + /// + internal bool HasCoalescedUnflushed => Volatile.Read(ref _coalescedUnflushed) > 0; + + /// + /// Writes a single-line response (code text), waiting until the ordered pump has + /// flushed it into the outbound pipe (subject to pipe backpressure). + /// + public ValueTask WriteLineAsync(int code, string text, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(code); + ArgumentNullException.ThrowIfNull(text); + return WriteAndAwaitFlushAsync(EncodeLine(code, text), cancellationToken); + } + + /// + /// Writes a complete pre-encoded wire response (including CRLF) and awaits pump flush. + /// + /// + /// Does not copy . The memory must remain unchanged until this + /// write has been flushed into the outbound pipe. Immortal static responses satisfy that + /// for the process lifetime. Dynamically composed responses must be an owned buffer + /// (not Pipe memory, not session scratch). Channel order, coalesce-flush-first, and TCS + /// barrier semantics match . + /// + public ValueTask WriteLineAsync(ReadOnlyMemory wireLine, CancellationToken cancellationToken = default) + { + return WriteAndAwaitFlushAsync(wireLine, cancellationToken); + } + + /// + /// Enqueues a single-line response without waiting for outbound pipe/network flush. + /// + /// + /// Ordering is preserved. Returns when the ordered queue accepts the line (may wait if the + /// response queue is saturated). Used by TAKETHIS so receive can proceed immediately. + /// + public ValueTask EnqueueLineAsync(int code, string text, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(code); + ArgumentNullException.ThrowIfNull(text); + return EnqueueAsync(new WriteRequest(EncodeLine(code, text), completed: null), cancellationToken); + } + + /// + /// Enqueues a complete pre-encoded wire response without waiting for outbound flush. + /// + /// + /// Does not copy . The memory must remain unchanged until the + /// coalesced batch that contains this item is flushed. Dynamically composed responses + /// must be an owned buffer (not Pipe memory, not session scratch). Coalesce/order + /// semantics match . + /// + public ValueTask EnqueueLineAsync(ReadOnlyMemory wireLine, CancellationToken cancellationToken = default) => + EnqueueAsync(new WriteRequest(wireLine, completed: null), cancellationToken); + + /// + /// Enqueues a pre-encoded wire line and flushes it without waiting for a coalesce batch. + /// Returns when the Channel accepts the line (may wait if the writer is saturated). + /// + /// + /// Used by TAKETHIS so 239/439 are not held for + /// siblings. Channel order and pump serialization are unchanged. CHECK still uses + /// . + /// + public ValueTask EnqueueLineImmediateAsync( + ReadOnlyMemory wireLine, + CancellationToken cancellationToken = default) => + EnqueueAsync(new WriteRequest(wireLine, completed: null, flushAfter: true), cancellationToken); + + /// + /// Flushes any held fire-and-forget lines without writing additional response bytes. + /// + /// + /// No-op when the coalesce buffer is empty so MODE READER / WriteLineAsync paths are not + /// delayed. Used when the session is about to wait for more inbound octets, and by tests. + /// + internal ValueTask FlushCoalescedAsync(CancellationToken cancellationToken = default) + { + if (!HasCoalescedUnflushed) + { + return ValueTask.CompletedTask; + } + + return WriteAndAwaitFlushAsync(ReadOnlyMemory.Empty, cancellationToken); + } + + /// Begins a multi-line response and writes the initial status line. + public async ValueTask WriteMultilineStartAsync( + int code, + string text, + CancellationToken cancellationToken = default) + { + await WriteLineAsync(code, text, cancellationToken).ConfigureAwait(false); + } + + /// Writes one multi-line body line (applies dot-stuffing) without terminating the block. + public ValueTask WriteMultilineDataAsync(string line, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(line); + if (line.StartsWith('.')) + { + line = "." + line; + } + + return WriteAndAwaitFlushAsync(EncodeAscii(line + "\r\n"), cancellationToken); + } + + /// Terminates a multi-line response with .CRLF and flushes. + public ValueTask WriteMultilineEndAsync(CancellationToken cancellationToken = default) => + WriteAndAwaitFlushAsync(DotCrlf, cancellationToken); + + /// + /// Writes a precomputed byte payload to the session output pipe and flushes (honouring pipe backpressure). + /// + /// + /// Does not copy . The memory must remain unchanged until this + /// write has been flushed. Intended for BENCHIT reuse of an immutable wire buffer. Still + /// uses the production ordered writer path; does not bypass transport pumps. + /// + public ValueTask WriteBytesAndFlushAsync( + ReadOnlyMemory payload, + CancellationToken cancellationToken = default) => + WriteAndAwaitFlushAsync(payload, cancellationToken); + + /// + /// Writes one framed article through the shared bounded article TX path (default 64 KiB chunks). + /// + /// + /// Destuffed article payload bytes supplied by the caller (no terminating .\r\n); + /// see . Not an NNTPD storage API. + /// + /// Mode-independent wire framing (e.g. 220 ARTICLE, 222 BODY, TAKETHIS). + /// Cancellation token. + /// + /// Materialized restuff → owned chunks → existing ordered Channel. Does not inspect session mode. + /// Awaits each chunk's flush into the outbound Pipe (not peer socket delivery). + /// + public ValueTask WriteArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedArticle, + framing, + NntpArticleTxChunkBudget.DefaultBytes, + cancellationToken); + + /// + /// Writes one framed article through the shared bounded article TX path with an explicit chunk budget. + /// + /// Caller-supplied destuffed article payload bytes. + /// Mode-independent wire framing. + /// Production chunk budget (64–256 KiB). + /// Cancellation token. + public ValueTask WriteArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + NntpArticleTxChunkBudget.ThrowIfNotProductionRange(chunkBytes); + return WriteArticleCoreAsync(storedDestuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// + /// Test-only article TX allowing non-production chunk sizes for boundary coverage. + /// + internal ValueTask WriteArticleForTestsAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + return WriteArticleCoreAsync(storedDestuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// + /// Convenience: ARTICLE-style (220) response from caller-supplied full destuffed article bytes. + /// + /// Full destuffed article (headers + body); external input. + /// Message-id for the 220 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + /// + /// Name reflects RFC 3977 ARTICLE wire shape, not an NNTPD customer repository. + /// + public ValueTask WriteCustomerArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedArticle, + NntpArticleTxFraming.CustomerArticle(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: BODY-style (222) response; strips headers at the first \r\n\r\n. + /// + /// + /// Full destuffed article; headers stripped at the first blank line. + /// When no blank line exists, an empty body is transmitted (still with 222 + terminator). + /// + /// Message-id for the 222 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + /// + /// Same Channel → pump → PipeWriter path as ARTICLE. Does not create a BODY-specific TX stack. + /// Name reflects RFC 3977 BODY wire shape, not storage ownership. + /// + public ValueTask WriteCustomerBodyAsync( + ReadOnlyMemory storedDestuffedArticle, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) + { + _ = ArticleWireReconstructor.TrySplitHeadersAndBody( + storedDestuffedArticle.Span, + out _, + out var bodySpan); + // Copy body span to owned memory before await (span cannot cross await). + var body = bodySpan.IsEmpty ? ReadOnlyMemory.Empty : bodySpan.ToArray(); + return WriteArticleAsync( + body, + NntpArticleTxFraming.CustomerBody(messageId, articleNumber), + cancellationToken); + } + + /// + /// Convenience: BODY-style (222) response from an already-separated destuffed body payload. + /// + /// Destuffed body bytes only (no headers; no terminator). + /// Message-id for the 222 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerBodyFromBodyAsync( + ReadOnlyMemory storedDestuffedBody, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedBody, + NntpArticleTxFraming.CustomerBody(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: HEAD-style (221) response from destuffed header bytes (including blank-line separator when present). + /// + /// Header block only; no body; no terminator. + /// Message-id for the 221 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerHeadAsync( + ReadOnlyMemory storedDestuffedHeaders, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedHeaders, + NntpArticleTxFraming.CustomerHead(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: STAT-style (223) single-line response (no multiline body). + /// + /// Message-id octets including angle brackets. + /// Article number, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerStatAsync( + ReadOnlySpan messageIdBytes, + long articleNumber = 0, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(articleNumber); + var number = articleNumber.ToString(System.Globalization.CultureInfo.InvariantCulture); + var prefix = System.Text.Encoding.ASCII.GetBytes("223 " + number + " "); + var owned = NntpResponseCompose.Concat(prefix, messageIdBytes, NntpResponses.Crlf.Span); + return WriteLineAsync(owned, cancellationToken); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _channel.Writer.TryComplete(); + await _pumpCts.CancelAsync().ConfigureAwait(false); + try + { + await _pump.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on dispose. + } + finally + { + _pumpCts.Dispose(); + } + } + + private async ValueTask WriteArticleCoreAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + var chunks = PrepareArticleChunks(storedDestuffedArticle, framing, chunkBytes); + foreach (var chunk in chunks) + { + // Ownership of `chunk` transfers to the Channel; do not mutate after enqueue. + await WriteAndAwaitFlushAsync(chunk, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Builds owned framed wire chunks (header + restuff) without touching the TX Channel. + /// + /// + /// Used by so preparation can run outside the + /// article-atomic enqueue gate. Caller owns the returned arrays until each is enqueued. + /// + internal static List PrepareArticleChunks( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(storedDestuffedArticle.Span); + var totalLength = header.Length + body.Length; + if (totalLength == 0) + { + return []; + } + + var chunks = new List((totalLength + chunkBytes - 1) / chunkBytes); + var offset = 0; + while (offset < totalLength) + { + var take = Math.Min(chunkBytes, totalLength - offset); + var chunk = new byte[take]; + CopyFramedWire(header, body, offset, chunk.AsSpan()); + chunks.Add(chunk); + offset += take; + } + + return chunks; + } + + /// + /// Enqueues an owned payload into the ordered TX Channel and returns a task that completes + /// when the pump has flushed that item into the outbound Pipe (not peer delivery). + /// + /// + /// Does not await flush before returning. Used with a short article-atomic enqueue gate so all + /// chunks of one article can be admitted contiguously before awaiting backpressure. + /// + internal async ValueTask EnqueueOwnedPayloadAsync( + byte[] payload, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(payload); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await EnqueueAsync(new WriteRequest(payload, completed), cancellationToken).ConfigureAwait(false); + return WaitForFlushAsync(completed, cancellationToken); + } + + private static async Task WaitForFlushAsync(TaskCompletionSource completed, CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => ((TaskCompletionSource)state!).TrySetCanceled(), + completed); + await completed.Task.ConfigureAwait(false); + } + + private static void CopyFramedWire(byte[] header, byte[] body, int absoluteOffset, Span destination) + { + var written = 0; + while (written < destination.Length) + { + if (absoluteOffset < header.Length) + { + var fromHeader = Math.Min(header.Length - absoluteOffset, destination.Length - written); + header.AsSpan(absoluteOffset, fromHeader).CopyTo(destination[written..]); + absoluteOffset += fromHeader; + written += fromHeader; + continue; + } + + var bodyOffset = absoluteOffset - header.Length; + var fromBody = Math.Min(body.Length - bodyOffset, destination.Length - written); + body.AsSpan(bodyOffset, fromBody).CopyTo(destination[written..]); + absoluteOffset += fromBody; + written += fromBody; + } + } + + private async ValueTask WriteAndAwaitFlushAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await EnqueueAsync(new WriteRequest(payload, completed), cancellationToken).ConfigureAwait(false); + using var registration = cancellationToken.Register(static state => + { + ((TaskCompletionSource)state!).TrySetCanceled(); + }, completed); + await completed.Task.ConfigureAwait(false); + } + + /// + /// DIAGNOSTIC-ONLY. Parks Channel/pump production and returns a lease that writes the + /// existing TX . SPEEDTEST is the only caller. + /// + internal async ValueTask AcquireDirectTxPipeAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + await FlushCoalescedAsync(cancellationToken).ConfigureAwait(false); + if (Interlocked.CompareExchange(ref _directExclusive, 1, 0) != 0) + { + throw new InvalidOperationException( + "TX PipeWriter is already exclusively leased; ResponseWriter and SPEEDTEST cannot produce concurrently."); + } + + return new DirectTxPipeLease(this); + } + + internal void ReleaseDirectTxPipe() => Volatile.Write(ref _directExclusive, 0); + + internal async ValueTask WriteDirectToOutputAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + if (Volatile.Read(ref _directExclusive) != 1) + { + throw new InvalidOperationException("Direct TX PipeWriter write requires an exclusive SPEEDTEST lease."); + } + + var remaining = payload; + while (!remaining.IsEmpty) + { + var memory = _output.GetMemory(Math.Min(remaining.Length, 64 * 1024)); + var toCopy = Math.Min(memory.Length, remaining.Length); + remaining.Span[..toCopy].CopyTo(memory.Span); + _output.Advance(toCopy); + Volatile.Read(ref _byteSink).ObserveCopied(toCopy); + remaining = remaining[toCopy..]; + + if (_output.UnflushedBytes >= 64 * 1024) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _directPipeFlushCount); + } + } + + if (_output.UnflushedBytes > 0) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _directPipeFlushCount); + } + } + + private async ValueTask EnqueueAsync(WriteRequest request, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + if (Volatile.Read(ref _directExclusive) == 1) + { + throw new InvalidOperationException( + "TX PipeWriter is exclusively leased to SPEEDTEST; ResponseWriter Channel cannot write concurrently."); + } + + try + { + await _channel.Writer.WriteAsync(request, cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _channelEnqueueCount); + Interlocked.Add(ref _ownedPayloadBytesEnqueued, request.Payload.Length); + if (request.Completed is null) + { + Interlocked.Increment(ref _coalescedUnflushed); + } + } + catch (ChannelClosedException ex) + { + throw new ObjectDisposedException(nameof(NntpResponseWriter), ex); + } + } + + private async Task PumpAsync(CancellationToken cancellationToken) + { + var batch = new WriteRequest[CoalesceResponseBatchSize + 1]; + var count = 0; + var coalesceCount = 0; + + void CancelHeld() + { + var coalesced = 0; + for (var i = 0; i < count; i++) + { + if (batch[i].Completed is null) + { + coalesced++; + } + + batch[i].Completed?.TrySetCanceled(); + } + + if (coalesced > 0) + { + Interlocked.Add(ref _coalescedUnflushed, -coalesced); + } + + count = 0; + coalesceCount = 0; + } + + try + { + await foreach (var request in _channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + if (request.Completed is not null) + { + batch[count++] = request; + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + continue; + } + + batch[count++] = request; + coalesceCount++; + + if (request.FlushAfter) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + continue; + } + + while (coalesceCount < CoalesceResponseBatchSize && + _channel.Reader.TryRead(out var queued)) + { + if (queued.Completed is not null || queued.FlushAfter) + { + batch[count++] = queued; + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + break; + } + + batch[count++] = queued; + coalesceCount++; + } + + if (coalesceCount >= CoalesceResponseBatchSize) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + for (var i = 0; i < count; i++) + { + batch[i].Completed?.TrySetException(ex); + } + + count = 0; + coalesceCount = 0; + if (cancellationToken.IsCancellationRequested) + { + break; + } + } + } + + if (count > 0 && !cancellationToken.IsCancellationRequested) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Dispose path. + } + finally + { + CancelHeld(); + while (_channel.Reader.TryRead(out var leftover)) + { + if (leftover.Completed is null) + { + Interlocked.Decrement(ref _coalescedUnflushed); + } + + leftover.Completed?.TrySetCanceled(); + } + } + } + + private async ValueTask FlushHeldAsync( + WriteRequest[] batch, + int count, + CancellationToken cancellationToken) + { + var coalesced = 0; + for (var i = 0; i < count; i++) + { + if (batch[i].Completed is null) + { + coalesced++; + } + } + + try + { + for (var i = 0; i < count; i++) + { + await CopyPayloadToPipeAsync(batch[i].Payload, cancellationToken).ConfigureAwait(false); + } + + if (_output.UnflushedBytes > 0) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + } + + for (var i = 0; i < count; i++) + { + batch[i].Completed?.TrySetResult(); + } + } + finally + { + if (coalesced > 0) + { + Interlocked.Add(ref _coalescedUnflushed, -coalesced); + } + } + } + + private async ValueTask CopyPayloadToPipeAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + var remaining = payload; + while (!remaining.IsEmpty) + { + var memory = _output.GetMemory(Math.Min(remaining.Length, 64 * 1024)); + var toCopy = Math.Min(memory.Length, remaining.Length); + remaining.Span[..toCopy].CopyTo(memory.Span); + _output.Advance(toCopy); + Volatile.Read(ref _byteSink).ObserveCopied(toCopy); + remaining = remaining[toCopy..]; + + if (_output.UnflushedBytes >= 64 * 1024) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + private async ValueTask FlushOutputAsync(CancellationToken cancellationToken) + { + var flush = _output.FlushAsync(cancellationToken); + var result = flush.IsCompletedSuccessfully + ? flush.Result + : await flush.ConfigureAwait(false); + + Interlocked.Increment(ref _pipeFlushCount); + + if (result.IsCanceled) + { + cancellationToken.ThrowIfCancellationRequested(); + } + + if (result.IsCompleted) + { + throw new InvalidOperationException("NNTP output pipe completed while writing."); + } + } + + private static byte[] EncodeLine(int code, string text) => + EncodeAscii($"{code} {text}\r\n"); + + private static byte[] EncodeAscii(string text) + { + var byteCount = Encoding.ASCII.GetByteCount(text); + var bytes = new byte[byteCount]; + Encoding.ASCII.GetBytes(text, bytes); + return bytes; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponses.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponses.cs new file mode 100644 index 00000000..0adb71ab --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpResponses.cs @@ -0,0 +1,440 @@ +using System.Text; + +using VectorNNTPD.ReaderService.Session.Commands; + +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Immortal pre-encoded NNTP wire responses. Each value is ASCII, CRLF-terminated, and + /// encoded once at type initialization. The TX pump may read the same backing array + /// repeatedly; callers must not mutate it. + /// + /// + /// Invariant replies are complete wire buffers. CHECK/TAKETHIS/DATE use the prefix/suffix + /// fields with (one owned copy of dynamic octets). + /// + internal static class NntpResponses + { + internal static readonly ReadOnlyMemory GreetingPostingPermitted = + Line("200 VectorNNTPD.ReaderService ready, posting permitted\r\n"u8); + + internal static readonly ReadOnlyMemory GreetingPostingProhibited = + Line("201 VectorNNTPD.ReaderService ready, posting prohibited\r\n"u8); + + internal static readonly ReadOnlyMemory ReaderModePostingPermitted = + Line("200 Reader mode, posting permitted\r\n"u8); + + internal static readonly ReadOnlyMemory ReaderModePostingProhibited = + Line("201 Reader mode, posting prohibited\r\n"u8); + + internal static readonly ReadOnlyMemory StreamingPermitted = + Line("203 Streaming permitted\r\n"u8); + + internal static readonly ReadOnlyMemory ConnectionClosing = + Line("205 Connection closing\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionActive = + Line("206 Compression active\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationAccepted = + Line("281 Authentication accepted\r\n"u8); + + internal static readonly ReadOnlyMemory PasswordRequired = + Line("381 Password required\r\n"u8); + + internal static readonly ReadOnlyMemory ContinueWithTls = + Line("382 Continue with TLS negotiation\r\n"u8); + + internal static readonly ReadOnlyMemory ServiceTemporarilyUnavailable = + Line("400 Service temporarily unavailable\r\n"u8); + + internal static readonly ReadOnlyMemory CommandFailed = + Line("403 Command failed\r\n"u8); + + internal static readonly ReadOnlyMemory UnableToActivateCompression = + Line("403 Unable to activate compression\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationRequired = + Line("480 Authentication required\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationFailed = + Line("481 Authentication failed\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationCancelled = + Line("481 Authentication cancelled\r\n"u8); + + internal static readonly ReadOnlyMemory TooManySessions = + Line("481 Too many sessions\r\n"u8); + + internal static readonly ReadOnlyMemory TooManySourceAddresses = + Line("481 Too many source addresses\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationOutOfSequence = + Line("482 Authentication commands issued out of sequence\r\n"u8); + + internal static readonly ReadOnlyMemory PrivacyRequired = + Line("483 Encryption or stronger authentication required\r\n"u8); + + internal static readonly ReadOnlyMemory UnknownCommand = + Line("500 Unknown command\r\n"u8); + + internal static readonly ReadOnlyMemory CommandNotImplemented = + Line("500 Command not implemented\r\n"u8); + + internal static readonly ReadOnlyMemory SyntaxError = + Line("501 Syntax error\r\n"u8); + + internal static readonly ReadOnlyMemory UnknownCommandVariant = + Line("501 Unknown command variant\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoUserRequiresUsername = + Line("501 AUTHINFO USER requires a username\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoPassRequiresPassword = + Line("501 AUTHINFO PASS requires a password\r\n"u8); + + internal static readonly ReadOnlyMemory CompressRequiresAlgorithm = + Line("501 COMPRESS requires a single algorithm argument\r\n"u8); + + internal static readonly ReadOnlyMemory CompressAlgorithmSyntaxInvalid = + Line("501 Syntactically incorrect compression algorithm\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoSaslNotImplemented = + Line("501 AUTHINFO SASL not implemented\r\n"u8); + + internal static readonly ReadOnlyMemory SaslMechanismRequired = + Line("501 SASL mechanism required\r\n"u8); + + internal static readonly ReadOnlyMemory SaslInvalidBase64 = + Line("504 Base64 encoding error\r\n"u8); + + internal static readonly ReadOnlyMemory SaslLoginUsernameChallenge = + Line("383 VXNlcm5hbWU6\r\n"u8); + + internal static readonly ReadOnlyMemory SaslLoginPasswordChallenge = + Line("383 UGFzc3dvcmQ6\r\n"u8); + + internal static readonly ReadOnlyMemory SaslEmptyChallenge = + Line("383 =\r\n"u8); + + internal static readonly ReadOnlyMemory SaslProtocolError = + Line("482 SASL protocol error\r\n"u8); + + internal static readonly ReadOnlyMemory SaslMechanismNotSupported = + Line("503 Mechanism not supported\r\n"u8); + + internal static readonly ReadOnlyMemory SaslContinuationNotSupported = + Line("503 SASL continuation not supported\r\n"u8); + + internal static readonly ReadOnlyMemory NoSaslExchangeInProgress = + Line("503 No SASL exchange in progress\r\n"u8); + + internal static readonly ReadOnlyMemory TemporaryAuthenticationFailure = + Line("503 Temporary authentication failure\r\n"u8); + + internal static readonly ReadOnlyMemory SaslScramUnavailable = + Line("503 SCRAM not available\r\n"u8); + + internal static readonly ReadOnlyMemory SaslCramUnavailable = + Line("503 CRAM-MD5 not available\r\n"u8); + + internal static readonly ReadOnlyMemory AlreadyAuthenticated = + Line("502 Already authenticated\r\n"u8); + + internal static readonly ReadOnlyMemory CommandUnavailableAfterAuthentication = + Line("502 Command unavailable after authentication\r\n"u8); + + internal static readonly ReadOnlyMemory CommandUnavailableAfterCompression = + Line("502 Command unavailable after compression\r\n"u8); + + internal static readonly ReadOnlyMemory DeflateAlreadyActive = + Line("502 DEFLATE compression already active\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionAlreadyActive = + Line("502 Compression already active\r\n"u8); + + internal static readonly ReadOnlyMemory TlsAlreadyActive = + Line("502 TLS already active\r\n"u8); + + internal static readonly ReadOnlyMemory TlsProviderUnavailable = + Line("502 TLS provider unavailable\r\n"u8); + + internal static readonly ReadOnlyMemory PostingNotPermitted = + Line("502 Posting not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory PermissionDenied = + Line("502 Permission denied\r\n"u8); + + internal static readonly ReadOnlyMemory CommandNotAvailable = + Line("502 Command not available on this server\r\n"u8); + + internal static readonly ReadOnlyMemory StreamingNotPermitted = + Line("502 Streaming not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory NotInReaderMode = + Line("502 Not in reader mode\r\n"u8); + + internal static readonly ReadOnlyMemory NotInStreamMode = + Line("502 Not in stream mode\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionAlgorithmNotSupported = + Line("503 Compression algorithm not supported\r\n"u8); + + /// RFC 3977 §7.6.1.2: recognized LIST keyword whose information is not maintained. + internal static readonly ReadOnlyMemory ListDataItemNotStored = + Line("503 Data item not stored\r\n"u8); + + internal static readonly ReadOnlyMemory HelpTextFollows = + Line("100 Help text follows\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityListFollows = + Line("101 Capability list:\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityVersion2 = + Multiline("VERSION 2\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityImplementation = + Multiline("IMPLEMENTATION VectorNNTPD.ReaderService\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityReader = + Multiline("READER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityModeReader = + Multiline("MODE-READER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityPost = + Multiline("POST\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfo = + Multiline("AUTHINFO\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfoUser = + Multiline("AUTHINFO USER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfoUserSasl = + Multiline("AUTHINFO USER SASL\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilitySasl = + Multiline("SASL PLAIN LOGIN SCRAM-SHA-256 CRAM-MD5\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityStartTls = + Multiline("STARTTLS\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityCompressDeflate = + Multiline("COMPRESS DEFLATE\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityStreaming = + Multiline("STREAMING\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilitySpeedTest = + Multiline("SPEEDTEST\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityList = + Multiline("LIST ACTIVE COUNTS HEADERS NEWSGROUPS OVERVIEW.FMT\r\n"u8); + + internal static readonly ReadOnlyMemory ListOfNewsgroupsFollows = + Line("215 list of newsgroups follows\r\n"u8); + + internal static readonly ReadOnlyMemory ListOverviewFmtFollows = + Line("215 Order of fields in overview database.\r\n"u8); + + internal static readonly ReadOnlyMemory ListHeadersFollows = + Line("215 headers and metadata items supported:\r\n"u8); + + internal static readonly ReadOnlyMemory NoSuchNewsgroup = + Line("411 No such newsgroup\r\n"u8); + + internal static readonly ReadOnlyMemory NoNewsgroupSelected = + Line("412 No newsgroup selected\r\n"u8); + + internal static readonly ReadOnlyMemory CurrentArticleNumberInvalid = + Line("420 Current article number is invalid\r\n"u8); + + internal static readonly ReadOnlyMemory NoArticleWithNumber = + Line("423 No article with that number\r\n"u8); + + internal static readonly ReadOnlyMemory NoArticleWithMessageId = + Line("430 No article with that message-id\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestUnknownPeer = + Line("502 UNKNOWN SPEEDTEST PEER\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestPeerMismatch = + Line("502 SPEEDTEST PEER MISMATCH\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestBusy = + Line("400 SPEEDTEST BUSY\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestNotSupported = + Line("503 SPEEDTEST NOT SUPPORTED\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestReadyPrefix = Prefix("290 SPEEDTEST "u8); + + internal static readonly ReadOnlyMemory SpeedTestReadySuffix = Suffix(" TX\r\n"u8); + + internal static readonly ReadOnlyMemory MultilineTerminator = + Line(".\r\n"u8); + + internal static readonly ReadOnlyMemory CheckPrefix = Prefix("238 "u8); + + internal static readonly ReadOnlyMemory CheckSuffix = + Suffix(" send article to be transferred\r\n"u8); + + internal static readonly ReadOnlyMemory CheckNotWantedPrefix = Prefix("438 "u8); + + internal static readonly ReadOnlyMemory CheckNotWantedSuffix = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory CheckTryLaterPrefix = Prefix("431 "u8); + + internal static readonly ReadOnlyMemory CheckTryLaterSuffix = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveSendArticle = + Line("335 Send article to be transferred\r\n"u8); + + internal static readonly ReadOnlyMemory ArticleReceivedOk = + Line("240 Article received OK\r\n"u8); + + internal static readonly ReadOnlyMemory PostSendArticle = + Line("340 Input article; end with .\r\n"u8); + + internal static readonly ReadOnlyMemory PostingProhibited = + Line("440 Posting not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory PostingFailed = + Line("441 Posting failed\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTransferredOk = + Line("235 Article transferred OK\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveNotWanted = + Line("435 Article not wanted\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTryLater = + Line("436 Transfer not possible; try again later\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTransferFailed = + Line("436 Transfer failed; try again later\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveRejected = + Line("437 Transfer rejected; do not retry\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityIhave = + Multiline("IHAVE\r\n"u8); + + internal static readonly ReadOnlyMemory ArticleTransferredOkPrefix = Prefix("239 "u8); + + internal static readonly ReadOnlyMemory TransferRejectedPrefix = Prefix("439 "u8); + + internal static readonly ReadOnlyMemory Crlf = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory DatePrefix = Prefix("111 "u8); + + internal static readonly ReadOnlyMemory[] HelpBodyLines = EncodeHelpBody(); + + /// Complete HELP multiline wire (status + body + terminator). Immortal. + internal static readonly ReadOnlyMemory HelpComplete = EncodeHelpComplete(); + + /// Complete LIST OVERVIEW.FMT multiline wire. Immortal; RFC 3977 §8.4 compatibility form. + internal static readonly ReadOnlyMemory ListOverviewFmtComplete = EncodeListOverviewFmtComplete(); + + /// Complete LIST HEADERS multiline wire. Immortal; same for MSGID and RANGE. + internal static readonly ReadOnlyMemory ListHeadersComplete = EncodeListHeadersComplete(); + + /// + /// Copies onto an immortal heap array once. + /// The UTF-8 literal span must not be retained. + /// + private static ReadOnlyMemory Line(ReadOnlySpan asciiIncludingCrlf) + { + if (asciiIncludingCrlf.Length < 2 + || asciiIncludingCrlf[^2] != (byte)'\r' + || asciiIncludingCrlf[^1] != (byte)'\n') + { + throw new ArgumentException("Static NNTP responses must end with a single CRLF."); + } + + var copy = asciiIncludingCrlf.ToArray(); + if (CrlfCount(copy) != 1) + { + throw new ArgumentException("Static single-line NNTP responses must contain exactly one CRLF."); + } + + return copy; + } + + private static ReadOnlyMemory Multiline(ReadOnlySpan asciiIncludingCrlf) => + Line(asciiIncludingCrlf); + + private static ReadOnlyMemory[] EncodeHelpBody() + { + var source = Help.BodyLines; + var encoded = new ReadOnlyMemory[source.Count]; + for (var i = 0; i < source.Count; i++) + { + encoded[i] = Line(Encoding.ASCII.GetBytes(source[i] + "\r\n")); + } + + return encoded; + } + + private static ReadOnlyMemory EncodeHelpComplete() + { + var parts = new ReadOnlyMemory[HelpBodyLines.Length + 2]; + parts[0] = HelpTextFollows; + HelpBodyLines.CopyTo(parts, 1); + parts[^1] = MultilineTerminator; + return NntpResponseCompose.Concatenate(parts); + } + + private static ReadOnlyMemory EncodeListOverviewFmtComplete() => + EncodeStaticMultiline( + ListOverviewFmtFollows, + Multiline("Subject:\r\n"u8), + Multiline("From:\r\n"u8), + Multiline("Date:\r\n"u8), + Multiline("Message-ID:\r\n"u8), + Multiline("References:\r\n"u8), + Multiline("Bytes:\r\n"u8), + Multiline("Lines:\r\n"u8)); + + private static ReadOnlyMemory EncodeListHeadersComplete() => + EncodeStaticMultiline( + ListHeadersFollows, + Multiline("Subject\r\n"u8), + Multiline("From\r\n"u8), + Multiline("Date\r\n"u8), + Multiline("Message-ID\r\n"u8), + Multiline("References\r\n"u8), + Multiline(":bytes\r\n"u8), + Multiline(":lines\r\n"u8)); + + private static ReadOnlyMemory EncodeStaticMultiline( + ReadOnlyMemory status, + params ReadOnlyMemory[] bodyLines) + { + var parts = new ReadOnlyMemory[bodyLines.Length + 2]; + parts[0] = status; + bodyLines.CopyTo(parts, 1); + parts[^1] = MultilineTerminator; + return NntpResponseCompose.Concatenate(parts); + } + + private static ReadOnlyMemory Prefix(ReadOnlySpan ascii) => ascii.ToArray(); + + private static ReadOnlyMemory Suffix(ReadOnlySpan ascii) => ascii.ToArray(); + + private static int CrlfCount(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs new file mode 100644 index 00000000..b93651de --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs @@ -0,0 +1,263 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Bounded STREAM outstanding article TX scheduler above the shared + /// TX path. + /// + /// + /// + /// Limits how many STREAM article operations may be admitted concurrently (default 8; valid 4–16). + /// Does not replace TX Channel capacity, Pipe thresholds, or transport backpressure. + /// + /// + /// Producer model (Phase 3B; evidence: + /// docs/STREAM-TX-PRODUCER-CONCURRENCY-EXPERIMENT.md): + /// + /// + /// Acquire STREAM depth slot. + /// Prepare framing/restuff/owned chunks outside the enqueue gate. + /// Acquire short FIFO enqueue gate; admit all chunks of this article + /// contiguously into the existing Channel; release gate. + /// Await existing per-chunk Pipe flush completion (backpressure) outside the gate. + /// Release depth slot in finally. + /// + /// + /// Callers must start operations in protocol order. The enqueue gate is FIFO among waiters and + /// prevents chunk interleaving (A1 B1 A2…). There is no second TX Channel or pump. + /// + /// + /// This type does not load or look up articles. Callers supply destuffed bytes (bench, peer feed, + /// or other external source). Article storage/catalogue is out of scope for NNTPD. + /// + /// + public sealed class NntpStreamArticleTxScheduler : IAsyncDisposable + { + /// Default STREAM outstanding article depth. + public const int DefaultDepth = 8; + + /// Minimum allowed STREAM outstanding article depth. + public const int MinDepth = 4; + + /// Maximum allowed STREAM outstanding article depth. + public const int MaxDepth = 16; + + private readonly SemaphoreSlim _depth; + private readonly SemaphoreSlim _enqueueGate = new(1, 1); + private int _outstanding; + private int _preparing; + private int _enqueuing; + private int _peakPreparing; + private int _disposed; + + /// Initializes a scheduler with . + public NntpStreamArticleTxScheduler() + : this(DefaultDepth) + { + } + + /// Initializes a scheduler with an explicit depth. + /// Outstanding article depth; must be in –. + public NntpStreamArticleTxScheduler(int depth) + { + ThrowIfNotValidDepth(depth); + Depth = depth; + _depth = new SemaphoreSlim(depth, depth); + } + + /// Gets the configured outstanding depth. + public int Depth { get; } + + /// Gets the number of currently admitted (not yet completed) operations. + public int Outstanding => Volatile.Read(ref _outstanding); + + /// Gets the peak number of concurrent preparation phases observed (diagnostics / tests). + internal int PeakPreparing => Volatile.Read(ref _peakPreparing); + + /// Gets the number of operations currently holding the enqueue gate (0 or 1). + internal int Enqueuing => Volatile.Read(ref _enqueuing); + + /// Test hook: invoked after preparation, before the enqueue gate (deterministic cancel tests). + internal Func? TestAfterPrepare { get; set; } + + /// Test hook: invoked while holding the enqueue gate, before chunk admission. + internal Func? TestWhileHoldingEnqueueGate { get; set; } + + /// Test hook: invoked after releasing the enqueue gate, before awaiting Pipe flush. + internal Func? TestBeforeAwaitFlush { get; set; } + + /// Returns when is in the valid range. + public static bool IsValidDepth(int depth) => depth is >= MinDepth and <= MaxDepth; + + /// Throws if is outside –. + public static void ThrowIfNotValidDepth(int depth) + { + if (!IsValidDepth(depth)) + { + throw new ArgumentOutOfRangeException( + nameof(depth), + depth, + $"Stream outstanding article depth must be between {MinDepth} and {MaxDepth}."); + } + } + + /// + /// Acquires one outstanding slot, runs , then releases the slot + /// (including on cancellation or fault). + /// + public async ValueTask ExecuteAsync( + Func operation, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + await _depth.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _outstanding); + try + { + await operation(cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _outstanding); + _depth.Release(); + } + } + + /// + /// Writes one framed article: prepare outside enqueue gate, admit all chunks under the gate, + /// await Pipe flush outside the gate. + /// + public ValueTask WriteArticleAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + writer, + destuffedArticle, + framing, + NntpArticleTxChunkBudget.DefaultBytes, + cancellationToken); + + /// + /// Writes one framed article with an explicit production chunk budget. + /// + public ValueTask WriteArticleAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(writer); + NntpArticleTxChunkBudget.ThrowIfNotProductionRange(chunkBytes); + return WriteArticleCoreAsync(writer, destuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// Test-only write allowing non-production chunk sizes. + internal ValueTask WriteArticleForTestsAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + return WriteArticleCoreAsync(writer, destuffedArticle, framing, chunkBytes, cancellationToken); + } + + private ValueTask WriteArticleCoreAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken) + { + return ExecuteAsync( + async token => + { + // 1) PREPARATION — outside enqueue gate (bounded by depth). + var preparing = Interlocked.Increment(ref _preparing); + UpdatePeakPreparing(preparing); + List chunks; + try + { + chunks = NntpResponseWriter.PrepareArticleChunks(destuffedArticle, framing, chunkBytes); + if (TestAfterPrepare is not null) + { + await TestAfterPrepare(token).ConfigureAwait(false); + } + } + finally + { + Interlocked.Decrement(ref _preparing); + } + + if (chunks.Count == 0) + { + return; + } + + // 2) ARTICLE-ATOMIC ENQUEUE — all chunks contiguous; gate may wait on Channel. + var flushTasks = new Task[chunks.Count]; + await _enqueueGate.WaitAsync(token).ConfigureAwait(false); + Interlocked.Increment(ref _enqueuing); + try + { + if (TestWhileHoldingEnqueueGate is not null) + { + await TestWhileHoldingEnqueueGate(token).ConfigureAwait(false); + } + + for (var i = 0; i < chunks.Count; i++) + { + flushTasks[i] = await writer + .EnqueueOwnedPayloadAsync(chunks[i], token) + .ConfigureAwait(false); + } + } + finally + { + Interlocked.Decrement(ref _enqueuing); + _enqueueGate.Release(); + } + + // 3) PIPE / TX BACKPRESSURE — outside enqueue gate. + if (TestBeforeAwaitFlush is not null) + { + await TestBeforeAwaitFlush(token).ConfigureAwait(false); + } + + await Task.WhenAll(flushTasks).ConfigureAwait(false); + }, + cancellationToken); + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return ValueTask.CompletedTask; + } + + _depth.Dispose(); + _enqueueGate.Dispose(); + return ValueTask.CompletedTask; + } + + private void UpdatePeakPreparing(int current) + { + int snap; + while (current > (snap = Volatile.Read(ref _peakPreparing))) + { + if (Interlocked.CompareExchange(ref _peakPreparing, current, snap) == snap) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerb.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerb.cs new file mode 100644 index 00000000..8c64505e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerb.cs @@ -0,0 +1,122 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Canonical NNTP verb or qualifier identity. Built-in dispatch uses this enum, not strings. + /// + public enum NntpVerb : byte + { + /// No verb (empty line). + None = 0, + + /// Unrecognized verb or qualifier. + Unknown, + + /// QUIT. + Quit, + + /// HELP. + Help, + + /// DATE. + Date, + + /// CAPABILITIES. + Capabilities, + + /// MODE. + Mode, + + /// STARTTLS. + StartTls, + + /// COMPRESS. + Compress, + + /// LIST. + List, + + /// LISTGROUP. + ListGroup, + + /// HDR. + Hdr, + + /// OVER. + Over, + + /// GROUP. + Group, + + /// ARTICLE. + Article, + + /// HEAD. + Head, + + /// BODY. + Body, + + /// STAT. + Stat, + + /// NEXT. + Next, + + /// LAST. + Last, + + /// POST. + Post, + + /// CHECK. + Check, + + /// IHAVE. + Ihave, + + /// TAKETHIS. + TakeThis, + + /// AUTHINFO. + AuthInfo, + + /// Internal BENCHIT facility. + BenchIt, + + /// VectorNNTP SPEEDTEST diagnostic extension. + SpeedTest, + + /// MODE READER qualifier. + Reader, + + /// MODE STREAM qualifier. + Stream, + + /// AUTHINFO USER qualifier. + User, + + /// AUTHINFO PASS qualifier. + Pass, + + /// AUTHINFO SASL qualifier. + Sasl, + + /// LIST ACTIVE qualifier. + Active, + + /// LIST COUNTS qualifier. + Counts, + + /// LIST HEADERS qualifier. + Headers, + + /// LIST MOTD qualifier. + Motd, + + /// LIST NEWSGROUPS qualifier. + Newsgroups, + + /// LIST OVERVIEW.FMT qualifier. + OverviewFmt, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerbClassifier.cs b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerbClassifier.cs new file mode 100644 index 00000000..15fee3c8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/CommandProcessor/NntpVerbClassifier.cs @@ -0,0 +1,109 @@ +namespace VectorNNTPD.ReaderService.Session.CommandProcessor +{ + /// + /// Zero-allocation verb classifier: length, then first byte, then remaining ASCII-folded compare. + /// + internal static class NntpVerbClassifier + { + public static NntpVerb Classify(ReadOnlySpan line, out int verbStart, out int verbEnd) + { + verbStart = NntpAscii.SkipWhitespace(line, 0); + if (verbStart >= line.Length) + { + verbEnd = verbStart; + return NntpVerb.None; + } + + verbEnd = NntpAscii.SkipToken(line, verbStart); + return ClassifyVerb(line.Slice(verbStart, verbEnd - verbStart)); + } + + private static NntpVerb ClassifyVerb(ReadOnlySpan verb) + { + if (verb.IsEmpty) + { + return NntpVerb.Unknown; + } + + byte first = NntpAscii.FoldUpper(verb[0]); + return verb.Length switch + { + 3 => first == (byte)'H' && NntpAscii.EqualsFolded(verb, "HDR"u8) ? NntpVerb.Hdr : NntpVerb.Unknown, + 4 => ClassifyLen4(verb, first), + 5 => ClassifyLen5(verb, first), + 7 => ClassifyLen7(verb, first), + 8 => ClassifyLen8(verb, first), + 9 => ClassifyLen9(verb, first), + 12 => first == (byte)'C' && NntpAscii.EqualsFolded(verb, "CAPABILITIES"u8) + ? NntpVerb.Capabilities + : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen4(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'B' => NntpAscii.EqualsFolded(verb, "BODY"u8) ? NntpVerb.Body : NntpVerb.Unknown, + (byte)'D' => NntpAscii.EqualsFolded(verb, "DATE"u8) ? NntpVerb.Date : NntpVerb.Unknown, + (byte)'H' => NntpAscii.EqualsFolded(verb, "HEAD"u8) ? NntpVerb.Head + : NntpAscii.EqualsFolded(verb, "HELP"u8) ? NntpVerb.Help + : NntpVerb.Unknown, + (byte)'L' => NntpAscii.EqualsFolded(verb, "LIST"u8) ? NntpVerb.List + : NntpAscii.EqualsFolded(verb, "LAST"u8) ? NntpVerb.Last + : NntpVerb.Unknown, + (byte)'M' => NntpAscii.EqualsFolded(verb, "MODE"u8) ? NntpVerb.Mode : NntpVerb.Unknown, + (byte)'N' => NntpAscii.EqualsFolded(verb, "NEXT"u8) ? NntpVerb.Next : NntpVerb.Unknown, + (byte)'O' => NntpAscii.EqualsFolded(verb, "OVER"u8) ? NntpVerb.Over : NntpVerb.Unknown, + (byte)'P' => NntpAscii.EqualsFolded(verb, "POST"u8) ? NntpVerb.Post : NntpVerb.Unknown, + (byte)'Q' => NntpAscii.EqualsFolded(verb, "QUIT"u8) ? NntpVerb.Quit : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "STAT"u8) ? NntpVerb.Stat : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen5(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'C' => NntpAscii.EqualsFolded(verb, "CHECK"u8) ? NntpVerb.Check : NntpVerb.Unknown, + (byte)'G' => NntpAscii.EqualsFolded(verb, "GROUP"u8) ? NntpVerb.Group : NntpVerb.Unknown, + (byte)'I' => NntpAscii.EqualsFolded(verb, "IHAVE"u8) ? NntpVerb.Ihave : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen7(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'A' => NntpAscii.EqualsFolded(verb, "ARTICLE"u8) ? NntpVerb.Article : NntpVerb.Unknown, + (byte)'B' => NntpAscii.EqualsFolded(verb, "BENCHIT"u8) ? NntpVerb.BenchIt : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen8(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'A' => NntpAscii.EqualsFolded(verb, "AUTHINFO"u8) ? NntpVerb.AuthInfo : NntpVerb.Unknown, + (byte)'C' => NntpAscii.EqualsFolded(verb, "COMPRESS"u8) ? NntpVerb.Compress : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "STARTTLS"u8) ? NntpVerb.StartTls : NntpVerb.Unknown, + (byte)'T' => NntpAscii.EqualsFolded(verb, "TAKETHIS"u8) ? NntpVerb.TakeThis : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen9(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'L' => NntpAscii.EqualsFolded(verb, "LISTGROUP"u8) ? NntpVerb.ListGroup : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "SPEEDTEST"u8) ? NntpVerb.SpeedTest : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Article.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Article.cs new file mode 100644 index 00000000..55d28ae4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Article.cs @@ -0,0 +1,796 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// ARTICLE, HEAD, BODY, and STAT as defined by RFC 3977, Sections 6.2.1–6.2.4. + /// + /// + /// + /// Message-id form asks first. A positive + /// StorageServer response is fetched with the existing VATP client + /// (). Only a completed lookup with no positive + /// response calls ArticleWork. When has no + /// entries, that client does not publish a StorageServer request and the same miss + /// continues to ArticleWork. An empty registry is not an article-not-found result. + /// Transport, malformed, and invalid StorageServer results stay on that path. A validated CanonicalV1 + /// is required before any 220/221/222/223 + /// reply. A StorageServer success is served without ingestion. A BackFiller success + /// admits a Path traversal whose current hop is this NNTPD, and the reply sends + /// that record. A StorageServer success traverses once on ingress and the reply + /// sends that record. A failed traversal still sends the fetched bytes. + /// + /// + /// Numeric article lookup and omitted current-article forms are unchanged: + /// message-id unavailable → 430, article number → 412/423, + /// omitted current article → 412/420. Temporary transfer failures use + /// 400 without closing the connection. Error replies are single-line. + /// + /// + /// An unsuccessful lookup MUST NOT change the selected group or current article + /// (RFC 3977 §6.2.1.2). There is no current-article pointer yet, so the omitted form + /// after a successful GROUP is 420. + /// + /// + internal static class Article + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Article)); + + private enum RetrievalKind + { + Article, + Head, + Body, + Stat, + } + + /// Handles ARTICLE (RFC 3977, Section 6.2.1). + public static ValueTask HandleArticleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "ARTICLE", static (c, ct) => ExecuteAsync(c, RetrievalKind.Article, ct), cancellationToken); + + /// Handles HEAD (RFC 3977, Section 6.2.2). + public static ValueTask HandleHeadAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "HEAD", static (c, ct) => ExecuteAsync(c, RetrievalKind.Head, ct), cancellationToken); + + /// Handles BODY (RFC 3977, Section 6.2.3). + public static ValueTask HandleBodyAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "BODY", static (c, ct) => ExecuteAsync(c, RetrievalKind.Body, ct), cancellationToken); + + /// Handles STAT (RFC 3977, Section 6.2.4). + public static ValueTask HandleStatAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "STAT", static (c, ct) => ExecuteAsync(c, RetrievalKind.Stat, ct), cancellationToken); + + private static async ValueTask ExecuteAsync( + NntpCommandContext context, + RetrievalKind kind, + CancellationToken cancellationToken) + { + var argument = context.ArgumentSpan; + if (argument.IsEmpty) + { + await WriteCurrentArticleFailureAsync(context, cancellationToken).ConfigureAwait(false); + return; + } + + if (argument[0] == (byte)'<') + { + await ExecuteMessageIdAsync(context, kind, cancellationToken).ConfigureAwait(false); + return; + } + + await WriteNumberLookupFailureAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ExecuteMessageIdAsync( + NntpCommandContext context, + RetrievalKind kind, + CancellationToken cancellationToken) + { + var resolved = await ResolveByMessageIdAsync(context, cancellationToken).ConfigureAwait(false); + switch (resolved.Kind) + { + case ResolveKind.NotFound: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoArticleWithMessageId, + NntpResponseStatus.NoArticleWithMessageId, + cancellationToken).ConfigureAwait(false); + return; + case ResolveKind.TemporaryFailure: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + case ResolveKind.Success: + break; + default: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + } + + var record = resolved.Record; + if (resolved.Source == RetrievalSource.BackFiller + && TryAdmitBackFiller(context, in record, out var admitted)) + { + record = admitted; + } + else if (resolved.Source == RetrievalSource.StorageServer + && TryTraverseStorageIngress(context, in record, out var entered)) + { + record = entered; + } + + var messageIdText = Encoding.ASCII.GetString(record.MessageId); + switch (kind) + { + case RetrievalKind.Article: + await context.Response.WriteCustomerArticleAsync( + record.ArtData, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "220 0 " + messageIdText); + return; + case RetrievalKind.Head: + if (!ArticleWireReconstructor.TrySplitHeadersAndBody(record.ArtData.Span, out var headersSpan, out _)) + { + headersSpan = record.ArtData.Span; + } + + var headers = record.ArtData[..headersSpan.Length]; + await context.Response.WriteCustomerHeadAsync( + headers, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "221 0 " + messageIdText); + return; + case RetrievalKind.Body: + _ = ArticleWireReconstructor.TrySplitHeadersAndBody(record.ArtData.Span, out var headerSpan, out var bodySpan); + var body = bodySpan.IsEmpty + ? ReadOnlyMemory.Empty + : record.ArtData.Slice(headerSpan.Length, bodySpan.Length); + await context.Response.WriteCustomerBodyFromBodyAsync( + body, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "222 0 " + messageIdText); + return; + case RetrievalKind.Stat: + await context.Response.WriteCustomerStatAsync(record.MessageId, articleNumber: 0, cancellationToken) + .ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "223 0 " + messageIdText); + return; + default: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + } + } + + private static async ValueTask ResolveByMessageIdAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var storage = context.Session.StorageArticleLookup; + if (storage is null) + { + return ResolveResult.NotFound(); + } + + var articleId = ArticleId.FromMessageId(messageId.Span); + var logger = Logger; + StorageArticleLookupResult fleet; + try + { + fleet = await storage.LookupAsync(articleId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "unavailable", + articleIdText, + Guid.Empty); + } + return ResolveResult.Temporary(); + } + + if (fleet.Outcome == StorageArticleLookupOutcome.Found) + { + return await FetchPositiveStorageAsync(context, messageId, articleId, fleet, cancellationToken) + .ConfigureAwait(false); + } + + var silence = string.Equals(fleet.Error, StorageLookupSilence, StringComparison.Ordinal); + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + silence ? "miss" : "unavailable", + articleIdText, + fleet.RequestId); + } + if (!silence) + { + return ResolveResult.Temporary(); + } + + return await ResolveFromArticleWorkAsync(context, messageId, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ResolveFromArticleWorkAsync( + NntpCommandContext context, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var rpc = context.Session.ArticleWorkRpc; + if (rpc is null) + { + return ResolveResult.NotFound(); + } + + ArticleWorkRpcResult lookup; + try + { + lookup = await rpc.LookupByMessageIdAsync(messageId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.ArticleLookupFailed( + Logger, + ex, + Encoding.ASCII.GetString(messageId.Span)); + return ResolveResult.Temporary(); + } + + if (lookup.Outcome is ArticleWorkOutcome.ArticleNotFound or ArticleWorkOutcome.InvalidArticle) + { + return ResolveResult.NotFound(); + } + + if (lookup.Outcome != ArticleWorkOutcome.Success + || lookup.ArticleId is not { } expectedArtId + || !VatpEndpointFields.IsCanonicalFqdn(lookup.Fqdn) + || lookup.VatpPort is not int vatpPort + || !VatpEndpointFields.IsCanonicalPort(vatpPort)) + { + ArticleRetrievalLogMessages.TransferUnavailable( + Logger, + lookup.Outcome.ToString(), + lookup.Error); + return ResolveResult.Temporary(); + } + + var vatp = context.Session.VatpArticleClient; + if (vatp is null) + { + ArticleRetrievalLogMessages.TransferUnavailable( + Logger, + "VatpClientMissing", + EndpointDetail(lookup.Fqdn, vatpPort)); + return ResolveResult.Temporary(); + } + + VatpFetchResult fetch; + try + { + fetch = await vatp.FetchArticleAsync( + lookup.Fqdn, + vatpPort, + lookup.RequestId, + expectedArtId, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleRetrievalLogMessages.VatpFetchFailed(Logger, ex, lookup.Fqdn, vatpPort, lookup.RequestId); + return ResolveResult.Temporary(); + } + + return WithSource(MapFetch(fetch, messageId.Span, expectedArtId), RetrievalSource.BackFiller); + } + + /// + /// Silence from the fleet lookup. Matches StorageArticleLookupService when no + /// positive response arrives before the existing lookup budget. + /// + private const string StorageLookupSilence = "Storage article lookup timed out with no positive response."; + + private static async ValueTask FetchPositiveStorageAsync( + NntpCommandContext context, + ReadOnlyMemory messageId, + ArticleId articleId, + StorageArticleLookupResult fleet, + CancellationToken cancellationToken) + { + var logger = Logger; + if (fleet.ArticleId != articleId || !TryBindStorageEndpoint(fleet.Fqdn, fleet.VatpPort, out var host, out var port)) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "unavailable", + articleIdText, + fleet.RequestId); + } + + ArticleRetrievalLogMessages.TransferUnavailable(Logger, "StorageEndpoint", EndpointDetail(fleet.Fqdn, fleet.VatpPort)); + return ResolveResult.Temporary(); + } + + var vatp = context.Session.VatpArticleClient; + if (vatp is null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "found", + articleIdText, + fleet.RequestId); + } + + ArticleRetrievalLogMessages.TransferUnavailable(Logger, "VatpClientMissing", EndpointDetail(host, port)); + return ResolveResult.Temporary(); + } + + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "found", + articleIdText, + fleet.RequestId); + } + + var serverId = fleet.ServerId ?? 0; + VatpFetchResult fetch; + try + { + fetch = await FetchStorageCandidateAsync( + vatp, + host, + port, + fleet.RequestId, + articleId, + serverId, + attempt: 1, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return ResolveResult.Temporary(); + } + + if (IsAlternateEligible(fetch) && fleet.Alternates is not null) + { + // Caller cancellation is terminal even when an alternate is already retained + // or the same cancel has already closed the lookup window. + cancellationToken.ThrowIfCancellationRequested(); + var alternate = await fleet.Alternates.WaitForAlternateAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (alternate is { } next) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + fetch = await FetchStorageCandidateAsync( + vatp, + next.Fqdn, + next.VatpPort, + fleet.RequestId, + articleId, + next.ServerId, + attempt: 2, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return ResolveResult.Temporary(); + } + } + else + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageCandidateExhausted( + logger, + articleIdText, + fleet.RequestId, + 1); + } + } + } + + return WithSource(MapFetch(fetch, messageId.Span, articleId), RetrievalSource.StorageServer); + } + + private static async ValueTask FetchStorageCandidateAsync( + IVatpArticleClient vatp, + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + int serverId, + int attempt, + CancellationToken cancellationToken) + { + var logger = Logger; + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageCandidateSelected( + logger, + articleIdText, + requestId, + serverId, + attempt); + } + try + { + var fetch = await vatp.FetchArticleAsync(fqdn, vatpPort, requestId, articleId, cancellationToken) + .ConfigureAwait(false); + if (fetch.Kind != VatpFetchKind.Success && logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + var kind = fetch.Kind.ToString(); + var failoverEligible = IsAlternateEligible(fetch); + ArticleRetrievalLogMessages.StorageCandidateFailed( + logger, + articleIdText, + requestId, + serverId, + attempt, + kind, + fetch.AcceptedDataBytes, + failoverEligible); + } + + return fetch; + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleRetrievalLogMessages.VatpFetchFailed(Logger, ex, fqdn, vatpPort, requestId); + throw; + } + } + + /// + /// A second candidate is allowed only when no DATA payload was copied and the failure + /// is connection, remote transfer, or protocol. Every other result is terminal. + /// + private static bool IsAlternateEligible(VatpFetchResult fetch) => + fetch.AcceptedDataBytes == 0 + && fetch.Kind is VatpFetchKind.ConnectionFailure + or VatpFetchKind.RemoteTransferFailure + or VatpFetchKind.ProtocolFailure; + + private static bool TryBindStorageEndpoint(string? fqdn, int? vatpPort, out string host, out int port) + { + host = string.Empty; + port = 0; + if (!VatpEndpointFields.IsCanonicalFqdn(fqdn) || vatpPort is not int parsed || !VatpEndpointFields.IsCanonicalPort(parsed)) + { + return false; + } + + host = fqdn; + port = parsed; + return true; + } + + private static string EndpointDetail(string? fqdn, int? vatpPort) => + FormattableString.Invariant($"fqdn={fqdn} port={vatpPort}"); + + private static ResolveResult MapFetch( + VatpFetchResult fetch, + ReadOnlySpan requestedMessageId, + ArticleId expectedArtId) + { + switch (fetch.Kind) + { + case VatpFetchKind.Success: + break; + case VatpFetchKind.Cancelled: + throw new OperationCanceledException(); + default: + ArticleRetrievalLogMessages.VatpFetchUnsuccessful( + Logger, + fetch.Kind.ToString(), + fetch.Error, + fetch.ErrorCode?.ToString()); + return ResolveResult.Temporary(); + } + + var record = fetch.Record; + if (record.ParseStatus != ArticleParseStatus.CanonicalV1 || record.ArtSize <= 0) + { + ArticleRetrievalLogMessages.VatpFetchUnsuccessful( + Logger, + "IncompleteOrMalformedArticle", + fetch.Error, + fetch.ErrorCode?.ToString()); + return ResolveResult.Temporary(); + } + + if (!record.ArtId.Equals(expectedArtId) + || !NntpArticleIdentity.MatchesRequest(record.MessageId, requestedMessageId)) + { + ArticleRetrievalLogMessages.IdentityMismatch(Logger, fetch.Error); + return ResolveResult.NotFound(); + } + + return ResolveResult.Success(record, RetrievalSource.None); + } + + /// + /// Keeps a failure result unchanged and stamps only on success. + /// + private static ResolveResult WithSource(ResolveResult result, RetrievalSource source) => + result.Kind == ResolveKind.Success + ? ResolveResult.Success(result.Record, source) + : result; + + /// + /// Records this NNTPD as the receiving hop for a StorageServer article. + /// + /// when traversal fails. The caller keeps the fetched record. + private static bool TryTraverseStorageIngress( + NntpCommandContext context, + in ArticleRecord record, + out ArticleRecord entered) + { + entered = default; + NntpArticlePolicyCapture.Capture( + context.Session, + ArticleResourceLimits.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + var created = siteNameUtf8 is null + ? ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse) + : ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted || created.Record.ArtId != record.ArtId) + { + return false; + } + + entered = created.Record; + return true; + } + + /// + /// Queues the Path-canonical copy and returns that same record for the current reply. + /// + /// when canonicalization or queue admission fails. + private static bool TryAdmitBackFiller( + NntpCommandContext context, + in ArticleRecord record, + out ArticleRecord admitted) + { + admitted = default; + try + { + var logger = Logger; + NntpArticlePolicyCapture.Capture( + context.Session, + ArticleResourceLimits.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + var created = siteNameUtf8 is null + ? ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse) + : ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted || created.Record.ArtId != record.ArtId) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var reason = created.ParseFailure != NntpArticleParseFailureCode.None + ? created.ParseFailure.ToString() + : created.MaterializeFailure.ToString(); + var rejectedMessageId = Encoding.ASCII.GetString(record.MessageId); + ArticleRetrievalLogMessages.IngestNotAdmitted( + logger, + reason, + rejectedMessageId); + } + return false; + } + + admitted = created.Record; + var messageIdText = Encoding.ASCII.GetString(admitted.MessageId); + var inbound = ArticleRecordIngress.CreateQueued( + messageIdText, + in admitted, + context.Session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.BackFiller); + + var enqueue = context.Session.ArticleIngestion.TryAdmit(inbound); + if (enqueue == ArticleEnqueueResult.Accepted) + { + // Copy Message-ID octets once for History; do not invent a second History path. + var midBytes = record.MessageId.ToArray(); + context.Session.HistoryDb?.Remember(midBytes); + return true; + } + + admitted = default; + if (logger.IsEnabled(LogLevel.Debug)) + { + var result = enqueue.ToString(); + ArticleRetrievalLogMessages.IngestNotAdmitted(logger, result, messageIdText); + } + return false; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + admitted = default; + ArticleRetrievalLogMessages.IngestFailed(Logger, ex); + return false; + } + } + + private static ValueTask WriteNumberLookupFailureAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!context.Session.HasSelectedGroup) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoNewsgroupSelected, + NntpResponseStatus.NoNewsgroupSelected, + cancellationToken); + } + + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoArticleWithNumber, + NntpResponseStatus.NoArticleWithNumber, + cancellationToken); + } + + private static ValueTask WriteCurrentArticleFailureAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!context.Session.HasSelectedGroup) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoNewsgroupSelected, + NntpResponseStatus.NoNewsgroupSelected, + cancellationToken); + } + + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CurrentArticleNumberInvalid, + NntpResponseStatus.CurrentArticleNumberInvalid, + cancellationToken); + } + + private enum RetrievalSource + { + None = 0, + StorageServer = 1, + BackFiller = 2, + } + + private enum ResolveKind + { + NotFound, + TemporaryFailure, + Success, + } + + private readonly struct ResolveResult + { + private ResolveResult(ResolveKind kind, ArticleRecord record, RetrievalSource source) + { + Kind = kind; + Record = record; + Source = source; + } + + public ResolveKind Kind { get; } + + public ArticleRecord Record { get; } + + public RetrievalSource Source { get; } + + public static ResolveResult NotFound() => new(ResolveKind.NotFound, default, RetrievalSource.None); + + public static ResolveResult Temporary() => new(ResolveKind.TemporaryFailure, default, RetrievalSource.None); + + public static ResolveResult Success(ArticleRecord record, RetrievalSource source) => + new(ResolveKind.Success, record, source); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/ArticleRetrievalLogMessages.cs b/src/VectorNNTPD.ReaderService/Session/Commands/ArticleRetrievalLogMessages.cs new file mode 100644 index 00000000..b15fe60f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/ArticleRetrievalLogMessages.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// Source-generated log messages for ARTICLE/HEAD/BODY/STAT BackFiller retrieval. + internal static partial class ArticleRetrievalLogMessages + { + [LoggerMessage( + EventId = 2610, + Level = LogLevel.Warning, + Message = "Article retrieval transfer unavailable ({Outcome}): {Detail}")] + public static partial void TransferUnavailable(ILogger logger, string Outcome, string? Detail); + + [LoggerMessage( + EventId = 2611, + Level = LogLevel.Warning, + Message = "VATP article fetch failed for {Fqdn}:{VatpPort} request {RequestId}")] + public static partial void VatpFetchFailed(ILogger logger, Exception ex, string Fqdn, int VatpPort, Guid RequestId); + + [LoggerMessage( + EventId = 2612, + Level = LogLevel.Warning, + Message = "VATP article fetch unsuccessful ({Kind}): {Detail} ({ErrorCode})")] + public static partial void VatpFetchUnsuccessful(ILogger logger, string Kind, string? Detail, string? ErrorCode); + + [LoggerMessage( + EventId = 2613, + Level = LogLevel.Warning, + Message = "Fetched ArticleRecord identity mismatch: {Detail}")] + public static partial void IdentityMismatch(ILogger logger, string? Detail); + + [LoggerMessage( + EventId = 2614, + Level = LogLevel.Debug, + Message = "BackFiller article ingest not admitted ({Result}) for {MessageId}")] + public static partial void IngestNotAdmitted(ILogger logger, string Result, string MessageId); + + [LoggerMessage( + EventId = 2615, + Level = LogLevel.Warning, + Message = "BackFiller article ingest failed after successful retrieval")] + public static partial void IngestFailed(ILogger logger, Exception ex); + + [LoggerMessage( + EventId = 2616, + Level = LogLevel.Debug, + Message = "Storage article lookup {Outcome} articleId={ArticleId} requestId={RequestId}")] + public static partial void StorageLookupCompleted( + ILogger logger, + string Outcome, + string ArticleId, + Guid RequestId); + + [LoggerMessage( + EventId = 2617, + Level = LogLevel.Debug, + Message = "Storage candidate selected articleId={ArticleId} requestId={RequestId} serverId={ServerId} attempt={Attempt}")] + public static partial void StorageCandidateSelected( + ILogger logger, + string ArticleId, + Guid RequestId, + int ServerId, + int Attempt); + + [LoggerMessage( + EventId = 2618, + Level = LogLevel.Debug, + Message = "Storage candidate failed articleId={ArticleId} requestId={RequestId} serverId={ServerId} attempt={Attempt} kind={Kind} acceptedDataBytes={AcceptedDataBytes} failoverEligible={FailoverEligible}")] + public static partial void StorageCandidateFailed( + ILogger logger, + string ArticleId, + Guid RequestId, + int ServerId, + int Attempt, + string Kind, + int AcceptedDataBytes, + bool FailoverEligible); + + [LoggerMessage( + EventId = 2619, + Level = LogLevel.Debug, + Message = "Storage candidate exhausted articleId={ArticleId} requestId={RequestId} attempt={Attempt}")] + public static partial void StorageCandidateExhausted( + ILogger logger, + string ArticleId, + Guid RequestId, + int Attempt); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/AuthInfo.cs b/src/VectorNNTPD.ReaderService/Session/Commands/AuthInfo.cs new file mode 100644 index 00000000..d574eedd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/AuthInfo.cs @@ -0,0 +1,420 @@ +using System.Text; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// AUTHINFO USER / PASS (RFC 4643, Section 2.3) and AUTHINFO SASL (RFC 4643, Section 2.4). + /// + /// + /// USER caches a username. PASS selects the credential authority from + /// (MODE), never from source IP. + /// Transit authority uses only. + /// Reader authority uses only (newsmaster then MySQL). + /// There is no Transit ↔ Reader fall-through. SASL is reader-authority only. + /// + internal static class AuthInfo + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(AuthInfo)); + + /// Handles AUTHINFO USER. + public static ValueTask HandleUserAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO USER", + (ctx, ct) => ExecuteUserAsync(ctx, authenticationProvider, ct), + cancellationToken); + + /// Handles AUTHINFO PASS. + public static ValueTask HandlePassAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO PASS", + (ctx, ct) => ExecutePassAsync(ctx, authenticationProvider, ct), + cancellationToken); + + /// Handles AUTHINFO SASL. + public static ValueTask HandleSaslAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO SASL", + ExecuteSaslAsync, + cancellationToken); + + private static async ValueTask ExecuteUserAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) + { + _ = authenticationProvider; + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + context.Session.AbandonSaslExchange(); + var username = Encoding.ASCII.GetString(context.ArgumentSpan); + context.Session.SetPendingAuthUsername(username); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PasswordRequired, + NntpResponseStatus.PasswordRequired, + cancellationToken) + .ConfigureAwait(false); + } + + private static async ValueTask ExecutePassAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) + { + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + var pending = context.Session.PendingAuthUsername; + if (pending is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthenticationOutOfSequence, + NntpResponseStatus.AuthenticationOutOfSequence, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var password = Encoding.ASCII.GetString(context.ArgumentSpan); + NntpAuthenticationResult result; + try + { + result = await AuthenticatePassAsync( + context.Session, + authenticationProvider, + pending, + password, + cancellationToken) + .ConfigureAwait(false); + } + finally + { + // codeql[cs/useless-assignment-to-local] Clear password reference after use; assignment is intentional. + password = null!; + } + + await CompleteAuthenticationAsync(context, result, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ExecuteSaslAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + if (context.Session.AuthenticationAuthority == NntpAuthenticationAuthority.Transit) + { + context.Session.ApplyFailedAuthentication(); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthenticationFailed, + NntpResponseStatus.AuthenticationFailed, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var sasl = context.Session.SaslService; + if (sasl is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthinfoSaslNotImplemented, + NntpResponseStatus.AuthinfoSaslNotImplemented, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var argument = context.ArgumentSpan; + if (argument.IsEmpty) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SaslMechanismRequired, + NntpResponseStatus.SaslMechanismRequired, + cancellationToken) + .ConfigureAwait(false); + return; + } + + SplitMechanism(argument, out var mechanism, out var initial); + context.Session.AbandonSaslExchange(); + var reply = await sasl + .StartAsync(mechanism, Encoding.ASCII.GetString(initial), context.Session.ClientAddress, cancellationToken) + .ConfigureAwait(false); + await ApplySaslReplyAsync(context, reply, cancellationToken).ConfigureAwait(false); + } + + /// Handles a SASL continuation line (base64, =, or *). + internal static async ValueTask ContinueSaslAsync( + NntpSession session, + NntpResponseWriter response, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + var sasl = session.SaslService; + var exchange = session.CurrentSaslExchange; + if (sasl is null || exchange is null) + { + session.AbandonSaslExchange(); + await response.WriteLineAsync(NntpResponses.NoSaslExchangeInProgress, cancellationToken).ConfigureAwait(false); + return; + } + + var reply = await sasl + .ContinueAsync(exchange, Encoding.ASCII.GetString(line.Span), session.ClientAddress, cancellationToken) + .ConfigureAwait(false); + await ApplySaslReplyCoreAsync(session, response, context: null, reply, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask AuthenticatePassAsync( + NntpSession session, + INntpAuthenticationProvider authenticationProvider, + string username, + string password, + CancellationToken cancellationToken) + { + if (session.AuthenticationAuthority == NntpAuthenticationAuthority.Transit) + { + return ValueTask.FromResult( + session.TransitAuthenticator.Authenticate( + session.Authorization.TransitPeerPolicy, + username, + password, + session.Authorization)); + } + + return authenticationProvider.AuthenticateAsync( + username, + password, + session.ClientAddress, + cancellationToken); + } + + private static bool TryBeginAuthCommand( + NntpCommandContext context, + CancellationToken cancellationToken, + out ValueTask blocked) + { + if (context.Session.Authentication.IsAuthenticated) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AlreadyAuthenticated, + NntpResponseStatus.AlreadyAuthenticated, + cancellationToken); + return false; + } + + if (context.Connection.IsCompressed) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.DeflateAlreadyActive, + NntpResponseStatus.DeflateAlreadyActive, + cancellationToken); + return false; + } + + if (!context.Session.IsAuthinfoPassPermitted) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PrivacyRequired, + NntpResponseStatus.PrivacyRequired, + cancellationToken); + return false; + } + + blocked = default; + return true; + } + + private static ValueTask ApplySaslReplyAsync( + NntpCommandContext context, + NntpSaslReply reply, + CancellationToken cancellationToken) => + ApplySaslReplyCoreAsync(context.Session, context.Response, context, reply, cancellationToken); + + private static async ValueTask ApplySaslReplyCoreAsync( + NntpSession session, + NntpResponseWriter response, + NntpCommandContext? context, + NntpSaslReply reply, + CancellationToken cancellationToken) + { + if (!reply.Completed && reply.Exchange is not null) + { + session.SetSaslExchange(reply.Exchange); + await WriteAsync(context, response, reply.Wire, reply.StatusLine, cancellationToken).ConfigureAwait(false); + return; + } + + session.AbandonSaslExchange(); + if (reply.Authentication is { Succeeded: true } result) + { + await CompleteAuthenticationCoreAsync( + session, + response, + context, + result, + reply.Wire, + reply.StatusLine, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (reply.Authentication is { } failed) + { + await CompleteAuthenticationCoreAsync( + session, + response, + context, + failed, + successWire: default, + successStatus: null, + cancellationToken) + .ConfigureAwait(false); + return; + } + + session.ApplyFailedAuthentication(); + await WriteAsync(context, response, reply.Wire, reply.StatusLine, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask CompleteAuthenticationAsync( + NntpCommandContext context, + NntpAuthenticationResult result, + CancellationToken cancellationToken) => + CompleteAuthenticationCoreAsync( + context.Session, + context.Response, + context, + result, + NntpResponses.AuthenticationAccepted, + NntpResponseStatus.AuthenticationAccepted, + cancellationToken); + + private static async ValueTask CompleteAuthenticationCoreAsync( + NntpSession session, + NntpResponseWriter response, + NntpCommandContext? context, + NntpAuthenticationResult result, + ReadOnlyMemory successWire, + string? successStatus, + CancellationToken cancellationToken) + { + if (!result.Succeeded || result.Username is null || result.Authorization is null) + { + session.ApplyFailedAuthentication(); + var (wire, status) = MapFailure(result.Failure); + await WriteAsync(context, response, wire, status, cancellationToken).ConfigureAwait(false); + return; + } + + var admitted = await session.AdmitAuthenticatedSessionAsync(result.Policy, cancellationToken) + .ConfigureAwait(false); + if (!admitted.Succeeded) + { + session.ApplyFailedAuthentication(); + var (wire, status) = MapFailure(admitted.Failure); + if (admitted.Failure is NntpAuthenticationFailureKind.TooManySessions + or NntpAuthenticationFailureKind.TooManySourceAddresses + or NntpAuthenticationFailureKind.TransientFailure) + { + AuthenticationLogMessages.AdmissionRejected( + Logger, + result.Username, + admitted.Failure.ToString(), + session.ClientAddress.ToString()); + } + + await WriteAsync(context, response, wire, status, cancellationToken).ConfigureAwait(false); + return; + } + + session.ApplySuccessfulAuthentication(result.Username, result.Authorization, result.Policy); + await session.ReleaseTransitAdmissionWhenNotTransitAsync().ConfigureAwait(false); + await session.AttachByteAccountingAsync(response, cancellationToken).ConfigureAwait(false); + var wireSuccess = successWire.IsEmpty ? NntpResponses.AuthenticationAccepted : successWire; + var statusSuccess = successStatus ?? NntpResponseStatus.AuthenticationAccepted; + await WriteAsync(context, response, wireSuccess, statusSuccess, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteAsync( + NntpCommandContext? context, + NntpResponseWriter response, + ReadOnlyMemory wire, + string status, + CancellationToken cancellationToken) + { + if (context is not null) + { + return NntpCommandReply.WriteAsync(context, Logger, wire, status, cancellationToken); + } + + return response.WriteLineAsync(wire, cancellationToken); + } + + private static (ReadOnlyMemory Wire, string Status) MapFailure(NntpAuthenticationFailureKind failure) => + failure switch + { + NntpAuthenticationFailureKind.TransientFailure => + (NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure), + NntpAuthenticationFailureKind.TooManySessions => + (NntpResponses.TooManySessions, NntpResponseStatus.TooManySessions), + NntpAuthenticationFailureKind.TooManySourceAddresses => + (NntpResponses.TooManySourceAddresses, NntpResponseStatus.TooManySourceAddresses), + _ => (NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed), + }; + + private static void SplitMechanism(ReadOnlySpan argument, out string mechanism, out ReadOnlySpan initial) + { + var end = NntpAscii.SkipToken(argument, 0); + mechanism = Encoding.ASCII.GetString(argument[..end]); + var next = NntpAscii.SkipWhitespace(argument, end); + initial = next >= argument.Length ? ReadOnlySpan.Empty : argument[next..]; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/BenchIt.cs b/src/VectorNNTPD.ReaderService/Session/Commands/BenchIt.cs new file mode 100644 index 00000000..a8572a11 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/BenchIt.cs @@ -0,0 +1,163 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// Internal benchmark facility: unadvertised BENCHIT command that serves one static + /// ~750 KiB ARTICLE-style multiline response over the production transport path. + /// + /// + /// + /// Not a production protocol feature and not advertised via CAPABILITIES, HELP, or the public + /// command inventory. Retained intentionally for transport performance baselines and regressions. + /// Registered so the real dispatcher/session stack can be exercised against actual TCP sockets. + /// + /// + /// The complete NNTP multiline wire representation (status line, CRLF article lines with dot-stuffing, + /// and terminating .CRLF) is generated exactly once at type initialization and reused immutably + /// for every request. Per-request work is: dispatch → write precomputed bytes → pipe/transport flush. + /// + /// + /// Logging: no per-request INFO TX completion (handler does not use ). + /// Session RX logging for BENCHIT is suppressed separately. + /// + /// + internal static class BenchIt + { + /// Target article content size before NNTP multiline framing (750 KiB). + public const int TargetArticleContentBytes = 750 * 1024; + + /// Stable Message-ID used in the static article and 220 status line. + private const string MessageId = ""; + + /// Immutable ASCII article bytes (headers + body), length . + public static readonly ReadOnlyMemory ArticleContent; + + /// Immutable complete multiline wire response including 220 status and terminating .CRLF. + public static readonly ReadOnlyMemory WireResponse; + + /// Exact article content length in bytes (before framing). + public static readonly int ArticleContentBytes; + + /// Exact wire response length in bytes (status + framed article + terminator). + public static readonly int WireResponseBytes; + + static BenchIt() + { + var article = BuildArticleContent(); + ArticleContent = article; + ArticleContentBytes = article.Length; + + var wire = BuildWireResponse(article); + WireResponse = wire; + WireResponseBytes = wire.Length; + } + + /// Handles BENCHIT (internal; public access; non-pipelined via normal session loop). + internal static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + // Intentionally skip NntpCommandExecution TX INFO logging — benchmark must not measure Serilog. + return context.Response.WriteBytesAndFlushAsync(WireResponse, cancellationToken); + } + + private static byte[] BuildArticleContent() + { + // Deterministic mandatory-style headers + blank line, then a fixed body pattern. + // One body line begins with '.' so the static wire form exercises dot-stuffing. + var headerText = + "From: benchit@vectornntp.local\r\n" + + "Newsgroups: vectornntp.bench\r\n" + + "Subject: VectorNNTP BENCHIT transport payload\r\n" + + "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" + + "Message-ID: " + MessageId + "\r\n" + + "Path: vectornntp.bench\r\n" + + "\r\n"; + + var headerBytes = Encoding.ASCII.GetBytes(headerText); + if (headerBytes.Length >= TargetArticleContentBytes) + { + throw new InvalidOperationException("BENCHIT headers alone exceed the target article size."); + } + + var article = new byte[TargetArticleContentBytes]; + headerBytes.CopyTo(article.AsSpan()); + + // 76-octet body lines (common Usenet width) of a repeating digit pattern, CRLF-terminated. + // First body line is ".DOTSTUFF-CHECK" so framing must emit "..DOTSTUFF-CHECK\r\n". + var offset = headerBytes.Length; + var firstLine = Encoding.ASCII.GetBytes(".DOTSTUFF-CHECK\r\n"); + if (offset + firstLine.Length > article.Length) + { + throw new InvalidOperationException("BENCHIT target size too small for required body line."); + } + + firstLine.CopyTo(article.AsSpan(offset)); + offset += firstLine.Length; + + // Pattern line without leading '.': 74 payload chars + CRLF = 76 bytes. + Span patternLine = stackalloc byte[76]; + for (var i = 0; i < 74; i++) + { + patternLine[i] = (byte)('0' + (i % 10)); + } + + patternLine[74] = (byte)'\r'; + patternLine[75] = (byte)'\n'; + + while (offset + patternLine.Length <= article.Length) + { + patternLine.CopyTo(article.AsSpan(offset)); + offset += patternLine.Length; + } + + // Fill any remainder with ASCII 'X' (no CR/LF) so content length is exact. + article.AsSpan(offset).Fill((byte)'X'); + return article; + } + + private static byte[] BuildWireResponse(ReadOnlySpan article) + { + var status = Encoding.ASCII.GetBytes($"220 0 {MessageId}\r\n"); + using var buffer = new MemoryStream(status.Length + article.Length + (article.Length / 64) + 16); + buffer.Write(status); + + var lineStart = 0; + for (var i = 0; i < article.Length; i++) + { + if (article[i] != (byte)'\n') + { + continue; + } + + // Line includes optional trailing CR before LF. + var lineEnd = i + 1; + var line = article[lineStart..lineEnd]; + if (line.Length > 0 && line[0] == (byte)'.') + { + buffer.WriteByte((byte)'.'); + } + + buffer.Write(line); + lineStart = lineEnd; + } + + if (lineStart < article.Length) + { + // Final unterminated fragment: emit as a line with CRLF (RFC multiline writer style). + var fragment = article[lineStart..]; + if (fragment.Length > 0 && fragment[0] == (byte)'.') + { + buffer.WriteByte((byte)'.'); + } + + buffer.Write(fragment); + buffer.Write("\r\n"u8); + } + + buffer.Write(".\r\n"u8); + return buffer.ToArray(); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Capabilities.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Capabilities.cs new file mode 100644 index 00000000..958a9e31 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Capabilities.cs @@ -0,0 +1,131 @@ +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// CAPABILITIES command as defined by RFC 3977, Section 5.2. + /// + /// + /// Returns the server's capability list (VERSION, READER, AUTHINFO, STARTTLS, COMPRESS, STREAMING, SPEEDTEST, and related labels). + /// Advertisement for AUTHINFO and MODE-READER follows RFC 4643 rules after authentication. + /// COMPRESS / STARTTLS / MODE-READER / AUTHINFO arguments follow RFC 8054 once a compression layer is active. + /// STREAMING (RFC 4644) is advertised when TAKETHIS/CHECK streaming transfer is implemented. + /// Individual lines are pre-encoded and immortal. Which lines appear depends on session state, + /// so the complete response is composed into one owned buffer and written once. + /// + internal static class Capabilities + { + private const int MaxCapabilityParts = 16; + + private static ILogger Logger => NntpCommandLoggers.For(typeof(Capabilities)); + + /// Handles CAPABILITIES. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "CAPABILITIES", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var parts = new ReadOnlyMemory[MaxCapabilityParts]; + var count = CollectLines(context, parts); + var owned = NntpResponseCompose.Concatenate(parts.AsSpan(0, count)); + return NntpCommandReply.WriteAsync( + context, + Logger, + owned, + NntpResponseStatus.CapabilityListFollows, + cancellationToken); + } + + private static int CollectLines(NntpCommandContext context, Span> parts) + { + var n = 0; + parts[n++] = NntpResponses.CapabilityListFollows; + parts[n++] = NntpResponses.CapabilityVersion2; + parts[n++] = NntpResponses.CapabilityImplementation; + + var authenticated = context.Session.Authentication.IsAuthenticated; + var compressed = context.Connection.IsCompressed; + var role = context.Session.ServerRole; + var advertiseReader = role is NntpServerRole.Reader; + var advertiseTransit = false /* transit not hosted */; + + if (advertiseReader && + context.Session.Mode is NntpSessionMode.Unspecified or NntpSessionMode.Reader) + { + parts[n++] = NntpResponses.CapabilityReader; + // RFC 3977 §7.6: LIST keywords that are actually implemented. + parts[n++] = NntpResponses.CapabilityList; + // RFC 4643: MUST NOT advertise MODE-READER after authentication. + // RFC 8054 §2.2.2: MUST NOT advertise MODE-READER once a compression layer is active. + if (!authenticated && !compressed) + { + parts[n++] = NntpResponses.CapabilityModeReader; + } + } + + if (advertiseReader && + context.Session.Authorization.PostingPermitted && + context.Session.Mode is NntpSessionMode.Unspecified or NntpSessionMode.Reader) + { + parts[n++] = NntpResponses.CapabilityPost; + } + + // RFC 4643: MUST NOT return AUTHINFO after successful authentication. + // RFC 8054 §2.2.2 / §7: after COMPRESS, advertise AUTHINFO with no arguments (or omit). + // SASL is advertised only when NntpSaslService is registered and AUTHINFO is permitted. + if (!authenticated) + { + var saslAvailable = context.Session.SaslService is not null; + if (compressed) + { + parts[n++] = NntpResponses.CapabilityAuthinfo; + } + else if (context.Session.IsAuthinfoPassPermitted) + { + if (saslAvailable) + { + parts[n++] = NntpResponses.CapabilityAuthinfoUserSasl; + parts[n++] = NntpResponses.CapabilitySasl; + } + else + { + parts[n++] = NntpResponses.CapabilityAuthinfoUser; + } + } + else + { + // Policy forbids cleartext AUTHINFO and TLS is inactive — do not advertise USER or SASL. + parts[n++] = NntpResponses.CapabilityAuthinfo; + } + } + + // RFC 8054 §2.2.2: MUST NOT advertise STARTTLS once a compression layer is active. + if (!context.Connection.IsTls && !compressed) + { + parts[n++] = NntpResponses.CapabilityStartTls; + } + + // RFC 8054 §2.1: advertise COMPRESS DEFLATE when available; MUST NOT once compression is active. + if (!compressed) + { + parts[n++] = NntpResponses.CapabilityCompressDeflate; + } + + if (advertiseTransit) + { + // RFC 3977 §3.3.2: advertise IHAVE when the command is implemented. + parts[n++] = NntpResponses.CapabilityIhave; + + // RFC 4644 §2.2: STREAMING capability for CHECK/TAKETHIS (MODE STREAM is legacy discovery). + parts[n++] = NntpResponses.CapabilityStreaming; + + // VectorNNTP private SPEEDTEST diagnostic (current-session TX; no outbound initiator). + parts[n++] = NntpResponses.CapabilitySpeedTest; + } + + parts[n++] = NntpResponses.MultilineTerminator; + return n; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Check.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Check.cs new file mode 100644 index 00000000..bfb5d137 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Check.cs @@ -0,0 +1,132 @@ +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// CHECK command as defined by RFC 4644, Section 2.4. + /// + /// + /// Advises whether the server wants an article (STREAMING) using HistoryDB + /// (no miss reservation): + /// local memory hit and Redis hit are and do not + /// record the identifier. A double miss is 238. Redis infrastructure failure returns 431 + /// and is never treated as a 238 miss. A result is + /// 431 when the connection's TransitPeerPolicy.DeferOnDuplicate is true, and 438 + /// otherwise. TAKETHIS does not read that flag. + /// The Message-ID is copied from session scratch into one owned wire buffer; it is not + /// converted to a string for response construction. A completed local-hit lookup + /// writes that CHECK reply without an extra async state machine. + /// Consecutive authorized CHECK commands may overlap Redis lookups in the per-session + /// (fixed depth ). + /// Every response passes through the pipeline emit gate and is enqueued in send order. + /// A slot stays occupied until the writer accepts the line. + /// + internal static class Check + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Check)); + + /// Handles CHECK when dispatched serially (gates / non-pipeline fallback). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "CHECK", ExecuteAsync, cancellationToken); + + /// + /// Looks up without composing a response or recording a miss. + /// + internal static ValueTask LookupAsync( + NntpSession session, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + if (session.HistoryDb is not { } history) + { + return new ValueTask(HistoryLookupResult.Unseen); + } + + return history.PeekAsync(messageId, cancellationToken); + } + + /// Composes the RFC 4644 CHECK wire line for . + /// History lookup outcome. is not rewritten. + /// Message-ID octets copied into the response. + /// + /// Connection TransitPeerPolicy.DeferOnDuplicate. When , a seen + /// identifier uses the existing 431 line. Misses and infrastructure failures ignore it. + /// + internal static ReadOnlyMemory Compose( + HistoryLookupResult result, + ReadOnlySpan messageId, + bool deferOnDuplicate) + { + ReadOnlyMemory prefix; + ReadOnlyMemory suffix; + switch (result) + { + case HistoryLookupResult.Seen when deferOnDuplicate: + prefix = NntpResponses.CheckTryLaterPrefix; + suffix = NntpResponses.CheckTryLaterSuffix; + break; + case HistoryLookupResult.Seen: + prefix = NntpResponses.CheckNotWantedPrefix; + suffix = NntpResponses.CheckNotWantedSuffix; + break; + case HistoryLookupResult.Unavailable: + prefix = NntpResponses.CheckTryLaterPrefix; + suffix = NntpResponses.CheckTryLaterSuffix; + break; + default: + prefix = NntpResponses.CheckPrefix; + suffix = NntpResponses.CheckSuffix; + break; + } + + return NntpResponseCompose.Concat(prefix.Span, messageId, suffix.Span); + } + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var lookup = LookupAsync(context.Session, messageId, cancellationToken); + if (lookup.IsCompletedSuccessfully) + { + return WriteResultAsync(context, lookup.Result, messageId, cancellationToken); + } + + return AwaitLookupAndWriteAsync(context, lookup, messageId, cancellationToken); + } + + private static async ValueTask AwaitLookupAndWriteAsync( + NntpCommandContext context, + ValueTask lookup, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var result = await lookup.ConfigureAwait(false); + await WriteResultAsync(context, result, messageId, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteResultAsync( + NntpCommandContext context, + HistoryLookupResult result, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var deferOnDuplicate = DeferOnDuplicate(context.Session); + var owned = Compose(result, messageId.Span, deferOnDuplicate); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= NntpCommandStatusText.FormatCheck(result, messageId.Span, deferOnDuplicate); + } + + return context.Response.WriteLineAsync(owned, cancellationToken); + } + + /// + /// Reads the connection's captured Receive DeferOnDuplicate flag. + /// A session without a peer policy does not defer. + /// + internal static bool DeferOnDuplicate(NntpSession session) => + session.Authorization.TransitPeerPolicy is { DeferOnDuplicate: true }; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Compress.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Compress.cs new file mode 100644 index 00000000..2e77b1ae --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Compress.cs @@ -0,0 +1,143 @@ +using System.IO.Pipelines; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// COMPRESS command as defined by RFC 8054, Section 2.2. + /// + /// + /// + /// Activates the NNTP compression layer using the mandatory DEFLATE algorithm (RFC 8054 §2.2 / §4). + /// The command MUST NOT be pipelined (RFC 8054 §2.2.1). Success reply 206 is the final + /// uncompressed NNTP response; compression takes effect immediately after its CRLF. + /// + /// + /// Transport owns raw DEFLATE wrapping and quiescence via UpgradeToDeflateAsync. This module + /// owns algorithm validation, capability-related session sequencing (discard pipelined input, pause + /// reads before 206, wait for outbound delivery, upgrade), and the TX completion record. + /// Layering follows RFC 8054 §2.2.2: application → DEFLATE → (TLS) → network. + /// + /// + internal static class Compress + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Compress)); + + /// Handles COMPRESS (algorithm argument validated per RFC 8054). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "COMPRESS", + ExecuteAsync, + cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionAlreadyActive, + NntpResponseStatus.CompressionAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §5.3: algorithm names are case-sensitive. Parser already rejected + // lowercase/illegal octets (501). This is the semantic DEFLATE check — no string. + if (!context.ArgumentSpan.SequenceEqual("DEFLATE"u8)) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionAlgorithmNotSupported, + NntpResponseStatus.CompressionAlgorithmNotSupported, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // Client MUST NOT pipeline (RFC 8054 §2.2.1). Discard any already-buffered application + // octets so they cannot be misinterpreted after the compression layer activates. + DiscardBufferedApplicationInput(context.Connection.Input); + + // Claim read ownership BEFORE 206 can reach the peer so the first compressed octets + // cannot enter application Input as plaintext. + try + { + await context.Connection.PauseReadsAsync(cancellationToken).ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + CommandLogMessages.CompressRefusedBeforeActivation( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.UnableToActivateCompression, + NntpResponseStatus.UnableToActivateCompression, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "failed"; + return; + } + + var idleVersionBefore206 = context.Connection.OutboundIdleVersion; + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionActive, + NntpResponseStatus.CompressionActive, + cancellationToken) + .ConfigureAwait(false); + + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore206, cancellationToken) + .ConfigureAwait(false); + + try + { + await context.Connection.UpgradeToDeflateAsync(cancellationToken).ConfigureAwait(false); + context.CompletionDetail = "DEFLATE active"; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // 206 already committed the transition. Do not write another NNTP status — the peer + // expects compressed traffic. Terminate so we never leave a half-compressed session. + CommandLogMessages.CompressActivationFailed( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + if (!context.Connection.IsCompleted) + { + await context.Connection.CompleteAsync(ex).ConfigureAwait(false); + } + + context.Session.RequestClose(); + } + } + + /// + /// Consumes any octets already buffered in application without waiting for more. + /// + private static void DiscardBufferedApplicationInput(PipeReader input) + { + while (input.TryRead(out var result)) + { + var isCompleted = result.IsCompleted; + var isEmpty = result.Buffer.IsEmpty; + input.AdvanceTo(result.Buffer.End); + if (isEmpty || isCompleted) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Date.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Date.cs new file mode 100644 index 00000000..62f9ca6e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Date.cs @@ -0,0 +1,71 @@ +using System.Globalization; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// DATE command as defined by RFC 3977, Section 7.1. + /// + /// + /// Returns the current UTC server date and time as YYYYMMDDhhmmss. + /// The stamp is formatted into an owned 20-octet wire line + /// (111 + 14 digits + CRLF) without a protocol string round-trip. + /// + internal static class Date + { + private const int DateWireLength = 20; + private const int StampLength = 14; + + private static ILogger Logger => NntpCommandLoggers.For(typeof(Date)); + + /// Handles DATE. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "DATE", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var utc = DateTime.UtcNow; + var owned = FormatUtcDateLine(utc, Logger, out var statusLine); + if (statusLine is not null) + { + context.StatusLine ??= statusLine; + } + + return context.Response.WriteLineAsync(owned, cancellationToken); + } + + /// + /// Builds 111 YYYYMMDDhhmmss\r\n as one owned buffer using + /// DateTime.TryFormat so UTC stamp semantics stay on the framework formatter. + /// + internal static byte[] FormatUtcDateLine(DateTime utc) => + FormatUtcDateLine(utc, logger: null, out _); + + /// + internal static byte[] FormatUtcDateLine(DateTime utc, ILogger? logger, out string? statusLine) + { + statusLine = null; + Span stamp = stackalloc char[StampLength]; + if (!utc.TryFormat(stamp, out var written, "yyyyMMddHHmmss", CultureInfo.InvariantCulture) + || written != StampLength) + { + throw new InvalidOperationException("DATE stamp formatting failed."); + } + + if (logger is not null && logger.IsEnabled(LogLevel.Debug)) + { + statusLine = NntpCommandStatusText.FormatDate(stamp); + } + + var wire = new byte[DateWireLength]; + NntpResponses.DatePrefix.Span.CopyTo(wire); + for (var i = 0; i < StampLength; i++) + { + wire[NntpResponses.DatePrefix.Length + i] = (byte)stamp[i]; + } + + NntpResponses.Crlf.Span.CopyTo(wire.AsSpan(NntpResponses.DatePrefix.Length + StampLength)); + return wire; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Group.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Group.cs new file mode 100644 index 00000000..a1da9097 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Group.cs @@ -0,0 +1,49 @@ +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// GROUP command as defined by RFC 3977, Section 6.1.1. + /// + /// + /// Group existence and water marks come from one captured + /// . The command path does not query MySQL. + /// Article-number sequences are not invented from the water marks. + /// + internal static class Group + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Group)); + + /// Handles GROUP. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "GROUP", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = context.Session.NewsgroupCatalogue?.Current; + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + if (!snapshot.TryGet(context.ArgumentSpan, out var group)) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoSuchNewsgroup, + NntpResponseStatus.NoSuchNewsgroup, + cancellationToken); + } + + context.Session.SelectGroup(group.GroupNameBytes); + return NntpCommandReply.WriteAsync( + context, + Logger, + group.GroupSelectedLine, + NntpResponseStatus.GroupSelected, + cancellationToken); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Hdr.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Hdr.cs new file mode 100644 index 00000000..5cf5b5f7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Hdr.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// HDR command as defined by RFC 3977, Section 8.5. + /// + /// + /// Returns a specific header field for articles. Deliberate placeholder until header/overview access is implemented. + /// + internal static class Hdr + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Hdr)); + + /// Handles HDR. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "HDR", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Help.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Help.cs new file mode 100644 index 00000000..5df8a6de --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Help.cs @@ -0,0 +1,141 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// HELP command as defined by RFC 3977, Section 7.2. + /// + /// + /// + /// Syntax: HELP (no arguments). Returns a multiline 100 static syntax reference + /// for registered NNTP commands, followed by concise RANGE and WILDMAT explanations + /// (RFC 3977 §6.1.2 / §8.3 / §8.5 and §4). Mandatory; not a substitute for CAPABILITIES. + /// + /// + /// The body is identical for every session state (TLS, auth, COMPRESS, mode). The complete + /// multiline wire (100 + body + terminator) is one immortal buffer + /// () written with a single awaited + /// . + /// Trailing arguments are a syntax error (501). Internal commands such as + /// BENCHIT are not listed. + /// + /// + /// Compact HELP syntax uses ABNF-style notation where applicable: + /// [element] denotes an optional element and / denotes alternatives. + /// Required arguments appear without brackets. These lines are compact human-readable + /// summaries of the command forms a client should use, not complete standalone ABNF productions. + /// + /// + internal static class Help + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Help)); + + /// + /// Static HELP syntax lines for the supported public command surface. + /// + /// + /// LIST keywords ACTIVE.TIMES, COUNTS, DISTRIB.PATS, DISTRIBUTIONS, MODERATORS, and + /// SUBSCRIPTIONS are intentionally omitted — VectorNNTPD.ReaderService does not support them. + /// + internal static readonly IReadOnlyList SyntaxLines = + [ + "ARTICLE [message-id / article-number]", + "AUTHINFO PASS password", + "AUTHINFO SASL mechanism [initial-response]", + "AUTHINFO USER username", + "BODY [message-id / article-number]", + "CAPABILITIES [keyword]", + "CHECK message-id", + "COMPRESS DEFLATE", + "DATE", + "GROUP newsgroup", + "HDR header [range / message-id]", + "HEAD [message-id / article-number]", + "HELP", + "IHAVE message-id", + "LAST", + "LIST", + "LIST ACTIVE [wildmat]", + "LIST COUNTS [wildmat]", + "LIST HEADERS [MSGID / RANGE]", + "LIST MOTD", + "LIST NEWSGROUPS [wildmat]", + "LIST OVERVIEW.FMT", + "LISTGROUP [newsgroup [range]]", + "MODE READER", + "MODE STREAM", + "NEXT", + "OVER [range / message-id]", + "POST", + "QUIT", + "SPEEDTEST ", + "STARTTLS", + "STAT [message-id / article-number]", + "TAKETHIS message-id", + "XHDR header [range / message-id]", + "XOVER [range]", + ]; + + /// + /// Static RANGE explanation (RFC 3977 §6.1.2, §8.3, §8.5; ABNF range in §9.2). + /// + internal static readonly IReadOnlyList RangeHelpLines = + [ + "Range formats:", + "Where HELP shows \"range\" (LISTGROUP, HDR, OVER, XOVER):", + "123 a single article number", + "123-456 articles from 123 through 456 inclusive", + "123- article 123 and all following article numbers", + "If the high number is less than the low number, the range contains no articles.", + ]; + + /// + /// Static WILDMAT explanation (RFC 3977 §4; LIST ACTIVE wildmat example in §7.6.3). + /// + /// + /// Character classes and backslash escapes are intentionally omitted: RFC 3977 §4.1 + /// excludes ,, \, [, and ] from wildmats. + /// + internal static readonly IReadOnlyList WildmatHelpLines = + [ + "Wildmat formats:", + "Where HELP shows \"wildmat\" (e.g. LIST ACTIVE, LIST NEWSGROUPS); matches a complete name:", + "* matches zero or more characters", + "? matches exactly one character", + "Other permitted characters match themselves (case-sensitive).", + "Patterns may be comma-separated; a leading \"!\" negates that pattern.", + "The rightmost matching pattern decides whether the wildmat matches.", + "The characters comma, backslash, and square brackets are not allowed in wildmats.", + "Examples: a* ; a*,!*b ; *.recovery", + ]; + + /// + /// Full HELP multiline body: command syntax, then RANGE and WILDMAT sections + /// separated by blank lines. + /// + internal static readonly IReadOnlyList BodyLines = CreateBodyLines(); + + private static IReadOnlyList CreateBodyLines() + { + var lines = new List(SyntaxLines.Count + RangeHelpLines.Count + WildmatHelpLines.Count + 2); + lines.AddRange(SyntaxLines); + lines.Add(string.Empty); + lines.AddRange(RangeHelpLines); + lines.Add(string.Empty); + lines.AddRange(WildmatHelpLines); + return lines; + } + + /// Handles HELP (RFC 3977 §7.2). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "HELP", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/IHave.cs b/src/VectorNNTPD.ReaderService/Session/Commands/IHave.cs new file mode 100644 index 00000000..e1572cb0 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/IHave.cs @@ -0,0 +1,295 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// IHAVE command as defined by RFC 3977, Section 6.3.2. + /// + /// + /// Not pipelined. Serial two-stage exchange: HistoryDB peek → non-blocking + /// Transit queue probe → 335/435/436 → raw article receive (frame terminator, + /// own stuffed wire) → destuff + ArticleRecordFactory → non-blocking + /// → 235/436/437. IHAVE never + /// waits for queue memory. TAKETHIS is not used and is not modified. Common + /// owns article parse/materialize. IHAVE-specific behaviour is History peek, + /// non-blocking probe/admit, and 335/235/435/436/437 timing. After CanonicalV1, + /// the connection's Receive article-type mask and newsgroup expression are applied. + /// The command + /// Message-ID is used for History and is not matched against the article + /// Message-ID (RFC 3977 §6.3.2 permits a mismatch). A History hit before + /// 335 is 436 Transfer not possible; try again later when the + /// connection's DeferOnDuplicate is true, and 435 Article not wanted + /// otherwise. Neither response reads an article. TAKETHIS does not read that flag. + /// Incomplete articles that cannot become CanonicalV1 are rejected with 437 after 335. + /// + internal static class IHave + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(IHave)); + + /// Handles IHAVE (RFC 3977 §6.3.2). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "IHAVE", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var history = context.Session.HistoryDb; + var peek = history is null + ? HistoryLookupResult.Unseen + : await history.PeekAsync(messageId, cancellationToken).ConfigureAwait(false); + + if (peek == HistoryLookupResult.Seen) + { + var defer = Check.DeferOnDuplicate(context.Session); + await NntpCommandReply.WriteAsync( + context, + Logger, + defer ? NntpResponses.IhaveTryLater : NntpResponses.IhaveNotWanted, + defer ? NntpResponseStatus.IhaveTryLater : NntpResponseStatus.IhaveNotWanted, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = defer ? "deferred duplicate" : "not wanted"; + return; + } + + if (peek == HistoryLookupResult.Unavailable) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTryLater, + NntpResponseStatus.IhaveTryLater, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "try later"; + return; + } + + var queue = context.Session.ArticleIngestion; + if (!queue.TryProbeCapacity()) + { + IHaveAdmissionLog.BudgetExhausted(Logger, queue); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTryLater, + NntpResponseStatus.IhaveTryLater, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "queue full"; + return; + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveSendArticle, + NntpResponseStatus.IhaveSendArticle, + cancellationToken) + .ConfigureAwait(false); + + NntpArticlePolicyCapture.Capture( + context.Session, + queue.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + IHaveArticleReadResult read; + try + { + read = await IHaveArticleReader + .ReadAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ArticleTooLarge, + read.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = tooLarge; + return; + } + + var created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromStuffedWire( + context.Session.ArticleParser, + read.Payload, + maxArticleBytes) + : ArticleRecordIngress.TryCreateFromStuffedWire( + context.Session.ArticleParser, + read.Payload, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted) + { + var recordReject = created.ParseFailure != VectorNNTPD.Common.Articles.Parsing.NntpArticleParseFailureCode.None + ? "rejected article record " + created.ParseFailure + : "rejected article record " + created.MaterializeFailure; + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ForArticleRecord(in created), + read.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = recordReject; + return; + } + + if (TransitReceivePolicy.TryReject( + context.Session.Authorization.TransitPeerPolicy, + created.Record, + out var policyReject)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ForExistingRejectDetail(policyReject), + created.Record.ArtSize); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = policyReject; + return; + } + + var messageIdText = System.Text.Encoding.ASCII.GetString(messageId.Span); + var inbound = ArticleRecordIngress.CreateQueued( + messageIdText, + created.Record, + context.Session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.IHave, + IngressNewsEvents.SnapshotInboundFeed(context.Session)); + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + context.Session.Transit, + context.Session.NewsgroupCatalogue, + out var uncarriedReason)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageIdText, + 437, + uncarriedReason, + inbound.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = IngressNewsEvents.NewsgroupNotCarried; + return; + } + + var enqueue = queue.TryAdmit(inbound); + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + context.Session, + messageIdText, + 437, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = budget; + return; + } + + if (enqueue != ArticleEnqueueResult.Accepted) + { + if (enqueue == ArticleEnqueueResult.Full) + { + IHaveAdmissionLog.BudgetExhausted(Logger, queue); + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTransferFailed, + NntpResponseStatus.IhaveTransferFailed, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = enqueue == ArticleEnqueueResult.Full + ? "queue full" + : "queue unavailable"; + return; + } + + history?.Remember(messageId); + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + IHaveLogMessages.Received( + logger, + client, + messageIdText, + read.Metrics.ArticleSize, + read.Metrics.PipeReads, + read.Metrics.ReceiveElapsed.TotalMilliseconds, + Queued: true); + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTransferredOk, + NntpResponseStatus.IhaveTransferredOk, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "accepted"; + } + + private static string CommandMessageId(ReadOnlyMemory messageId) => + System.Text.Encoding.ASCII.GetString(messageId.Span); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Last.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Last.cs new file mode 100644 index 00000000..019f2f5c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Last.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// LAST command as defined by RFC 3977, Section 6.1.3. + /// + /// + /// Moves the current article pointer backward. Deliberate placeholder until article selection state is implemented. + /// + internal static class Last + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Last)); + + /// Handles LAST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "LAST", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/List.cs b/src/VectorNNTPD.ReaderService/Session/Commands/List.cs new file mode 100644 index 00000000..10ba9baf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/List.cs @@ -0,0 +1,192 @@ +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// LIST command as defined by RFC 3977, Section 7.6.1 (see also RFC 6048). + /// + /// + /// + /// Implemented keywords: default/ACTIVE, COUNTS, and NEWSGROUPS (catalogue + /// snapshot), plus static OVERVIEW.FMT and HEADERS field lists. LIST ACTIVE + /// and LIST COUNTS status is the stored y/n/m/x/j + /// octet; the RFC 6048 =<newsgroup> form is not supported. LIST + /// COUNTS estimated counts reuse the GROUP watermark estimate + /// (RFC 3977 §6.1.1 / RFC 6048 §2.2). The command path does not query MySQL. + /// + /// + /// LIST OVERVIEW.FMT and LIST HEADERS write immortal static 215 multiline + /// responses (RFC 3977 §8.4 / §8.6). They do not read articles or the + /// catalogue. LIST HEADERS MSGID and RANGE return the same list because HDR + /// forms are not distinguished. LIST MOTD remains recognized without stored + /// information and returns 503. Unsupported keywords (ACTIVE.TIMES, + /// DISTRIB.PATS, DISTRIBUTIONS, MODERATORS, SUBSCRIPTIONS) are not registered + /// and resolve as unknown variants. + /// + /// + internal static class List + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(List)); + + /// Handles LIST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, DisplayName(context), ExecuteAsync, cancellationToken); + + private static string DisplayName(NntpCommandContext context) => + DefaultNntpCommandCatalog.DisplayName(context.Command.Verb, context.Command.Qualifier); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + return context.Command.Qualifier switch + { + NntpVerb.None or NntpVerb.Active => WriteActiveAsync(context, cancellationToken), + NntpVerb.Counts => WriteCountsAsync(context, cancellationToken), + NntpVerb.Newsgroups => WriteNewsgroupsAsync(context, cancellationToken), + NntpVerb.OverviewFmt => WriteOverviewFmtAsync(context, cancellationToken), + NntpVerb.Headers => WriteHeadersAsync(context, cancellationToken), + NntpVerb.Motd => WriteDataItemNotStoredAsync(context, cancellationToken), + _ => NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken), + }; + } + + private static ValueTask WriteOverviewFmtAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListOverviewFmtComplete, + NntpResponseStatus.ListOverviewFmtFollows, + cancellationToken); + + private static ValueTask WriteHeadersAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListHeadersComplete, + NntpResponseStatus.ListHeadersFollows, + cancellationToken); + + private static ValueTask WriteDataItemNotStoredAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListDataItemNotStored, + NntpResponseStatus.ListDataItemNotStored, + cancellationToken); + + private static ValueTask WriteActiveAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListActiveComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Active, cancellationToken); + } + + private static ValueTask WriteCountsAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListCountsComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Counts, cancellationToken); + } + + private static ValueTask WriteNewsgroupsAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListNewsgroupsComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Newsgroups, cancellationToken); + } + + private static async ValueTask WriteFilteredAsync( + NntpCommandContext context, + NewsgroupSnapshot snapshot, + ReadOnlyMemory pattern, + ListLineKind kind, + CancellationToken cancellationToken) + { + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ListOfNewsgroupsFollows); + await context.Response + .WriteLineAsync(NntpResponses.ListOfNewsgroupsFollows, cancellationToken) + .ConfigureAwait(false); + + var groups = snapshot.Groups; + for (var i = 0; i < groups.Count; i++) + { + var group = groups[i]; + if (!NntpWildmat.IsMatchValidated(group.GroupNameBytes.Span, pattern.Span)) + { + continue; + } + + var line = kind switch + { + ListLineKind.Active => group.ListActiveLine, + ListLineKind.Counts => group.ListCountsLine, + _ => group.ListNewsgroupsLine, + }; + await context.Response.WriteLineAsync(line, cancellationToken).ConfigureAwait(false); + } + + await context.Response.WriteMultilineEndAsync(cancellationToken).ConfigureAwait(false); + } + + private static NewsgroupSnapshot? Capture(NntpCommandContext context) => + context.Session.NewsgroupCatalogue?.Current; + + private enum ListLineKind + { + Active, + Counts, + Newsgroups, + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/ListGroup.cs b/src/VectorNNTPD.ReaderService/Session/Commands/ListGroup.cs new file mode 100644 index 00000000..cfad459c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/ListGroup.cs @@ -0,0 +1,26 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// LISTGROUP command as defined by RFC 3977, Section 6.1.2. + /// + /// + /// Lists article numbers in a newsgroup. Deliberate placeholder until an article-number + /// source exists. Do not invent article numbers from catalogue water marks. + /// + internal static class ListGroup + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(ListGroup)); + + /// Handles LISTGROUP. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "LISTGROUP", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Mode.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Mode.cs new file mode 100644 index 00000000..fe7e4c61 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Mode.cs @@ -0,0 +1,124 @@ +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// MODE READER (RFC 3977, Section 5.3) and MODE STREAM (RFC 4644, Section 2.3). + /// + /// + /// + /// MODE READER selects reader mode and reports posting permission. After authentication, + /// MODE READER is rejected (RFC 4643). + /// + /// + /// MODE STREAM (RFC 4644 §2.3) is retained for legacy clients. It returns 203 and + /// MUST NOT change receive/capability session mode. It does select Transit AUTHINFO + /// authority (no MySQL). Authorization is + /// (peer or authenticated streaming privilege). CHECK/TAKETHIS are gated by transit authorization + /// and the advertised STREAMING capability — not by issuing MODE STREAM first. + /// + /// + internal static class Mode + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Mode)); + + /// Handles MODE READER. + public static ValueTask HandleReaderAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "MODE READER", ExecuteReaderAsync, cancellationToken); + + /// Handles MODE STREAM. + public static ValueTask HandleStreamAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "MODE STREAM", ExecuteStreamAsync, cancellationToken); + + private static async ValueTask ExecuteReaderAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Session.ServerRole == NntpServerRole.Transit) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 4643: client MUST NOT issue MODE READER after authentication; reject consistently. + if (context.Session.Authentication.IsAuthenticated) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandUnavailableAfterAuthentication, + NntpResponseStatus.CommandUnavailableAfterAuthentication, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §2.2.2: client MUST NOT issue MODE READER after a compression layer is active. + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandUnavailableAfterCompression, + NntpResponseStatus.CommandUnavailableAfterCompression, + cancellationToken) + .ConfigureAwait(false); + return; + } + + context.Session.SetMode(NntpSessionMode.Reader); + if (context.Session.Authorization.PostingPermitted) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ReaderModePostingPermitted, + NntpResponseStatus.ReaderModePostingPermitted, + cancellationToken) + .ConfigureAwait(false); + } + else + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ReaderModePostingProhibited, + NntpResponseStatus.ReaderModePostingProhibited, + cancellationToken) + .ConfigureAwait(false); + } + } + + private static async ValueTask ExecuteStreamAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Session.ServerRole == NntpServerRole.Reader) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 4644 §2.3.2: MUST return 203 and MUST NOT change receive/capability mode. + // AUTHINFO authority is selected here so STREAM credentials are Transit-only. + // StreamingPermitted is enforced by the dispatcher (RequiresStreaming). + context.Session.SetAuthenticationAuthority(NntpAuthenticationAuthority.Transit); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.StreamingPermitted, + NntpResponseStatus.StreamingPermitted, + cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Next.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Next.cs new file mode 100644 index 00000000..89a3bbfc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Next.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// NEXT command as defined by RFC 3977, Section 6.1.4. + /// + /// + /// Moves the current article pointer forward. Deliberate placeholder until article selection state is implemented. + /// + internal static class Next + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Next)); + + /// Handles NEXT. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "NEXT", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Over.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Over.cs new file mode 100644 index 00000000..54c17699 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Over.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// OVER command as defined by RFC 3977, Section 8.3. + /// + /// + /// Returns overview information for articles. Deliberate placeholder until overview database is implemented. + /// + internal static class Over + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Over)); + + /// Handles OVER. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "OVER", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Post.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Post.cs new file mode 100644 index 00000000..e27214dc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Post.cs @@ -0,0 +1,528 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// POST command as defined by RFC 3977, Section 6.3.1. + /// + /// + /// Not pipelined. First-stage 440 does not consume an article. After 340 + /// the article is streamed from the session PipeReader into one destuffed output + /// buffer (client headers destuffed write-through, server-owned headers at the + /// header/body boundary for ordinary injection, body destuffed). Unapproved + /// moderated proto-articles are submitted through + /// without Injection-Info/Injection-Date and without History Peek / TryAdmit / Remember. + /// Authorized moderator reinjection follows the ordinary injection path. History Peek, + /// , Remember, and 240 stay + /// after the terminator and after injection authorization. Downstream spool/worker + /// processing is not awaited. + /// + internal static class Post + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Post)); + + /// Handles POST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "POST", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (!context.Session.Authorization.PostingPermitted) + { + await WriteStatusAsync( + context, + NntpResponses.PostingProhibited, + NntpResponseStatus.PostingProhibited, + "posting not permitted", + cancellationToken) + .ConfigureAwait(false); + return; + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PostSendArticle, + NntpResponseStatus.PostSendArticle, + cancellationToken) + .ConfigureAwait(false); + + NntpArticlePolicyCapture.Capture( + context.Session, + context.Session.MaxArticleSize, + out var maxArticleBytes, + out var siteNameUtf8); + StreamingPostReadResult read; + try + { + read = await StreamingPostArticleReader + .ReadAsync( + context.Connection.Input, + new StreamingPostReadOptions + { + MaxArticleSize = maxArticleBytes, + Time = context.Session.Time, + NewsgroupPolicy = context.Session.NewsgroupPostingPolicy, + InjectionIdentity = context.Session.InjectionIdentity, + ClientIdentity = context.Session.ClientIdentity, + MailComplaintsTo = context.Session.MailComplaintsTo, + TraceProtector = context.Session.PostingTraceProtector, + ControlCancelPermitted = context.Session.Authorization.ControlCancelPermitted, + AuthenticatedUsername = context.Session.Authentication.IsAuthenticated + ? context.Session.Authentication.Username + : null, + ModeratorAuthorization = context.Session.CaptureModeratorAuthorization(), + }, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (read.Status == StreamingPostReadStatus.Incomplete) + { + return; + } + + if (read.Status != StreamingPostReadStatus.Completed) + { + var failure = read.Status == StreamingPostReadStatus.TooLarge + ? new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded") + : read.Failure; + await RejectAsync( + context, + failure, + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (read.Disposition == StreamingPostDisposition.SubmitForModeration) + { + await SubmitForModerationAsync(context, read, cancellationToken).ConfigureAwait(false); + return; + } + + var messageIdBytes = Encoding.ASCII.GetBytes(read.MessageId!); + var history = context.Session.HistoryDb; + if (history is not null) + { + var peek = await history.PeekAsync(messageIdBytes, cancellationToken).ConfigureAwait(false); + if (peek == HistoryLookupResult.Seen) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.DuplicateArticle, "Message-ID already known"), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (peek == HistoryLookupResult.Unavailable) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PersistenceFailure, "history unavailable"), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + } + + if (cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + var created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromDestuffed( + context.Session.ArticleParser, + read.Wire) + : ArticleRecordIngress.TryCreateFromDestuffed( + context.Session.ArticleParser, + read.Wire, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted) + { + await RejectAsync( + context, + MapArticleRecordFailure(in created), + read.MessageId, + FormatGroups(read.Newsgroups), + read.Wire.Length, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (!ArticleTypeAccessPolicy.CanPostArticleType(context.Session, created.Record.ArtType)) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PolicyRejected, "arttype-capability"), + read.MessageId, + FormatGroups(read.Newsgroups), + created.Record.ArtSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var filterRequest = new PostFilterRequest( + created.Record, + context.Session.Authentication.Username, + context.Session.ClientIdentity, + context.Session.AccountPolicy, + read.InjectionUtc, + read.Newsgroups, + context.Session.Time.GetUtcNow(), + context.Session.InjectionIdentity); + + PostFilterResult filter; + try + { + filter = await context.Session.PostFilter + .EvaluateAsync(filterRequest, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (filter.Decision == PostFilterDecision.Reject) + { + EnqueueRejectionEvidence( + context, + PostFilterRejectionEvidence.FromEvaluation( + filterRequest, + filter, + context.Session.Time.GetUtcNow())); + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PolicyRejected, filter.Reason), + read.MessageId, + FormatGroups(read.Newsgroups), + created.Record.ArtSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var lease = filter.Lease; + if (cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + await ReleaseLeaseAsync(context, lease).ConfigureAwait(false); + return; + } + + var inbound = ArticleRecordIngress.CreateQueued( + read.MessageId!, + created.Record, + context.Session.ClientIdentity, + read.InjectionUtc, + InboundArticleProducer.Post, + IngressNewsEvents.SnapshotInboundFeed(context.Session)); + + var enqueue = context.Session.ArticleIngestion.TryAdmit(inbound); + if (enqueue != ArticleEnqueueResult.Accepted) + { + await ReleaseLeaseAsync(context, lease).ConfigureAwait(false); + EnqueueRejectionEvidence( + context, + PostFilterRejectionEvidence.FromAdmissionFailure( + filterRequest, + filter, + enqueue.ToString(), + context.Session.Time.GetUtcNow())); + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PersistenceFailure, enqueue.ToString()), + read.MessageId, + FormatGroups(read.Newsgroups), + inbound.Payload.Length, + cancellationToken) + .ConfigureAwait(false); + return; + } + + await CommitLeaseAsync(context, lease, read.MessageId!).ConfigureAwait(false); + + history?.Remember(messageIdBytes); + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + var groups = FormatGroups(read.Newsgroups); + PostLogMessages.Accepted( + logger, + client, + read.MessageId!, + groups, + read.Wire.Length); + } + + await WriteStatusAsync( + context, + NntpResponses.ArticleReceivedOk, + NntpResponseStatus.ArticleReceivedOk, + "accepted", + cancellationToken) + .ConfigureAwait(false); + } + + private static void EnqueueRejectionEvidence( + NntpCommandContext context, + PostFilterRejectionEvidence evidence) + { + if (context.Session.PostFilterEvidence.TryEnqueue(evidence)) + { + return; + } + + PostFilterLogMessages.EvidenceQueueFull(Logger, evidence.Stage, evidence.Reason); + } + + private static async ValueTask ReleaseLeaseAsync(NntpCommandContext context, PostFilterLease? lease) + { + if (lease is null) + { + return; + } + + var status = await lease + .ReleaseAsync(context.Session.Time.GetUtcNow(), CancellationToken.None) + .ConfigureAwait(false); + PostFilterLogMessages.Released( + Logger, + status, + lease.AccountName, + lease.Token); + } + + private static async ValueTask CommitLeaseAsync( + NntpCommandContext context, + PostFilterLease? lease, + string messageId) + { + if (lease is null) + { + return; + } + + var status = await lease + .CommitAsync(context.Session.Time.GetUtcNow(), CancellationToken.None) + .ConfigureAwait(false); + if (status == PostFilterQuotaCommitStatus.Committed) + { + return; + } + + if (status == PostFilterQuotaCommitStatus.Noop) + { + context.Session.PostFilterMetrics.RecordCommitNoop(); + PostFilterLogMessages.CommitNoop( + Logger, + lease.AccountName, + lease.Token, + messageId); + return; + } + + context.Session.PostFilterMetrics.RecordCommitUnavailable(); + PostFilterLogMessages.CommitUnavailable( + Logger, + lease.AccountName, + lease.Token, + messageId); + } + + private static async ValueTask SubmitForModerationAsync( + NntpCommandContext context, + StreamingPostReadResult read, + CancellationToken cancellationToken) + { + var submission = new ModerationSubmission + { + ProtoArticle = ArticleWireReconstructor.RestuffArticle(read.Wire.Span, includeTerminator: false), + MessageId = read.MessageId ?? string.Empty, + Newsgroups = read.Newsgroups, + TargetModeratedGroup = read.TargetModeratedGroup ?? string.Empty, + ModeratorAddress = read.ModeratorAddress ?? string.Empty, + AuthenticatedUsername = context.Session.Authentication.IsAuthenticated + ? context.Session.Authentication.Username + : null, + Sender = context.Session.ClientIdentity, + }; + + ModerationSubmissionResult result; + try + { + result = await context.Session.ModerationSubmission + .SubmitAsync(submission, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (result.Status != ModerationSubmissionStatus.Accepted) + { + await RejectAsync( + context, + new PostingFailure( + PostingFailureCategory.ModerationForwardingFailed, + result.Detail), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + var groups = FormatGroups(read.Newsgroups); + PostLogMessages.SubmittedForModeration( + logger, + client, + read.MessageId ?? "-", + groups, + read.TargetModeratedGroup ?? "-", + read.ModeratorAddress ?? "-", + context.Session.Authentication.Username ?? "-", + read.DestuffedSize); + } + + IngressNewsEvents.TryWriteModerated(context.Session, read.MessageId, read.DestuffedSize); + await WriteStatusAsync( + context, + NntpResponses.ArticleReceivedOk, + NntpResponseStatus.ArticleReceivedOk, + "accepted for moderation", + cancellationToken) + .ConfigureAwait(false); + } + + private static async ValueTask RejectAsync( + NntpCommandContext context, + PostingFailure failure, + string? messageId, + string? newsgroups, + int size, + CancellationToken cancellationToken) + { + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + PostLogMessages.Rejected( + logger, + client, + failure.Category, + messageId ?? "-", + newsgroups ?? "-", + size, + failure.Detail); + } + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId ?? "-", + 441, + IngressNewsReasons.ForPostingFailure(failure), + size); + await WriteFailedAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteFailedAsync(NntpCommandContext context, CancellationToken cancellationToken) => + WriteStatusAsync( + context, + NntpResponses.PostingFailed, + NntpResponseStatus.PostingFailed, + "posting failed", + cancellationToken); + + private static ValueTask WriteStatusAsync( + NntpCommandContext context, + ReadOnlyMemory wire, + string statusLine, + string detail, + CancellationToken cancellationToken) + { + context.StatusLine = statusLine; + context.CompletionDetail = detail; + return context.Response.WriteLineAsync(wire, cancellationToken); + } + + private static string FormatGroups(string[] groups) => + groups.Length == 0 ? "-" : string.Join(',', groups); + + private static PostingFailure MapArticleRecordFailure(in ArticleRecordCreateResult created) + { + if (created.ParseFailure != NntpArticleParseFailureCode.None) + { + return new PostingFailure(MapParseFailure(created.ParseFailure), created.ParseFailure.ToString()); + } + + return new PostingFailure(PostingFailureCategory.InvalidPath, created.MaterializeFailure.ToString()); + } + + private static PostingFailureCategory MapParseFailure(NntpArticleParseFailureCode code) => + code switch + { + NntpArticleParseFailureCode.ArticleTooLarge => PostingFailureCategory.ArticleTooLarge, + NntpArticleParseFailureCode.MalformedHeader + or NntpArticleParseFailureCode.MalformedHeaderContinuation + or NntpArticleParseFailureCode.MissingHeaderBodySeparator + or NntpArticleParseFailureCode.ContainsNul + or NntpArticleParseFailureCode.ContainsIllegalControlByte => PostingFailureCategory.MalformedHeader, + NntpArticleParseFailureCode.MissingMessageId + or NntpArticleParseFailureCode.MissingNewsgroups + or NntpArticleParseFailureCode.EmptyArticle => PostingFailureCategory.MissingRequiredHeader, + NntpArticleParseFailureCode.DuplicateMessageId + or NntpArticleParseFailureCode.DuplicateNewsgroups + or NntpArticleParseFailureCode.DuplicatePath => PostingFailureCategory.DuplicateHeader, + NntpArticleParseFailureCode.InvalidMessageId => PostingFailureCategory.InvalidMessageId, + NntpArticleParseFailureCode.MissingOrInvalidDate => PostingFailureCategory.InvalidDate, + NntpArticleParseFailureCode.InvalidNewsgroups => PostingFailureCategory.InvalidNewsgroups, + NntpArticleParseFailureCode.InvalidPath => PostingFailureCategory.InvalidPath, + _ => PostingFailureCategory.PolicyRejected, + }; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs new file mode 100644 index 00000000..bd9b2fdf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs @@ -0,0 +1,351 @@ +using System.Buffers.Text; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// AES-256-GCM protector for POST X-Trace (envelope version v1). + /// + /// + /// Token envelope: v1. followed by Base64url of nonce, ciphertext, and tag. + /// Envelope version is independent of the inner payload version and of key rotation. + /// Inner payload version 1 is the pre-username layout (still decryptable). + /// Inner payload version 2 appends a big-endian u16 length and UTF-8 username. + /// always writes payload version 2 with the current key. + /// tries the current key, then the optional previous key. + /// This type does not log keys, tokens, or decrypted payloads. + /// + public sealed class AesGcmPostingTraceProtector : IPostingTraceProtector + { + /// Header-safe envelope prefix for the current token version. + public const string TokenPrefix = "v1."; + + private const int NonceLength = 12; + private const int TagLength = 16; + private const byte LegacyPayloadVersion = 1; + private const byte CurrentPayloadVersion = 2; + internal const int MaxAuthenticatedUsernameBytes = 256; + private static readonly byte[] AssociatedData = "VectorNNTP.XTrace.v1"u8.ToArray(); + private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + private readonly byte[] _currentKey; + private readonly byte[]? _previousKey; + + /// Creates a protector from already-decoded AES-256 keys. + public AesGcmPostingTraceProtector(byte[] currentKey, byte[]? previousKey = null) + { + ArgumentNullException.ThrowIfNull(currentKey); + if (currentKey.Length != XTraceKeyParser.KeyLength) + { + throw new ArgumentException("X-Trace key must be 32 bytes.", nameof(currentKey)); + } + + if (previousKey is not null && previousKey.Length != XTraceKeyParser.KeyLength) + { + throw new ArgumentException("X-Trace previous key must be 32 bytes.", nameof(previousKey)); + } + + _currentKey = currentKey; + _previousKey = previousKey; + } + + /// Creates a protector from validated . + public static AesGcmPostingTraceProtector Create(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!XTraceKeyParser.TryDecode(options.XTraceKey, out var current)) + { + throw new InvalidOperationException("XTraceKey is missing or is not a 32-byte AES-256 key."); + } + + byte[]? previous = null; + if (!string.IsNullOrWhiteSpace(options.XTracePreviousKey) + && !XTraceKeyParser.TryDecode(options.XTracePreviousKey, out previous)) + { + throw new InvalidOperationException("XTracePreviousKey is not a 32-byte AES-256 key."); + } + + return new AesGcmPostingTraceProtector(current, previous); + } + + /// Creates a protector from options monitored at resolve time. + public static AesGcmPostingTraceProtector Create(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return Create(options.Value); + } + + /// + public string Protect(PostingTracePayload payload) => EncryptPacked(EncodePayload(payload)); + + /// + public bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload) + { + payload = default; + if (!token.StartsWith(TokenPrefix, StringComparison.Ordinal)) + { + return false; + } + + var encoded = token[TokenPrefix.Length..]; + var max = Base64Url.GetMaxDecodedLength(encoded.Length); + if (max < NonceLength + TagLength + 1) + { + return false; + } + + Span packed = max <= 256 ? stackalloc byte[max] : new byte[max]; + int written; + try + { + if (!Base64Url.TryDecodeFromChars(encoded, packed, out written) + || written < NonceLength + TagLength + 1) + { + return false; + } + } + catch (FormatException) + { + return false; + } + + packed = packed[..written]; + var nonce = packed[..NonceLength]; + var tag = packed[^TagLength..]; + var ciphertext = packed[NonceLength..^TagLength]; + return TryDecrypt(nonce, ciphertext, tag, _currentKey, out payload) + || (_previousKey is not null && TryDecrypt(nonce, ciphertext, tag, _previousKey, out payload)); + } + + private static bool TryDecrypt( + ReadOnlySpan nonce, + ReadOnlySpan ciphertext, + ReadOnlySpan tag, + byte[] key, + out PostingTracePayload payload) + { + payload = default; + Span plaintext = stackalloc byte[ciphertext.Length]; + try + { + using var aes = new AesGcm(key, TagLength); + aes.Decrypt(nonce, ciphertext, tag, plaintext, AssociatedData); + } + catch (CryptographicException) + { + return false; + } + + return TryDecodePayload(plaintext, out payload); + } + + /// + /// Protects a pre-username payload (inner version 1) for backward-compatibility tests. + /// Production always writes inner version 2. + /// + internal string ProtectLegacyV1(PostingTracePayload payload) => + EncryptPacked(EncodeLegacyV1(payload)); + + /// Encrypts caller-supplied plaintext for malformed-payload tests. + internal string ProtectRawForTests(ReadOnlySpan plaintext) => EncryptPacked(plaintext); + + private string EncryptPacked(ReadOnlySpan plaintext) + { + Span nonce = stackalloc byte[NonceLength]; + RandomNumberGenerator.Fill(nonce); + Span ciphertext = stackalloc byte[plaintext.Length]; + Span tag = stackalloc byte[TagLength]; + using (var aes = new AesGcm(_currentKey, TagLength)) + { + aes.Encrypt(nonce, plaintext, ciphertext, tag, AssociatedData); + } + + Span packed = stackalloc byte[NonceLength + plaintext.Length + TagLength]; + nonce.CopyTo(packed); + ciphertext.CopyTo(packed[NonceLength..]); + tag.CopyTo(packed[(NonceLength + plaintext.Length)..]); + return TokenPrefix + Base64Url.EncodeToString(packed); + } + + private static byte[] EncodePayload(PostingTracePayload payload) + { + var usernameBytes = EncodeUsername(payload.AuthenticatedUsername); + var address = payload.Address.GetAddressBytes(); + var length = FixedFieldLength(address.Length) + 2 + usernameBytes.Length; + var buffer = new byte[length]; + var offset = WriteFixedFields(buffer, CurrentPayloadVersion, payload, address); + System.Buffers.Binary.BinaryPrimitives.WriteUInt16BigEndian( + buffer.AsSpan(offset, 2), + (ushort)usernameBytes.Length); + offset += 2; + usernameBytes.CopyTo(buffer.AsSpan(offset)); + return buffer; + } + + private static byte[] EncodeLegacyV1(PostingTracePayload payload) + { + var address = payload.Address.GetAddressBytes(); + var buffer = new byte[FixedFieldLength(address.Length)]; + WriteFixedFields(buffer, LegacyPayloadVersion, payload, address); + return buffer; + } + + private static int FixedFieldLength(int addressLength) => 1 + 1 + addressLength + 2 + 8 + 16; + + private static int WriteFixedFields( + Span buffer, + byte version, + PostingTracePayload payload, + ReadOnlySpan address) + { + var offset = 0; + buffer[offset++] = version; + buffer[offset++] = (byte)address.Length; + address.CopyTo(buffer[offset..]); + offset += address.Length; + System.Buffers.Binary.BinaryPrimitives.WriteUInt16BigEndian(buffer[offset..(offset + 2)], (ushort)payload.Port); + offset += 2; + System.Buffers.Binary.BinaryPrimitives.WriteInt64BigEndian( + buffer[offset..(offset + 8)], + payload.InjectedAtUtc.ToUnixTimeSeconds()); + offset += 8; + payload.TraceId.TryWriteBytes(buffer[offset..(offset + 16)]); + return offset + 16; + } + + private static byte[] EncodeUsername(string? username) + { + if (string.IsNullOrEmpty(username)) + { + return []; + } + + var bytes = Utf8Strict.GetBytes(username); + if (bytes.Length > MaxAuthenticatedUsernameBytes || ContainsNul(bytes)) + { + throw new ArgumentException("Authenticated username is not encodable in X-Trace.", nameof(username)); + } + + return bytes; + } + + private static bool ContainsNul(ReadOnlySpan value) + { + foreach (var b in value) + { + if (b == 0) + { + return true; + } + } + + return false; + } + + private static bool TryDecodePayload(ReadOnlySpan plaintext, out PostingTracePayload payload) + { + payload = default; + if (plaintext.Length < 1 + 1 + 4 + 2 + 8 + 16) + { + return false; + } + + var version = plaintext[0]; + var addressLength = plaintext[1]; + if (addressLength is not (4 or 16)) + { + return false; + } + + var fixedLength = FixedFieldLength(addressLength); + if (plaintext.Length < fixedLength) + { + return false; + } + + var offset = 2; + IPAddress address; + try + { + address = new IPAddress(plaintext.Slice(offset, addressLength)); + } + catch (ArgumentException) + { + return false; + } + + offset += addressLength; + var port = System.Buffers.Binary.BinaryPrimitives.ReadUInt16BigEndian(plaintext.Slice(offset, 2)); + offset += 2; + var unix = System.Buffers.Binary.BinaryPrimitives.ReadInt64BigEndian(plaintext.Slice(offset, 8)); + offset += 8; + var traceId = new Guid(plaintext.Slice(offset, 16)); + offset += 16; + DateTimeOffset injected; + try + { + injected = DateTimeOffset.FromUnixTimeSeconds(unix); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + + string? username = null; + if (version == LegacyPayloadVersion) + { + if (plaintext.Length != fixedLength) + { + return false; + } + } + else if (version == CurrentPayloadVersion) + { + if (plaintext.Length < fixedLength + 2) + { + return false; + } + + var userLength = System.Buffers.Binary.BinaryPrimitives.ReadUInt16BigEndian(plaintext.Slice(offset, 2)); + offset += 2; + if (userLength > MaxAuthenticatedUsernameBytes || plaintext.Length != fixedLength + 2 + userLength) + { + return false; + } + + if (userLength > 0) + { + var userBytes = plaintext.Slice(offset, userLength); + if (ContainsNul(userBytes)) + { + return false; + } + + try + { + username = Utf8Strict.GetString(userBytes); + } + catch (DecoderFallbackException) + { + return false; + } + catch (ArgumentException) + { + return false; + } + } + } + else + { + return false; + } + + payload = new PostingTracePayload(address, port, injected, traceId, username); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ApprovedHeaderParser.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ApprovedHeaderParser.cs new file mode 100644 index 00000000..4d54c5a1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ApprovedHeaderParser.cs @@ -0,0 +1,240 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// Parses Approved: as RFC 5536 mailbox-list identities. + /// + /// + /// Header names compare case-insensitively. Values are mailbox identities, not booleans. + /// Multiple fields or mailbox-list entries are collected; identical identities collapse. + /// Conflicting identities are rejected rather than silently choosing one. + /// Comparison of the extracted mailbox is ASCII case-insensitive. + /// + internal static class ApprovedHeaderParser + { + /// + /// Collects every Approved: identity from . + /// + /// + /// when there is no Approved field, or when every field is a + /// well-formed mailbox-list whose identities do not conflict. + /// + public static bool TryRead( + ParsedPostArticle article, + out string[] identities, + out PostingFailure failure) + { + ArgumentNullException.ThrowIfNull(article); + identities = []; + failure = default; + + List? collected = null; + foreach (var header in article.Headers) + { + if (!PostFieldSyntax.EqualsFolded(header.Name.Span, "APPROVED"u8)) + { + continue; + } + + if (!TryParseMailboxList(header.UnfoldedValue.Span, out var parsed, out failure)) + { + identities = []; + return false; + } + + collected ??= new List(parsed.Length); + foreach (var identity in parsed) + { + if (ContainsIdentity(collected, identity)) + { + continue; + } + + collected.Add(identity); + } + } + + identities = collected is null ? [] : collected.ToArray(); + return true; + } + + internal static bool TryParseMailboxList( + ReadOnlySpan value, + out string[] identities, + out PostingFailure failure) + { + identities = []; + failure = default; + var collected = new List(2); + var i = 0; + var sawToken = false; + while (i < value.Length) + { + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "malformed Approved"); + return false; + } + + var start = i; + var angle = 0; + while (i < value.Length) + { + var b = value[i]; + if (b == (byte)'<') + { + angle++; + } + else if (b == (byte)'>') + { + angle--; + } + else if (b == (byte)',' && angle == 0) + { + break; + } + + i++; + } + + if (angle != 0 || !TryExtractMailbox(value[start..i], out var mailbox)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "malformed Approved"); + return false; + } + + if (!ContainsIdentity(collected, mailbox)) + { + collected.Add(mailbox); + } + + sawToken = true; + if (i >= value.Length) + { + break; + } + + i++; + } + + if (!sawToken) + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "empty Approved"); + return false; + } + + identities = collected.ToArray(); + return true; + } + + internal static bool TryExtractMailbox(ReadOnlySpan value, out string mailbox) + { + mailbox = string.Empty; + var trimmed = Trim(value); + if (trimmed.IsEmpty) + { + return false; + } + + var open = LastIndexOf(trimmed, (byte)'<'); + var close = LastIndexOf(trimmed, (byte)'>'); + ReadOnlySpan candidate; + if (open >= 0 || close >= 0) + { + if (open < 0 || close <= open) + { + return false; + } + + candidate = Trim(trimmed[(open + 1)..close]); + } + else + { + candidate = trimmed; + } + + if (!PostFieldSyntax.IsMailbox(candidate)) + { + return false; + } + + mailbox = Encoding.ASCII.GetString(candidate); + return true; + } + + internal static bool IdentitiesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (Fold(left[i]) != Fold(right[i])) + { + return false; + } + } + + return true; + } + + private static bool ContainsIdentity(List identities, string candidate) + { + foreach (var existing in identities) + { + if (IdentitiesEqual(existing, candidate)) + { + return true; + } + } + + return false; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && PostFieldSyntax.IsWsp(value[start])) + { + start++; + } + + while (end > start && PostFieldSyntax.IsWsp(value[end - 1])) + { + end--; + } + + return value[start..end]; + } + + private static int LastIndexOf(ReadOnlySpan value, byte needle) + { + for (var i = value.Length - 1; i >= 0; i--) + { + if (value[i] == needle) + { + return i; + } + } + + return -1; + } + + private static char Fold(char c) => + c is >= 'a' and <= 'z' ? (char)(c - 32) : c; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs new file mode 100644 index 00000000..2d17b83b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs @@ -0,0 +1,94 @@ +using System.Text; +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// POST newsgroup policy backed by one captured snapshot. + /// + /// + /// + /// Every syntax-validated Newsgroups: target must exist in the snapshot. + /// Local POST is rejected for unknown groups and for n/x/j + /// (RFC 3977 §7.6.3, RFC 6048 §3.1). Status y is ordinary local posting. + /// Status m is classified as ; + /// this policy does not authorize Approved: and does not forward to a moderator. + /// + /// + /// The catalogue snapshot is captured once per evaluation. Lookups do not query MySQL. + /// + /// + public sealed class CatalogueNewsgroupPostingPolicy : INewsgroupPostingPolicy + { + private readonly INewsgroupCatalogue _catalogue; + + /// Initializes a new instance of the class. + /// Process-wide catalogue whose is captured once per evaluation. + public CatalogueNewsgroupPostingPolicy(INewsgroupCatalogue catalogue) + { + ArgumentNullException.ThrowIfNull(catalogue); + _catalogue = catalogue; + } + + /// + public NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups) + { + if (groups.IsEmpty) + { + return Reject(PostingFailureCategory.InvalidNewsgroups, "empty Newsgroups"); + } + + var snapshot = _catalogue.Current; + Span nameBytes = stackalloc byte[128]; + string[]? moderated = null; + var moderatedCount = 0; + for (var i = 0; i < groups.Length; i++) + { + var group = groups[i]; + if (group.Length is < 1 or > 128) + { + return Reject(PostingFailureCategory.InvalidNewsgroups, "malformed newsgroup name"); + } + + var written = Encoding.ASCII.GetBytes(group, nameBytes); + if (!snapshot.TryGet(nameBytes[..written], out var record)) + { + return Reject(PostingFailureCategory.PolicyRejected, "unknown newsgroup"); + } + + switch (record.PostingStatus) + { + case NewsgroupPostingStatus.Allowed: + continue; + case NewsgroupPostingStatus.Prohibited: + return Reject(PostingFailureCategory.PolicyRejected, "posting prohibited"); + case NewsgroupPostingStatus.Moderated: + moderated ??= new string[groups.Length]; + moderated[moderatedCount++] = group; + continue; + case NewsgroupPostingStatus.NoPostingOrPeerArticles: + return Reject(PostingFailureCategory.PolicyRejected, "newsgroup closed"); + case NewsgroupPostingStatus.PeerOnly: + return Reject(PostingFailureCategory.PolicyRejected, "local posting not accepted"); + default: + return Reject(PostingFailureCategory.PolicyRejected, "unsupported posting status"); + } + } + + if (moderatedCount == 0) + { + return new NewsgroupPostingEvaluation(NewsgroupCatalogStatus.Accepted, Failure: null); + } + + var names = new string[moderatedCount]; + Array.Copy(moderated!, names, moderatedCount); + return new NewsgroupPostingEvaluation( + NewsgroupCatalogStatus.RequiresModeration, + Failure: null, + names); + } + + private static NewsgroupPostingEvaluation Reject(PostingFailureCategory category, string detail) => + new(NewsgroupCatalogStatus.Rejected, new PostingFailure(category, detail)); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/INewsgroupPostingPolicy.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/INewsgroupPostingPolicy.cs new file mode 100644 index 00000000..5b76f121 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/INewsgroupPostingPolicy.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// Newsgroup existence, posting-authorization, and moderation boundary used by POST. + /// + /// + /// Production sessions that have an + /// use . + /// remains the stand-in when no catalogue is attached (tests). + /// + public interface INewsgroupPostingPolicy + { + /// + /// Evaluates posting authorization for already syntax-validated newsgroup names. + /// + /// Distinct, syntactically valid newsgroup names. + /// + /// Catalogue classification. Does not re-validate group-name syntax and does not + /// treat Approved: as authorization. + /// + NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups); + } + + /// Result of . + /// Whether a catalogue decision was available. + /// Set when posting is rejected by policy. + /// + /// Moderated targets in Newsgroups: order when + /// ; otherwise empty. + /// + public readonly record struct NewsgroupPostingEvaluation( + NewsgroupCatalogStatus Status, + PostingFailure? Failure, + string[]? ModeratedGroups = null) + { + /// Gets moderated targets, or an empty array when none were classified. + public string[] ModeratedGroupNames => ModeratedGroups ?? []; + } + + /// Availability of newsgroup existence/moderation state. + public enum NewsgroupCatalogStatus + { + /// + /// Group existence and moderation state are not available. This is not a confirmed + /// existence match and must not be logged or treated as such. + /// + CatalogUnavailable = 0, + + /// Policy rejected posting to one or more groups. + Rejected = 1, + + /// + /// Every target exists in the captured snapshot and permits local POST + /// (y). This is a confirmed catalogue decision. + /// + Accepted = 2, + + /// + /// At least one target is moderated (m) and every other target permits + /// local POST (y). Approval authorization or moderator forwarding is required. + /// + RequiresModeration = 3, + } + + /// + /// POST newsgroup policy used when no catalogue is attached: syntax-valid groups + /// are accepted without asserting that the groups exist or that moderation state + /// is known. + /// + /// + /// Production sessions with a catalogue use . + /// This stand-in does not consult the catalogue and does not treat Approved: + /// as authorization to post to a moderated group. + /// + public sealed class SyntaxOnlyNewsgroupPostingPolicy : INewsgroupPostingPolicy + { + /// Shared instance. + public static SyntaxOnlyNewsgroupPostingPolicy Instance { get; } = new(); + + /// + public NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups) + { + _ = groups; + return new NewsgroupPostingEvaluation(NewsgroupCatalogStatus.CatalogUnavailable, Failure: null); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/IPostingTraceProtector.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/IPostingTraceProtector.cs new file mode 100644 index 00000000..69831465 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/IPostingTraceProtector.cs @@ -0,0 +1,26 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// Authenticated-encryption boundary for server-owned POST X-Trace values. + /// + /// + /// Implementations must provide confidentiality, integrity, and tamper detection. + /// Do not log keys or decrypted payloads. + /// + public interface IPostingTraceProtector + { + /// + /// Protects as an opaque header-safe token. + /// + string Protect(PostingTracePayload payload); + + /// + /// Attempts to recover a payload from a previously protected token. + /// + /// + /// when authentication succeeds and the payload is well-formed; + /// otherwise (tamper, unknown version, or wrong key). + /// + bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ParsedPostArticle.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ParsedPostArticle.cs new file mode 100644 index 00000000..210bd11e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/ParsedPostArticle.cs @@ -0,0 +1,92 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// One structurally parsed header field from a destuffed POST article. + /// Original field-name octets (case preserved). + /// Logical field body with folding CRLF removed. + /// Original field octets including folding and the terminating CRLF. + internal readonly record struct ParsedPostHeader( + ReadOnlyMemory Name, + ReadOnlyMemory UnfoldedValue, + ReadOnlyMemory RawField); + + /// Destuffed POST article after one structural header parse. + internal sealed class ParsedPostArticle + { + /// Initializes a new instance of the class. + public ParsedPostArticle( + ReadOnlyMemory payload, + IReadOnlyList headers, + ReadOnlyMemory body, + int headerBlockBytes) + { + Payload = payload; + Headers = headers; + Body = body; + HeaderBlockBytes = headerBlockBytes; + } + + /// Gets the destuffed received payload (headers + separator + body). + public ReadOnlyMemory Payload { get; } + + /// Gets parsed headers in wire order. + public IReadOnlyList Headers { get; } + + /// Gets the destuffed body (may be empty). + public ReadOnlyMemory Body { get; } + + /// Gets destuffed header-block length including the blank separator CRLF. + public int HeaderBlockBytes { get; } + + /// Gets destuffed article size (payload length; terminator excluded). + public int ArticleSize => Payload.Length; + + /// Gets or sets the article Message-ID (supplied or synthesized). + public string? MessageId { get; set; } + + /// Gets or sets whether was synthesized by the server. + public bool MessageIdSynthesized { get; set; } + + /// Gets or sets syntax-validated newsgroup names. + public string[] Newsgroups { get; set; } = []; + + /// Gets or sets the parsed author Date:, or the injection time when synthesized. + public DateTimeOffset AuthorDate { get; set; } + + /// Gets or sets whether was generated because the client omitted Date:. + public bool DateSynthesized { get; set; } + + /// Gets or sets whether a structurally valid Approved: is present. + public bool ApprovedPresent { get; set; } + + /// + /// Gets or sets the distinct mailbox identities parsed from Approved:. + /// Empty when the header is absent. + /// + public string[] ApprovedIdentities { get; set; } = []; + + /// Gets or sets the catalogue classification for this article's newsgroups. + public NewsgroupCatalogStatus CatalogStatus { get; set; } + + /// + /// Gets or sets moderated Newsgroups: targets in header order when + /// is . + /// + public string[] ModeratedGroups { get; set; } = []; + + /// Returns the first header whose name matches . + public bool TryGetHeader(ReadOnlySpan upperName, out ParsedPostHeader header) + { + foreach (var candidate in Headers) + { + if (PostFieldSyntax.EqualsFolded(candidate.Name.Span, upperName)) + { + header = candidate; + return true; + } + } + + header = default; + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostArticleValidator.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostArticleValidator.cs new file mode 100644 index 00000000..be8d049d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostArticleValidator.cs @@ -0,0 +1,423 @@ +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Strict POST header-field validation after one structural parse. + internal static class PostArticleValidator + { + private static readonly byte[][] SingletonNames = + [ + "DATE"u8.ToArray(), + "FROM"u8.ToArray(), + "MESSAGE-ID"u8.ToArray(), + "NEWSGROUPS"u8.ToArray(), + "PATH"u8.ToArray(), + "SUBJECT"u8.ToArray(), + "CONTROL"u8.ToArray(), + "DISTRIBUTION"u8.ToArray(), + "FOLLOWUP-TO"u8.ToArray(), + "INJECTION-DATE"u8.ToArray(), + "INJECTION-INFO"u8.ToArray(), + "LINES"u8.ToArray(), + "ORGANIZATION"u8.ToArray(), + "REFERENCES"u8.ToArray(), + "SUMMARY"u8.ToArray(), + "SUPERSEDES"u8.ToArray(), + "USER-AGENT"u8.ToArray(), + "XREF"u8.ToArray(), + "NNTP-POSTING-DATE"u8.ToArray(), + "NNTP-POSTING-HOST"u8.ToArray(), + "X-TRACE"u8.ToArray(), + "IN-REPLY-TO"u8.ToArray(), + ]; + + /// Validates parsed headers and fills Message-ID, Newsgroups, and Date on . + public static bool TryValidate( + ParsedPostArticle article, + DateTimeOffset injectionUtc, + INewsgroupPostingPolicy newsgroupPolicy, + out PostingFailure failure, + bool controlCancelPermitted = false) + { + ArgumentNullException.ThrowIfNull(article); + ArgumentNullException.ThrowIfNull(newsgroupPolicy); + + if (!TryCheckSingletonsAndForbidden(article, out failure)) + { + return false; + } + + if (!TryRequire(article, "FROM"u8, "From", out var fromHeader, out failure) + || !TryRequire(article, "NEWSGROUPS"u8, "Newsgroups", out var groupsHeader, out failure) + || !TryRequire(article, "SUBJECT"u8, "Subject", out _, out failure)) + { + return false; + } + + if (!PostFieldSyntax.IsMailbox(fromHeader.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "invalid From"); + return false; + } + + if (article.TryGetHeader("DATE"u8, out var dateHeader)) + { + if (!PostRfcDate.TryParse(dateHeader.UnfoldedValue.Span, out var authorDate)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDate, "malformed Date"); + return false; + } + + if (!PostRfcDate.IsWithinPolicy(authorDate, injectionUtc)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDate, "Date outside policy window"); + return false; + } + + article.AuthorDate = authorDate; + article.DateSynthesized = false; + } + else + { + article.AuthorDate = injectionUtc; + article.DateSynthesized = true; + } + + var groups = new List(4); + if (!PostFieldSyntax.TryParseNewsgroupList( + groupsHeader.UnfoldedValue.Span, + groups, + PostingLimits.MaxNewsgroups, + allowPoster: false)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidNewsgroups, "malformed Newsgroups"); + return false; + } + + article.Newsgroups = groups.ToArray(); + + if (article.TryGetHeader("MESSAGE-ID"u8, out var messageIdHeader)) + { + if (!NntpMessageIdValidation.IsValidMessageId(messageIdHeader.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidMessageId, "malformed Message-ID"); + return false; + } + + article.MessageId = System.Text.Encoding.ASCII.GetString(messageIdHeader.UnfoldedValue.Span); + article.MessageIdSynthesized = false; + } + else + { + article.MessageId = PostMessageIdFactory.Create(); + article.MessageIdSynthesized = true; + } + + if (!TryValidateOptionalFields(article, controlCancelPermitted, out failure)) + { + return false; + } + + var evaluation = newsgroupPolicy.Evaluate(article.Newsgroups); + article.CatalogStatus = evaluation.Status; + article.ModeratedGroups = evaluation.ModeratedGroupNames; + if (evaluation.Status == NewsgroupCatalogStatus.Rejected) + { + failure = evaluation.Failure + ?? new PostingFailure(PostingFailureCategory.PolicyRejected, "newsgroup policy rejected"); + return false; + } + + return true; + } + + private static bool TryCheckSingletonsAndForbidden(ParsedPostArticle article, out PostingFailure failure) + { + failure = default; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var header in article.Headers) + { + var name = System.Text.Encoding.ASCII.GetString(header.Name.Span); + if (IsForbiddenMailRouting(header.Name.Span)) + { + failure = new PostingFailure(PostingFailureCategory.PolicyRejected, "mail routing header"); + return false; + } + + if (!IsSingleton(header.Name.Span)) + { + continue; + } + + if (!seen.Add(name)) + { + failure = new PostingFailure(PostingFailureCategory.DuplicateHeader, name); + return false; + } + } + + return true; + } + + private static bool TryValidateOptionalFields( + ParsedPostArticle article, + bool controlCancelPermitted, + out PostingFailure failure) + { + failure = default; + var ids = new List>(8); + + if (article.TryGetHeader("FOLLOWUP-TO"u8, out var followup)) + { + var followGroups = new List(4); + if (!PostFieldSyntax.TryParseNewsgroupList( + followup.UnfoldedValue.Span, + followGroups, + PostingLimits.MaxFollowupToGroups, + allowPoster: true)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidFollowupTo, "malformed Followup-To"); + return false; + } + } + + if (article.TryGetHeader("REFERENCES"u8, out var references) + && !PostFieldSyntax.TryParseMessageIdList(references.UnfoldedValue.Span, ids, maxCount: 32)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidReferences, "malformed References"); + return false; + } + + var referenceIds = ids.ToArray(); + if (article.TryGetHeader("IN-REPLY-TO"u8, out var inReplyTo)) + { + if (!PostFieldSyntax.TryParseMessageIdList(inReplyTo.UnfoldedValue.Span, ids, maxCount: 8)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidReferences, "malformed In-Reply-To"); + return false; + } + + if (referenceIds.Length > 0 && !ContainsMessageId(referenceIds, ids[^1].Span)) + { + failure = new PostingFailure( + PostingFailureCategory.InvalidReferences, + "In-Reply-To is not in References"); + return false; + } + } + + if (article.TryGetHeader("DISTRIBUTION"u8, out var distribution) + && !TryValidateDistribution(distribution.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDistribution, "malformed Distribution"); + return false; + } + + if (!ApprovedHeaderParser.TryRead(article, out var approvedIdentities, out failure)) + { + return false; + } + + article.ApprovedIdentities = approvedIdentities; + article.ApprovedPresent = approvedIdentities.Length > 0; + + if (article.TryGetHeader("CONTROL"u8, out var control) + && (!controlCancelPermitted + || !IsAuthorizedCancelControl(control.UnfoldedValue.Span))) + { + failure = new PostingFailure(PostingFailureCategory.InvalidControl, "unsupported Control"); + return false; + } + + if (article.TryGetHeader("SUPERSEDES"u8, out var supersedes) + && !PostFieldSyntax.TryParseMessageIdList(supersedes.UnfoldedValue.Span, ids, maxCount: 8)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidControl, "malformed Supersedes"); + return false; + } + + if (article.TryGetHeader("LINES"u8, out var lines) + && !IsUnsignedInteger(lines.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "malformed Lines"); + return false; + } + + return true; + } + + private static bool TryValidateDistribution(ReadOnlySpan value) + { + var i = 0; + var count = 0; + while (i < value.Length) + { + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + return false; + } + + var start = i; + while (i < value.Length && value[i] != (byte)',' && !PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (!PostFieldSyntax.IsDistributionToken(value[start..i])) + { + return false; + } + + count++; + if (count > 16) + { + return false; + } + + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)',') + { + return false; + } + + i++; + } + + return count > 0; + } + + /// + /// Accepts only cancel <message-id> (optional surrounding WSP). + /// Other Control verbs stay rejected even for the newsmaster. + /// + internal static bool IsAuthorizedCancelControl(ReadOnlySpan value) + // codeql[cs/complex-block] Protocol validation clarity for cancel Control parsing. + { + var i = 0; + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i + 6 > value.Length || !PostFieldSyntax.EqualsFolded(value.Slice(i, 6), "CANCEL"u8)) + { + return false; + } + + i += 6; + if (i >= value.Length || !PostFieldSyntax.IsWsp(value[i])) + { + return false; + } + + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + var idStart = i; + while (i < value.Length && !PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + var messageId = value[idStart..i]; + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + return i == value.Length && NntpMessageIdValidation.IsValidMessageId(messageId); + } + + private static bool TryRequire( + ParsedPostArticle article, + ReadOnlySpan upperName, + string display, + out ParsedPostHeader header, + out PostingFailure failure) + { + if (article.TryGetHeader(upperName, out header)) + { + failure = default; + return true; + } + + header = default; + failure = new PostingFailure(PostingFailureCategory.MissingRequiredHeader, display); + return false; + } + + private static bool IsSingleton(ReadOnlySpan name) + { + foreach (var candidate in SingletonNames) + { + if (PostFieldSyntax.EqualsFolded(name, candidate)) + { + return true; + } + } + + return false; + } + + private static bool IsForbiddenMailRouting(ReadOnlySpan name) => + PostFieldSyntax.EqualsFolded(name, "RECEIVED"u8) + || PostFieldSyntax.EqualsFolded(name, "DELIVERED-TO"u8) + || PostFieldSyntax.EqualsFolded(name, "BCC"u8) + || PostFieldSyntax.EqualsFolded(name, "CC"u8) + || PostFieldSyntax.EqualsFolded(name, "TO"u8) + || PostFieldSyntax.EqualsFolded(name, "X-FORWARDED-FOR"u8) + || PostFieldSyntax.EqualsFolded(name, "FORWARDED"u8); + + private static bool ContainsMessageId(ReadOnlyMemory[] haystack, ReadOnlySpan needle) + { + foreach (var item in haystack) + { + if (item.Span.SequenceEqual(needle)) + { + return true; + } + } + + return false; + } + + private static bool IsUnsignedInteger(ReadOnlySpan value) + { + if (value.IsEmpty) + { + return false; + } + + foreach (var b in value) + { + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostFieldSyntax.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostFieldSyntax.cs new file mode 100644 index 00000000..36b5deb2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostFieldSyntax.cs @@ -0,0 +1,287 @@ +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Structural syntax checks for Netnews header field values (byte-oriented). + internal static class PostFieldSyntax + { + /// RFC 5536 newsgroup-name: component *( "." component ). + public static bool IsNewsgroupName(ReadOnlySpan value) + { + if (value.IsEmpty || value.Length > 128) + { + return false; + } + + var componentLength = 0; + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b == (byte)'.') + { + if (componentLength == 0) + { + return false; + } + + componentLength = 0; + continue; + } + + if (!IsComponentChar(b)) + { + return false; + } + + componentLength++; + } + + return componentLength > 0; + } + + /// One or more Message-IDs separated by WSP. + public static bool TryParseMessageIdList( + ReadOnlySpan value, + List> ids, + int maxCount) + { + ids.Clear(); + var i = 0; + while (i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)'<') + { + return false; + } + + var end = value[i..].IndexOf((byte)'>'); + if (end < 0) + { + return false; + } + + var token = value.Slice(i, end + 1); + if (!NntpMessageIdValidation.IsValidMessageId(token)) + { + return false; + } + + if (ids.Count >= maxCount) + { + return false; + } + + ids.Add(token.ToArray()); + i += end + 1; + } + + return ids.Count > 0; + } + + /// Comma-separated newsgroup names with optional WSP around commas. + public static bool TryParseNewsgroupList( + ReadOnlySpan value, + List groups, + int maxCount, + bool allowPoster) + { + groups.Clear(); + var i = 0; + var sawToken = false; + while (i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + return false; + } + + var start = i; + while (i < value.Length && value[i] != (byte)',' && !IsWsp(value[i])) + { + i++; + } + + var token = value[start..i]; + if (token.IsEmpty) + { + return false; + } + + if (allowPoster && EqualsFolded(token, "POSTER"u8)) + { + if (sawToken || i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i < value.Length) + { + return false; + } + } + + groups.Add("poster"); + return groups.Count == 1; + } + + if (!IsNewsgroupName(token)) + { + return false; + } + + var name = System.Text.Encoding.ASCII.GetString(token); + foreach (var existing in groups) + { + if (existing.Equals(name, StringComparison.Ordinal)) + { + return false; + } + } + + if (groups.Count >= maxCount) + { + return false; + } + + groups.Add(name); + sawToken = true; + + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)',') + { + return false; + } + + i++; + if (i >= value.Length) + { + return false; + } + } + + return groups.Count > 0; + } + + /// Requires a mailbox-shaped value containing exactly one @ in a printable region. + public static bool IsMailbox(ReadOnlySpan value) + { + if (value.IsEmpty || value.Length > 512) + { + return false; + } + + var at = -1; + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b == 0 || (b < 32 && b != (byte)'\t')) + { + return false; + } + + if (b == (byte)'@') + { + if (at >= 0) + { + return false; + } + + at = i; + } + } + + return at > 0 && at < value.Length - 1; + } + + /// Distribution token: world, local, or a newsgroup-name. + public static bool IsDistributionToken(ReadOnlySpan value) => + EqualsFolded(value, "WORLD"u8) + || EqualsFolded(value, "LOCAL"u8) + || IsNewsgroupName(value); + + public static bool EqualsFolded(ReadOnlySpan value, ReadOnlySpan upperAscii) + { + if (value.Length != upperAscii.Length) + { + return false; + } + + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b >= (byte)'a' && b <= (byte)'z') + { + b = (byte)(b - 32); + } + + if (b != upperAscii[i]) + { + return false; + } + } + + return true; + } + + public static bool HeaderNamesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (FoldUpper(left[i]) != FoldUpper(right[i])) + { + return false; + } + } + + return true; + } + + public static byte FoldUpper(byte b) => + b is >= (byte)'a' and <= (byte)'z' ? (byte)(b - 32) : b; + + public static bool IsWsp(byte b) => b is (byte)' ' or (byte)'\t'; + + private static bool IsComponentChar(byte b) => + // codeql[cs/complex-condition] Protocol validation clarity for component character class. + (b >= (byte)'A' && b <= (byte)'Z') + || (b >= (byte)'a' && b <= (byte)'z') + || (b >= (byte)'0' && b <= (byte)'9') + || b is (byte)'+' or (byte)'-' or (byte)'_'; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderNormalizer.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderNormalizer.cs new file mode 100644 index 00000000..0e2e0a9d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderNormalizer.cs @@ -0,0 +1,173 @@ +using System.Buffers; +using System.Net; +using System.Text; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Replaces server-owned POST headers and emits one destuffed article buffer. + internal static class PostHeaderNormalizer + { + /// Server-controlled Path for locally POSTed articles. + public const string PostedPath = ".POSTED"; + + /// + /// Builds the accepted article: client Path / injection metadata / Xref discarded, + /// server-owned values written once, a valid client Date preserved, and a Date + /// written from the injection time only when the client omitted Date. + /// + public static ReadOnlyMemory Normalize( + ParsedPostArticle article, + DateTimeOffset injectionUtc, + string injectionIdentity, + ConnectionClientIdentity clientIdentity, + string mailComplaintsTo, + IPostingTraceProtector traceProtector, + string? authenticatedUsername = null) + { + ArgumentNullException.ThrowIfNull(article); + ArgumentException.ThrowIfNullOrWhiteSpace(article.MessageId); + ArgumentException.ThrowIfNullOrWhiteSpace(injectionIdentity); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(mailComplaintsTo); + ArgumentNullException.ThrowIfNull(traceProtector); + + var writer = new ArrayBufferWriter(article.Payload.Length + 256); + foreach (var header in article.Headers) + { + if (IsServerOwned(header.Name.Span)) + { + continue; + } + + writer.Write(header.RawField.Span); + } + + WriteServerOwnedHeaders( + writer, + article.MessageIdSynthesized, + article.MessageId, + article.DateSynthesized, + injectionUtc, + injectionIdentity, + clientIdentity, + mailComplaintsTo, + traceProtector, + authenticatedUsername); + if (!article.Body.IsEmpty) + { + writer.Write(article.Body.Span); + } + + return writer.WrittenMemory.ToArray(); + } + + /// Formats the transport peer for trusted diagnostics (never written as a header). + public static string FormatPostingHost(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return address.IsIPv4MappedToIPv6 ? address.MapToIPv4().ToString() : address.ToString(); + } + + internal static bool IsServerOwned(ReadOnlySpan name) => + PostFieldSyntax.EqualsFolded(name, "PATH"u8) + || PostFieldSyntax.EqualsFolded(name, "INJECTION-DATE"u8) + || PostFieldSyntax.EqualsFolded(name, "INJECTION-INFO"u8) + || PostFieldSyntax.EqualsFolded(name, "NNTP-POSTING-DATE"u8) + || PostFieldSyntax.EqualsFolded(name, "NNTP-POSTING-HOST"u8) + || PostFieldSyntax.EqualsFolded(name, "X-TRACE"u8) + || PostFieldSyntax.EqualsFolded(name, "XREF"u8); + + /// + /// Writes synthesized Message-ID and Date (when needed), Path, Injection-Date, Injection-Info, + /// X-Trace, and the header/body separator into . + /// + /// + /// writes of + /// . A client Date already copied into the article is not rewritten. + /// + internal static void WriteServerOwnedHeaders( + IBufferWriter writer, + bool writeMessageId, + string messageId, + bool writeDate, + DateTimeOffset injectionUtc, + string injectionIdentity, + ConnectionClientIdentity clientIdentity, + string mailComplaintsTo, + IPostingTraceProtector traceProtector, + string? authenticatedUsername = null) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentException.ThrowIfNullOrWhiteSpace(injectionIdentity); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(mailComplaintsTo); + ArgumentNullException.ThrowIfNull(traceProtector); + + if (writeMessageId) + { + WriteHeader(writer, "Message-ID: "u8, messageId); + } + + if (writeDate) + { + WriteHeader(writer, "Date: "u8, PostRfcDate.Format(injectionUtc)); + } + + WriteHeader(writer, "Path: "u8, PostedPath); + WriteHeader(writer, "Injection-Date: "u8, PostRfcDate.Format(injectionUtc)); + WriteHeader(writer, "Injection-Info: "u8, BuildInjectionInfo(injectionIdentity, messageId, mailComplaintsTo)); + WriteHeader( + writer, + "X-Trace: "u8, + traceProtector.Protect( + new PostingTracePayload( + clientIdentity.ClientAddress, + clientIdentity.ClientPort, + injectionUtc, + Guid.NewGuid(), + authenticatedUsername))); + writer.Write("\r\n"u8); + } + + /// + /// Writes a synthesized Message-ID and Date when required, then the header/body separator. + /// Does not write Path, Injection-Date, Injection-Info, or X-Trace (RFC 5537 §3.5 step 7). + /// + internal static void WriteProtoArticleBoundary( + IBufferWriter writer, + bool writeMessageId, + string messageId, + bool writeDate, + DateTimeOffset injectionUtc) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + if (writeMessageId) + { + WriteHeader(writer, "Message-ID: "u8, messageId); + } + + if (writeDate) + { + WriteHeader(writer, "Date: "u8, PostRfcDate.Format(injectionUtc)); + } + + writer.Write("\r\n"u8); + } + + private static void WriteHeader(IBufferWriter writer, ReadOnlySpan prefix, string value) + { + writer.Write(prefix); + Encoding.ASCII.GetBytes(value, writer); + writer.Write("\r\n"u8); + } + + private static string BuildInjectionInfo( + string identity, + string messageId, + string mailComplaintsTo) => + $"{identity}; logging-data=\"{messageId}\"; mail-complaints-to=\"{mailComplaintsTo}\""; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderParser.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderParser.cs new file mode 100644 index 00000000..7270468e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostHeaderParser.cs @@ -0,0 +1,301 @@ +using System.Buffers; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Structural destuffed-article header parser. Does not use string.Split. + internal static class PostHeaderParser + { + /// + /// Parses destuffed article bytes (headers + blank separator + body). + /// Size has already been enforced by the multiline reader. + /// + public static bool TryParse( + ReadOnlyMemory payload, + out ParsedPostArticle? article, + out PostingFailure failure) + { + article = null; + var span = payload.Span; + if (span.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty article"); + return false; + } + + for (var i = 0; i < span.Length; i++) + { + if (span[i] == 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "embedded NUL"); + return false; + } + } + + if (!TryFindSeparator(span, out var separator, out failure)) + { + return false; + } + + if (separator + 2 > PostingLimits.MaxHeaderBlockBytes) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + return false; + } + + var headerBlock = span[..separator]; + var body = payload[(separator + 2)..]; + if (!TryParseHeaderBlock(payload, headerBlock, out var headers, out failure)) + { + return false; + } + + article = new ParsedPostArticle(payload, headers, body, separator + 2); + return true; + } + + private static bool TryFindSeparator( + ReadOnlySpan span, + out int separator, + out PostingFailure failure) + { + separator = -1; + var i = 0; + while (i < span.Length) + { + if (span[i] == (byte)'\n' && (i == 0 || span[i - 1] != (byte)'\r')) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "LF without CR"); + return false; + } + + if (span[i] == (byte)'\r') + { + if (i + 1 >= span.Length || span[i + 1] != (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "CR without LF"); + return false; + } + + if (i + 3 < span.Length && span[i + 2] == (byte)'\r' && span[i + 3] == (byte)'\n') + { + separator = i + 2; + failure = default; + return true; + } + + i += 2; + continue; + } + + i++; + } + + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "missing header/body separator"); + return false; + } + + private static bool TryParseHeaderBlock( + ReadOnlyMemory payload, + ReadOnlySpan headerBlock, + out List headers, + out PostingFailure failure) + { + headers = []; + failure = default; + if (headerBlock.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header block"); + return false; + } + + if (headerBlock.Length < 2 + || headerBlock[^2] != (byte)'\r' + || headerBlock[^1] != (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block is not CRLF terminated"); + return false; + } + + var unfolded = new ArrayBufferWriter(256); + var fieldStart = 0; + var nameStart = -1; + var nameLength = 0; + var hasField = false; + var lineStart = 0; + var i = 0; + while (i < headerBlock.Length) + { + if (i + 1 >= headerBlock.Length + || headerBlock[i] != (byte)'\r' + || headerBlock[i + 1] != (byte)'\n') + { + i++; + continue; + } + + var line = headerBlock[lineStart..i]; + var isFold = line.Length > 0 && PostFieldSyntax.IsWsp(line[0]); + if (isFold) + { + if (!hasField) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "leading fold"); + return false; + } + + if (line.Length < 2) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty fold"); + return false; + } + + unfolded.Write(line); + } + else + { + if (hasField + && !FlushHeader(payload, fieldStart, lineStart, nameStart, nameLength, unfolded, headers, out failure)) + { + return false; + } + + if (line.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header line"); + return false; + } + + if (!TryReadFieldName(line, out nameLength, out failure)) + { + return false; + } + + fieldStart = lineStart; + nameStart = lineStart; + hasField = true; + unfolded.Clear(); + unfolded.Write(line[(nameLength + 2)..]); + } + + i += 2; + lineStart = i; + } + + if (lineStart != headerBlock.Length) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "truncated header line"); + return false; + } + + if (!hasField) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "no header fields"); + return false; + } + + if (!FlushHeader(payload, fieldStart, headerBlock.Length, nameStart, nameLength, unfolded, headers, out failure)) + { + return false; + } + + if (headers.Count > PostingLimits.MaxHeaderCount) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "too many headers"); + return false; + } + + return true; + } + + internal static bool TryReadFieldName( + ReadOnlySpan line, + out int nameLength, + out PostingFailure failure) + { + nameLength = 0; + var colon = line.IndexOf((byte)':'); + if (colon <= 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header line missing field name"); + return false; + } + + if (colon + 1 >= line.Length || line[colon + 1] != (byte)' ') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header requires colon-space"); + return false; + } + + for (var i = 0; i < colon; i++) + { + var b = line[i]; + if (b is < 33 or > 126 || b == (byte)':') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "malformed field name"); + return false; + } + } + + nameLength = colon; + failure = default; + return true; + } + + private static bool FlushHeader( + ReadOnlyMemory payload, + int fieldStart, + int fieldEnd, + int nameStart, + int nameLength, + ArrayBufferWriter unfolded, + List headers, + out PostingFailure failure) + { + failure = default; + var raw = payload.Slice(fieldStart, fieldEnd - fieldStart); + var name = payload.Slice(nameStart, nameLength); + var value = unfolded.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : unfolded.WrittenMemory.ToArray(); + + if (!TryCreateHeader(name, value, raw, out var header, out failure)) + { + return false; + } + + headers.Add(header); + return true; + } + + /// Validates unfolded field length/control characters and builds a parsed header. + internal static bool TryCreateHeader( + ReadOnlyMemory name, + ReadOnlyMemory unfoldedValue, + ReadOnlyMemory rawField, + out ParsedPostHeader header, + out PostingFailure failure) + { + header = default; + if (name.Length + 2 + unfoldedValue.Length > PostingLimits.MaxSingleHeaderBytes) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header too long"); + return false; + } + + var valueSpan = unfoldedValue.Span; + for (var i = 0; i < valueSpan.Length; i++) + { + var b = valueSpan[i]; + if (b == 0 || (b < 32 && b != (byte)'\t')) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "forbidden control character"); + return false; + } + } + + header = new ParsedPostHeader(name, unfoldedValue, rawField); + failure = default; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostLogMessages.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostLogMessages.cs new file mode 100644 index 00000000..4a004f2d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostLogMessages.cs @@ -0,0 +1,44 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Source-generated POST accept/reject diagnostics. Does not log article bodies. + internal static partial class PostLogMessages + { + [LoggerMessage( + EventId = 1720, + Level = LogLevel.Information, + Message = "[{Client}] POST accepted Message-ID={MessageId} newsgroups={Newsgroups} size={Size}")] + public static partial void Accepted( + ILogger logger, + string Client, + string MessageId, + string Newsgroups, + int Size); + + [LoggerMessage( + EventId = 1722, + Level = LogLevel.Information, + Message = "[{Client}] POST submitted for moderation Message-ID={MessageId} newsgroups={Newsgroups} target={TargetGroup} moderator={ModeratorAddress} user={Username} size={Size}")] + public static partial void SubmittedForModeration( + ILogger logger, + string Client, + string MessageId, + string Newsgroups, + string TargetGroup, + string ModeratorAddress, + string Username, + int Size); + + [LoggerMessage( + EventId = 1721, + Level = LogLevel.Information, + Message = "[{Client}] POST rejected category={Category} Message-ID={MessageId} newsgroups={Newsgroups} size={Size} detail={Detail}")] + public static partial void Rejected( + ILogger logger, + string Client, + PostingFailureCategory Category, + string MessageId, + string Newsgroups, + int Size, + string Detail); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostMessageIdFactory.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostMessageIdFactory.cs new file mode 100644 index 00000000..2923bc6a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostMessageIdFactory.cs @@ -0,0 +1,31 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Synthesizes a POST Message-ID as <MD5(UUID())@usenet.ninja>. + internal static class PostMessageIdFactory + { + /// Right-hand side of a synthesized Message-ID. + public const string Domain = "usenet.ninja"; + + /// + /// Creates a Message-ID for one posting attempt. The UUID is generated once; + /// MD5 is taken over its canonical D-format text (not the raw UUID). + /// + public static string Create() + { + var uuid = Guid.NewGuid().ToString("D"); + var hash = MD5.HashData(Encoding.ASCII.GetBytes(uuid)); + var hex = Convert.ToHexStringLower(hash); + return string.Create(hex.Length + Domain.Length + 3, hex, static (span, id) => + { + span[0] = '<'; + id.CopyTo(span[1..]); + span[1 + id.Length] = '@'; + Domain.CopyTo(span[(2 + id.Length)..]); + span[^1] = '>'; + }); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostRfcDate.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostRfcDate.cs new file mode 100644 index 00000000..8128eaa4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostRfcDate.cs @@ -0,0 +1,255 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// RFC 5322 / Netnews date-time format used for POST Date: and injection timestamps. + internal static class PostRfcDate + { + private static readonly string[] DayNames = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + private static readonly string[] MonthNames = + ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + + /// Formats as Wed, 25 Sep 2026 10:49:00 +0000. + public static string Format(DateTimeOffset utc) + { + var value = utc.ToUniversalTime(); + return string.Create( + CultureInfo.InvariantCulture, + $"{DayNames[(int)value.DayOfWeek]}, {value.Day:00} {MonthNames[value.Month - 1]} {value.Year:0000} {value.Hour:00}:{value.Minute:00}:{value.Second:00} +0000"); + } + + /// Parses an RFC-compatible date (numeric zone, GMT, UT, or UTC). + public static bool TryParse(ReadOnlySpan value, out DateTimeOffset date) + { + date = default; + if (value.IsEmpty || value.Length > 128) + { + return false; + } + + Span chars = stackalloc char[value.Length]; + if (Encoding.ASCII.GetChars(value, chars) != value.Length) + { + return false; + } + + return TryParse(chars, out date); + } + + /// Parses an RFC-compatible date from already-decoded ASCII. + public static bool TryParse(ReadOnlySpan value, out DateTimeOffset date) + { + date = default; + var span = value.Trim(); + if (span.IsEmpty) + { + return false; + } + + if (span.Length >= 5 && span[3] == ',') + { + if (!IsDayName(span[..3])) + { + return false; + } + + span = span[4..].TrimStart(); + } + + if (!TryReadNumber(ref span, minDigits: 1, maxDigits: 2, out var day) || day is < 1 or > 31) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadMonth(ref span, out var month)) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadNumber(ref span, minDigits: 4, maxDigits: 4, out var year) + || year < 1900) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out var hour) + || hour > 23) + { + return false; + } + + if (span.IsEmpty || span[0] != ':') + { + return false; + } + + span = span[1..]; + if (!TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out var minute) || minute > 59) + { + return false; + } + + var second = 0; + if (!span.IsEmpty && span[0] == ':') + { + span = span[1..]; + if (!TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out second) || second > 60) + { + return false; + } + } + + if (!TrySkipWsp(ref span) || !TryReadZone(ref span, out var offset)) + { + return false; + } + + if (!span.IsEmpty) + { + return false; + } + + try + { + date = new DateTimeOffset(year, month, day, hour, minute, Math.Min(second, 59), offset); + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + } + + /// Returns whether is inside the POST date window relative to . + public static bool IsWithinPolicy(DateTimeOffset date, DateTimeOffset nowUtc) + { + var utc = date.ToUniversalTime(); + var now = nowUtc.ToUniversalTime(); + if (utc > now + PostingLimits.MaxDateSkewFuture) + { + return false; + } + + return utc >= now - PostingLimits.MaxDateAge; + } + + private static bool IsDayName(ReadOnlySpan value) + { + foreach (var name in DayNames) + { + if (value.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static bool TryReadMonth(ref ReadOnlySpan span, out int month) + { + month = 0; + if (span.Length < 3) + { + return false; + } + + var token = span[..3]; + for (var i = 0; i < MonthNames.Length; i++) + { + if (token.Equals(MonthNames[i], StringComparison.OrdinalIgnoreCase)) + { + month = i + 1; + span = span[3..]; + return true; + } + } + + return false; + } + + private static bool TryReadZone(ref ReadOnlySpan span, out TimeSpan offset) + { + offset = TimeSpan.Zero; + if (span.IsEmpty) + { + return false; + } + + if (span.StartsWith("GMT", StringComparison.OrdinalIgnoreCase) + || span.StartsWith("UTC", StringComparison.OrdinalIgnoreCase)) + { + span = span[3..]; + return true; + } + + if (span.StartsWith("UT", StringComparison.OrdinalIgnoreCase) + && (span.Length == 2 || !char.IsAsciiLetter(span[2]))) + { + span = span[2..]; + return true; + } + + var sign = span[0]; + if (sign is not ('+' or '-') || span.Length < 5) + { + return false; + } + + if (!char.IsAsciiDigit(span[1]) || !char.IsAsciiDigit(span[2]) + || !char.IsAsciiDigit(span[3]) || !char.IsAsciiDigit(span[4])) + { + return false; + } + + var hours = ((span[1] - '0') * 10) + (span[2] - '0'); + var minutes = ((span[3] - '0') * 10) + (span[4] - '0'); + if (hours > 14 || minutes > 59) + { + return false; + } + + var ticks = TimeSpan.FromHours(hours) + TimeSpan.FromMinutes(minutes); + offset = sign == '-' ? -ticks : ticks; + span = span[5..]; + return true; + } + + private static bool TryReadNumber(ref ReadOnlySpan span, int minDigits, int maxDigits, out int value) + { + value = 0; + var taken = 0; + while (taken < maxDigits && taken < span.Length && char.IsAsciiDigit(span[taken])) + { + value = (value * 10) + (span[taken] - '0'); + taken++; + } + + if (taken < minDigits) + { + return false; + } + + span = span[taken..]; + return true; + } + + private static bool TrySkipWsp(ref ReadOnlySpan span) + { + var i = 0; + while (i < span.Length && (span[i] == ' ' || span[i] == '\t')) + { + i++; + } + + if (i == 0) + { + return false; + } + + span = span[i..]; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingFailureCategory.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingFailureCategory.cs new file mode 100644 index 00000000..acf30a5d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingFailureCategory.cs @@ -0,0 +1,71 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Internal POST validation/persistence failure category for structured logs. + /// Never copied onto the NNTP wire. Clients always receive 441 Posting failed. + public enum PostingFailureCategory + { + /// Destuffed article exceeded Nntpd:MaxArticleSize during receive. + ArticleTooLarge = 0, + + /// Header line, folding, field-name, or CR/LF structure is invalid. + MalformedHeader = 1, + + /// A required header is absent. + MissingRequiredHeader = 2, + + /// A singleton header occurred more than once. + DuplicateHeader = 3, + + /// Message-ID syntax is invalid. + InvalidMessageId = 4, + + /// Date syntax is invalid or outside server date policy. + InvalidDate = 5, + + /// Newsgroups syntax is invalid. + InvalidNewsgroups = 6, + + /// Path generation/normalization failed. + InvalidPath = 7, + + /// Followup-To syntax is invalid. + InvalidFollowupTo = 8, + + /// References / In-Reply-To syntax or consistency is invalid. + InvalidReferences = 9, + + /// Distribution syntax or policy is invalid. + InvalidDistribution = 10, + + /// Control / unsupported control semantics. + InvalidControl = 11, + + /// Approved syntax is invalid. + InvalidApproved = 12, + + /// HistoryDB already knows this Message-ID. + DuplicateArticle = 13, + + /// Session or isolated policy rejected the article. + PolicyRejected = 14, + + /// Ingestion/history persistence did not accept the article. + PersistenceFailure = 15, + + /// + /// An Approved: identity was present but the authenticated principal is not + /// authorized to approve every moderated target. + /// + UnauthorizedApproval = 16, + + /// + /// The proto-article required moderator forwarding and submission was unavailable or failed. + /// + ModerationForwardingFailed = 17, + } + + /// A POST validation or persistence failure. + /// Structured failure category. + /// Short internal detail (not sent to the client). + public readonly record struct PostingFailure(PostingFailureCategory Category, string Detail); +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingLimits.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingLimits.cs new file mode 100644 index 00000000..b3eb23b1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingLimits.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// Fixed POST header and newsgroup bounds (distinct from Nntpd:MaxArticleSize). + internal static class PostingLimits + { + /// Maximum destuffed header block including the blank separator line. + public const int MaxHeaderBlockBytes = 64 * 1024; + + /// Maximum unfolded single header (name + colon + value). + public const int MaxSingleHeaderBytes = 8192; + + /// Maximum header fields in one article. + public const int MaxHeaderCount = 256; + + /// Maximum distinct newsgroups in Newsgroups:. + public const int MaxNewsgroups = 12; + + /// Maximum distinct groups in Followup-To:. + public const int MaxFollowupToGroups = 12; + + /// Articles whose Date: is more than this far in the future are rejected. + public static readonly TimeSpan MaxDateSkewFuture = TimeSpan.FromHours(24); + + /// Articles whose Date: is older than this relative to injection time are rejected. + public static readonly TimeSpan MaxDateAge = TimeSpan.FromDays(14); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingTracePayload.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingTracePayload.cs new file mode 100644 index 00000000..6dcdb1a3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/PostingTracePayload.cs @@ -0,0 +1,67 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// Transport and authentication identity recovered from a server-owned POST X-Trace token. + /// + /// + /// Intended for trusted server-side diagnostics only. Do not log this value. + /// is the AUTHINFO identity captured at POST admission; + /// it is never taken from article headers. Null or empty means the session was unauthenticated. + /// + public readonly struct PostingTracePayload + { + /// Initializes a payload without an authenticated username (unauthenticated POST). + public PostingTracePayload(IPAddress address, int port, DateTimeOffset injectedAtUtc, Guid traceId) + : this(address, port, injectedAtUtc, traceId, authenticatedUsername: null) + { + } + + /// Initializes a payload, optionally including the authenticated session username. + public PostingTracePayload( + IPAddress address, + int port, + DateTimeOffset injectedAtUtc, + Guid traceId, + string? authenticatedUsername) + { + ArgumentNullException.ThrowIfNull(address); + ArgumentOutOfRangeException.ThrowIfLessThan(port, 0); + ArgumentOutOfRangeException.ThrowIfGreaterThan(port, 65535); + Address = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + Port = port; + InjectedAtUtc = injectedAtUtc.ToUniversalTime(); + TraceId = traceId; + AuthenticatedUsername = NormalizeUsername(authenticatedUsername); + } + + /// Gets the effective transport peer address. + public IPAddress Address { get; } + + /// Gets the effective transport peer port. + public int Port { get; } + + /// Gets the injection-boundary UTC timestamp. + public DateTimeOffset InjectedAtUtc { get; } + + /// Gets the unique identifier for this POST attempt. + public Guid TraceId { get; } + + /// + /// Gets the AUTHINFO username captured at POST admission, or + /// when the session was unauthenticated or the token is a pre-username v1 payload. + /// + public string? AuthenticatedUsername { get; } + + internal static string? NormalizeUsername(string? username) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + return username; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostArticleReader.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostArticleReader.cs new file mode 100644 index 00000000..337dbb92 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostArticleReader.cs @@ -0,0 +1,694 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// + /// Streams a POST multiline article from a into one destuffed + /// output buffer for . + /// + /// + /// Header lines are destuffed for validation and written destuffed (client Path / + /// injection metadata discarded). Body lines drop one NNTP stuffing layer + /// (RFC 3977 §3.1.1). Destuffed byte counting enforces Nntpd:MaxArticleSize. + /// History Peek is not performed here; the caller peeks after the terminator. + /// + internal static class StreamingPostArticleReader + { + /// Initial capacity of the single destuffed output buffer. + public const int InitialCapacity = 64 * 1024; + + /// + /// Reads one POST article from into one owned destuffed buffer. + /// + public static async ValueTask ReadAsync( + PipeReader reader, + StreamingPostReadOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(options); + ArgumentOutOfRangeException.ThrowIfLessThan(options.MaxArticleSize, 1); + ArgumentNullException.ThrowIfNull(options.Time); + ArgumentNullException.ThrowIfNull(options.NewsgroupPolicy); + ArgumentException.ThrowIfNullOrWhiteSpace(options.InjectionIdentity); + ArgumentNullException.ThrowIfNull(options.ClientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(options.MailComplaintsTo); + + var session = new ReadSession(options); + return await session.RunAsync(reader, cancellationToken).ConfigureAwait(false); + } + + private sealed class ReadSession(StreamingPostReadOptions options) + { + private readonly GrowingArticleBuffer _output = new(InitialCapacity); + private readonly List _headers = []; + private readonly ArrayBufferWriter _unfolded = new(256); + private bool _inHeaders = true; + private bool _hasField; + private bool _writeCurrentField; + private bool _drain; + private byte[]? _currentName; + private int _destuffedBytes; + private int _headerBlockBytes; + // Incomplete until HandleTerminator assigns Completed/Rejected/TooLarge. Defaulting to + // Completed would make Finish()'s success path look reachable with a null _parsed. + private StreamingPostReadStatus _status = StreamingPostReadStatus.Incomplete; + private PostingFailure _failure; + private ParsedPostArticle? _parsed; + private DateTimeOffset _injectionUtc; + private StreamingPostDisposition _disposition = StreamingPostDisposition.Inject; + private string? _moderatorAddress; + private string? _targetModeratedGroup; + + public async ValueTask RunAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examined = buffer.End; + + while (NntpDelimiterSearch.TryReadLine(ref buffer, out var line)) + { + var finished = HandleLine(line); + if (finished) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return Finish(); + } + } + + reader.AdvanceTo(buffer.Start, examined); + if (result.IsCompleted) + { + return Incomplete(); + } + } + } + + private bool HandleLine(ReadOnlySequence stuffedLine) + { + if (NntpDelimiterSearch.IsTerminatorLine(stuffedLine)) + { + return HandleTerminator(); + } + + if (_drain) + { + return false; + } + + var leadingDot = NntpDelimiterSearch.FirstByte(stuffedLine) == (byte)'.'; + var destuffedLength = (int)stuffedLine.Length - (leadingDot ? 1 : 0); + var destuffedWithCrlf = destuffedLength + 2; + if (_destuffedBytes + destuffedWithCrlf > options.MaxArticleSize) + { + MarkTooLarge(); + return false; + } + + if (_inHeaders) + { + return HandleHeaderLine(stuffedLine, destuffedLength, destuffedWithCrlf); + } + + _destuffedBytes += destuffedWithCrlf; + WriteDestuffedLine(_output, stuffedLine, destuffedLength); + return false; + } + + private bool HandleHeaderLine( + ReadOnlySequence stuffedLine, + int destuffedLength, + int destuffedWithCrlf) + { + var rented = ArrayPool.Shared.Rent(Math.Max(destuffedLength, 1)); + try + { + CopyDestuffed(stuffedLine, rented.AsSpan(0, destuffedLength)); + var destuffed = rented.AsSpan(0, destuffedLength); + if (!TryValidateLineOctets(destuffed, out _failure)) + { + MarkRejected(); + return false; + } + + if (destuffed.IsEmpty) + { + return CompleteHeaderSection(destuffedWithCrlf); + } + + if (_headerBlockBytes + destuffedWithCrlf > PostingLimits.MaxHeaderBlockBytes) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + MarkRejected(); + return false; + } + + _destuffedBytes += destuffedWithCrlf; + _headerBlockBytes += destuffedWithCrlf; + + var isFold = destuffed.Length > 0 && PostFieldSyntax.IsWsp(destuffed[0]); + if (isFold) + { + return HandleFold(destuffed); + } + + if (_hasField && !FlushCurrentField()) + { + MarkRejected(); + return false; + } + + if (!PostHeaderParser.TryReadFieldName(destuffed, out var nameLength, out _failure)) + { + MarkRejected(); + return false; + } + + _currentName = destuffed[..nameLength].ToArray(); + _writeCurrentField = !PostHeaderNormalizer.IsServerOwned(_currentName); + _hasField = true; + _unfolded.Clear(); + _unfolded.Write(destuffed[(nameLength + 2)..]); + if (_writeCurrentField) + { + _output.Write(destuffed); + _output.Write(NntpDelimiterSearch.Crlf); + } + + return false; + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + private bool HandleFold(ReadOnlySpan destuffed) + { + if (!_hasField) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "leading fold"); + MarkRejected(); + return false; + } + + if (destuffed.Length < 2) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty fold"); + MarkRejected(); + return false; + } + + _unfolded.Write(destuffed); + if (_writeCurrentField) + { + _output.Write(destuffed); + _output.Write(NntpDelimiterSearch.Crlf); + } + + return false; + } + + private bool CompleteHeaderSection(int separatorBytes) + { + if (_headerBlockBytes + separatorBytes > PostingLimits.MaxHeaderBlockBytes) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + MarkRejected(); + return false; + } + + if (!_hasField) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header block"); + MarkRejected(); + return false; + } + + if (!FlushCurrentField()) + { + MarkRejected(); + return false; + } + + if (_headers.Count > PostingLimits.MaxHeaderCount) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "too many headers"); + MarkRejected(); + return false; + } + + _destuffedBytes += separatorBytes; + _headerBlockBytes += separatorBytes; + + _injectionUtc = options.Time.GetUtcNow(); + _parsed = new ParsedPostArticle( + ReadOnlyMemory.Empty, + _headers, + ReadOnlyMemory.Empty, + _headerBlockBytes); + + if (!PostArticleValidator.TryValidate( + _parsed, + _injectionUtc, + options.NewsgroupPolicy, + out _failure, + options.ControlCancelPermitted)) + { + MarkRejected(); + return false; + } + + if (!TryCompletePostingDecision(_parsed)) + { + MarkRejected(); + return false; + } + + _inHeaders = false; + return false; + } + + private bool TryCompletePostingDecision(ParsedPostArticle article) + { + if (article.CatalogStatus == NewsgroupCatalogStatus.RequiresModeration) + { + if (!article.ApprovedPresent) + { + return TryBeginModerationSubmission(article); + } + + var authorization = options.ModeratorAuthorization ?? EmptyModeratorAuthorization.Instance; + if (!authorization.TryAuthorizeApproval( + options.AuthenticatedUsername, + article.ApprovedIdentities, + article.ModeratedGroups, + out var detail)) + { + _failure = new PostingFailure( + detail == "unresolved moderator" + ? PostingFailureCategory.ModerationForwardingFailed + : PostingFailureCategory.UnauthorizedApproval, + detail ?? "unauthorized approval"); + return false; + } + } + + return TryWriteInjectionHeaders(article); + } + + private bool TryBeginModerationSubmission(ParsedPostArticle article) + { + if (article.ModeratedGroups.Length == 0) + { + _failure = new PostingFailure(PostingFailureCategory.PolicyRejected, "moderated newsgroup"); + return false; + } + + var target = article.ModeratedGroups[0]; + if (target.Length is < 1 or > 128) + { + _failure = new PostingFailure(PostingFailureCategory.InvalidNewsgroups, "malformed newsgroup name"); + return false; + } + + Span nameBytes = stackalloc byte[128]; + var written = Encoding.ASCII.GetBytes(target, nameBytes); + var authorization = options.ModeratorAuthorization ?? EmptyModeratorAuthorization.Instance; + if (!authorization.TryResolve(nameBytes[..written], out var identity)) + { + _failure = new PostingFailure( + PostingFailureCategory.ModerationForwardingFailed, + "unresolved moderator"); + return false; + } + + try + { + PostHeaderNormalizer.WriteProtoArticleBoundary( + _output, + article.MessageIdSynthesized, + article.MessageId!, + article.DateSynthesized, + _injectionUtc); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "proto-article header generation failed"); + return false; + } + + _disposition = StreamingPostDisposition.SubmitForModeration; + _targetModeratedGroup = target; + _moderatorAddress = identity.Address; + return true; + } + + private bool TryWriteInjectionHeaders(ParsedPostArticle article) + { + if (options.TraceProtector is null) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "trace protector unavailable"); + return false; + } + + try + { + PostHeaderNormalizer.WriteServerOwnedHeaders( + _output, + article.MessageIdSynthesized, + article.MessageId!, + article.DateSynthesized, + _injectionUtc, + options.InjectionIdentity, + options.ClientIdentity, + options.MailComplaintsTo, + options.TraceProtector, + options.AuthenticatedUsername); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "server header generation failed"); + return false; + } + + _disposition = StreamingPostDisposition.Inject; + return true; + } + + private bool HandleTerminator() + { + if (_drain) + { + return true; + } + + if (_inHeaders) + { + _failure = _destuffedBytes == 0 + ? new PostingFailure(PostingFailureCategory.MalformedHeader, "empty article") + : new PostingFailure(PostingFailureCategory.MalformedHeader, "missing header/body separator"); + _status = StreamingPostReadStatus.Rejected; + return true; + } + + _status = StreamingPostReadStatus.Completed; + return true; + } + + private bool FlushCurrentField() + { + var name = _currentName is null + ? ReadOnlyMemory.Empty + : _currentName.AsMemory(); + var value = _unfolded.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : _unfolded.WrittenMemory.ToArray(); + if (!PostHeaderParser.TryCreateHeader(name, value, ReadOnlyMemory.Empty, out var header, out _failure)) + { + return false; + } + + _headers.Add(header); + _hasField = false; + _currentName = null; + _unfolded.Clear(); + return true; + } + + private void MarkTooLarge() + { + _status = StreamingPostReadStatus.TooLarge; + _failure = new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded"); + _drain = true; + } + + private void MarkRejected() + { + _status = StreamingPostReadStatus.Rejected; + _drain = true; + } + + private StreamingPostReadResult Finish() + { + if (_status != StreamingPostReadStatus.Completed) + { + return new StreamingPostReadResult( + _status, + _failure, + ReadOnlyMemory.Empty, + _parsed?.MessageId, + _parsed?.Newsgroups ?? [], + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + _parsed?.ApprovedIdentities); + } + + // Completed is assigned only after CompleteHeaderSection materializes _parsed. + var parsed = _parsed + ?? throw new InvalidOperationException( + "Streaming POST completed without a parsed article; header completion must precede the terminator."); + + return new StreamingPostReadResult( + StreamingPostReadStatus.Completed, + default, + _output.Take(), + parsed.MessageId, + parsed.Newsgroups, + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + parsed.ApprovedIdentities); + } + + private StreamingPostReadResult Incomplete() => + new( + StreamingPostReadStatus.Incomplete, + default, + ReadOnlyMemory.Empty, + _parsed?.MessageId, + _parsed?.Newsgroups ?? [], + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + _parsed?.ApprovedIdentities); + } + + private static bool TryValidateLineOctets(ReadOnlySpan destuffed, out PostingFailure failure) + { + for (var i = 0; i < destuffed.Length; i++) + { + var b = destuffed[i]; + if (b == 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "embedded NUL"); + return false; + } + + if (b == (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "LF without CR"); + return false; + } + + if (b == (byte)'\r') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "CR without LF"); + return false; + } + } + + failure = default; + return true; + } + + private static void CopyDestuffed(ReadOnlySequence stuffed, Span destuffed) + { + if (destuffed.IsEmpty) + { + return; + } + + var leadingDot = NntpDelimiterSearch.FirstByte(stuffed) == (byte)'.'; + if (stuffed.IsSingleSegment) + { + var src = stuffed.FirstSpan; + if (leadingDot) + { + src = src[1..]; + } + + src.CopyTo(destuffed); + return; + } + + var rented = ArrayPool.Shared.Rent((int)stuffed.Length); + try + { + stuffed.CopyTo(rented); + var src = leadingDot + ? rented.AsSpan(1, destuffed.Length) + : rented.AsSpan(0, destuffed.Length); + src.CopyTo(destuffed); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + /// + /// Writes one destuffed content line plus CRLF (one NNTP stuffing layer removed). + /// + private static void WriteDestuffedLine( + GrowingArticleBuffer output, + ReadOnlySequence stuffedLine, + int destuffedLength) + { + if (destuffedLength > 0) + { + if (stuffedLine.IsSingleSegment) + { + var src = stuffedLine.FirstSpan; + if (src.Length > 0 && src[0] == (byte)'.') + { + src = src[1..]; + } + + output.Write(src); + } + else + { + var rented = ArrayPool.Shared.Rent(destuffedLength); + try + { + CopyDestuffed(stuffedLine, rented.AsSpan(0, destuffedLength)); + output.Write(rented.AsSpan(0, destuffedLength)); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + } + + output.Write(NntpDelimiterSearch.Crlf); + } + + /// One owned destuffed output buffer transferred to the caller on success. + internal sealed class GrowingArticleBuffer : IBufferWriter + { + private byte[] _buffer; + private int _written; + + public GrowingArticleBuffer(int initialCapacity) + { + _buffer = new byte[initialCapacity]; + } + + public int Written => _written; + + public void Advance(int count) + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + if (_written + count > _buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + _written += count; + } + + public Memory GetMemory(int sizeHint = 0) + { + Ensure(_written + Math.Max(sizeHint, 1)); + return _buffer.AsMemory(_written); + } + + public Span GetSpan(int sizeHint = 0) + { + Ensure(_written + Math.Max(sizeHint, 1)); + return _buffer.AsSpan(_written); + } + + public void Write(ReadOnlySpan bytes) + { + if (bytes.IsEmpty) + { + return; + } + + Ensure(_written + bytes.Length); + bytes.CopyTo(_buffer.AsSpan(_written)); + _written += bytes.Length; + } + + public void Write(ReadOnlySequence bytes) + { + if (bytes.IsEmpty) + { + return; + } + + if (bytes.IsSingleSegment) + { + Write(bytes.FirstSpan); + return; + } + + foreach (var segment in bytes) + { + Write(segment.Span); + } + } + + public ReadOnlyMemory Take() + { + var buffer = _buffer; + var length = _written; + _buffer = []; + _written = 0; + return buffer.AsMemory(0, length); + } + + private void Ensure(int needed) + { + if (_buffer.Length >= needed) + { + return; + } + + var next = _buffer.Length < 256 ? 256 : _buffer.Length * 2; + if (next < needed) + { + next = needed; + } + + var grown = new byte[next]; + if (_written > 0) + { + _buffer.AsSpan(0, _written).CopyTo(grown); + } + + _buffer = grown; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostReadResult.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostReadResult.cs new file mode 100644 index 00000000..c80610eb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Posting/StreamingPostReadResult.cs @@ -0,0 +1,99 @@ +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Session.Commands.Posting +{ + /// What the POST command should do with a completed receive. + internal enum StreamingPostDisposition + { + /// Article is ready for normal History Peek / TryAdmit injection. + Inject = 0, + + /// + /// Unapproved moderated proto-article; caller must submit for moderation and must + /// not Peek, TryAdmit, or Remember. + /// + SubmitForModeration = 1, + } + + /// Outcome of one streaming POST article receive. + internal enum StreamingPostReadStatus + { + /// Terminator seen; destuffed article is ready for the caller-selected disposition. + Completed = 0, + + /// Peer disconnected or cancelled before the terminating dot line. + Incomplete = 1, + + /// Destuffed article exceeded Nntpd:MaxArticleSize; remaining bytes were drained. + TooLarge = 2, + + /// Header parse or POST validation failed; remaining bytes were drained when needed. + Rejected = 3, + } + + /// Inputs required to stream one POST article into a destuffed article buffer. + internal sealed class StreamingPostReadOptions + { + /// Gets the destuffed POST size limit (Nntpd:MaxArticleSize). + public required int MaxArticleSize { get; init; } + + /// Gets the clock used for the single injection timestamp. + public required TimeProvider Time { get; init; } + + /// Gets the newsgroup posting policy. + public required INewsgroupPostingPolicy NewsgroupPolicy { get; init; } + + /// Gets the server injection identity (nntpd{ServerId:00}.{DnsSuffix}). + public required string InjectionIdentity { get; init; } + + /// Gets the session client identity used only inside protected X-Trace. + public required ConnectionClientIdentity ClientIdentity { get; init; } + + /// Gets the configured mail-complaints-to mailbox. + public required string MailComplaintsTo { get; init; } + + /// Gets the X-Trace protector, or when unset. + public IPostingTraceProtector? TraceProtector { get; init; } + + /// + /// Gets whether this receive may accept a well-formed Control: cancel header. + /// Ordinary posters remain . + /// + public bool ControlCancelPermitted { get; init; } + + /// + /// Gets the AUTHINFO username captured at POST admission, or + /// when the session is unauthenticated. Never taken from article headers. + /// + public string? AuthenticatedUsername { get; init; } + + /// Gets the moderator authorization table used for Approved: decisions. + public IModeratorAuthorization ModeratorAuthorization { get; init; } = EmptyModeratorAuthorization.Instance; + } + + /// Result of . + /// Receive / validation status. + /// Set when is or TooLarge. + /// Destuffed article (terminator omitted) when Completed; otherwise empty. + /// Article Message-ID when known. + /// Validated newsgroup names when known. + /// Logical destuffed client article size (stuffing dots excluded). + /// Single captured UTC timestamp written into server-owned headers. + /// Inject or submit-for-moderation once the terminator is seen. + /// Resolved leftmost moderator mailbox on the moderation path. + /// Leftmost moderated group on the moderation path. + /// Parsed Approved: identities when present. + internal readonly record struct StreamingPostReadResult( + StreamingPostReadStatus Status, + PostingFailure Failure, + ReadOnlyMemory Wire, + string? MessageId, + string[] Newsgroups, + int DestuffedSize, + DateTimeOffset InjectionUtc, + StreamingPostDisposition Disposition = StreamingPostDisposition.Inject, + string? ModeratorAddress = null, + string? TargetModeratedGroup = null, + string[]? ApprovedIdentities = null); +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/Quit.cs b/src/VectorNNTPD.ReaderService/Session/Commands/Quit.cs new file mode 100644 index 00000000..c1b75cf6 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/Quit.cs @@ -0,0 +1,93 @@ +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// QUIT command as defined by RFC 3977, Section 5.4. + /// + /// + /// + /// Syntax: QUIT (no arguments). Returns 205 Connection closing then ends the session. + /// Trailing arguments are a syntax error (501) and do not close the connection. + /// + /// + /// Real clients often close the TCP socket immediately after sending QUIT, before observing 205. + /// After a valid QUIT is accepted, peer disappearance during response write/delivery or close is + /// treated as a normal termination race — not an unexpected application failure. Unrelated + /// transport failures are still logged and rethrown. + /// + /// + internal static class Quit + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Quit)); + + /// Handles QUIT (RFC 3977 §5.4). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "QUIT", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + // Valid QUIT accepted: after this point the session must become terminal and no further + // NNTP commands/responses should be processed (aside from finishing this 205 write). + var responseWritten = false; + var idleVersionBefore205 = context.Connection.OutboundIdleVersion; + try + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ConnectionClosing, + NntpResponseStatus.ConnectionClosing, + cancellationToken) + .ConfigureAwait(false); + responseWritten = true; + + // Prefer observing send-pump idle so 205 has left the outbound pipe before teardown, + // matching STARTTLS/COMPRESS delivery waits. Peer-gone during this wait is expected. + if (context.Connection is { IsCompleted: false, ConnectionClosed.IsCancellationRequested: false }) + { + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore205, cancellationToken) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (context.Connection.ConnectionClosed.IsCancellationRequested) + { + // Connection became terminal (often peer FIN/RST) while writing or waiting for 205. + context.CompletionDetail = "peer disconnected"; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (NntpPeerDisconnect.IsPeerDisconnect(ex, context.Connection)) + { + context.CompletionDetail = "peer disconnected"; + CommandLogMessages.QuitPeerDisconnected( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // TEMP DIAG — remove after capture + CommandLogMessages.QuitUncaught( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session), + ex.GetType().FullName, + (ex as SocketException)?.SocketErrorCode + ?? (ex.InnerException as SocketException)?.SocketErrorCode); + throw; + } + + // Always terminate after a valid QUIT — even when the peer vanished mid-205. + context.Session.RequestClose(); + + // Do not claim a clean delivery when we never successfully wrote the status line. + if (!responseWritten && context.CompletionDetail is null) + { + context.CompletionDetail = "peer disconnected"; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/SpeedTest.cs b/src/VectorNNTPD.ReaderService/Session/Commands/SpeedTest.cs new file mode 100644 index 00000000..973a0dff --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/SpeedTest.cs @@ -0,0 +1,273 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.SpeedTest; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// VectorNNTP private extension: SPEEDTEST identifier. + /// + /// + /// + /// Syntax: SPEEDTEST identifier. identifier is one NNTP token and must be an + /// existing Transit dictionary key (exact ordinal match). It is not PeerName, a + /// hostname, IP, port, URL, or socket endpoint. Multi-token display names are not accepted. + /// + /// + /// Access: transit-authorized sessions only (dispatcher 480/502). When the + /// session was identified as a named Transit peer, the argument must match that name. + /// + /// + /// V1 measures TX on the current NNTP session only (this host writes a synthetic multiline + /// payload to the connected client). Outbound Transit initiation is not implemented; the + /// command never connects to ConnectTo and never reports a measurement that was not + /// performed. RX, RTT, and packet loss are NOT-MEASURED. + /// + /// + /// Responses use private-extension x9x codes (RFC 3977 §3.2): 290 starts the + /// TX payload multiline; 291 is followed by machine-readable result fields. + /// Unknown peers use the project 502 convention. The command is serial (not pipelined) + /// and does not use article ingestion, HistoryDB, or Redis. + /// + /// + internal static class SpeedTest + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(SpeedTest)); + + /// Handles SPEEDTEST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "SPEEDTEST", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var coordinator = context.Session.SpeedTest; + if (coordinator is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestNotSupported, + NntpResponseStatus.SpeedTestNotSupported, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "not supported"; + return; + } + + var peerToken = context.ArgumentMemory; + if (!coordinator.TryResolvePeer(peerToken.Span, out var peer) || peer is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestUnknownPeer, + NntpResponseStatus.SpeedTestUnknownPeer, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + FormatPeerForLog(peerToken.Span), + string.Empty, + "unknown peer"); + context.CompletionDetail = "unknown peer"; + return; + } + + var sessionPeer = context.Session.Authorization.TransitPeerName; + if (sessionPeer is not null + && !SpeedTestCoordinator.NameEquals(sessionPeer, peerToken.Span)) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestPeerMismatch, + NntpResponseStatus.SpeedTestPeerMismatch, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "peer mismatch"); + context.CompletionDetail = "peer mismatch"; + return; + } + + if (!coordinator.TryAcquire(peer.Identifier, out var lease) || lease is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestBusy, + NntpResponseStatus.SpeedTestBusy, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "busy"); + context.CompletionDetail = "busy"; + return; + } + + using (lease) + using (var workCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + context.Session.Connection.ConnectionClosed)) + { + CommandLogMessages.SpeedTestStarted( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName); + + var ready = NntpResponseCompose.Concat( + NntpResponses.SpeedTestReadyPrefix.Span, + peerToken.Span, + NntpResponses.SpeedTestReadySuffix.Span); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= + "290 SPEEDTEST " + System.Text.Encoding.ASCII.GetString(peerToken.Span) + " TX"; + } + + await context.Response.WriteLineAsync(ready, workCts.Token).ConfigureAwait(false); + + if (!ReferenceEquals(context.Response.UnderlyingOutput, context.Session.Connection.Output)) + { + throw new InvalidOperationException( + "SPEEDTEST direct path must use the existing session TX PipeWriter."); + } + + long bytes; + TimeSpan elapsed; + long directFlushes; + try + { + (bytes, elapsed, directFlushes) = await WritePayloadAsync( + context.Response, + lease.Limits, + workCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Session.Connection.ConnectionClosed.IsCancellationRequested) + { + CommandLogMessages.SpeedTestCancelled( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "cancelled"); + context.CompletionDetail = "cancelled"; + return; + } + + var result = FormatResult(peer.Identifier, peer.PeerName, bytes, elapsed, directFlushes); + await context.Response.WriteLineAsync(result, workCts.Token).ConfigureAwait(false); + + CommandLogMessages.SpeedTestCompleted( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + bytes, + elapsed.TotalMilliseconds); + context.CompletionDetail = "tx complete"; + } + } + + internal static async ValueTask<(long Bytes, TimeSpan Elapsed, long DirectFlushes)> WritePayloadAsync( + NntpResponseWriter response, + Configuration.SpeedTestLimits limits, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + await using var pipeLease = await response.AcquireDirectTxPipeAsync(cancellationToken).ConfigureAwait(false); + var started = Stopwatch.GetTimestamp(); + long bytes = 0; + while (bytes < limits.MaxBytes) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Stopwatch.GetElapsedTime(started) >= limits.MaxDuration) + { + break; + } + + var slice = SpeedTestPayload.Take(limits.MaxBytes - bytes); + if (slice.IsEmpty) + { + break; + } + + await pipeLease.WriteAndFlushAsync(slice, cancellationToken).ConfigureAwait(false); + bytes += slice.Length; + } + + await pipeLease.WriteAndFlushAsync(NntpResponses.MultilineTerminator, cancellationToken).ConfigureAwait(false); + return (bytes, Stopwatch.GetElapsedTime(started), pipeLease.FlushCount); + } + + /// Builds the 291 multiline result (status + fields + terminator). + internal static byte[] FormatResult( + string identifier, + string peerName, + long bytes, + TimeSpan elapsed, + long directPipeFlushes = 0) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + var durationMs = Math.Max(0, elapsed.TotalMilliseconds); + var seconds = elapsed.TotalSeconds; + var measured = bytes > 0 && seconds > 0; + var mbps = measured + ? (bytes * 8d / seconds / 1_000_000d).ToString("F3", CultureInfo.InvariantCulture) + : "NOT-MEASURED"; + var gbit = measured + ? (bytes * 8d / seconds / 1_000_000_000d).ToString("F6", CultureInfo.InvariantCulture) + : "NOT-MEASURED"; + + var text = + "291 SPEEDTEST COMPLETE\r\n" + + "PEER=" + identifier + "\r\n" + + "PEERNAME=" + peerName + "\r\n" + + "IMPLEMENTATION=VectorNNTP\r\n" + + "DIRECTION=TX\r\n" + + "BYTES=" + bytes.ToString(CultureInfo.InvariantCulture) + "\r\n" + + "DURATION_MS=" + durationMs.ToString("F3", CultureInfo.InvariantCulture) + "\r\n" + + "THROUGHPUT_MBPS=" + mbps + "\r\n" + + "THROUGHPUT_GBIT=" + gbit + "\r\n" + + "RTT_US=NOT-MEASURED\r\n" + + "RX=NOT-MEASURED\r\n" + + "OUTBOUND=NOT-AVAILABLE\r\n" + + "PATH=DIRECT-PIPE\r\n" + + "DIRECT_PIPE_FLUSHES=" + directPipeFlushes.ToString(CultureInfo.InvariantCulture) + "\r\n" + + "COMPLETE\r\n" + + ".\r\n"; + return Encoding.UTF8.GetBytes(text); + } + + private static string FormatPeerForLog(ReadOnlySpan peerToken) + { + var unsafeControl = false; + for (var i = 0; i < peerToken.Length; i++) + { + if (peerToken[i] < 0x20 || peerToken[i] > 0x7E) + { + unsafeControl = true; + break; + } + } + + return unsafeControl ? "" : Encoding.ASCII.GetString(peerToken); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/StartTls.cs b/src/VectorNNTPD.ReaderService/Session/Commands/StartTls.cs new file mode 100644 index 00000000..e150b01b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/StartTls.cs @@ -0,0 +1,138 @@ +using System.IO.Pipelines; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// STARTTLS command as defined by RFC 4642, Section 2.2 (updated by RFC 8143). + /// + /// + /// Negotiates TLS on an existing plaintext NNTP connection. Transport owns the handshake and + /// quiescence; this module owns command sequencing (discard pipelined input, pause reads before + /// 382, wait for outbound delivery, upgrade) and the TX completion record. + /// + internal static class StartTls + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(StartTls)); + + /// Handles STARTTLS. + public static ValueTask HandleAsync( + NntpCommandContext context, + ITlsCertificateContextProvider? certificateProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "STARTTLS", + (ctx, ct) => ExecuteAsync(ctx, certificateProvider, ct), + cancellationToken); + + private static async ValueTask ExecuteAsync( + NntpCommandContext context, + ITlsCertificateContextProvider? certificateProvider, + CancellationToken cancellationToken) + { + if (context.Connection.IsTls) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.TlsAlreadyActive, + NntpResponseStatus.TlsAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §2.2.2: MUST reply 502 to STARTTLS while a compression layer is already active. + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.DeflateAlreadyActive, + NntpResponseStatus.DeflateAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (certificateProvider is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.TlsProviderUnavailable, + NntpResponseStatus.TlsProviderUnavailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8143 §4: discard any NNTP command pipelined between STARTTLS and TLS negotiation. + DiscardBufferedApplicationInput(context.Connection.Input); + + // Claim read ownership BEFORE 382 can reach the peer. Writes remain allowed while reads + // are paused, so ClientHello completions stash for PrefixedStream instead of Input. + await context.Connection.PauseReadsAsync(cancellationToken).ConfigureAwait(false); + + var idleVersionBefore382 = context.Connection.OutboundIdleVersion; + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ContinueWithTls, + NntpResponseStatus.ContinueWithTls, + cancellationToken) + .ConfigureAwait(false); + + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore382, cancellationToken) + .ConfigureAwait(false); + + try + { + await context.Connection.UpgradeToTlsAsync(certificateProvider, cancellationToken).ConfigureAwait(false); + if (context.Connection.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)) + { + context.CompletionDetail = TlsNegotiationLogging.FormatDetail(tlsVersion, cipher); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + CommandLogMessages.StartTlsHandshakeFailed( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + context.Session.RequestClose(); + // Handshake failures complete the connection; do not rethrow into the dispatcher error path. + // Precondition failures leave the connection usable — rethrow so the dispatcher can respond. + if (context.Connection.IsCompleted) + { + return; + } + + throw; + } + } + + /// + /// Consumes any octets already buffered in application without waiting for more. + /// + private static void DiscardBufferedApplicationInput(PipeReader input) + { + while (input.TryRead(out var result)) + { + var isCompleted = result.IsCompleted; + var isEmpty = result.Buffer.IsEmpty; + input.AdvanceTo(result.Buffer.End); + if (isEmpty || isCompleted) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Commands/TakeThis.cs b/src/VectorNNTPD.ReaderService/Session/Commands/TakeThis.cs new file mode 100644 index 00000000..0dbd07ba --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Commands/TakeThis.cs @@ -0,0 +1,437 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.Commands +{ + /// + /// TAKETHIS command as defined by RFC 4644, Section 2.5. + /// + /// + /// + /// Syntax: TAKETHIS message-id followed immediately by an NNTP multiline article. + /// Responses: 239 message-id (accepted into ingestion) or 439 message-id + /// (permanent reject). Temporary infrastructure failure uses 400 and closes the connection. + /// + /// + /// STREAM path: the session RX task starts HistoryDB peek at the Message-ID, frames + /// the article with (one owned stuffed-wire buffer, + /// terminator omitted, no destuff on RX), attaches that buffer to + /// , and returns. Peek and destuff/ArticleRecord + /// run concurrently per occupied slot. Enqueue / Remember / ordered 239/439 stay + /// on the emit gate so publication remains command-ordered. Depth bounds how + /// many owned articles stay in flight. TAKETHIS responses flush immediately + /// (no coalesce batch). ArticleRecord construction failure is a permanent + /// 439. After CanonicalV1, the connection's Receive article-type mask and + /// newsgroup expression are applied. CHECK does not use them. + /// + /// + /// MODE READER fallback still destuffs via and + /// overlaps HistoryDB peek with that receive. + /// + /// + internal static class TakeThis + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(TakeThis)); + + /// Handles TAKETHIS (RFC 4644 §2.5) on the serial (non-pipeline) path. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "TAKETHIS", ExecuteAsync, cancellationToken); + + /// HistoryDB peek without miss reservation (same contract as CHECK and IHAVE). + internal static ValueTask PeekAsync( + NntpSession session, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + if (session.HistoryDb is not { } history) + { + return new ValueTask(HistoryLookupResult.Unseen); + } + + return history.PeekAsync(messageId, cancellationToken); + } + + internal static void WriteCompletion( + ILogger logger, + NntpSession session, + long startedTimestamp, + string? detail = null, + string? statusLine = null) => + NntpCommandExecution.WriteCompletion( + logger, + session, + "TAKETHIS", + System.Diagnostics.Stopwatch.GetElapsedTime(startedTimestamp), + detail, + statusLine); + + internal static async ValueTask FailTemporaryAsync( + NntpSession session, + NntpResponseWriter response, + CancellationToken cancellationToken) + { + try + { + await response + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when ( + cancellationToken.IsCancellationRequested + || session.Connection.ConnectionClosed.IsCancellationRequested + || ex is ObjectDisposedException or InvalidOperationException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + session.RequestClose(); + CommandLogMessages.TakeThisTemporaryFailure( + Logger, + NntpCommandLogFormat.Client(session)); + } + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var probe = context.Session.FeedProbe; + probe?.RecordCommand(); + context.Session.SetActivityState(FeedSessionState.Receiving); + var messageIdBytes = context.ArgumentSpan.ToArray(); + var lookup = PeekAsync(context.Session, messageIdBytes, cancellationToken); + var queue = context.Session.ArticleIngestion; + + NntpArticlePolicyCapture.Capture( + context.Session, + queue.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + NntpMultilineReadStatus status; + ReadOnlyMemory payload; + var receiveStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + if (context.PreReadArticle is { } preRead) + { + status = preRead.Status; + payload = preRead.Payload; + } + else if (context.Session.ReceiveStrategy == NntpReceiveStrategy.StreamDataPlane) + { + var read = await IHaveArticleReader + .ReadAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + status = read.Status; + payload = read.Payload; + } + else + { + var article = await NntpMultilineDataReader + .ReadArticleAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + status = article.Status; + payload = article.Payload; + } + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + probe?.RecordArticleReceived( + payload.Length, + System.Diagnostics.Stopwatch.GetTimestamp() - receiveStart); + context.Session.RecordPeerArticleReceived(payload.Length); + context.Session.SetActivityState(FeedSessionState.WaitingHistory); + + HistoryLookupResult peek; + var peekStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + peek = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + peek = HistoryLookupResult.Unavailable; + } + + probe?.RecordHistory(peek, System.Diagnostics.Stopwatch.GetTimestamp() - peekStart); + + if (status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + context.Session, + System.Text.Encoding.ASCII.GetString(messageIdBytes), + 439, + IngressNewsReasons.ArticleTooLarge, + payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = tooLarge; + return; + } + + if (peek == HistoryLookupResult.Unavailable || !queue.IsAccepting) + { + await FailTemporaryAsync(context.Session, context.Response, cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ServiceTemporarilyUnavailable); + context.CompletionDetail = "temporary failure"; + return; + } + + if (peek == HistoryLookupResult.Seen) + { + context.Session.SetActivityState(FeedSessionState.Completing); + var seenStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueTransferReplyAsync(context, rejected: false, cancellationToken) + .ConfigureAwait(false); + probe?.RecordArticleCompleted( + duplicate: true, + System.Diagnostics.Stopwatch.GetTimestamp() - seenStart); + context.Session.SetActivityState(FeedSessionState.Idle); + context.CompletionDetail = "accepted duplicate"; + return; + } + + var messageId = System.Text.Encoding.ASCII.GetString(messageIdBytes); + var stuffed = context.Session.ReceiveStrategy == NntpReceiveStrategy.StreamDataPlane + && context.PreReadArticle is null; + if (!TryCreateQueuedRecord( + context.Session, + payload, + stuffed, + messageId, + maxArticleBytes, + siteNameUtf8, + out var inbound, + out var recordReject, + out var canonicalSize)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + IngressNewsReasons.ForExistingRejectDetail(recordReject), + canonicalSize != 0 ? canonicalSize : payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = recordReject; + return; + } + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + context.Session.Transit, + context.Session.NewsgroupCatalogue, + out var uncarriedReason)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + uncarriedReason, + inbound.Payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = IngressNewsEvents.NewsgroupNotCarried; + return; + } + + ArticleEnqueueResult enqueue; + try + { + context.Session.SetActivityState(FeedSessionState.WaitingQueue); + var queueStart = System.Diagnostics.Stopwatch.GetTimestamp(); + enqueue = await queue.EnqueueAsync(inbound, cancellationToken).ConfigureAwait(false); + probe?.RecordQueue(enqueue, System.Diagnostics.Stopwatch.GetTimestamp() - queueStart, payload.Length); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (enqueue == ArticleEnqueueResult.Unavailable) + { + await FailTemporaryAsync(context.Session, context.Response, cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ServiceTemporarilyUnavailable); + context.CompletionDetail = "temporary failure"; + return; + } + + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = budget; + return; + } + + context.Session.HistoryDb?.Remember(messageIdBytes); + context.Session.SetActivityState(FeedSessionState.Completing); + var doneStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueTransferReplyAsync(context, rejected: false, cancellationToken) + .ConfigureAwait(false); + probe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - doneStart); + context.Session.SetActivityState(FeedSessionState.Idle); + context.CompletionDetail = "accepted"; + } + + private static ValueTask EnqueueTransferReplyAsync( + NntpCommandContext context, + bool rejected, + CancellationToken cancellationToken) + { + var prefix = rejected + ? NntpResponses.TransferRejectedPrefix + : NntpResponses.ArticleTransferredOkPrefix; + var owned = NntpResponseCompose.Concat( + prefix.Span, + context.ArgumentSpan, + NntpResponses.Crlf.Span); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= rejected + ? NntpCommandStatusText.FormatTakeThisRejected(context.ArgumentSpan) + : NntpCommandStatusText.FormatTakeThisAccepted(context.ArgumentSpan); + } + + return context.Response.EnqueueLineImmediateAsync(owned, cancellationToken); + } + + /// + /// Destuffs when required, builds , and returns a queue item + /// that references ArtData. Does not enqueue. + /// + internal static bool TryCreateQueuedRecord( + NntpSession session, + ReadOnlyMemory payload, + bool stuffed, + string messageId, + out InboundArticle inbound, + out string rejectDetail, + out int canonicalSize) + => TryCreateQueuedRecord( + session, + payload, + stuffed, + messageId, + session.ArticleIngestion.MaxArticleBytes, + siteNameUtf8: null, + out inbound, + out rejectDetail, + out canonicalSize); + + internal static bool TryCreateQueuedRecord( + NntpSession session, + ReadOnlyMemory payload, + bool stuffed, + string messageId, + int maxArticleBytes, + byte[]? siteNameUtf8, + out InboundArticle inbound, + out string rejectDetail, + out int canonicalSize) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + canonicalSize = 0; + + ArticleRecordCreateResult created; + if (stuffed) + { + created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromStuffedWire( + session.ArticleParser, + payload, + maxArticleBytes) + : ArticleRecordIngress.TryCreateFromStuffedWire( + session.ArticleParser, + payload, + maxArticleBytes, + siteNameUtf8); + } + else if (siteNameUtf8 is null) + { + created = ArticleRecordIngress.TryCreateFromDestuffed( + session.ArticleParser, + payload, + maxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + } + else + { + created = ArticleRecordIngress.TryCreateFromDestuffed( + session.ArticleParser, + payload, + maxArticleBytes, + siteNameUtf8); + } + + if (!created.IsAccepted) + { + inbound = null!; + rejectDetail = created.ParseFailure != NntpArticleParseFailureCode.None + ? "rejected article record " + created.ParseFailure + : "rejected article record " + created.MaterializeFailure; + return false; + } + + canonicalSize = created.Record.ArtSize; + if (!ArticleTypeAccessPolicy.CanPostArticleType(session, created.Record.ArtType)) + { + inbound = null!; + rejectDetail = TransitReceivePolicy.ArticleTypeRejectDetail; + return false; + } + + if (TransitReceivePolicy.TryReject(session.Authorization.TransitPeerPolicy, created.Record, out rejectDetail)) + { + inbound = null!; + return false; + } + + inbound = ArticleRecordIngress.CreateQueued( + messageId, + created.Record, + session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis, + IngressNewsEvents.SnapshotInboundFeed(session)); + rejectDetail = string.Empty; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/ArticleWireReconstructor.cs b/src/VectorNNTPD.ReaderService/Session/Framing/ArticleWireReconstructor.cs new file mode 100644 index 00000000..dc988f15 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/ArticleWireReconstructor.cs @@ -0,0 +1,155 @@ +using System.Buffers; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// + /// Reconstructs NNTP multiline wire bytes from a destuffed article payload (wire-framing utility). + /// + /// + /// + /// Destuffed payload convention (aligned with ingest output). + /// This type does not own article storage or catalogues; callers supply bytes (bench corpus, spool + /// snapshot, peer feed, etc.). + /// + /// + /// Terminator .\r\n is not part of the destuffed payload. + /// Dot-stuffing is removed on ingest: a wire line starting with .. is kept with one leading . removed. + /// Payload lines use \r\n (CRLF). A final line without CRLF is still emitted with CRLF on the wire. + /// + /// + /// Re-stuffing (RFC 3977 §3.1.1): any destuffed line whose content begins with . gets an extra + /// . prepended on the wire, then \r\n, then the article is terminated with .\r\n. + /// + /// + public static class ArticleWireReconstructor + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + private static readonly byte[] Terminator = ".\r\n"u8.ToArray(); + + /// + /// Re-stuffs a de-stuffed stored article into multiline wire form (no status line). + /// + /// Destuffed article bytes (terminator not present). + /// + /// When , appends the NNTP multiline terminator .CRLF + /// (TX / client transfer). When , omits it so the result matches + /// the IHAVE/TAKETHIS ingestion-queue contract. + /// + public static byte[] RestuffArticle(ReadOnlySpan storedDestuffed, bool includeTerminator = true) + { + if (storedDestuffed.IsEmpty) + { + return includeTerminator ? Terminator.ToArray() : []; + } + + var estimated = EstimateRestuffedWireBytes(storedDestuffed, includeTerminator); + var writer = new ArrayBufferWriter((int)Math.Min(estimated, int.MaxValue)); + var offset = 0; + while (offset < storedDestuffed.Length) + { + var remaining = storedDestuffed[offset..]; + var crlfAt = remaining.IndexOf(Crlf); + ReadOnlySpan line; + if (crlfAt < 0) + { + line = remaining; + offset = storedDestuffed.Length; + } + else + { + line = remaining[..crlfAt]; + offset += crlfAt + 2; + } + + if (line.Length > 0 && line[0] == (byte)'.') + { + writer.Write([(byte)'.']); + } + + writer.Write(line); + writer.Write(Crlf); + } + + if (includeTerminator) + { + writer.Write(Terminator); + } + + return writer.WrittenSpan.ToArray(); + } + + /// + /// Estimates wire article bytes (restuffed body + terminator) without allocating the full buffer. + /// + public static long EstimateRestuffedWireBytes( + ReadOnlySpan storedDestuffed, + bool includeTerminator = true) + { + if (storedDestuffed.IsEmpty) + { + return includeTerminator ? Terminator.Length : 0; + } + + long extraDots = 0; + var offset = 0; + while (offset < storedDestuffed.Length) + { + var remaining = storedDestuffed[offset..]; + var crlfAt = remaining.IndexOf(Crlf); + ReadOnlySpan line; + if (crlfAt < 0) + { + line = remaining; + // Missing trailing CRLF will be added on the wire. + extraDots += line.Length > 0 && line[0] == (byte)'.' ? 1 : 0; + var withoutTerminator = storedDestuffed.Length + 2 + extraDots; + return includeTerminator ? withoutTerminator + Terminator.Length : withoutTerminator; + } + + line = remaining[..crlfAt]; + if (line.Length > 0 && line[0] == (byte)'.') + { + extraDots++; + } + + offset += crlfAt + 2; + } + + var complete = storedDestuffed.Length + extraDots; + return includeTerminator ? complete + Terminator.Length : complete; + } + + /// + /// Splits a destuffed article into header block and body at the first blank line (\r\n\r\n). + /// + /// Full destuffed article (headers + body; no NNTP terminator). + /// Header bytes including the blank-line separator when present. + /// Body bytes after the blank line (maybe empty). + /// + /// when a blank line was found; otherwise and + /// is empty (entire payload treated as headers-only / no body). + /// + /// + /// Used by BODY TX to transmit only the body portion. Does not invent alternate separators; + /// matches CRLF blank-line semantics of the destuffed payload convention. + /// + public static bool TrySplitHeadersAndBody( + ReadOnlySpan storedDestuffedArticle, + out ReadOnlySpan headers, + out ReadOnlySpan body) + { + var separator = "\r\n\r\n"u8; + var at = storedDestuffedArticle.IndexOf(separator); + if (at < 0) + { + headers = storedDestuffedArticle; + body = ReadOnlySpan.Empty; + return false; + } + + headers = storedDestuffedArticle[..(at + separator.Length)]; + body = storedDestuffedArticle[(at + separator.Length)..]; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReadResult.cs b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReadResult.cs new file mode 100644 index 00000000..0d0b81bd --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReadResult.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// Outcome of . + /// + /// is owned NNTP wire bytes (dot-stuffing preserved, terminator omitted) + /// or empty when is not . + /// + public readonly record struct IHaveArticleReadResult( + NntpMultilineReadStatus Status, + ReadOnlyMemory Payload, + IHaveReceiveMetrics Metrics); +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReader.cs b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReader.cs new file mode 100644 index 00000000..7c69441d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveArticleReader.cs @@ -0,0 +1,285 @@ +using System.Buffers; +using System.Diagnostics; +using System.IO.Pipelines; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// + /// IHAVE article receive: locate CRLF . CRLF and copy stuffed wire octets into one owned buffer. + /// + /// + /// IHAVE is not pipelined (RFC 3977 §6.3.2). TAKETHIS STREAM receive reuses this reader + /// on the session RX task so the queued payload is one owned stuffed-wire copy + /// (OwnedWireBuffer.Take()). After that take, Pipe sequences are not retained + /// and the buffer is handed to . + /// This reader does not build an article record. The retained buffer is stuffed + /// wire with the terminator omitted. The size ceiling is the destuffed article: + /// one leading stuffing dot removed per line, content CRLF included, terminator + /// excluded, using . + /// A line that would exceed that ceiling stops further retention; the reader still + /// consumes through the terminator and returns + /// with an empty payload. IHAVE destuff and ArticleRecordFactory run after + /// this read, before queue admission. Pipeline workers never call this reader. + /// + public static class IHaveArticleReader + { + /// Initial owned-buffer capacity when the incoming article size is unknown. + public const int InitialCapacity = 64 * 1024; + + /// + /// Reads one IHAVE article from as owned stuffed wire. + /// + /// Connection input. Advanced through the article terminator. + /// + /// Destuffed article ceiling. Stuffing dots and the terminator are not counted. + /// + /// Cancels the pipe read. + /// + /// Stuffed wire without the terminator when the destuffed size fits; otherwise + /// or + /// with an empty payload. + /// + public static async ValueTask ReadAsync( + PipeReader reader, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var session = new ReadSession(maxArticleBytes); + return await session.RunAsync(reader, cancellationToken).ConfigureAwait(false); + } + + private sealed class ReadSession(int maxArticleBytes) + { + private readonly OwnedWireBuffer _article = new(InitialCapacity); + private int _pipeReads; + private bool _exceeded; + private int _destuffed; + private int _lineStart; + + public async ValueTask RunAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + var started = Stopwatch.GetTimestamp(); + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + _pipeReads++; + var unread = result.Buffer; + var examined = unread.End; + var atStart = _article.Written == 0; + + if (NntpDelimiterSearch.TryFindArticleTerminator( + unread, + atStart, + out var payloadBytes, + out var consumedBytes)) + { + if (payloadBytes > 0) + { + Append(unread.Slice(0, payloadBytes)); + } + + unread = unread.Slice(consumedBytes); + reader.AdvanceTo(unread.Start, unread.Start); + return Finish( + _exceeded ? NntpMultilineReadStatus.TooLarge : NntpMultilineReadStatus.Completed, + started); + } + + var hold = (int)Math.Min(NntpDelimiterSearch.ArticleTerminatorLookbehind, unread.Length); + var copyBytes = (int)unread.Length - hold; + if (copyBytes > 0) + { + Append(unread.Slice(0, copyBytes)); + unread = unread.Slice(copyBytes); + } + + reader.AdvanceTo(unread.Start, examined); + if (result.IsCompleted) + { + return Finish(NntpMultilineReadStatus.Incomplete, started); + } + } + } + + private void Append(ReadOnlySequence bytes) + { + if (_exceeded || bytes.IsEmpty) + { + return; + } + + foreach (var segment in bytes) + { + AppendSpan(segment.Span); + if (_exceeded) + { + return; + } + } + } + + private void AppendSpan(ReadOnlySpan span) + { + var offset = 0; + while (offset < span.Length && !_exceeded) + { + if (_article.Written > _lineStart + && _article.EndsWithCr + && span[offset] == (byte)'\n') + { + _article.AppendRaw(span.Slice(offset, 1)); + offset++; + TryCommitLine(); + continue; + } + + var slice = span[offset..]; + var crlf = slice.IndexOf("\r\n"u8); + if (crlf < 0) + { + TryAppendOpenLine(slice); + return; + } + + _article.AppendRaw(slice[..(crlf + 2)]); + offset += crlf + 2; + TryCommitLine(); + } + } + + private void TryAppendOpenLine(ReadOnlySpan slice) + { + if (slice.IsEmpty) + { + return; + } + + var lineLength = (_article.Written - _lineStart) + slice.Length; + var prospective = NntpArticleDestuffer.DestuffedLineBytes(lineLength, LineStartsWithDot(slice)); + if (_destuffed + prospective > maxArticleBytes) + { + RejectOpenLine(); + return; + } + + _article.AppendRaw(slice); + } + + private void TryCommitLine() + { + var contentLength = _article.Written - _lineStart - 2; + var add = NntpArticleDestuffer.DestuffedLineBytes(_article.LineAt(_lineStart, contentLength)); + if (_destuffed + add > maxArticleBytes) + { + RejectOpenLine(); + return; + } + + _destuffed += add; + _lineStart = _article.Written; + } + + private void RejectOpenLine() + { + _article.Truncate(_lineStart); + _exceeded = true; + } + + private bool LineStartsWithDot(ReadOnlySpan upcoming) + { + if (_article.Written > _lineStart) + { + return _article.FirstAt(_lineStart) == (byte)'.'; + } + + return upcoming.Length > 0 && upcoming[0] == (byte)'.'; + } + + private IHaveArticleReadResult Finish(NntpMultilineReadStatus status, long started) + { + var elapsed = Stopwatch.GetElapsedTime(started); + if (status != NntpMultilineReadStatus.Completed) + { + return new IHaveArticleReadResult( + status, + ReadOnlyMemory.Empty, + new IHaveReceiveMetrics(_pipeReads, 0, elapsed)); + } + + var payload = _article.Take(); + return new IHaveArticleReadResult( + status, + payload, + new IHaveReceiveMetrics(_pipeReads, payload.Length, elapsed)); + } + } + + private sealed class OwnedWireBuffer + { + private byte[] _buffer; + private int _written; + + public OwnedWireBuffer(int initialCapacity) + { + _buffer = new byte[initialCapacity]; + } + + public int Written => _written; + + public bool EndsWithCr => _written > 0 && _buffer[_written - 1] == (byte)'\r'; + + public byte FirstAt(int index) => _buffer[index]; + + public ReadOnlySpan LineAt(int start, int length) => _buffer.AsSpan(start, length); + + public void Truncate(int written) => _written = written; + + public void AppendRaw(ReadOnlySpan bytes) + { + if (bytes.IsEmpty) + { + return; + } + + Ensure(_written + bytes.Length); + bytes.CopyTo(_buffer.AsSpan(_written, bytes.Length)); + _written += bytes.Length; + } + + public ReadOnlyMemory Take() + { + var buffer = _buffer; + var length = _written; + _buffer = []; + _written = 0; + return buffer.AsMemory(0, length); + } + + private void Ensure(int needed) + { + if (_buffer.Length >= needed) + { + return; + } + + var next = _buffer.Length < 256 ? 256 : _buffer.Length * 2; + if (next < needed) + { + next = needed; + } + + var grown = new byte[next]; + if (_written > 0) + { + _buffer.AsSpan(0, _written).CopyTo(grown); + } + + _buffer = grown; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/IHaveReceiveMetrics.cs b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveReceiveMetrics.cs new file mode 100644 index 00000000..af7092d9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/IHaveReceiveMetrics.cs @@ -0,0 +1,8 @@ +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// IHAVE receive instrumentation (frame + own; no destuff). + public readonly record struct IHaveReceiveMetrics( + int PipeReads, + int ArticleSize, + TimeSpan ReceiveElapsed); +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/NntpArticleDestuffer.cs b/src/VectorNNTPD.ReaderService/Session/Framing/NntpArticleDestuffer.cs new file mode 100644 index 00000000..631fd1bb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/NntpArticleDestuffer.cs @@ -0,0 +1,191 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// + /// Dot-unstuffs NNTP multiline article lines into an owned buffer (RFC 3977 §3.1.1). + /// + /// + /// Shared by and the continuous TAKETHIS scanner so destuff + /// semantics stay identical. Pipe sequences must not be retained after AdvanceTo; callers + /// copy into before releasing Pipe memory. + /// + public static class NntpArticleDestuffer + { + /// + /// Appends one destuffed content line plus CRLF, or marks when + /// the destuffed article would exceed . + /// + public static void AppendUnstuffedLine( + ArrayBufferWriter output, + ReadOnlySequence lineBytes, + int maxArticleBytes, + ref bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return; + } + + var length = (int)lineBytes.Length; + var leadingDot = length > 0 && NntpDelimiterSearch.FirstByte(lineBytes) == (byte)'.'; + var needed = DestuffedLineBytes(length, leadingDot); + var contentLength = needed - 2; + if (output.WrittenCount + needed > maxArticleBytes) + { + exceeded = true; + return; + } + + var span = output.GetSpan(needed); + if (lineBytes.IsSingleSegment) + { + var src = lineBytes.FirstSpan; + if (leadingDot) + { + src = src[1..]; + } + + src.CopyTo(span); + } + else + { + var rented = ArrayPool.Shared.Rent(length); + try + { + lineBytes.CopyTo(rented); + var src = leadingDot ? rented.AsSpan(1, contentLength) : rented.AsSpan(0, contentLength); + src.CopyTo(span); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + span[contentLength] = (byte)'\r'; + span[contentLength + 1] = (byte)'\n'; + output.Advance(needed); + } + + /// + /// Appends one destuffed content line plus CRLF from a contiguous span. + /// + public static void AppendUnstuffedLine( + ArrayBufferWriter output, + ReadOnlySpan lineBytes, + int maxArticleBytes, + ref bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return; + } + + var needed = DestuffedLineBytes(lineBytes); + var content = lineBytes.Length > 0 && lineBytes[0] == (byte)'.' ? lineBytes[1..] : lineBytes; + if (output.WrittenCount + needed > maxArticleBytes) + { + exceeded = true; + return; + } + + var span = output.GetSpan(needed); + content.CopyTo(span); + span[content.Length] = (byte)'\r'; + span[content.Length + 1] = (byte)'\n'; + output.Advance(needed); + } + + /// + /// Returns the destuffed size of one content line plus CRLF. + /// + /// One stuffed line with the CRLF already removed. + /// + /// Content bytes after removing a single leading stuffing dot, plus the two CRLF octets. + /// A line that does not begin with . contributes its full length plus CRLF. + /// + public static int DestuffedLineBytes(ReadOnlySpan lineWithoutCrlf) => + DestuffedLineBytes( + lineWithoutCrlf.Length, + lineWithoutCrlf.Length > 0 && lineWithoutCrlf[0] == (byte)'.'); + + /// + /// Returns the destuffed size of one content line plus CRLF. + /// + /// Stuffed line length with the CRLF already removed. + /// when the line begins with a stuffing dot. + /// Destuffed content length plus two CRLF octets. + public static int DestuffedLineBytes(int lineLength, bool leadingDot) + { + var contentLength = leadingDot ? lineLength - 1 : lineLength; + return contentLength + 2; + } + + /// + /// Completes a destuffed article: with an empty + /// payload, or with an owned copy. + /// + public static NntpMultilineReadResult Complete(ArrayBufferWriter output, bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return new NntpMultilineReadResult(NntpMultilineReadStatus.TooLarge, ReadOnlyMemory.Empty); + } + + return new NntpMultilineReadResult( + NntpMultilineReadStatus.Completed, + output.WrittenMemory.ToArray()); + } + + /// + /// Destuffs terminator-omitted stuffed wire into one owned destuffed article buffer. + /// + /// + /// Uses the same line destuff as . + /// Does not classify, parse, or rewrite Date/Path. A line without a CRLF terminator is + /// ignored, matching . + /// + /// Stuffed article bytes without the NNTP terminator. + /// Destuffed size ceiling. + /// Owned destuffed bytes when the method returns . + /// when destuff completed within the limit. + public static bool TryDestuffStuffedWire( + ReadOnlySpan stuffedWire, + int maxArticleBytes, + [NotNullWhen(true)] out byte[]? destuffed) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var output = new ArrayBufferWriter(Math.Min(Math.Max(stuffedWire.Length, 1), maxArticleBytes)); + var exceeded = false; + var offset = 0; + while (offset < stuffedWire.Length && !exceeded) + { + var remaining = stuffedWire[offset..]; + var crlf = remaining.IndexOf(NntpDelimiterSearch.Crlf); + if (crlf < 0) + { + break; + } + + AppendUnstuffedLine(output, remaining[..crlf], maxArticleBytes, ref exceeded); + offset += crlf + 2; + } + + if (exceeded) + { + destuffed = null; + return false; + } + + destuffed = output.WrittenCount == 0 + ? [] + : output.WrittenMemory.ToArray(); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxParser.cs b/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxParser.cs new file mode 100644 index 00000000..dff42a9f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxParser.cs @@ -0,0 +1,256 @@ +using System.Buffers; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// Scan mode for . + public enum NntpContinuousRxMode + { + /// Waiting for a command line terminated by CRLF. + Command = 0, + + /// Inside a TAKETHIS multiline article; only the terminator ends the unit. + Article = 1, + } + + /// Kind of protocol unit produced by the continuous RX parser. + public enum NntpContinuousRxKind + { + /// Need more Pipe bytes (incomplete command or article). + NeedMore = 0, + + /// A complete non-article command line (without CRLF). + Command = 1, + + /// A complete TAKETHIS command plus framed article bytes (no destuff). + TakeThis = 2, + } + + /// One consumed protocol unit. Pipe memory is not retained. + public readonly struct NntpContinuousRxUnit + { + /// Initializes a new instance of the struct. + public NntpContinuousRxUnit( + NntpContinuousRxKind kind, + NntpCommand command = default, + NntpMultilineReadResult article = default) + { + Kind = kind; + Command = command; + Article = article; + } + + /// Gets the unit kind. + public NntpContinuousRxKind Kind { get; } + + /// Gets the parsed command. Indexes refer to . + public NntpCommand Command { get; } + + /// + /// Gets the article result for . + /// STREAM framing copies wire bytes without destuffing; the terminator is omitted. + /// + public NntpMultilineReadResult Article { get; } + + /// Need-more sentinel. + public static NntpContinuousRxUnit NeedMore { get; } = new(NntpContinuousRxKind.NeedMore); + } + + /// + /// Stateful command/article scanner over . + /// + /// + /// Command CRLF ends a command. Inside an article, the terminator is the five-octet + /// \r\n.\r\n (or a leading .\r\n for an empty article). CHECK/QUIT/TAKETHIS + /// text in the body is payload. Incomplete tails of at most four octets stay in the Pipe. + /// STREAM article bytes are copied as received (no destuff) into an owned buffer before the + /// caller advances the Pipe. The terminator is framing, not payload. MODE READER destuff + /// remains line-oriented in . + /// + public sealed class NntpContinuousRxParser + { + private readonly ArrayBufferWriter _article = new(64 * 1024); + private readonly byte[] _commandScratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + private int _commandLength; + private bool _articleExceeded; + private int _maxArticleBytes = int.MaxValue; + private NntpCommand _pendingCommand; + + /// Gets the current scan mode. + public NntpContinuousRxMode Mode { get; private set; } + + /// + /// Gets the current command-line bytes. Valid until the next command line is consumed. + /// + public ReadOnlyMemory CurrentCommandLine => _commandScratch.AsMemory(0, _commandLength); + + /// Resets command/article state (does not release writer capacity). + public void Reset() + { + Mode = NntpContinuousRxMode.Command; + _pendingCommand = default; + // Keep _commandLength/_commandScratch until the next command is copied so + // CurrentCommandLine remains valid for dispatch after a TAKETHIS article. + _articleExceeded = false; + _maxArticleBytes = int.MaxValue; + _article.ResetWrittenCount(); + } + + /// + /// Attempts to consume one complete protocol unit from . + /// Slices to the unconsumed remainder. + /// + /// Unread Pipe bytes. Sliced to the unconsumed tail. + /// + /// When , a TAKETHIS verb consumes the following article in this + /// parser. When , TAKETHIS is returned as a command line so session + /// gates can reject it without consuming a body (existing 480/502 behaviour). + /// + /// Framed (non-destuffed) payload cap for TAKETHIS articles. + public NntpContinuousRxUnit TryConsume( + ref ReadOnlySequence buffer, + bool consumeTakeThisArticle, + int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (Mode == NntpContinuousRxMode.Article) + { + return TryConsumeArticle(ref buffer); + } + + if (!NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + return NntpContinuousRxUnit.NeedMore; + } + + if (!TryCopyCommandLine(lineBytes)) + { + return new NntpContinuousRxUnit( + NntpContinuousRxKind.Command, + NntpCommand.OverlongLine); + } + + var command = NntpCommandParser.Parse(CurrentCommandLine.Span); + + if (consumeTakeThisArticle && command.Verb == NntpVerb.TakeThis) + { + _pendingCommand = command; + Mode = NntpContinuousRxMode.Article; + _maxArticleBytes = maxArticleBytes; + _articleExceeded = false; + _article.ResetWrittenCount(); + return TryConsumeArticle(ref buffer); + } + + return new NntpContinuousRxUnit(NntpContinuousRxKind.Command, command); + } + + private NntpContinuousRxUnit TryConsumeArticle(ref ReadOnlySequence buffer) + { + var atArticleStart = _article.WrittenCount == 0 && !_articleExceeded; + if (NntpDelimiterSearch.TryFindArticleTerminator( + buffer, + atArticleStart, + out var payloadBytes, + out var consumedBytes)) + { + if (payloadBytes > 0) + { + AppendFramedWire(buffer.Slice(0, payloadBytes)); + } + + buffer = buffer.Slice(consumedBytes); + var article = NntpArticleDestuffer.Complete(_article, _articleExceeded); + var unit = new NntpContinuousRxUnit( + NntpContinuousRxKind.TakeThis, + _pendingCommand, + article); + Reset(); + return unit; + } + + // Incomplete terminator: copy the prefix that cannot be a delimiter lookbehind + // so Pipe memory can be released. The last 0–4 octets stay unconsumed. + var hold = (int)Math.Min(NntpDelimiterSearch.ArticleTerminatorLookbehind, buffer.Length); + var copyBytes = (int)buffer.Length - hold; + if (copyBytes > 0) + { + AppendFramedWire(buffer.Slice(0, copyBytes)); + buffer = buffer.Slice(copyBytes); + } + + return NntpContinuousRxUnit.NeedMore; + } + + /// + /// Destuffs article lines from up to . + /// Used by (MODE READER multiline semantics). + /// + public NntpContinuousRxUnit TryConsumeArticleOnly( + ref ReadOnlySequence buffer, + int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (Mode != NntpContinuousRxMode.Article) + { + Mode = NntpContinuousRxMode.Article; + _maxArticleBytes = maxArticleBytes; + _articleExceeded = false; + _article.ResetWrittenCount(); + } + + while (NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + if (NntpDelimiterSearch.IsTerminatorLine(lineBytes)) + { + var article = NntpArticleDestuffer.Complete(_article, _articleExceeded); + Reset(); + return new NntpContinuousRxUnit(NntpContinuousRxKind.TakeThis, article: article); + } + + NntpArticleDestuffer.AppendUnstuffedLine(_article, lineBytes, maxArticleBytes, ref _articleExceeded); + } + + return NntpContinuousRxUnit.NeedMore; + } + + /// + /// Copies framed STREAM wire octets (already including content CRLFs) without destuffing. + /// + private void AppendFramedWire(ReadOnlySequence wireBytes) + { + if (_articleExceeded || wireBytes.IsEmpty) + { + return; + } + + var needed = checked((int)wireBytes.Length); + if (_article.WrittenCount + needed > _maxArticleBytes) + { + _articleExceeded = true; + return; + } + + var span = _article.GetSpan(needed); + wireBytes.CopyTo(span); + _article.Advance(needed); + } + + /// + /// Copies a command line that fits. An overlong line is consumed by the caller and is not copied. + /// + private bool TryCopyCommandLine(ReadOnlySequence lineBytes) + { + if (lineBytes.Length > _commandScratch.Length) + { + _commandLength = 0; + return false; + } + + var length = checked((int)lineBytes.Length); + lineBytes.CopyTo(_commandScratch); + _commandLength = length; + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxReader.cs b/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxReader.cs new file mode 100644 index 00000000..37d7649d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/NntpContinuousRxReader.cs @@ -0,0 +1,106 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// + /// Long-lived loop over . + /// + /// + /// + /// Consumed/examined positions follow parser progress: complete units are consumed; incomplete + /// tails are examined but not consumed. AdvanceTo(buffer.End) is never used as a generic + /// progress mechanism. + /// + /// + /// Article payloads are owned byte[] copies taken before AdvanceTo. + /// STREAM units copy framed wire bytes (no destuff). destuffs + /// for . Pipe sequences are not enqueued. + /// + /// + public static class NntpContinuousRxReader + { + /// + /// Reads one protocol unit. Returns only when + /// the pipe completes mid-command (treated as end of stream by the caller) or mid-article. + /// + public static async ValueTask ReadUnitAsync( + PipeReader reader, + NntpContinuousRxParser parser, + bool consumeTakeThisArticle, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examinedOrigin = buffer.End; + + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle, maxArticleBytes); + if (unit.Kind != NntpContinuousRxKind.NeedMore) + { + // Consume the unit; remaining bytes stay in the Pipe for the next call. + reader.AdvanceTo(buffer.Start, buffer.Start); + return unit; + } + + // Incomplete command or article: consume copied/complete prefix (buffer.Start), + // examine everything so the next read waits for new bytes. + reader.AdvanceTo(buffer.Start, examinedOrigin); + if (result.IsCompleted) + { + if (parser.Mode == NntpContinuousRxMode.Article) + { + parser.Reset(); + return new NntpContinuousRxUnit( + NntpContinuousRxKind.TakeThis, + article: new NntpMultilineReadResult( + NntpMultilineReadStatus.Incomplete, + ReadOnlyMemory.Empty)); + } + + parser.Reset(); + return new NntpContinuousRxUnit(NntpContinuousRxKind.NeedMore); + } + } + } + + /// + /// Reads one destuffed article from the current Pipe position (MODE READER / multiline body). + /// + public static async ValueTask ReadArticleAsync( + PipeReader reader, + NntpContinuousRxParser parser, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examinedOrigin = buffer.End; + var unit = parser.TryConsumeArticleOnly(ref buffer, maxArticleBytes); + if (unit.Kind == NntpContinuousRxKind.TakeThis) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return unit.Article; + } + + reader.AdvanceTo(buffer.Start, examinedOrigin); + if (result.IsCompleted) + { + parser.Reset(); + return new NntpMultilineReadResult(NntpMultilineReadStatus.Incomplete, ReadOnlyMemory.Empty); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/NntpDelimiterSearch.cs b/src/VectorNNTPD.ReaderService/Session/Framing/NntpDelimiterSearch.cs new file mode 100644 index 00000000..0cd6355a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/NntpDelimiterSearch.cs @@ -0,0 +1,193 @@ +using System.Buffers; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// + /// Runtime-vectorized delimiter search for NNTP CRLF framing and multiline terminators. + /// + /// + /// Uses runtime ReadOnlySpan<byte>.IndexOf so hardware acceleration is provided + /// by the runtime. A scalar fallback is implicit when vectorization is unavailable. + /// Correctness does not depend on SIMD. + /// + public static class NntpDelimiterSearch + { + /// CRLF sequence that terminates an NNTP command or article line. + public static ReadOnlySpan Crlf => "\r\n"u8; + + /// Five-byte article terminator delimiter after a preceding content line. + public static ReadOnlySpan FiveByteTerminator => "\r\n.\r\n"u8; + + /// Empty-article terminator at the start of a multiline block. + public static ReadOnlySpan EmptyTerminator => ".\r\n"u8; + + /// Finds the first CRLF in , or -1. + public static int IndexOfCrlf(ReadOnlySpan span) => span.IndexOf(Crlf); + + /// Finds the first \r\n.\r\n in , or -1. + public static int IndexOfFiveByteTerminator(ReadOnlySpan span) => span.IndexOf(FiveByteTerminator); + + /// + /// Bytes that must remain unconsumed when a STREAM article terminator is incomplete. + /// + /// + /// \r\n.\r\n is five octets; four may already be present at the tail of a Pipe read. + /// + public const int ArticleTerminatorLookbehind = 4; + + /// + /// Locates the STREAM article terminator in without walking CRLF lines. + /// + /// Unread article octets (not previously copied into the parser buffer). + /// + /// when no article payload has been accepted yet, so a leading + /// .\r\n is the empty-article terminator. + /// + /// + /// Count of octets that are article payload (includes the last + /// content line's CRLF; excludes the terminator). + /// + /// + /// Count of octets to consume (payload plus terminator). + /// + /// when a complete terminator is present. + /// + /// RFC 3977 §3.1.1: a non-empty multiline block ends with the five octets CRLF "." CRLF; + /// an empty block is "." CRLF. Payload excludes the terminating line. STREAM copies those + /// payload octets as received (no destuff). + /// + public static bool TryFindArticleTerminator( + ReadOnlySequence buffer, + bool atArticleStart, + out int payloadBytes, + out int consumedBytes) + { + payloadBytes = 0; + consumedBytes = 0; + + if (atArticleStart && StartsWithEmptyTerminator(buffer)) + { + consumedBytes = EmptyTerminator.Length; + return true; + } + + if (buffer.Length < FiveByteTerminator.Length) + { + return false; + } + + if (buffer.IsSingleSegment) + { + var index = IndexOfFiveByteTerminator(buffer.FirstSpan); + if (index < 0) + { + return false; + } + + payloadBytes = index + Crlf.Length; + consumedBytes = index + FiveByteTerminator.Length; + return true; + } + + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence before, FiveByteTerminator, advancePastDelimiter: true)) + { + return false; + } + + payloadBytes = checked((int)before.Length + Crlf.Length); + consumedBytes = checked((int)reader.Consumed); + return true; + } + + /// Returns whether begins with the empty-article terminator. + public static bool StartsWithEmptyTerminator(ReadOnlySequence buffer) + { + if (buffer.Length < EmptyTerminator.Length) + { + return false; + } + + if (buffer.IsSingleSegment) + { + return buffer.FirstSpan.StartsWith(EmptyTerminator); + } + + Span prefix = stackalloc byte[3]; + buffer.Slice(0, 3).CopyTo(prefix); + return prefix.SequenceEqual(EmptyTerminator); + } + + /// + /// Reads one CRLF-terminated line from (CRLF not included). + /// + /// when a complete line was found. + public static bool TryReadLine(ref ReadOnlySequence buffer, out ReadOnlySequence lineBytes) + { + if (buffer.IsSingleSegment) + { + var span = buffer.FirstSpan; + var crlf = IndexOfCrlf(span); + if (crlf < 0) + { + lineBytes = default; + return false; + } + + lineBytes = buffer.Slice(0, crlf); + buffer = buffer.Slice(crlf + 2); + return true; + } + + var seqReader = new SequenceReader(buffer); + if (!seqReader.TryReadTo(out lineBytes, Crlf)) + { + lineBytes = default; + return false; + } + + buffer = buffer.Slice(seqReader.Position); + return true; + } + + /// Returns whether is the NNTP multiline terminator line. + public static bool IsTerminatorLine(ReadOnlySequence lineBytes) + { + if (lineBytes.Length != 1) + { + return false; + } + + return FirstByte(lineBytes) == (byte)'.'; + } + + /// Returns whether is the NNTP multiline terminator line. + public static bool IsTerminatorLine(ReadOnlySpan line) => + line.Length == 1 && line[0] == (byte)'.'; + + /// Returns the first byte of , or -1 when empty. + public static int FirstByte(ReadOnlySequence sequence) + { + if (sequence.IsEmpty) + { + return -1; + } + + var first = sequence.FirstSpan; + if (!first.IsEmpty) + { + return first[0]; + } + + foreach (var segment in sequence) + { + if (!segment.IsEmpty) + { + return segment.Span[0]; + } + } + + return -1; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/Framing/NntpMultilineDataReader.cs b/src/VectorNNTPD.ReaderService/Session/Framing/NntpMultilineDataReader.cs new file mode 100644 index 00000000..76d3cbfa --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/Framing/NntpMultilineDataReader.cs @@ -0,0 +1,62 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.ReaderService.Session.Framing +{ + /// Result of reading one NNTP multiline data block from a . + public enum NntpMultilineReadStatus + { + /// + /// Complete article consumed (terminator seen). Payload meaning depends on the reader: + /// STREAM framing copies wire bytes; destuffs. + /// + Completed = 0, + + /// Peer disconnected before the terminating dot line; nothing should be enqueued. + Incomplete = 1, + + /// + /// Article exceeded the configured size limit; remaining bytes through the terminator were discarded. + /// + TooLarge = 2, + } + + /// Outcome of . + /// Read status. + /// + /// Article bytes when (STREAM: framed wire + /// without terminator; multiline reader: destuffed); otherwise empty. + /// + public readonly record struct NntpMultilineReadResult( + NntpMultilineReadStatus Status, + ReadOnlyMemory Payload); + + /// + /// Reads an NNTP multiline data block (RFC 3977 §3.1.1) with dot-unstuffing. + /// + /// + /// Terminator is a line containing only . (i.e. .CRLF). + /// A leading doubled dot on a content line is reduced to a single leading dot. + /// The terminator line is not included in the payload. + /// Line boundaries are located with (runtime-vectorized + /// IndexOf on contiguous spans, sequence scan across segments). + /// + public static class NntpMultilineDataReader + { + /// + /// Reads one multiline article from up to . + /// + public static async ValueTask ReadArticleAsync( + PipeReader reader, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + var parser = new NntpContinuousRxParser(); + return await NntpContinuousRxReader + .ReadArticleAsync(reader, parser, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/INntpSessionCensus.cs b/src/VectorNNTPD.ReaderService/Session/INntpSessionCensus.cs new file mode 100644 index 00000000..b69bde92 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/INntpSessionCensus.cs @@ -0,0 +1,99 @@ +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.Session +{ + /// Point-in-time counts of every established NNTP session on this process. + public readonly struct NntpSessionCensusSnapshot + { + /// Initializes a census snapshot. + public NntpSessionCensusSnapshot( + int active, + int established, + int idle, + int receiving, + int waitingHistory, + int waitingQueue, + int waitingWindow, + int completing, + int takeThisOccupied = 0, + int takeThisOccupiedMax = 0, + int takeThisFullSessions = 0, + int takeThisProcessing = 0, + int takeThisProcessingMax = 0) + { + Active = active; + Established = established; + Idle = idle; + Receiving = receiving; + WaitingHistory = waitingHistory; + WaitingQueue = waitingQueue; + WaitingWindow = waitingWindow; + Completing = completing; + TakeThisOccupied = takeThisOccupied; + TakeThisOccupiedMax = takeThisOccupiedMax; + TakeThisFullSessions = takeThisFullSessions; + TakeThisProcessing = takeThisProcessing; + TakeThisProcessingMax = takeThisProcessingMax; + } + + /// Gets established session count (complete server population). + public int Active { get; } + + /// Gets established session count (same population as ). + public int Established { get; } + + /// Gets sessions in . + public int Idle { get; } + + /// Gets sessions in . + public int Receiving { get; } + + /// Gets sessions in . + public int WaitingHistory { get; } + + /// Gets sessions in . + public int WaitingQueue { get; } + + /// Gets sessions in . + public int WaitingWindow { get; } + + /// Gets sessions in . + public int Completing { get; } + + /// Gets the sum of TAKETHIS Occupied slots across sessions. + public int TakeThisOccupied { get; } + + /// Gets the highest TAKETHIS Occupied count on any one session. + public int TakeThisOccupiedMax { get; } + + /// Gets how many sessions currently have Occupied equal to TAKETHIS Depth (16). + public int TakeThisFullSessions { get; } + + /// Gets the sum of concurrent ProcessArticleAsync tasks across sessions. + public int TakeThisProcessing { get; } + + /// Gets the highest concurrent ProcessArticleAsync count on any one session. + public int TakeThisProcessingMax { get; } + } + + /// + /// Authoritative registry of established NNTP sessions (readers, streamers, and transit). + /// + public interface INntpSessionCensus + { + /// Registers a session that has entered . + void Register(NntpSession session); + + /// Unregisters a session that is leaving . + void Unregister(NntpSession session); + + /// + /// Stops attributing to Transit peer telemetry after + /// it authenticates as a reader. Does not remove it from the process census. + /// + void ReleasePeerAttribution(NntpSession session); + + /// Captures the current complete session population. + NntpSessionCensusSnapshot Capture(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpArticleNumber.cs b/src/VectorNNTPD.ReaderService/Session/NntpArticleNumber.cs new file mode 100644 index 00000000..959ee846 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpArticleNumber.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// Validates RFC 3977 article-number tokens (1*16DIGIT). + public static class NntpArticleNumber + { + /// Maximum article-number length in octets (RFC 3977 §9.2). + public const int MaxDigits = 16; + + /// + /// Returns whether is a syntactically valid article number. + /// Does not imply that an article exists. + /// + public static bool IsSyntax(ReadOnlySpan value) + { + if (value.Length is < 1 or > MaxDigits) + { + return false; + } + + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationAuthority.cs b/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationAuthority.cs new file mode 100644 index 00000000..ca819ff7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationAuthority.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Selects which AUTHINFO credential authority is active. + /// Independent of receive/capability state. + /// + /// + /// Source IP never selects this value. MODE READER and the unspecified default + /// use . A successful MODE STREAM (or an explicit + /// ) uses . + /// There is no Transit ↔ Reader fallback. + /// + public enum NntpAuthenticationAuthority + { + /// Newsmaster then MySQL nntpusers. Never Transit peer credentials. + Reader = 0, + + /// Configured Transit peer credentials only. Never MySQL or newsmaster. + Transit = 1, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationState.cs b/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationState.cs new file mode 100644 index 00000000..9e48dec1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpAuthenticationState.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// Immutable authentication identity for an NNTP session (distinct from authorization). + public sealed class NntpAuthenticationState + { + /// Unauthenticated session (no identity). + public static NntpAuthenticationState Unauthenticated { get; } = new(isAuthenticated: false, username: null); + + private NntpAuthenticationState(bool isAuthenticated, string? username) + { + IsAuthenticated = isAuthenticated; + Username = username; + } + + /// Gets a value indicating whether the peer completed AUTHINFO successfully. + public bool IsAuthenticated { get; } + + /// Gets the authenticated username when is . + public string? Username { get; } + + /// Creates an authenticated identity snapshot. + public static NntpAuthenticationState ForUser(string username) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + return new(isAuthenticated: true, username: username); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpAuthorization.cs b/src/VectorNNTPD.ReaderService/Session/NntpAuthorization.cs new file mode 100644 index 00000000..d47315ba --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpAuthorization.cs @@ -0,0 +1,134 @@ +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Immutable authorization snapshot for an NNTP session. + /// + /// + /// Authentication and authorization are distinct: a session may be authenticated without + /// reader, transit, posting, streaming, or control-cancel privileges. Production default for + /// new sessions is unauthenticated with false. A Transit + /// AllowFrom match retains (the Transit identifier) and + /// . + /// + public sealed class NntpAuthorization + { + /// Default privileges before authentication (no reader/transit/posting/streaming/control-cancel). + public static NntpAuthorization Unauthenticated { get; } = new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + /// + /// Unauthenticated transit/streaming peer privileges (ACL match). + /// Does not grant reader access, posting, or . + /// Nameless; production identification uses . + /// + public static NntpAuthorization TrustedTransitPeer { get; } = new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + /// Initializes a new instance of the class. + public NntpAuthorization( + bool isAuthenticated, + bool authorizedReader, + bool authorizedTransit, + bool postingPermitted, + bool streamingPermitted, + string? transitPeerName = null, + TransitPeerPolicy? transitPeerPolicy = null, + bool controlCancelPermitted = false) + { + IsAuthenticated = isAuthenticated; + AuthorizedReader = authorizedReader; + AuthorizedTransit = authorizedTransit; + PostingPermitted = postingPermitted; + StreamingPermitted = streamingPermitted; + TransitPeerName = transitPeerName; + TransitPeerPolicy = transitPeerPolicy; + ControlCancelPermitted = controlCancelPermitted; + } + + /// + /// Creates streaming privileges bound to a named peer policy. + /// + /// + /// is always set so MODE STREAM can select Transit AUTHINFO. + /// is set only when receive authentication is disabled + /// ( is false: username or password empty). + /// When both are non-empty, feed commands stay unauthorized until AUTHINFO matches that pair. + /// The peer name and policy are retained either way, including for inbound connection limits. + /// + public static NntpAuthorization ForTransitPeer(TransitPeerPolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + return new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: !policy.HasPeerCredentials, + postingPermitted: false, + streamingPermitted: true, + transitPeerName: policy.Identifier, + transitPeerPolicy: policy); + } + + /// Gets a value indicating whether the peer has completed authentication. + public bool IsAuthenticated { get; } + + /// Gets a value indicating whether reader commands are authorized. + public bool AuthorizedReader { get; } + + /// Gets a value indicating whether transit/streaming feed commands are authorized. + public bool AuthorizedTransit { get; } + + /// Gets a value indicating whether POST is permitted (affects greeting / MODE READER). + public bool PostingPermitted { get; } + + /// Gets a value indicating whether streaming feed mode may be entered. + public bool StreamingPermitted { get; } + + /// + /// Gets a value indicating whether this session may POST a well-formed + /// Control: cancel <message-id> article. + /// + /// + /// Ordinary posting remains forbidden from supplying Control. This flag is granted + /// only by an authentication provider (the configured newsmaster account). It is not a + /// generic control-header bypass. + /// + public bool ControlCancelPermitted { get; } + + /// + /// Gets the Transit identifier when this session was identified as a named peer. + /// + /// This is the dictionary-key identifier, not . + public string? TransitPeerName { get; } + + /// Gets the identified Transit peer policy, or when not a named peer. + public TransitPeerPolicy? TransitPeerPolicy { get; } + + /// Returns a copy with updated flags, preserving Transit peer identity. + public NntpAuthorization With( + bool? isAuthenticated = null, + bool? authorizedReader = null, + bool? authorizedTransit = null, + bool? postingPermitted = null, + bool? streamingPermitted = null, + bool? controlCancelPermitted = null) => + new( + isAuthenticated ?? IsAuthenticated, + authorizedReader ?? AuthorizedReader, + authorizedTransit ?? AuthorizedTransit, + postingPermitted ?? PostingPermitted, + streamingPermitted ?? StreamingPermitted, + TransitPeerName, + TransitPeerPolicy, + controlCancelPermitted ?? ControlCancelPermitted); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpReaderCommandRx.cs b/src/VectorNNTPD.ReaderService/Session/NntpReaderCommandRx.cs new file mode 100644 index 00000000..4e3796de --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpReaderCommandRx.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// MODE READER RX: one command line at a time via . + /// + /// + /// Does not use the continuous TAKETHIS scanner. TAKETHIS on this path is dispatched as a + /// normal command; the handler reads the article after authorization gates. + /// + internal sealed class NntpReaderCommandRx + { + private readonly NntpSession _session; + private readonly NntpCommandDispatcher _dispatcher; + private readonly ILogger _logger; + private readonly byte[] _commandScratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + /// Initializes a new instance of the class. + public NntpReaderCommandRx(NntpSession session, NntpCommandDispatcher dispatcher, ILogger logger) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(logger); + _session = session; + _dispatcher = dispatcher; + _logger = logger; + } + + /// + /// Reads and dispatches one command. Returns on EOF. + /// + public async ValueTask ProcessOneAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + await _session.WaitForCheckCapacityAsync(cancellationToken).ConfigureAwait(false); + var length = await NntpCommandLineReader + .ReadLineBytesAsync(_session.Connection.Input, _commandScratch, cancellationToken) + .ConfigureAwait(false); + if (length == NntpCommandLineReader.OverlongLine) + { + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + NntpCommand.OverlongLine, + ReadOnlyMemory.Empty, + preReadArticle: null, + cancellationToken) + .ConfigureAwait(false); + return true; + } + + if (length < 0) + { + return false; + } + + var line = _commandScratch.AsMemory(0, length); + if (_session.HasSaslExchange) + { + _session.BeginCommandWork(); + try + { + if (_session.TakeThisWindow is not null) + { + await _session.TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + } + + if (_session.Pipeline is not null) + { + await _session.Pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + } + + await Commands.AuthInfo + .ContinueSaslAsync(_session, response, line, cancellationToken) + .ConfigureAwait(false); + } + finally + { + _session.EndCommandWork(); + } + + return true; + } + + var command = NntpCommandParser.Parse(line.Span); + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + command, + line, + preReadArticle: null, + cancellationToken) + .ConfigureAwait(false); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpReceiveStrategy.cs b/src/VectorNNTPD.ReaderService/Session/NntpReceiveStrategy.cs new file mode 100644 index 00000000..5a7f63ab --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpReceiveStrategy.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Per-connection RX strategy selected from existing . + /// + /// + /// RFC 4644 §2.3: MODE STREAM MUST NOT change server state, so this is not set by that + /// command. is the default (unspecified) path used by transit/feed + /// peers. is selected only after a successful MODE READER. + /// + internal enum NntpReceiveStrategy + { + /// Command-oriented RX: ReadLineAsync → parser → dispatcher. + ReaderCommand = 0, + + /// Continuous CHECK/TAKETHIS data-plane RX. + StreamDataPlane = 1, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpReplyCodes.cs b/src/VectorNNTPD.ReaderService/Session/NntpReplyCodes.cs new file mode 100644 index 00000000..819ded90 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpReplyCodes.cs @@ -0,0 +1,117 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// Common NNTP reply codes used by the session and command layer (RFC 3977 / 4643). + public static class NntpReplyCodes + { + /// Help text follows (multiline). + public const int HelpTextFollows = 100; + + /// Capability list follows (multiline). + public const int CapabilityListFollows = 101; + + /// Server date stamp. + public const int ServerDate = 111; + + /// Posting allowed (greeting / MODE READER). + public const int PostingAllowed = 200; + + /// Posting prohibited (greeting / MODE READER). + public const int PostingProhibited = 201; + + /// Connection closing (QUIT). + public const int ConnectionClosing = 205; + + /// Compression layer activated (COMPRESS; RFC 8054). + public const int CompressionActive = 206; + + /// Streaming permitted (MODE STREAM; RFC 4644 §2.3). + public const int StreamingPermitted = 203; + + /// Article follows (ARTICLE; RFC 3977 §6.2.1). + public const int ArticleFollows = 220; + + /// Headers follow (HEAD; RFC 3977 §6.2.2). + public const int HeadFollows = 221; + + /// Body follows (BODY; RFC 3977 §6.2.3). + public const int BodyFollows = 222; + + /// Article exists / selected (STAT; RFC 3977 §6.2.4). + public const int ArticleExists = 223; + + /// Article received OK (POST second stage; RFC 3977 §6.3.1). + public const int ArticleReceivedOk = 240; + + /// Send article to be posted (POST first stage; RFC 3977 §6.3.1). + public const int PostSendArticle = 340; + + /// Posting not permitted (POST first stage; RFC 3977 §6.3.1). + public const int PostingNotPermitted = 440; + + /// Posting failed (POST second stage; RFC 3977 §6.3.1). + public const int PostingFailed = 441; + + /// Article transferred OK (IHAVE second stage; RFC 3977 §6.3.2). + public const int IhaveTransferredOk = 235; + + /// Send article to be transferred (IHAVE first stage; RFC 3977 §6.3.2). + public const int IhaveSendArticle = 335; + + /// Article not wanted (IHAVE first stage; RFC 3977 §6.3.2). + public const int IhaveNotWanted = 435; + + /// Transfer not possible or failed; try again later (IHAVE; RFC 3977 §6.3.2). + public const int IhaveTryLater = 436; + + /// Transfer rejected; do not retry (IHAVE second stage; RFC 3977 §6.3.2). + public const int IhaveRejected = 437; + + /// Send article to be transferred (CHECK; RFC 4644 §2.4). + public const int SendArticleToBeTransferred = 238; + + /// Article transferred OK (TAKETHIS; RFC 4644 §2.5). + public const int ArticleTransferredOk = 239; + + /// Service temporarily unavailable (close connection; RFC 4644 §2.5). + public const int ServiceTemporarilyUnavailable = 400; + + /// Transfer rejected; do not retry (TAKETHIS; RFC 4644 §2.5). + public const int TransferRejected = 439; + + /// Authentication accepted (AUTHINFO). + public const int AuthenticationAccepted = 281; + + /// Continue with TLS negotiation (STARTTLS). + public const int ContinueWithTls = 382; + + /// Password required to complete AUTHINFO USER/PASS. + public const int PasswordRequired = 381; + + /// Command failed due to temporary/internal fault. + public const int CommandFailed = 403; + + /// Feature exists but is unavailable or not supported in this form (e.g. unsupported COMPRESS algorithm). + public const int FeatureUnavailable = 503; + + /// Authentication required before the facility can be used. + public const int AuthenticationRequired = 480; + + /// Authentication failed or rejected. + public const int AuthenticationRejected = 481; + + /// Authentication commands issued out of sequence. + public const int AuthenticationOutOfSequence = 482; + + /// Privacy/encryption required for the command. + public const int PrivacyRequired = 483; + + /// Unknown or unimplemented command. + public const int UnknownCommand = 500; + + /// Syntax error or unknown command variant. + public const int SyntaxError = 501; + + /// Command not permitted in the current state / permanently unavailable. + public const int CommandUnavailable = 502; + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpSession.cs b/src/VectorNNTPD.ReaderService/Session/NntpSession.cs new file mode 100644 index 00000000..00b4efac --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpSession.cs @@ -0,0 +1,1361 @@ +using System.IO.Pipelines; +using System.Net; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Session.SpeedTest; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Session +{ + using VectorNNTPD.Common.Core; + /// + /// NNTP session boundary: greeting, command loop, mode, authentication, and authorization state over an + /// . + /// + /// + /// Transport owns sockets, TLS, DEFLATE, PROXY identity, and connection lifecycle. + /// This type owns NNTP protocol sequencing and does not touch sockets or stream wrappers directly. + /// Authentication and authorization are distinct; AUTHINFO success applies only provider-returned privileges. + /// Connection-time may grant transit/streaming peer privileges + /// without authentication and retains the named Transit peer policy. + /// An established session that executes no NNTP command for Nntpd:IdleTime is disconnected + /// via the existing close path (reason IdleTimeout). In-flight commands including + /// pipelined CHECK and TAKETHIS keep the session non-idle. + /// Any command-loop exit (QUIT, EOF, reset, timeout, cancel, exception, shutdown) + /// takes the same one-shot finalization path and releases distributed admission at most once. + /// + public sealed class NntpSession + { + private readonly ILogger _logger; + private readonly NntpCommandDispatcher _dispatcher; + private readonly NntpAuthorization _connectionAuthorization; + private NntpResponseWriter? _response; + private NntpAuthorization _authorization; + private NntpAuthenticationState _authentication; + private string? _pendingAuthUsername; + private NntpAccountPolicy? _accountPolicy; + private string? _admittedAccountName; + private TransitInboundConnectionLease? _transitAdmission; + private NntpSaslExchange? _saslExchange; + private NntpSessionMode _mode; + private NntpAuthenticationAuthority _authenticationAuthority; + private int _closeRequested; + private int _lifetime; + private int _activityState; + private int _commandWork; + private long _lastCommandTimestamp; + /// + /// Low 63 bits: activity epoch, incremented when commits. + /// High bit set: idle waiter has committed . + /// + private long _idleState; + private readonly TimeSpan _commandIdleTimeout; + private readonly TimeProvider _timeProvider; + private readonly TaskCompletionSource _idleWatchArmed = new(TaskCreationOptions.RunContinuationsAsynchronously); + // codeql[cs/missed-using-statement] Owned by NntpSession until Dispose; a using at this declaration would cancel session lifetime when the constructor returns. + private readonly CancellationTokenSource _closeCts = new(); + private Task? _idleWatchTask; + private int _closeReason; + private ReadOnlyMemory _selectedGroupName; + + /// Initializes a new instance of the class. + public NntpSession( + INntpConnection connection, + ILogger logger, + ITlsCertificateContextProvider? certificateProvider = null, + INntpAuthenticationProvider? authenticationProvider = null, + bool allowCleartextAuth = true, + ILoggerFactory? loggerFactory = null, + IArticleIngestionQueue? articleIngestion = null, + ITransitPeerAuthorization? transitPeerAuthorization = null, + int streamOutstandingArticleDepth = NntpStreamArticleTxScheduler.DefaultDepth, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + TimeSpan? commandIdleTimeout = null, + TimeProvider? timeProvider = null, + int? maxArticleSize = null, + string? injectionIdentity = null, + Commands.Posting.INewsgroupPostingPolicy? newsgroupPostingPolicy = null, + string? mailComplaintsTo = null, + Commands.Posting.IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + ITransitPeerAuthenticator? transitAuthenticator = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + TransitOptions? transit = null, + NntpServerRole serverRole = NntpServerRole.Reader) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(logger); + ServerRole = serverRole; + Connection = connection; + ClientIdentity = connection.ClientIdentity; + _logger = logger; + CertificateProvider = certificateProvider; + _connectionAuthorization = (transitPeerAuthorization ?? TransitPeerAuthorization.Disabled) + .Resolve(ClientIdentity.ClientAddress); + _authorization = _connectionAuthorization; + _authentication = NntpAuthenticationState.Unauthenticated; + _mode = NntpSessionMode.Unspecified; + _authenticationAuthority = NntpAuthenticationAuthority.Reader; + AllowCleartextAuth = allowCleartextAuth; + AuthenticationProvider = authenticationProvider ?? DenyAllNntpAuthenticationProvider.Instance; + TransitAuthenticator = transitAuthenticator ?? TransitPeerAuthenticator.Instance; + ArticleIngestion = articleIngestion ?? DisabledArticleIngestionQueue.Instance; + HistoryDb = historyDb; + SpeedTest = speedTest; + SessionCensus = sessionCensus; + PeerMetrics = peerMetrics; + StreamArticleTx = new NntpStreamArticleTxScheduler(streamOutstandingArticleDepth); + _dispatcher = new NntpCommandDispatcher(loggerFactory); + _timeProvider = timeProvider ?? TimeProvider.System; + _commandIdleTimeout = commandIdleTimeout ?? TimeSpan.FromSeconds(NntpdOptions.DefaultIdleTime); + if (_commandIdleTimeout < TimeSpan.FromSeconds(NntpdOptions.MinIdleTime) + || _commandIdleTimeout > TimeSpan.FromSeconds(NntpdOptions.MaxIdleTime)) + { + throw new ArgumentOutOfRangeException( + nameof(commandIdleTimeout), + commandIdleTimeout, + $"Idle timeout must be between {NntpdOptions.MinIdleTime} and {NntpdOptions.MaxIdleTime} seconds."); + } + + MaxArticleSize = maxArticleSize ?? NntpdOptions.DefaultMaxArticleSize; + if (MaxArticleSize < NntpdOptions.MinMaxArticleSize + || MaxArticleSize > NntpdOptions.MaxMaxArticleSize) + { + throw new ArgumentOutOfRangeException( + nameof(maxArticleSize), + maxArticleSize, + $"Max article size must be between {NntpdOptions.MinMaxArticleSize} and {NntpdOptions.MaxMaxArticleSize} bytes."); + } + + InjectionIdentity = string.IsNullOrWhiteSpace(injectionIdentity) + ? NntpdOptions.FormatFqdn(1, "usenet.ninja") + : injectionIdentity.Trim(); + ArticleParser = new NntpArticleParser(InjectionIdentity); + NewsgroupPostingPolicy = newsgroupPostingPolicy + ?? (newsgroupCatalogue is not null + ? new Commands.Posting.CatalogueNewsgroupPostingPolicy(newsgroupCatalogue) + : Commands.Posting.SyntaxOnlyNewsgroupPostingPolicy.Instance); + MailComplaintsTo = string.IsNullOrWhiteSpace(mailComplaintsTo) + ? NntpdOptions.DefaultMailComplaintsTo + : mailComplaintsTo.Trim(); + PostingTraceProtector = postingTraceProtector; + NewsgroupCatalogue = newsgroupCatalogue; + ModeratorCatalogue = moderatorCatalogue; + ModeratorAuthorization = moderatorAuthorization + ?? (IModeratorAuthorization?)moderatorCatalogue + ?? EmptyModeratorAuthorization.Instance; + ModerationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + SessionAdmission = sessionAdmission; + SaslService = saslService; + AccountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + AccountRates = accountRates ?? NullAccountRateAllocator.Instance; + ArticleWorkRpc = articleWorkRpc; + VatpArticleClient = vatpArticleClient; + StorageArticleLookup = storageArticleLookup; + PostFilter = postFilter ?? DisabledPostFilter.Instance; + PostFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + PostFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + NewsLog = newsLog ?? NullNewsLogWriter.Instance; + Transit = transit ?? new TransitOptions(); + SessionId = Guid.NewGuid().ToString("N"); + } + + /// Gets the underlying transport connection. + public INntpConnection Connection { get; } + + /// Gets the immutable client identity established at connection start. + public ConnectionClientIdentity ClientIdentity { get; } + + /// + /// Gets the article ingestion queue used by transfer commands (TAKETHIS, IHAVE, POST). + /// + public IArticleIngestionQueue ArticleIngestion { get; } + + /// + /// Gets the article-work RPC client used by ARTICLE/HEAD/BODY/STAT message-id lookup, or + /// when tests construct a session without RabbitMQ. + /// + internal IArticleWorkRpcClient? ArticleWorkRpc { get; } + + /// + /// Gets the VATP article client used after ArticleWork Success, or + /// when tests construct a session without VATP. + /// + internal IVatpArticleClient? VatpArticleClient { get; } + + /// + /// Gets the StorageServer fleet lookup used after a definitive ArticleWork miss, or + /// when tests construct a session without fleet lookup. + /// + internal IStorageArticleLookupClient? StorageArticleLookup { get; } + + /// Gets the POST-only PostFilter. IHAVE/TAKETHIS must not call this. + internal IPostFilter PostFilter { get; } + + /// Gets process counters for PostFilter COMMIT outcomes. + internal PostFilterMetrics PostFilterMetrics { get; } + + /// Gets the bounded PostFilter rejection-evidence queue. + internal IPostFilterRejectionEvidenceQueue PostFilterEvidence { get; } + + /// Gets the dedicated INN news writer. + internal INewsLogWriter NewsLog { get; } + + /// Gets site-wide transit options including WantTrash/LogTrash. + internal TransitOptions Transit { get; } + + /// Gets the destuffed POST article size limit (Nntpd:MaxArticleSize). + public int MaxArticleSize { get; } + + /// + /// Published nntpsharedconfig. Null when the session was constructed without the catalogue. + /// Article commands capture one policy for the whole operation. + /// + internal INntpArticlePolicySource? SharedConfiguration { get; set; } + + /// + /// Gets the server injection identity used for POST metadata + /// (nntpd{ServerId:00}.{DnsSuffix}). + /// + public string InjectionIdentity { get; } + + /// + /// Gets the session-scoped Common article parser (local identity is + /// ). Used by TAKETHIS and POST before queue admission. + /// + internal NntpArticleParser ArticleParser { get; } + + /// + /// Gets the newsgroup existence/posting-authorization boundary used by POST. + /// Catalogue-backed when is set and no policy was injected. + /// + public Commands.Posting.INewsgroupPostingPolicy NewsgroupPostingPolicy { get; } + + /// Gets the configured POST mail-complaints-to mailbox. + public string MailComplaintsTo { get; } + + /// Gets the POST X-Trace protector, or when unset (tests). + public Commands.Posting.IPostingTraceProtector? PostingTraceProtector { get; } + + /// + /// Gets the in-memory newsgroup catalogue, or when unset (tests). + /// + public INewsgroupCatalogue? NewsgroupCatalogue { get; } + + /// + /// Gets the process-wide moderator catalogue, or when unset (tests). + /// + public IModeratorCatalogue? ModeratorCatalogue { get; } + + /// + /// Gets the moderator authorization table used by POST for Approved: decisions. + /// + public IModeratorAuthorization ModeratorAuthorization { get; } + + /// Captures the current moderator snapshot once for a POST command. + public IModeratorAuthorization CaptureModeratorAuthorization() => + ModeratorCatalogue?.Current ?? ModeratorAuthorization; + + /// + /// Gets the moderation submission boundary used when an unapproved moderated + /// proto-article must be forwarded. + /// + public IModerationSubmissionService ModerationSubmission { get; } + + /// Gets whether a newsgroup is currently selected. + internal bool HasSelectedGroup => !_selectedGroupName.IsEmpty; + + /// Gets the original stored name of the currently selected newsgroup. + internal ReadOnlySpan SelectedGroupName => _selectedGroupName.Span; + + /// Gets the clock used for idle accounting and POST injection timestamps. + public TimeProvider Time => _timeProvider; + + /// Gets the HistoryDB used by CHECK, IHAVE, TAKETHIS, and POST, or when unset (tests). + public IHistoryDb? HistoryDb { get; } + + /// + /// Gets the SPEEDTEST coordinator, or when the diagnostic is not registered. + /// + public ISpeedTestCoordinator? SpeedTest { get; } + + /// + /// Gets the opt-in feed-diagnostics probe for this session, or when disabled. + /// + public FeedSessionProbe? FeedProbe { get; set; } + + /// Gets the process-wide session census, or when unset (tests). + internal INntpSessionCensus? SessionCensus { get; } + + /// Gets always-on Transit peer counters, or when unset (tests). + internal ITransitPeerMetrics? PeerMetrics { get; } + + /// Gets the existing article-ingest activity state for this session. + public FeedSessionState ActivityState => (FeedSessionState)Volatile.Read(ref _activityState); + + /// Gets the per-session CHECK pipeline once has started. + internal CheckPipeline? Pipeline { get; private set; } + + /// Gets the per-session TAKETHIS pipeline once has started. + internal TakeThisPipeline? TakeThisWindow { get; private set; } + + /// + /// Gets the bounded STREAM article TX scheduler (depth gate above + /// ). + /// + /// + /// Available for outbound STREAM article producers that already hold destuffed bytes. + /// No production command currently supplies those bytes (storage/catalogue is out of scope). + /// + public NntpStreamArticleTxScheduler StreamArticleTx { get; } + + /// Gets the effective client IP address for this session. + public IPAddress ClientAddress => ClientIdentity.ClientAddress; + + /// Gets the effective client TCP source port for this session. + public int ClientPort => ClientIdentity.ClientPort; + + /// Gets the actual TCP peer endpoint of the accepted socket. + public IPEndPoint TcpPeer => ClientIdentity.TcpPeer; + + /// Gets the reader AUTHINFO provider (newsmaster / MySQL). Never used in Transit authority. + public INntpAuthenticationProvider AuthenticationProvider { get; } + + /// Gets the Transit AUTHINFO authenticator. Never used in Reader authority. + public ITransitPeerAuthenticator TransitAuthenticator { get; } + + /// + /// Gets the AUTHINFO credential authority selected by MODE, not by source IP. + /// + public NntpAuthenticationAuthority AuthenticationAuthority => _authenticationAuthority; + + /// Gets the authenticated-session admission tracker, if registered. + public ISessionStateTracker? SessionAdmission { get; } + + /// Gets the AUTHINFO SASL service, if registered. + public NntpSaslService? SaslService { get; } + + /// Gets the account byte-quota accountant. No-op when unauthenticated. + public IAccountByteAccountant AccountBytes { get; } + + /// Gets the account rate allocator. No-op when the rate is unlimited. + public IAccountRateAllocator AccountRates { get; } + + /// Gets the unique id used for admission tracking. + public string SessionId { get; } + + /// Gets the authenticated account policy, or when none applies. + public NntpAccountPolicy? AccountPolicy => _accountPolicy; + + /// Gets whether an AUTHINFO SASL exchange is waiting for a continuation line. + internal bool HasSaslExchange => _saslExchange is not null; + + /// Gets the in-progress SASL exchange, or . + internal NntpSaslExchange? CurrentSaslExchange => _saslExchange; + + /// Gets the TLS certificate provider used by STARTTLS, if configured. + public ITlsCertificateContextProvider? CertificateProvider { get; } + + /// + /// Gets a value indicating whether AUTHINFO USER/PASS is permitted without TLS + /// (Nntpd:AllowCleartextAuth). TLS connections always permit AUTHINFO USER/PASS. + /// + public bool AllowCleartextAuth { get; } + + /// + /// Gets a value indicating whether AUTHINFO USER/PASS may proceed on this connection + /// given TLS state and . + /// + public bool IsAuthinfoPassPermitted => Connection.IsTls || AllowCleartextAuth; + + /// Gets the current authentication identity snapshot. + public NntpAuthenticationState Authentication => _authentication; + + /// Gets the current authorization snapshot. + public NntpAuthorization Authorization => _authorization; + + /// Gets the current NNTP operating mode. + public NntpSessionMode Mode => _mode; + + /// Gets the host process role that created this session. + public NntpServerRole ServerRole { get; } + + /// + /// Gets the RX strategy implied by . + /// + /// + /// only after MODE READER. + /// MODE STREAM does not change this (RFC 4644 §2.3). + /// + internal NntpReceiveStrategy ReceiveStrategy => + _mode == NntpSessionMode.Reader + ? NntpReceiveStrategy.ReaderCommand + : NntpReceiveStrategy.StreamDataPlane; + + /// + /// Gets the pending username from AUTHINFO USER awaiting AUTHINFO PASS, + /// or when no USER is cached. + /// + public string? PendingAuthUsername => _pendingAuthUsername; + + /// Selects as the current newsgroup (immortal snapshot bytes). + internal void SelectGroup(ReadOnlyMemory groupName) => _selectedGroupName = groupName; + + /// Replaces the authorization snapshot (tests / advanced handlers). + public void SetAuthorization(NntpAuthorization authorization) + { + ArgumentNullException.ThrowIfNull(authorization); + _authorization = authorization; + if (!authorization.IsAuthenticated) + { + _authentication = NntpAuthenticationState.Unauthenticated; + _accountPolicy = null; + _saslExchange = null; + } + } + + /// + /// Sets the session operating mode and the AUTHINFO authority that accompanies it. + /// selects Transit credentials; any other mode + /// selects reader (newsmaster / MySQL) credentials. + /// + public void SetMode(NntpSessionMode mode) + { + _mode = mode; + _authenticationAuthority = mode == NntpSessionMode.Stream + ? NntpAuthenticationAuthority.Transit + : NntpAuthenticationAuthority.Reader; + } + + /// + /// Selects the AUTHINFO authority without changing . + /// Used by MODE STREAM, which must not change RFC 4644 receive state. + /// + public void SetAuthenticationAuthority(NntpAuthenticationAuthority authority) => + _authenticationAuthority = authority; + + /// Caches the username from AUTHINFO USER (does not authenticate). + public void SetPendingAuthUsername(string username) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + _pendingAuthUsername = username; + } + + /// Clears any pending AUTHINFO USER username. + public void ClearPendingAuthUsername() => _pendingAuthUsername = null; + + /// + /// Applies a successful authentication atomically: identity + authorization, pending USER cleared. + /// Forces to . + /// + public void ApplySuccessfulAuthentication( + string username, + NntpAuthorization authorization, + NntpAccountPolicy? policy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(authorization); + var previousPeer = _authorization.TransitPeerName; + _pendingAuthUsername = null; + _saslExchange = null; + _authentication = NntpAuthenticationState.ForUser(username); + _authorization = authorization.With(isAuthenticated: true); + _accountPolicy = policy; + if (previousPeer is not null && _authorization.TransitPeerName is null) + { + SessionCensus?.ReleasePeerAttribution(this); + PeerMetrics?.RetractAccepted(previousPeer); + } + } + + /// + /// Attaches the inbound TransitPeerState lease taken at connection identification. + /// The listener still disposes the same lease on disconnect; the lease is one-shot. + /// + internal void AttachTransitAdmission(TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + _transitAdmission = lease; + } + + /// + /// Releases the connection-time TransitPeerState slot when AUTHINFO replaced + /// named-peer identity with reader privileges. No-op when the session is still + /// a named Transit peer or no slot was taken. + /// + internal ValueTask ReleaseTransitAdmissionWhenNotTransitAsync() + { + if (_authorization.TransitPeerName is not null) + { + return ValueTask.CompletedTask; + } + + return _transitAdmission?.DisposeAsync() ?? ValueTask.CompletedTask; + } + + /// + /// Attaches output accounting after AUTHINFO success and before the 281. + /// Live remaining is observed from Redis/MySQL, not from the cached policy snapshot. + /// Every authenticated account participates. + /// + internal async ValueTask AttachByteAccountingAsync( + NntpResponseWriter response, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(response); + var policy = _accountPolicy; + if (policy is null) + { + return; + } + + long? remaining; + try + { + remaining = await AccountBytes.ObserveRemainingAsync(policy.Username, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + remaining = null; + } + + if (remaining is <= 0) + { + AccountBytes.MarkExhausted(policy.Username); + } + else if (remaining is > 0) + { + AccountBytes.ClearExhausted(policy.Username); + } + + response.SetByteSink(AccountBytes.CreateSink(policy.Username)); + } + + /// Gets whether this authenticated session must reject the next command. + internal bool IsByteQuotaExhausted + { + get + { + var policy = _accountPolicy; + return policy is not null && AccountBytes.IsExhausted(policy.Username); + } + } + + /// + /// Records a failed authentication attempt: session remains unauthenticated. + /// Connection-time peer privileges (if any) are restored; pending USER is retained so the + /// client may retry AUTHINFO PASS or issue a new AUTHINFO USER. + /// + public void ApplyFailedAuthentication() + { + _authentication = NntpAuthenticationState.Unauthenticated; + _authorization = _connectionAuthorization; + _accountPolicy = null; + _saslExchange = null; + ReleaseAdmission(); + } + + /// Abandons an in-progress SASL exchange without changing identity. + internal void AbandonSaslExchange() => _saslExchange = null; + + /// Stores in-progress SASL state for the next continuation line. + internal void SetSaslExchange(NntpSaslExchange exchange) + { + ArgumentNullException.ThrowIfNull(exchange); + _saslExchange = exchange; + } + + /// Admits this session for after credential success. + internal async ValueTask AdmitAuthenticatedSessionAsync( + NntpAccountPolicy? policy, + CancellationToken cancellationToken = default) + { + if (policy is null + || !policy.RequiresAdmission + || SessionAdmission is null + || !string.Equals(_admittedAccountName, policy.Username, StringComparison.Ordinal)) + { + await ReleaseAdmissionAsync().ConfigureAwait(false); + } + + if (policy is null || !policy.RequiresAdmission || SessionAdmission is null) + { + return NntpAuthenticationResult.Success( + policy?.Username ?? Authentication.Username ?? "unknown", + Authorization, + policy); + } + + var outcome = await SessionAdmission.TryAdmitAsync( + policy.Username, + SessionId, + ClientAddress, + policy.SessionLimit, + policy.SrcIpLimit, + policy.RequiresRateTracking ? policy.RateLimitBps : 0, + cancellationToken).ConfigureAwait(false); + switch (outcome) + { + case SessionAdmissionResult.Success: + _admittedAccountName = policy.Username; + if (policy.RequiresRateTracking && Connection.OutboundRate is { } cap) + { + AccountRates.Register(policy.Username, SessionId, cap, policy.RateLimitBps); + } + + return NntpAuthenticationResult.Success(policy.Username, Authorization, policy); + case SessionAdmissionResult.SessionLimitExceeded: + return NntpAuthenticationResult.TooManySessions; + case SessionAdmissionResult.SourceAddressLimitExceeded: + return NntpAuthenticationResult.TooManySourceAddresses; + case SessionAdmissionResult.Unavailable: + return NntpAuthenticationResult.TransientFailure; + default: + return NntpAuthenticationResult.TransientFailure; + } + } + + /// Gets the current connection lifetime (tests and diagnostics). + internal NntpSessionLifetime Lifetime => (NntpSessionLifetime)Volatile.Read(ref _lifetime); + + /// + /// Claims the one-shot → + /// transition. + /// + internal bool TryBeginFinalization() + { + var current = Volatile.Read(ref _lifetime); + if (current is (int)NntpSessionLifetime.Finalizing or (int)NntpSessionLifetime.Finalized) + { + return false; + } + + if (current == (int)NntpSessionLifetime.Running + && Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Finalizing, + (int)NntpSessionLifetime.Running) == (int)NntpSessionLifetime.Running) + { + return true; + } + + return Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Finalizing, + (int)NntpSessionLifetime.Created) == (int)NntpSessionLifetime.Created; + } + + /// + /// Releases distributed admission at most once and marks the session finalized. + /// Safe to call from QUIT, cancellation, timeout, and exception paths. + /// + internal async ValueTask FinalizeAdmissionAsync() + { + if (!TryBeginFinalization()) + { + return; + } + + try + { + await ReleaseAdmissionAsync().ConfigureAwait(false); + } + finally + { + Volatile.Write(ref _lifetime, (int)NntpSessionLifetime.Finalized); + } + } + + /// Releases any admission slot held by this session without awaiting Redis. + internal void ReleaseAdmission() + { + var pending = ReleaseAdmissionAsync(); + if (pending.IsCompletedSuccessfully) + { + return; + } + + if (pending.IsCompleted) + { + _ = pending.AsTask().Exception; + return; + } + + _ = ObserveAdmissionReleaseAsync(pending); + } + + /// Releases any admission slot held by this session. + internal async ValueTask ReleaseAdmissionAsync() + { + var account = Interlocked.Exchange(ref _admittedAccountName, null); + if (account is not null) + { + AccountRates.Unregister(account, SessionId); + if (SessionAdmission is not null) + { + await SessionAdmission.ReleaseAsync(account, SessionId).ConfigureAwait(false); + } + } + } + + private async Task ObserveAdmissionReleaseAsync(ValueTask pending) + { + try + { + await pending.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed( + _logger, + ex, + "session", + ClientAddress.ToString()); + } + } + + /// Records a successful inbound admission for this session's Transit peer, if any. + public void RecordPeerAccepted() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordAccepted(peerId); + } + } + + /// Records an inbound admission rejection for this session's Transit peer, if any. + public void RecordPeerRejected() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordRejected(peerId); + } + } + + /// Records one CHECK for this session's Transit peer, if any. + public void RecordPeerCheck() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordCheck(peerId); + } + } + + /// Records one framed article received from this session's Transit peer, if any. + public void RecordPeerArticleReceived(int bytes) + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordArticleReceived(peerId, bytes); + } + } + + /// + /// Updates using the existing feed-session state machine + /// and forwards to when attached. + /// + public void SetActivityState(FeedSessionState state) + { + Volatile.Write(ref _activityState, (int)state); + FeedProbe?.SetState(state); + } + + /// + /// Requests the command loop to exit after the current response (e.g. QUIT / TAKETHIS 400). + /// Cancels a pending RX ReadAsync so a close requested off the RX stack still ends + /// the session. + /// + public void RequestClose() => RequestCloseWithReason(TcpDisconnectReason.ProtocolClose); + + /// + /// Notes (first-wins on the transport) and cancels the session loop. + /// + internal void RequestCloseWithReason(TcpDisconnectReason reason) + { + if (Connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(reason); + } + + Interlocked.CompareExchange( + ref _closeReason, + (int)reason, + (int)TcpDisconnectReason.Unspecified); + + Interlocked.Exchange(ref _closeRequested, 1); + try + { + _closeCts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + /// Marks that an NNTP command has been accepted for processing. + /// Timestamp and work count are published before the idle-epoch CAS so a waiter + /// that has not yet committed close observes activity and cannot close on stale idle. + /// + /// + /// Ownership: if this method increments the epoch while the high bit of + /// is still clear, idle-timeout close must not win. + /// If the high bit is already set, this command loses and runs on the closing session. + /// + internal void BeginCommandWork() + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + Interlocked.Increment(ref _commandWork); + CommitCommandActivityEpoch(); + } + + /// + /// Increments the idle activity epoch unless idle-timeout close has already committed. + /// + private void CommitCommandActivityEpoch() + { + while (true) + { + var current = Interlocked.Read(ref _idleState); + if ((current & IdleClosedFlag) != 0) + { + return; + } + + var next = current + 1; + if (Interlocked.CompareExchange(ref _idleState, next, current) == current) + { + return; + } + } + } + + /// High bit of : idle waiter committed close. + private const long IdleClosedFlag = unchecked((long)0x8000_0000_0000_0000); + + /// + /// Marks that one accepted NNTP command (or pipeline slot) has finished. + /// Timestamp is published before the work count drops to zero so a waiter + /// cannot treat a just-finished long command as already idle. + /// + internal void EndCommandWork() + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + Interlocked.Decrement(ref _commandWork); + } + + /// Signaled once the per-session idle waiter is armed after the greeting. + internal Task IdleWatchArmed => _idleWatchArmed.Task; + + /// The idle-watch task, or before starts it. + internal Task? IdleWatchTaskForTests => _idleWatchTask; + + /// Session-requested close reason (first-wins); tests and idle timeout. + internal TcpDisconnectReason CloseReasonForTests => + (TcpDisconnectReason)Volatile.Read(ref _closeReason); + + /// In-flight accepted command / pipeline-slot count (tests). + internal int CommandWorkForTests => Volatile.Read(ref _commandWork); + + /// Gets whether the idle waiter has committed . + internal bool IdleCloseCommittedForTests => + (Interlocked.Read(ref _idleState) & IdleClosedFlag) != 0; + + /// + /// Test-only: set when the idle waiter has observed a due deadline and is about to + /// CAS-commit close. Production never sets this. + /// + internal TaskCompletionSource? IdleAboutToCommit { get; set; } + + /// + /// Test-only: when set, the idle waiter awaits this after + /// and before the close CAS. Production never sets this. + /// + internal Task? IdleCommitHold { get; set; } + + /// + /// Sends the initial greeting and runs the command loop until QUIT, EOF, or cancellation. + /// + public async Task RunAsync(CancellationToken cancellationToken = default) + { + if (Connection is NntpConnection nntpConnection) + { + nntpConnection.AttachInputReaderConsumer(); + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + Connection.ConnectionClosed, + _closeCts.Token); + var token = linked.Token; + + try + { + Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Running, + (int)NntpSessionLifetime.Created); + SessionCensus?.Register(this); + var response = _response ??= new NntpResponseWriter(Connection.Output); + Pipeline = new CheckPipeline(this, response); + TakeThisWindow = new TakeThisPipeline(this, response); + await SendGreetingAsync(response, token).ConfigureAwait(false); + _idleWatchTask = WatchIdleAsync(token); + + var readerRx = new NntpReaderCommandRx(this, _dispatcher, _logger); + var streamRx = new NntpStreamDataPlaneRx(this, _dispatcher, _logger); + + while (!token.IsCancellationRequested && Volatile.Read(ref _closeRequested) == 0) + { + var progressed = ReceiveStrategy == NntpReceiveStrategy.ReaderCommand + ? await readerRx.ProcessOneAsync(response, token).ConfigureAwait(false) + : await streamRx.ProcessOneAsync(response, token).ConfigureAwait(false); + if (!progressed) + { + break; + } + + // Coalesced fire-and-forget lines (CHECK EnqueueLineAsync) flush when the + // inbound pipe has nothing ready. TAKETHIS uses EnqueueLineImmediateAsync + // and does not wait for this idle flush. STREAM leftover stays unconsumed + // so CHECK can still reach its coalesce batch. + if (response.HasCoalescedUnflushed && !HasUnconsumedInput(Connection.Input)) + { + await response.FlushCoalescedAsync(token).ConfigureAwait(false); + } + } + } + catch (OperationCanceledException ex) when (token.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown or connection close. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (NntpPeerDisconnect.IsPeerDisconnect(ex, Connection)) + { + SessionLogMessages.SessionEndedByPeerDisconnect(_logger, ex, ClientAddress); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionLogMessages.SessionEndedWithError(_logger, ex, ClientAddress); + } + finally + { + try + { + _closeCts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + if (_idleWatchTask is not null) + { + try + { + await _idleWatchTask.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + SessionCensus?.Unregister(this); + try + { + await FinalizeAdmissionAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed( + _logger, + ex, + "session", + ClientAddress.ToString()); + } + if (TakeThisWindow is not null) + { + try + { + await TakeThisWindow.ShutdownAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; peeks use the session token. + } + } + + if (Pipeline is not null) + { + try + { + await Pipeline.ShutdownAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; lookups use the session token. + } + } + + if (_response is not null) + { + try + { + await _response.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + try + { + await StreamArticleTx.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + try + { + if (Connection is NntpConnection inputOwner) + { + await inputOwner.CompleteInputReaderAsync().ConfigureAwait(false); + } + else + { + await Connection.Input.CompleteAsync().ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; reader may already be completed. + } + + try + { + await Connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _closeCts.Dispose(); + } + } + + private ValueTask SendGreetingAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + if (_authorization.PostingPermitted) + { + return response.WriteLineAsync(NntpResponses.GreetingPostingPermitted, cancellationToken); + } + + return response.WriteLineAsync(NntpResponses.GreetingPostingProhibited, cancellationToken); + } + + /// + /// Logs and dispatches one already-parsed command (shared by both RX strategies). + /// + internal async ValueTask DispatchCommandAsync( + NntpCommandDispatcher dispatcher, + NntpResponseWriter response, + ILogger logger, + NntpCommand command, + ReadOnlyMemory line, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(logger); + + if (command.Verb != NntpVerb.BenchIt + && !NntpCommandLogFormat.SuppressHotPathCommand(command.Verb) + && logger.IsEnabled(LogLevel.Debug)) + { + var client = NntpCommandLogFormat.Client(this); + var commandText = NntpCommandLogFormat.RedactRxCommand(command, line.Span); + CommandLogMessages.CommandRx(logger, client, commandText); + } + + await dispatcher + .DispatchAsync(this, command, line, response, cancellationToken, preReadArticle) + .ConfigureAwait(false); + } + + /// + /// Dispatches one parsed unit: authorized CHECK enters the CHECK pipeline; every other + /// command drains outstanding CHECK and TAKETHIS windows first and then runs serially. + /// + internal async ValueTask ProcessParsedCommandAsync( + NntpCommandDispatcher dispatcher, + NntpResponseWriter response, + ILogger logger, + NntpCommand command, + ReadOnlyMemory line, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(Pipeline); + ArgumentNullException.ThrowIfNull(TakeThisWindow); + + BeginCommandWork(); + try + { + if (IsByteQuotaExhausted) + { + await response + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + RequestClose(); + return; + } + + // Transit CHECK fast-path: only on a Transit host with transit authority. + // Reader hosts must fall through to the dispatcher role gate (502). + if (command.IsValid + && command.Verb == NntpVerb.Check + && ServerRole == NntpServerRole.Transit + && Authorization.AuthorizedTransit) + { + await TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + if (command.Verb != NntpVerb.BenchIt + && !NntpCommandLogFormat.SuppressHotPathCommand(command.Verb) + && logger.IsEnabled(LogLevel.Debug)) + { + var client = NntpCommandLogFormat.Client(this); + var commandText = NntpCommandLogFormat.RedactRxCommand(command, line.Span); + CommandLogMessages.CommandRx(logger, client, commandText); + } + + await Pipeline.SubmitAsync(command, line, cancellationToken).ConfigureAwait(false); + return; + } + + await TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + await Pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await DispatchCommandAsync(dispatcher, response, logger, command, line, preReadArticle, cancellationToken) + .ConfigureAwait(false); + } + finally + { + EndCommandWork(); + } + } + + /// + /// Waits until lastCommandActivity + IdleTime with no accepted command work, + /// then closes the session. One Delay per wait; no per-command timer or lock. + /// + private async Task WatchIdleAsync(CancellationToken cancellationToken) + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + _idleWatchArmed.TrySetResult(); + try + { + while (!cancellationToken.IsCancellationRequested && Volatile.Read(ref _closeRequested) == 0) + { + if (Volatile.Read(ref _commandWork) > 0) + { + await Task.Delay(_commandIdleTimeout, _timeProvider, cancellationToken).ConfigureAwait(false); + continue; + } + + var last = Volatile.Read(ref _lastCommandTimestamp); + var epochAtIdle = Interlocked.Read(ref _idleState); + if ((epochAtIdle & IdleClosedFlag) != 0) + { + return; + } + + var elapsed = _timeProvider.GetElapsedTime(last, _timeProvider.GetTimestamp()); + var remaining = _commandIdleTimeout - elapsed; + if (remaining > TimeSpan.Zero) + { + await Task.Delay(remaining, _timeProvider, cancellationToken).ConfigureAwait(false); + continue; + } + + // Ownership snapshot: CAS this epoch later. BeginCommandWork increments + // the epoch before publishing work; if that increment lands first, + // CompareExchange(epochAtIdle) fails and this stale idle decision cannot close. + // Re-reading _idleState after the hold would observe the new epoch and + // incorrectly close after activity had already committed. + var aboutToCommit = IdleAboutToCommit; + aboutToCommit?.TrySetResult(); + if (IdleCommitHold is { } hold) + { + await hold.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Volatile.Read(ref _commandWork) > 0 + || Volatile.Read(ref _lastCommandTimestamp) != last) + { + continue; + } + + if (Interlocked.CompareExchange(ref _idleState, epochAtIdle | IdleClosedFlag, epochAtIdle) + != epochAtIdle) + { + continue; + } + + RequestCloseWithReason(TcpDisconnectReason.IdleTimeout); + return; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// Waits until the CHECK window can accept another command (RX backpressure). + internal ValueTask WaitForCheckCapacityAsync(CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(Pipeline); + return Pipeline.IsFull + ? Pipeline.WaitForCapacityAsync(cancellationToken) + : ValueTask.CompletedTask; + } + + /// Waits until the TAKETHIS window can accept another command (RX backpressure). + internal ValueTask WaitForTakeThisCapacityAsync(CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(TakeThisWindow); + return TakeThisWindow.IsFull + ? TakeThisWindow.WaitForCapacityAsync(cancellationToken) + : ValueTask.CompletedTask; + } + + /// + /// Emits an already-decided news event. Failures are reported through diagnostics + /// and do not change protocol state. + /// + internal void TryWriteNews(in NewsLogEvent evt) + { + try + { + NewsLog.Write(in evt); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + var id = evt.MessageId.IsEmpty + ? "-" + : System.Text.Encoding.ASCII.GetString(evt.MessageId.Span); + SpoolLogMessages.NewsLogFailed(_logger, ex, id); + } + } + + /// Logs a parser rejection without converting the full command line to a string. + internal void LogCommandRejected(NntpCommand command, string detail) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + var client = NntpCommandLogFormat.Client(this); + CommandLogMessages.CommandRejected( + _logger, + client, + command.Verb, + command.Qualifier, + command.Status, + detail); + } + + /// + /// Peeks the application input pipe without consuming. Used to decide whether a partial + /// coalesced TX batch must flush before the session waits for more inbound octets. + /// + private static bool HasUnconsumedInput(PipeReader input) + { + try + { + if (!input.TryRead(out var result)) + { + return false; + } + + var has = !result.Buffer.IsEmpty; + // Peek only. Examined must stay at Start: AdvanceTo(Start, End) tells the Pipe + // the reader needs more data, so the next ReadAsync waits even though leftover + // TAKETHIS bytes are already buffered (same rule as NntpContinuousRxReader + // after a complete unit, and NntpConnection.EnsureApplicationInputDrainedForUpgrade). + input.AdvanceTo(result.Buffer.Start, result.Buffer.Start); + return has; + } + catch (InvalidOperationException) + { + return false; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpSessionCensus.cs b/src/VectorNNTPD.ReaderService/Session/NntpSessionCensus.cs new file mode 100644 index 00000000..537d32e7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpSessionCensus.cs @@ -0,0 +1,145 @@ +using System.Collections.Concurrent; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Process-wide established-session registry. Independent of FeedDiagnostics. + /// + public sealed class NntpSessionCensus : INntpSessionCensus + { + private readonly ConcurrentDictionary _sessions = new(); + private readonly ITransitPeerMetrics? _peerMetrics; + + /// Initializes a census that optionally updates on register/unregister. + public NntpSessionCensus(ITransitPeerMetrics? peerMetrics = null) + { + _peerMetrics = peerMetrics; + } + + /// + public void Register(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var peerId = session.Authorization.TransitPeerName; + _sessions[session] = peerId; + if (peerId is not null) + { + _peerMetrics?.RecordSessionRegistered(peerId); + } + } + + /// + public void Unregister(NntpSession session) + { + if (session is null) + { + return; + } + + if (_sessions.TryRemove(session, out var peerId) && peerId is not null) + { + _peerMetrics?.RecordSessionUnregistered(peerId); + } + } + + /// + public void ReleasePeerAttribution(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + if (!_sessions.TryGetValue(session, out var peerId) || peerId is null) + { + return; + } + + if (_sessions.TryUpdate(session, null, peerId)) + { + _peerMetrics?.RecordSessionUnregistered(peerId); + } + } + + /// + public NntpSessionCensusSnapshot Capture() + { + var idle = 0; + var receiving = 0; + var waitingHistory = 0; + var waitingQueue = 0; + var waitingWindow = 0; + var completing = 0; + var active = 0; + var takeThisOccupied = 0; + var takeThisOccupiedMax = 0; + var takeThisFullSessions = 0; + var takeThisProcessing = 0; + var takeThisProcessingMax = 0; + foreach (var session in _sessions.Keys) + { + active++; + switch (session.ActivityState) + { + case FeedSessionState.Receiving: + receiving++; + break; + case FeedSessionState.WaitingHistory: + waitingHistory++; + break; + case FeedSessionState.WaitingQueue: + waitingQueue++; + break; + case FeedSessionState.WaitingWindow: + waitingWindow++; + break; + case FeedSessionState.Completing: + completing++; + break; + default: + idle++; + break; + } + + var window = session.TakeThisWindow; + if (window is null) + { + continue; + } + + var occupied = window.Occupied; + takeThisOccupied += occupied; + if (occupied > takeThisOccupiedMax) + { + takeThisOccupiedMax = occupied; + } + + if (occupied >= TakeThisPipeline.Depth) + { + takeThisFullSessions++; + } + + var processing = window.ActiveArticleProcessing; + takeThisProcessing += processing; + var processingPeak = window.MaxActiveArticleProcessing; + if (processingPeak > takeThisProcessingMax) + { + takeThisProcessingMax = processingPeak; + } + } + + return new NntpSessionCensusSnapshot( + active, + active, + idle, + receiving, + waitingHistory, + waitingQueue, + waitingWindow, + completing, + takeThisOccupied, + takeThisOccupiedMax, + takeThisFullSessions, + takeThisProcessing, + takeThisProcessingMax); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpSessionLifetime.cs b/src/VectorNNTPD.ReaderService/Session/NntpSessionLifetime.cs new file mode 100644 index 00000000..c4d09ffc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpSessionLifetime.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// Observable lifetime of an connection. + /// + /// → is the one-shot transition that may + /// release distributed admission. Later teardown signals observe + /// or and do nothing. + /// + public enum NntpSessionLifetime + { + /// Constructed; has not started. + Created = 0, + + /// Greeting/command loop is running. + Running = 1, + + /// The first teardown path has claimed cleanup. + Finalizing = 2, + + /// Admission release and connection cleanup have finished. + Finalized = 3, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpSessionMode.cs b/src/VectorNNTPD.ReaderService/Session/NntpSessionMode.cs new file mode 100644 index 00000000..3d5af596 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpSessionMode.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.ReaderService.Session +{ + /// NNTP operating mode (distinct from authentication/authorization). + public enum NntpSessionMode + { + /// No MODE command has successfully selected an operating mode yet. + Unspecified = 0, + + /// Reader mode after a successful MODE READER. + Reader = 1, + + /// + /// Transit/streaming operating mode. MODE STREAM does not set this (RFC 4644 §2.3); + /// AUTHINFO Transit authority is selected separately via + /// . + /// + Stream = 2, + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/NntpStreamDataPlaneRx.cs b/src/VectorNNTPD.ReaderService/Session/NntpStreamDataPlaneRx.cs new file mode 100644 index 00000000..60715666 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/NntpStreamDataPlaneRx.cs @@ -0,0 +1,131 @@ +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Default / transit STREAM RX: continuous command/article units from the connection input pipe. + /// + /// + /// Authorized TAKETHIS is admitted into : HistoryDB peek starts + /// at the command line, then this RX task frames one stuffed-wire copy with + /// , attaches the owned buffer to a slot, and returns so + /// the next command can be parsed. Pipeline workers never read Connection.Input. + /// Unauthorized TAKETHIS is returned as a command line so 480/502 gates do not consume a + /// following QUIT. + /// + internal sealed class NntpStreamDataPlaneRx + { + private readonly NntpSession _session; + private readonly NntpCommandDispatcher _dispatcher; + private readonly ILogger _logger; + private readonly NntpContinuousRxParser _parser = new(); + + /// Initializes a new instance of the class. + public NntpStreamDataPlaneRx(NntpSession session, NntpCommandDispatcher dispatcher, ILogger logger) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(logger); + _session = session; + _dispatcher = dispatcher; + _logger = logger; + } + + /// + /// Reads and dispatches one protocol unit. Returns on EOF or an + /// incomplete TAKETHIS article. + /// + public async ValueTask ProcessOneAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + if (_session.Pipeline?.IsFull == true || _session.TakeThisWindow?.IsFull == true) + { + _session.SetActivityState(FeedSessionState.WaitingWindow); + } + + await _session.WaitForCheckCapacityAsync(cancellationToken).ConfigureAwait(false); + await _session.WaitForTakeThisCapacityAsync(cancellationToken).ConfigureAwait(false); + _session.SetActivityState(FeedSessionState.Idle); + + // Command line only. Authorized TAKETHIS starts HistoryDB peek, frames the + // article on this RX task, hands the owned buffer to the pipeline, and returns. + // Unauthorized TAKETHIS stays a command so 480/502 gates do not consume a + // following QUIT. + var unit = await NntpContinuousRxReader + .ReadUnitAsync( + _session.Connection.Input, + _parser, + consumeTakeThisArticle: false, + _session.ArticleIngestion.MaxArticleBytes, + cancellationToken) + .ConfigureAwait(false); + + if (unit.Kind == NntpContinuousRxKind.NeedMore) + { + return false; + } + + if (_session.HasSaslExchange) + { + _session.BeginCommandWork(); + try + { + var takeThisWindow = _session.TakeThisWindow + ?? throw new InvalidOperationException( + "SASL exchange requires an initialized TakeThis admission window."); + var pipeline = _session.Pipeline + ?? throw new InvalidOperationException( + "SASL exchange requires an initialized CHECK pipeline."); + await takeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + await pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await Commands.AuthInfo + .ContinueSaslAsync(_session, response, _parser.CurrentCommandLine, cancellationToken) + .ConfigureAwait(false); + } + finally + { + _session.EndCommandWork(); + } + + return true; + } + + // Transit TAKETHIS admit: only on a Transit host with transit authority. + // Reader hosts leave TAKETHIS as a command line so the dispatcher returns 502. + if (unit.Command.IsValid + && unit.Command.Verb == NntpVerb.TakeThis + && _session.ServerRole == Hosting.NntpServerRole.Transit + && _session.Authorization.AuthorizedTransit) + { + // Capture once: nullable session properties must not be re-read between check and use + // (concurrent teardown can clear them after a prior null check). + var takeThisWindow = _session.TakeThisWindow + ?? throw new InvalidOperationException( + "Transit TAKETHIS admit requires an initialized TakeThis admission window."); + var pipeline = _session.Pipeline + ?? throw new InvalidOperationException( + "Transit TAKETHIS admit requires an initialized CHECK pipeline."); + await pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await takeThisWindow + .AdmitAuthorizedAsync(unit.Command, _parser.CurrentCommandLine, cancellationToken) + .ConfigureAwait(false); + return true; + } + + var preRead = unit.Kind == NntpContinuousRxKind.TakeThis ? unit.Article : (NntpMultilineReadResult?)null; + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + unit.Command, + _parser.CurrentCommandLine, + preRead, + cancellationToken) + .ConfigureAwait(false); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/SessionLogMessages.cs b/src/VectorNNTPD.ReaderService/Session/SessionLogMessages.cs new file mode 100644 index 00000000..b4e36e88 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/SessionLogMessages.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Session +{ + /// Source-generated session and transit-identification log messages. + internal static partial class SessionLogMessages + { + [LoggerMessage( + EventId = 1611, + Level = LogLevel.Debug, + Message = "NNTP session ended with an error for {Client}")] + public static partial void SessionEndedWithError(ILogger logger, Exception exception, IPAddress Client); + + [LoggerMessage( + EventId = 1612, + Level = LogLevel.Debug, + Message = "NNTP session ended; peer or transport disconnected for {Client}")] + public static partial void SessionEndedByPeerDisconnect(ILogger logger, Exception exception, IPAddress Client); + + [LoggerMessage( + EventId = 1700, + Level = LogLevel.Information, + Message = "Trusted Transit peers configured ({Count}): {Peers}")] + public static partial void TransitPeersConfigured(ILogger logger, int Count, string Peers); + + [LoggerMessage( + EventId = 1701, + Level = LogLevel.Warning, + Message = "Transit peer identification is ambiguous for {ClientAddress}; matching peers: {PeerNames}")] + public static partial void TransitPeerAmbiguous(ILogger logger, IPAddress ClientAddress, string PeerNames); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/SpeedTest/ISpeedTestCoordinator.cs b/src/VectorNNTPD.ReaderService/Session/SpeedTest/ISpeedTestCoordinator.cs new file mode 100644 index 00000000..75b7a26f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/SpeedTest/ISpeedTestCoordinator.cs @@ -0,0 +1,66 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.SpeedTest +{ + /// + /// Read-only Transit lookup and concurrency gate for SPEEDTEST. + /// + /// + /// Resolves Transit identifiers against the current immutable snapshot. Does not open + /// outbound sockets and does not retain a mutable options object across a reload. + /// + public interface ISpeedTestCoordinator + { + /// + /// Resolves against the current Transit snapshot + /// using exact ordinal identifier match (the configuration dictionary key). + /// + /// + /// when a configured Transit peer exists with that exact identifier. + /// + bool TryResolvePeer(ReadOnlySpan identifier, out TransitPeerPolicy? peer); + + /// + /// Attempts to acquire a SPEEDTEST slot for using + /// the current configured limits. + /// + bool TryAcquire(string identifier, out SpeedTestLease? lease); + } + + /// + /// Held SPEEDTEST concurrency slot. Dispose releases host and per-peer counters. + /// + public sealed class SpeedTestLease : IDisposable + { + private readonly Action _release; + private readonly string _peerName; + private int _disposed; + + internal SpeedTestLease(string peerName, SpeedTestLimits limits, Action release) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + ArgumentNullException.ThrowIfNull(release); + _peerName = peerName; + Limits = limits; + _release = release; + } + + /// Gets the limits captured when this slot was acquired. + public SpeedTestLimits Limits { get; } + + /// Gets the Transit identifier this slot is counted against. + public string PeerName => _peerName; + + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _release(_peerName); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestCoordinator.cs b/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestCoordinator.cs new file mode 100644 index 00000000..526e5751 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestCoordinator.cs @@ -0,0 +1,205 @@ +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session.SpeedTest +{ + /// + /// Snapshot-backed peer lookup and bounded SPEEDTEST concurrency. + /// + /// + /// Each resolve reads . Limits are read from + /// at acquire time. In-flight tests keep the limits + /// captured on the lease. This type does not initiate outbound Transit connections. + /// + public sealed class SpeedTestCoordinator : ISpeedTestCoordinator + { + private readonly TransitConfigurationStore _store; + private readonly IOptionsMonitor _options; + private readonly object _gate = new(); + private readonly ConcurrentDictionary _perPeer = new(StringComparer.Ordinal); + private int _global; + + /// Creates a coordinator bound to the live Transit store and NNTPD options. + public SpeedTestCoordinator(TransitConfigurationStore store, IOptionsMonitor options) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(options); + _store = store; + _options = options; + } + + /// Creates a coordinator for tests with a fixed options snapshot. + public static SpeedTestCoordinator Create( + TransitConfigurationStore store, + NntpdOptions? options = null) => + new(store, new StaticOptions(options ?? new NntpdOptions())); + + /// + public bool TryResolvePeer(ReadOnlySpan peerName, out TransitPeerPolicy? peer) + { + peer = null; + if (peerName.IsEmpty) + { + return false; + } + + var snapshot = _store.Current; + foreach (var candidate in snapshot.Peers.Values) + { + if (NameEquals(candidate.Identifier, peerName)) + { + peer = candidate; + return true; + } + } + + return false; + } + + /// + public bool TryAcquire(string peerName, out SpeedTestLease? lease) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + lease = null; + var speed = _options.CurrentValue.SpeedTest ?? new SpeedTestOptions(); + var limits = speed.Snapshot(); + + lock (_gate) + { + if (_global >= limits.MaxConcurrent) + { + return false; + } + + var peerCount = _perPeer.GetValueOrDefault(peerName); + if (peerCount >= limits.MaxConcurrentPerPeer) + { + return false; + } + + _global++; + _perPeer[peerName] = peerCount + 1; + } + + lease = new SpeedTestLease(peerName, limits, Release); + return true; + } + + /// Gets the current host-wide in-flight count (tests). + internal int GlobalInFlight + { + get + { + lock (_gate) + { + return _global; + } + } + } + + /// Gets the current in-flight count for (tests). + internal int InFlightFor(string peerName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + lock (_gate) + { + return _perPeer.GetValueOrDefault(peerName); + } + } + + private void Release(string peerName) + { + lock (_gate) + { + _global = Math.Max(0, _global - 1); + if (_perPeer.TryGetValue(peerName, out var count)) + { + if (count <= 1) + { + _perPeer.TryRemove(peerName, out _); + } + else + { + _perPeer[peerName] = count - 1; + } + } + } + } + + /// + /// Exact Transit name match. ASCII names compare as ordinal bytes; non-ASCII names + /// compare as UTF-8 of the configured string (the dictionary key representation). + /// + internal static bool NameEquals(string name, ReadOnlySpan token) + { + ArgumentNullException.ThrowIfNull(name); + if (IsAscii(name)) + { + if (name.Length != token.Length) + { + return false; + } + + for (var i = 0; i < name.Length; i++) + { + if ((byte)name[i] != token[i]) + { + return false; + } + } + + return true; + } + + var max = Encoding.UTF8.GetMaxByteCount(name.Length); + if (max > 1024) + { + return false; + } + + Span utf8 = stackalloc byte[max]; + if (!Encoding.UTF8.TryGetBytes(name, utf8, out var written)) + { + return false; + } + + return utf8[..written].SequenceEqual(token); + } + + private static bool IsAscii(string name) + { + foreach (var ch in name) + { + if (ch > 0x7F) + { + return false; + } + } + + return true; + } + + private sealed class StaticOptions : IOptionsMonitor + { + public StaticOptions(NntpdOptions value) => CurrentValue = value; + + public NntpdOptions CurrentValue { get; } + + public NntpdOptions Get(string? name) => CurrentValue; + + public IDisposable OnChange(Action listener) => EmptyDisposable.Instance; + } + + private sealed class EmptyDisposable : IDisposable + { + public static readonly EmptyDisposable Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestPayload.cs b/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestPayload.cs new file mode 100644 index 00000000..87226b80 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/SpeedTest/SpeedTestPayload.cs @@ -0,0 +1,71 @@ +namespace VectorNNTPD.ReaderService.Session.SpeedTest +{ + /// + /// Deterministic synthetic SPEEDTEST payload. One immortal chunk is reused for every write. + /// + /// + /// + /// The payload is not an article, is not read from disk, and is not customer or feed data. + /// It is a fixed ASCII pattern chosen so NNTP dot-stuffing never applies: no line begins + /// with .. Content characteristics are therefore irrelevant to compression or + /// article-pipeline behaviour. + /// + /// + /// Each line is octets: one #, then repeating digits + /// 0–9, then CRLF. is concatenated + /// lines (). Callers write a CRLF-aligned slice of that buffer + /// and must not retain a reference after the awaited flush when they later reuse the + /// same immortal memory for the next write (writes are sequential and flush-awaited). + /// + /// + internal static class SpeedTestPayload + { + /// One payload line including CRLF. + public const int LineBytes = 1024; + + /// Lines packed into the reusable chunk. + public const int LinesPerChunk = 64; + + /// Reusable chunk size (64 KiB). + public const int ChunkBytes = LineBytes * LinesPerChunk; + + /// Immortal reusable chunk. The TX pump only reads it. + public static readonly ReadOnlyMemory Chunk = BuildChunk(); + + /// + /// Returns a CRLF-aligned prefix of no larger than + /// . Empty when fewer than remain. + /// + public static ReadOnlyMemory Take(long remainingBytes) + { + if (remainingBytes < LineBytes) + { + return ReadOnlyMemory.Empty; + } + + var take = remainingBytes >= ChunkBytes + ? ChunkBytes + : (int)(remainingBytes - (remainingBytes % LineBytes)); + return Chunk[..take]; + } + + private static byte[] BuildChunk() + { + var chunk = new byte[ChunkBytes]; + for (var line = 0; line < LinesPerChunk; line++) + { + var offset = line * LineBytes; + chunk[offset] = (byte)'#'; + for (var i = 1; i < LineBytes - 2; i++) + { + chunk[offset + i] = (byte)('0' + (i % 10)); + } + + chunk[offset + LineBytes - 2] = (byte)'\r'; + chunk[offset + LineBytes - 1] = (byte)'\n'; + } + + return chunk; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/TakeThisPipeline.cs b/src/VectorNNTPD.ReaderService/Session/TakeThisPipeline.cs new file mode 100644 index 00000000..71c56087 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/TakeThisPipeline.cs @@ -0,0 +1,1141 @@ +using Microsoft.Extensions.Logging; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.Session +{ + using VectorNNTPD.Common.Core; + /// + /// Per-session bounded TAKETHIS window: one RX owner frames articles, then detached + /// slots complete Peek / enqueue / ordered 239 off the RX stack. + /// + /// + /// + /// The session RX task is the only consumer of Connection.Input. It parses the + /// TAKETHIS command, starts HistoryDB peek, frames the article with + /// (one owned stuffed-wire buffer, terminator omitted), + /// attaches that buffer to a slot, and returns. Each occupied slot destuffs and + /// builds on its own completion + /// task. They never call ReadAsync on the session pipe. Depth matches the + /// CHECK window and bounds how many owned article buffers and process results + /// may be retained. + /// + /// + /// After detach, waits for Peek if needed, then + /// destuffs and builds ArticleRecord on that slot's task (concurrent across + /// occupied slots). Enqueue / Remember / news - / ordered 239/439/400 stay + /// behind _emitGate so publication remains command-ordered. Emit never runs + /// on the RX task. A slot is occupied from Message-ID copy / peek start until the + /// response writer accepts the ordered line (or cancel). + /// + /// + internal sealed class TakeThisPipeline + { + /// + /// Outstanding TAKETHIS executions per session. Architectural constant; not configurable. + /// + public const int Depth = CheckPipeline.Depth; + + private readonly NntpSession _session; + private readonly NntpResponseWriter _response; + private readonly ILogger _logger; + private readonly Slot?[] _slots; + private readonly Lock _gate = new(); + // codeql[cs/missed-using-statement] Owned by TakeThisPipeline until ShutdownAsync; a using at this declaration would release it when the constructor returns. + private readonly SemaphoreSlim _emitGate = new(1, 1); + // codeql[cs/missed-using-statement] Owned by TakeThisPipeline until ShutdownAsync; a using at this declaration would cancel in-flight work when the constructor returns. + private readonly CancellationTokenSource _lifetimeCts = new(); + private readonly TakeThisStageSessionLog? _stageLog; + private int _head; + private int _count; + private int _peakOccupied; + private int _shutDown; + private int _inFlightCompletions; + private int _activeArticleReads; + private int _maxActiveArticleReads; + private int _activeArticleProcessing; + private int _maxActiveArticleProcessing; + private TaskCompletionSource _progress = NewProgress(); + + /// Initializes a new instance of the class. + public TakeThisPipeline(NntpSession session, NntpResponseWriter response) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + _session = session; + _response = response; + _logger = NntpCommandLoggers.For(typeof(TakeThis)); + _slots = new Slot?[Depth]; + _stageLog = TakeThisStageProbe.CreateSessionLog(); + } + + /// Gets the number of occupied slots. + public int Occupied + { + get + { + lock (_gate) + { + return _count; + } + } + } + + /// Gets whether the window cannot accept another TAKETHIS without waiting. + public bool IsFull => Occupied >= Depth; + + /// Gets the peak occupied slot count observed on this session. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Gets the number of completion tasks still running (at most ). + internal int InFlightCompletions => Volatile.Read(ref _inFlightCompletions); + + /// + /// Test-only: set after HistoryDB peek is started and before article receive begins. + /// + internal Action? AfterPeekStarted { get; set; } + + /// + /// Test-only: set immediately before is awaited. + /// + internal Action? AfterReceiveStarted { get; set; } + + /// + /// Test-only: set after returns (including cancel). + /// + internal Action? AfterReceiveCompleted { get; set; } + + /// + /// Test-only: when set, awaits this task after + /// and before . + /// Production never sets this. + /// + internal Task? HoldReceive { get; set; } + + /// + /// Test-only: invoked after the owned article is stored on the slot and before + /// admission returns to the RX loop. Production never sets this. + /// + internal Action>? AfterArticleDetached { get; set; } + + /// + /// Test-only: awaited after response composition and before + /// . + /// The slot remains occupied. Production never sets this. + /// + internal Func? BeforeEnqueueProbe { get; set; } + + /// Gets how many calls are on this session's RX stack. + internal int ActiveArticleReads => Volatile.Read(ref _activeArticleReads); + + /// Gets the peak concurrent article-receive count observed on this session (must stay 1). + internal int MaxActiveArticleReads => Volatile.Read(ref _maxActiveArticleReads); + + /// + /// Gets how many destuff/parse/CRC tasks are running on this session (at most ). + /// + internal int ActiveArticleProcessing => Volatile.Read(ref _activeArticleProcessing); + + /// Gets the peak concurrent destuff/parse/CRC count observed on this session. + internal int MaxActiveArticleProcessing => Volatile.Read(ref _maxActiveArticleProcessing); + + /// + /// Test-only: awaited on a slot after Peek is Unseen and before destuff/parse/CRC. + /// Production never sets this. + /// + internal Func, Task>? HoldProcess { get; set; } + + /// + /// Test-only: invoked when destuff/parse/CRC starts for a slot. Production never sets this. + /// + internal Action>? AfterProcessStarted { get; set; } + + /// + /// Test-only: invoked when destuff/parse/CRC finishes for a slot. Production never sets this. + /// + internal Action>? AfterProcessCompleted { get; set; } + + /// Waits until a slot is free. The caller must not read RX while this is pending. + public async ValueTask WaitForCapacityAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfShutDown(); + if (_count < Depth) + { + return; + } + + wait = _progress.Task; + } + + var waitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordOccupiedFullWait(waitStart); + } + } + + /// + /// Starts HistoryDB peek, frames the STREAM article on the RX task, attaches the + /// owned buffer to a slot, and returns. Does not wait for Peek, destuff, enqueue, + /// Remember, or 239 after the article is detached from the PipeReader. + /// + public async ValueTask AdmitAuthorizedAsync( + NntpCommand command, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ThrowIfShutDown(); + var probe = _session.FeedProbe; + probe?.RecordCommand(); + _session.SetActivityState(FeedSessionState.Receiving); + var ownedId = command.ArgumentMemory(line).ToArray(); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var marks = _stageLog is null ? null : new TakeThisStageMarks + { + CommandParsedTs = started, + PeekStartTs = started, + }; + var lookup = marks is not null + ? marks.ObservePeek(TakeThis.PeekAsync(_session, ownedId, cancellationToken)) + : TakeThis.PeekAsync(_session, ownedId, cancellationToken); + + AfterPeekStarted?.Invoke(); + + Slot slot; + lock (_gate) + { + ThrowIfShutDown(); + if (_count >= Depth) + { + throw new InvalidOperationException("TAKETHIS pipeline accepted a command without a free slot."); + } + + NntpArticlePolicyCapture.Capture( + _session, + _session.ArticleIngestion.MaxArticleBytes, + out var capturedMaxArticleBytes, + out var capturedSiteName); + slot = new Slot + { + MessageId = ownedId, + StartedTimestamp = started, + Index = (_head + _count) % Depth, + Marks = marks, + MaxArticleBytes = capturedMaxArticleBytes, + SiteNameUtf8 = capturedSiteName, + }; + _slots[slot.Index] = slot; + _count++; + _session.BeginCommandWork(); + if (_count > _peakOccupied) + { + _peakOccupied = _count; + } + + _stageLog?.NoteOccupied(_count); + IngestionPipelineMetrics.Shared.ObserveTakeThis( + _count, + Volatile.Read(ref _activeArticleProcessing)); + } + + marks?.ReceiveStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + + AfterReceiveStarted?.Invoke(); + var reads = Interlocked.Increment(ref _activeArticleReads); + ObservePeak(ref _maxActiveArticleReads, reads); + + try + { + IHaveArticleReadResult read; + try + { + if (HoldReceive is { } holdReceive) + { + await holdReceive.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + read = await IHaveArticleReader + .ReadAsync(_session.Connection.Input, slot.MaxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + Interlocked.Decrement(ref _activeArticleReads); + AfterReceiveCompleted?.Invoke(); + CancelAndRelease(slot); + return; + } + + Interlocked.Decrement(ref _activeArticleReads); + AfterReceiveCompleted?.Invoke(); + + if (marks is not null) + { + marks.ReceiveCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + marks.ArticleBytes = read.Payload.Length; + } + + if (probe is not null) + { + var receiveStart = marks?.ReceiveStartTs ?? started; + probe.RecordArticleReceived( + read.Payload.Length, + System.Diagnostics.Stopwatch.GetTimestamp() - receiveStart); + } + + _session.RecordPeerArticleReceived(read.Payload.Length); + _session.SetActivityState(FeedSessionState.WaitingHistory); + + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.Read = read; + slot.ArticleReady = true; + SignalProgressNoLock(); + } + + AfterArticleDetached?.Invoke(read.Payload); + _ = CompleteWhenAsync(slot, lookup, cancellationToken); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + CancelAndRelease(slot); + throw; + } + } + + /// Waits until every slot has been accepted by the TX writer (or cancelled). + public async ValueTask DrainAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// Stops emission and waits until every slot is released. + public async ValueTask ShutdownAsync() + { + if (Interlocked.Exchange(ref _shutDown, 1) == 1) + { + return; + } + + try + { + await _lifetimeCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + lock (_gate) + { + for (var i = 0; i < Depth; i++) + { + if (_slots[i] is { } slot) + { + slot.Cancelled = true; + } + } + + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + try + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.ConfigureAwait(false); + } + } + finally + { + _lifetimeCts.Dispose(); + WriteStageLog(); + } + } + + private async Task CompleteWhenAsync( + Slot slot, + ValueTask lookup, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _inFlightCompletions); + try + { + // Leave the RX stack before Peek/enqueue/239, including when Peek already completed. + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + + HistoryLookupResult peek; + var peekWaitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + peek = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) + { + peek = HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + peek = HistoryLookupResult.Unavailable; + } + + _session.FeedProbe?.RecordHistory( + peek, + System.Diagnostics.Stopwatch.GetTimestamp() - peekWaitStart); + + if (Volatile.Read(ref _shutDown) == 1 + || cancellationToken.IsCancellationRequested) + { + CancelAndRelease(slot); + return; + } + + ArticleProcessOutcome process = default; + try + { + using var processCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _lifetimeCts.Token); + process = await ProcessArticleAsync(slot, peek, processCts.Token).ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + + var emit = false; + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + if (Volatile.Read(ref _shutDown) == 1) + { + slot.PeekCompleted = true; + slot.Processed = true; + slot.Cancelled = true; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot.PeekCompleted = true; + slot.Peek = peek; + slot.Processed = true; + slot.Process = process; + emit = HeadIsReadyNoLock(); + SignalProgressNoLock(); + } + + if (emit) + { + try + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + finally + { + Interlocked.Decrement(ref _inFlightCompletions); + } + } + + private async ValueTask EmitReadySerializedAsync(CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetimeCts.Token); + var token = linked.Token; + var emitGateStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await _emitGate.WaitAsync(token).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordEmitGateWait(emitGateStart); + try + { + while (true) + { + Slot? slot; + lock (_gate) + { + if (Volatile.Read(ref _shutDown) == 1) + { + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot = TryBeginEmitHeadNoLock(); + } + + if (slot is null) + { + return; + } + + try + { + await EmitOccupiedAsync(slot, token).ConfigureAwait(false); + lock (_gate) + { + ReleaseHeadIfNoLock(slot); + SignalProgressNoLock(); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + lock (_gate) + { + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + throw; + } + } + } + finally + { + _emitGate.Release(); + } + } + + private async ValueTask EmitOccupiedAsync(Slot slot, CancellationToken cancellationToken) + { + if (slot.Marks is { } emitMarks) + { + emitMarks.EmitStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + emitMarks.OccupiedAtEmit = Occupied; + } + + var read = slot.Read; + if (read.Status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + _session, + SlotMessageId(slot), + 439, + IngressNewsReasons.ArticleTooLarge, + read.Payload.Length); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, tooLarge, TransferLogKind.Rejected); + return; + } + + if (slot.Peek == HistoryLookupResult.Unavailable) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + if (slot.Peek == HistoryLookupResult.Seen) + { + if (slot.Marks is { } seenMarks) + { + seenMarks.Outcome = "seen"; + } + + var probe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var responseStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.ArticleTransferredOkPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, "accepted duplicate", TransferLogKind.Accepted); + probe?.RecordArticleCompleted( + duplicate: true, + System.Diagnostics.Stopwatch.GetTimestamp() - responseStart); + _session.SetActivityState(FeedSessionState.Idle); + RecordAccepted(slot); + return; + } + + var queue = _session.ArticleIngestion; + if (!queue.IsAccepting) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + var messageIdText = System.Text.Encoding.ASCII.GetString(slot.MessageId); + var process = slot.Process; + if (process.RecordRejected) + { + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + IngressNewsReasons.ForExistingRejectDetail(process.RejectDetail), + process.Size); + var rejectProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var rejectStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, process.RejectDetail ?? string.Empty, TransferLogKind.Rejected); + rejectProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - rejectStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + if (process.UncarriedRejected) + { + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + process.UncarriedReason ?? IngressNewsEvents.NewsgroupNotCarried, + process.Inbound?.Payload.Length ?? read.Payload.Length); + var uncarriedProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var uncarriedStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, IngressNewsEvents.NewsgroupNotCarried, TransferLogKind.Rejected); + uncarriedProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - uncarriedStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + var inbound = process.Inbound; + if (inbound is null) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + ArticleEnqueueResult enqueue; + try + { + if (slot.Marks is { } enqueueMarks) + { + enqueueMarks.EnqueueStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + var probe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.WaitingQueue); + var queueStart = System.Diagnostics.Stopwatch.GetTimestamp(); + enqueue = await queue.EnqueueAsync(inbound, cancellationToken).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordTakeThisEnqueue(queueStart); + probe?.RecordQueue(enqueue, System.Diagnostics.Stopwatch.GetTimestamp() - queueStart, read.Payload.Length); + if (slot.Marks is { } acceptedMarks) + { + acceptedMarks.EnqueueAcceptedTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (enqueue == ArticleEnqueueResult.Unavailable) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + var rejectProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var rejectStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, budget, TransferLogKind.Rejected); + rejectProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - rejectStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + if (slot.Marks is { } rememberMarks) + { + rememberMarks.Outcome = "unseen"; + rememberMarks.RememberStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + _session.HistoryDb?.Remember(slot.MessageId); + if (slot.Marks is { } rememberedMarks) + { + rememberedMarks.RememberCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + var completeProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var completeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.ArticleTransferredOkPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, "accepted", TransferLogKind.Accepted); + completeProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - completeStart); + _session.SetActivityState(FeedSessionState.Idle); + RecordAccepted(slot); + } + + private enum TransferLogKind + { + Accepted, + Rejected, + Temporary, + } + + private void WriteTxIfDebug(Slot slot, string detail, TransferLogKind kind) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + var status = kind switch + { + TransferLogKind.Rejected => NntpCommandStatusText.FormatTakeThisRejected(slot.MessageId), + TransferLogKind.Temporary => NntpResponseStatus.ServiceTemporarilyUnavailable, + _ => NntpCommandStatusText.FormatTakeThisAccepted(slot.MessageId), + }; + TakeThis.WriteCompletion(_logger, _session, slot.StartedTimestamp, detail, status); + } + + private ValueTask EnqueueReplyAsync( + Slot slot, + ReadOnlyMemory prefix, + CancellationToken cancellationToken) + { + var owned = NntpResponseCompose.Concat(prefix.Span, slot.MessageId, NntpResponses.Crlf.Span); + if (slot.Marks is { } marks) + { + return EnqueueReplyTimedAsync(marks, owned, cancellationToken); + } + + return EnqueueReplyImmediateAsync(owned, cancellationToken); + } + + private async ValueTask EnqueueReplyImmediateAsync( + ReadOnlyMemory owned, + CancellationToken cancellationToken) + { + if (BeforeEnqueueProbe is { } probe) + { + await probe().ConfigureAwait(false); + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + } + + await _response.EnqueueLineImmediateAsync(owned, cancellationToken).ConfigureAwait(false); + } + + private async ValueTask EnqueueReplyTimedAsync( + TakeThisStageMarks marks, + ReadOnlyMemory owned, + CancellationToken cancellationToken) + { + marks.ReplyEnqueueStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyImmediateAsync(owned, cancellationToken).ConfigureAwait(false); + marks.ReplyEnqueueCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + private void RecordAccepted(Slot slot) + { + if (_stageLog is null || slot.Marks is not { } marks) + { + return; + } + + _stageLog.Record(marks.ToSample(_peakOccupied)); + } + + private void WriteStageLog() + { + if (_stageLog is null || !TakeThisStageProbe.IsEnabled) + { + return; + } + + var dir = TakeThisStageProbe.OutputDirectory; + if (dir is null) + { + return; + } + + try + { + var remote = _session.ClientIdentity.TcpPeer?.ToString() ?? "session"; + var path = _stageLog.Write(dir, remote); + if (_logger.IsEnabled(LogLevel.Information)) + { + var samples = _stageLog.Count; + _logger.LogInformation( + "TAKETHIS stage timing wrote {Path} samples={Samples} peak={Peak}", + path, + samples, + _stageLog.PeakOccupied); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Diagnostic dump must not affect shutdown. + } + } + + private bool HeadIsReadyNoLock() + { + if (_count == 0) + { + return false; + } + + var slot = _slots[_head]; + return slot is + { + ArticleReady: true, + PeekCompleted: true, + Processed: true, + Cancelled: false, + Emitting: false, + }; + } + + /// + /// Destuff/parse/CRC for an unseen completed article. Independent per slot. + /// History Seen/Unavailable and receive failures skip this work. + /// + /// Occupied slot that already owns stuffed wire. + /// Already-awaited HistoryDB result for this slot. + /// Session or pipeline lifetime cancellation. + /// The destuff/parse outcome for the ordered publisher. + private async ValueTask ProcessArticleAsync( + Slot slot, + HistoryLookupResult peek, + CancellationToken cancellationToken) + { + var read = slot.Read; + if (read.Status != NntpMultilineReadStatus.Completed + || peek != HistoryLookupResult.Unseen) + { + return default; + } + + var processing = Interlocked.Increment(ref _activeArticleProcessing); + ObservePeak(ref _maxActiveArticleProcessing, processing); + + AfterProcessStarted?.Invoke(slot.MessageId); + try + { + if (HoldProcess is { } hold) + { + await hold(slot.MessageId).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + var messageIdText = SlotMessageId(slot); + if (!TakeThis.TryCreateQueuedRecord( + _session, + read.Payload, + stuffed: true, + messageIdText, + slot.MaxArticleBytes, + slot.SiteNameUtf8, + out var inbound, + out var recordReject, + out var canonicalSize)) + { + return new ArticleProcessOutcome + { + RecordRejected = true, + RejectDetail = recordReject, + Size = canonicalSize != 0 ? canonicalSize : read.Payload.Length, + }; + } + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + _session.Transit, + _session.NewsgroupCatalogue, + out var uncarriedReason)) + { + return new ArticleProcessOutcome + { + UncarriedRejected = true, + UncarriedReason = uncarriedReason, + Inbound = inbound, + }; + } + + return new ArticleProcessOutcome { Inbound = inbound }; + } + finally + { + Interlocked.Decrement(ref _activeArticleProcessing); + AfterProcessCompleted?.Invoke(slot.MessageId); + } + } + + private Slot? TryBeginEmitHeadNoLock() + { + if (!HeadIsReadyNoLock()) + { + return null; + } + + var slot = _slots[_head]!; + slot.Emitting = true; + return slot; + } + + private void ReleaseHeadIfNoLock(Slot slot) + { + if (_count == 0 || !ReferenceEquals(_slots[_head], slot)) + { + return; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + slot.Emitting = false; + } + + private void CancelAndRelease(Slot slot) + { + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.ArticleReady = true; + slot.PeekCompleted = true; + slot.Processed = true; + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + } + + private void ReleaseCancelledPrefixNoLock() + { + while (_count > 0) + { + var slot = _slots[_head]; + if (slot is null || slot.Emitting || !slot.Cancelled) + { + break; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + } + } + + private bool IsPresentNoLock(Slot slot) => + (uint)slot.Index < Depth && ReferenceEquals(_slots[slot.Index], slot); + + private void ThrowIfShutDown() + { + if (Volatile.Read(ref _shutDown) == 1) + { + throw new OperationCanceledException("TAKETHIS pipeline has shut down."); + } + } + + private void SignalProgressNoLock() + { + var previous = _progress; + _progress = NewProgress(); + previous.TrySetResult(); + } + + /// + /// Records when it exceeds . + /// Concurrent observers cannot publish a lower peak over a higher one. + /// + private static void ObservePeak(ref int peak, int observed) + { + while (true) + { + var current = Volatile.Read(ref peak); + if (observed <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref peak, observed, current) == current) + { + return; + } + } + } + + private static string SlotMessageId(Slot slot) => + System.Text.Encoding.ASCII.GetString(slot.MessageId); + + private static TaskCompletionSource NewProgress() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class Slot + { + public required byte[] MessageId { get; init; } + + public required int Index { get; init; } + + public required long StartedTimestamp { get; init; } + + public TakeThisStageMarks? Marks { get; init; } + + public IHaveArticleReadResult Read { get; set; } + + public int MaxArticleBytes { get; init; } + + public byte[]? SiteNameUtf8 { get; init; } + + public HistoryLookupResult Peek { get; set; } + + public bool ArticleReady { get; set; } + + public bool PeekCompleted { get; set; } + + public bool Processed { get; set; } + + public ArticleProcessOutcome Process { get; set; } + + public bool Cancelled { get; set; } + + public bool Emitting { get; set; } + } + + /// + /// Per-slot destuff/parse/CRC outcome consumed by the ordered publisher. + /// + private readonly struct ArticleProcessOutcome + { + public bool RecordRejected { get; init; } + + public bool UncarriedRejected { get; init; } + + public string? RejectDetail { get; init; } + + public string? UncarriedReason { get; init; } + + public InboundArticle? Inbound { get; init; } + + public int Size { get; init; } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/TakeThisStageProbe.cs b/src/VectorNNTPD.ReaderService/Session/TakeThisStageProbe.cs new file mode 100644 index 00000000..e4fba7ba --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/TakeThisStageProbe.cs @@ -0,0 +1,468 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.History; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Off-by-default TAKETHIS pipeline stage sampler. Disabled unless + /// VECTORNNTP_TAKETHIS_TIMING is set or is called. + /// When disabled, callers must not allocate per-article state. + /// + internal static class TakeThisStageProbe + { + /// Environment variable that enables server-side TAKETHIS stage timing. + public const string EnvironmentVariableName = "VECTORNNTP_TAKETHIS_TIMING"; + + private static ProbeState? _state; + + static TakeThisStageProbe() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var dir = value.Equals("1", StringComparison.OrdinalIgnoreCase) || + value.Equals("true", StringComparison.OrdinalIgnoreCase) + ? DefaultDirectory() + : value; + Enable(dir); + } + + /// Gets whether stage timestamps should be recorded. + public static bool IsEnabled => Volatile.Read(ref _state) is not null; + + /// Gets the directory that receives session dumps, or . + public static string? OutputDirectory => Volatile.Read(ref _state)?.Directory; + + /// Enables sampling and writes dumps under . + public static void Enable(string outputDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(outputDirectory); + Directory.CreateDirectory(outputDirectory); + Volatile.Write(ref _state, new ProbeState(outputDirectory)); + } + + /// Disables sampling (tests). + public static void Disable() => Volatile.Write(ref _state, null); + + /// Creates a per-session log when enabled; otherwise . + public static TakeThisStageSessionLog? CreateSessionLog() => + IsEnabled ? new TakeThisStageSessionLog() : null; + + /// + /// Repo-local diagnostic dump directory. + /// + /// + /// Deliberately walks from the process working directory toward a folder that + /// contains VectorNNTPD.ReaderService.sln or .artifacts/. This is + /// development-only output, off unless is set. + /// It is not application-local runtime state and must not be used for certs/logs. + /// + internal static string DefaultDirectory() + { + var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.ReaderService.sln")) || + Directory.Exists(Path.Join(dir.FullName, ".artifacts"))) + { + return Path.Join(dir.FullName, ".artifacts", "takethis-timing"); + } + + dir = dir.Parent; + } + + return Path.Join(Directory.GetCurrentDirectory(), ".artifacts", "takethis-timing"); + } + + private sealed class ProbeState(string directory) + { + public string Directory { get; } = directory; + } + } + + /// Per-session collection of TAKETHIS stage samples. + internal sealed class TakeThisStageSessionLog + { + private readonly ConcurrentBag _samples = []; + private int _peakOccupied; + private int _accepted; + + /// Gets the number of recorded accepted samples. + public int Count => _samples.Count; + + /// Gets the peak occupied slot count observed while sampling. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Records occupied-window observations (all admits, not only accepted samples). + public void NoteOccupied(int occupied) + { + var current = Volatile.Read(ref _peakOccupied); + while (occupied > current) + { + var original = Interlocked.CompareExchange(ref _peakOccupied, occupied, current); + if (original == current) + { + break; + } + + current = original; + } + } + + /// Adds one accepted TAKETHIS stage sample. + public void Record(in TakeThisStageSample sample) + { + Interlocked.Increment(ref _accepted); + _samples.Add(sample); + } + + /// Writes CSV and a percentile report for this session. + public string Write(string directory, string sessionLabel) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionLabel); + Directory.CreateDirectory(directory); + var samples = _samples.ToArray(); + Array.Sort(samples, static (a, b) => a.CommandParsedTs.CompareTo(b.CommandParsedTs)); + var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture); + var safe = Sanitize(sessionLabel); + var csvPath = Path.Join(directory, $"{stamp}-{safe}.csv"); + var txtPath = Path.Join(directory, $"{stamp}-{safe}.txt"); + File.WriteAllText(csvPath, TakeThisStageReport.FormatCsv(samples), Encoding.UTF8); + var report = TakeThisStageReport.Format( + samples, + PeakOccupied, + sessionLabel, + csvPath); + File.WriteAllText(txtPath, report, Encoding.UTF8); + return txtPath; + } + + private static string Sanitize(string value) + { + var chars = value.ToCharArray(); + for (var i = 0; i < chars.Length; i++) + { + if (!char.IsAsciiLetterOrDigit(chars[i]) && chars[i] is not '-' and not '_') + { + chars[i] = '_'; + } + } + + return new string(chars); + } + } + + /// Monotonic timestamps for one TAKETHIS pipeline slot. + internal sealed class TakeThisStageMarks + { + public long CommandParsedTs; + public long PeekStartTs; + public long PeekCompleteTs; + public long ReceiveStartTs; + public long ReceiveCompleteTs; + public long EmitStartTs; + public long EnqueueStartTs; + public long EnqueueAcceptedTs; + public long RememberStartTs; + public long RememberCompleteTs; + public long ReplyEnqueueStartTs; + public long ReplyEnqueueCompleteTs; + public int OccupiedAtEmit; + public int ArticleBytes; + public string Outcome = "accepted"; + + /// Wraps so peek completion is stamped when Redis returns. + public ValueTask ObservePeek(ValueTask lookup) => + ObservePeekAsync(this, lookup); + + public TakeThisStageSample ToSample(int peakOccupied) => + new( + CommandParsedTs, + PeekStartTs, + PeekCompleteTs, + ReceiveStartTs, + ReceiveCompleteTs, + EmitStartTs, + EnqueueStartTs, + EnqueueAcceptedTs, + RememberStartTs, + RememberCompleteTs, + ReplyEnqueueStartTs, + ReplyEnqueueCompleteTs, + OccupiedAtEmit, + peakOccupied, + ArticleBytes, + Outcome); + + private static async ValueTask ObservePeekAsync( + TakeThisStageMarks marks, + ValueTask lookup) + { + try + { + var result = await lookup.ConfigureAwait(false); + marks.PeekCompleteTs = Stopwatch.GetTimestamp(); + return result; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + marks.PeekCompleteTs = Stopwatch.GetTimestamp(); + throw; + } + } + } + + /// One accepted TAKETHIS pipeline transaction (monotonic timestamps). + internal readonly record struct TakeThisStageSample( + long CommandParsedTs, + long PeekStartTs, + long PeekCompleteTs, + long ReceiveStartTs, + long ReceiveCompleteTs, + long EmitStartTs, + long EnqueueStartTs, + long EnqueueAcceptedTs, + long RememberStartTs, + long RememberCompleteTs, + long ReplyEnqueueStartTs, + long ReplyEnqueueCompleteTs, + int OccupiedAtEmit, + int PeakOccupied, + int ArticleBytes, + string Outcome); + + /// Percentile report for TAKETHIS stage samples. + internal static class TakeThisStageReport + { + public static string FormatCsv(IReadOnlyList samples) + { + var sb = new StringBuilder(); + sb.AppendLine( + "command_parsed_ts,peek_us,receive_us,ready_to_emit_us,enqueue_us,remember_us,reply_enqueue_us,server_txn_us,occupied_at_emit,peak_occupied,article_bytes,outcome,dominant"); + foreach (var sample in samples) + { + var i = Intervals(sample); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"{sample.CommandParsedTs},{i.PeekUs},{i.ReceiveUs},{i.ReadyToEmitUs},{i.EnqueueUs},{i.RememberUs},{i.ReplyEnqueueUs},{i.ServerTxnUs},{sample.OccupiedAtEmit},{sample.PeakOccupied},{sample.ArticleBytes},{sample.Outcome},{i.Dominant}")); + } + + return sb.ToString(); + } + + public static string Format( + IReadOnlyList samples, + int peakOccupied, + string sessionLabel, + string? csvPath) + { + var sb = new StringBuilder(); + sb.AppendLine("TAKETHIS PIPELINE STAGE TIMING — DIAGNOSTIC"); + sb.AppendLine(); + sb.AppendLine("Off-by-default server probe. Production path is unchanged when disabled."); + sb.AppendLine("Clock: Stopwatch.GetTimestamp (monotonic). 239 enqueue is Channel accept, not socket flush."); + sb.AppendLine(); + sb.AppendLine($"Session: {sessionLabel}"); + sb.AppendLine($"Samples: {samples.Count:N0}"); + var seen = samples.Count(static s => s.Outcome == "seen"); + var unseen = samples.Count(static s => s.Outcome == "unseen"); + sb.AppendLine($"Seen / unseen: {seen:N0} / {unseen:N0}"); + sb.AppendLine($"Peak occupied: {peakOccupied} (window {TakeThisPipeline.Depth})"); + if (samples.Count == 0) + { + sb.AppendLine(); + sb.AppendLine("No accepted samples."); + return sb.ToString(); + } + + var maxAtEmit = samples.Max(static s => s.OccupiedAtEmit); + sb.AppendLine($"Max occupied@emit: {maxAtEmit}"); + sb.AppendLine($"Average article: {samples.Average(static s => (double)s.ArticleBytes):N0} bytes"); + sb.AppendLine(); + + WriteWindow(sb, "ALL SAMPLES", samples); + if (samples.Count >= 20) + { + var mid = samples.Count / 2; + WriteWindow(sb, "FIRST HALF", samples.Take(mid).ToArray()); + WriteWindow(sb, "SECOND HALF", samples.Skip(mid).ToArray()); + } + + if (csvPath is not null) + { + sb.AppendLine($"CSV: {csvPath}"); + } + + return sb.ToString(); + } + + private static void WriteWindow(StringBuilder sb, string title, IReadOnlyList samples) + { + sb.AppendLine(title); + sb.AppendLine($"{"Interval",-28} {"Mean",10} {"P50",10} {"P95",10} {"P99",10} {"Max",10}"); + foreach (var row in Rows(samples)) + { + sb.AppendLine(row); + } + + sb.AppendLine(); + var votes = new Dictionary(StringComparer.Ordinal); + foreach (var name in samples.Select(static sample => Intervals(sample).Dominant)) + { + votes[name] = votes.TryGetValue(name, out var n) ? n + 1 : 1; + } + + sb.AppendLine("Dominant interval (largest of peek / receive / ready→emit / enqueue / remember / 239 enqueue):"); + foreach (var pair in votes.OrderByDescending(static p => p.Value)) + { + var pct = 100.0 * pair.Value / samples.Count; + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" {pair.Key,-22} {pair.Value,8:N0} {pct,6:F1}%")); + } + + sb.AppendLine(); + } + + private static IEnumerable Rows(IReadOnlyList samples) + { + yield return Stats("msgid → peek start", samples, static i => i.MsgidToPeekUs); + yield return Stats("HistoryDB Peek", samples, static i => i.PeekUs); + yield return Stats("article receive", samples, static i => i.ReceiveUs); + yield return Stats("max(peek, receive)", samples, static i => i.OverlapUs); + yield return Stats("peek leftover after RX", samples, static i => i.PeekAfterReceiveUs); + yield return Stats("ready → emit (order)", samples, static i => i.ReadyToEmitUs); + yield return Stats("queue EnqueueAsync", samples, static i => i.EnqueueUs); + yield return Stats("HistoryDB Remember", samples, static i => i.RememberUs); + yield return Stats("239 Channel enqueue", samples, static i => i.ReplyEnqueueUs); + yield return Stats("server txn (parse→239)", samples, static i => i.ServerTxnUs); + } + + private static string Stats( + string name, + IReadOnlyList samples, + Func selector) + { + var values = new double[samples.Count]; + double sum = 0; + for (var i = 0; i < samples.Count; i++) + { + var value = selector(Intervals(samples[i])); + values[i] = value; + sum += value; + } + + Array.Sort(values); + var mean = sum / values.Length; + return string.Create( + CultureInfo.InvariantCulture, + $"{name,-28} {mean,10:F1} {Percentile(values, 0.50),10:F1} {Percentile(values, 0.95),10:F1} {Percentile(values, 0.99),10:F1} {values[^1],10:F1}"); + } + + internal static double Percentile(double[] sorted, double p) + { + if (sorted.Length == 1) + { + return sorted[0]; + } + + var position = p * (sorted.Length - 1); + var lo = (int)position; + var hi = Math.Min(lo + 1, sorted.Length - 1); + var frac = position - lo; + return sorted[lo] + ((sorted[hi] - sorted[lo]) * frac); + } + + internal static StageIntervals Intervals(in TakeThisStageSample sample) + { + var peek = Us(sample.PeekStartTs, sample.PeekCompleteTs); + var receive = Us(sample.ReceiveStartTs, sample.ReceiveCompleteTs); + var ready = Math.Max(sample.PeekCompleteTs, sample.ReceiveCompleteTs); + var readyToEmit = Us(ready, sample.EmitStartTs); + var enqueue = Us(sample.EnqueueStartTs, sample.EnqueueAcceptedTs); + var remember = Us(sample.RememberStartTs, sample.RememberCompleteTs); + var reply = Us(sample.ReplyEnqueueStartTs, sample.ReplyEnqueueCompleteTs); + var txn = Us(sample.CommandParsedTs, sample.ReplyEnqueueCompleteTs); + var msgidToPeek = Us(sample.CommandParsedTs, sample.PeekStartTs); + var leftover = sample.PeekCompleteTs > sample.ReceiveCompleteTs + ? Us(sample.ReceiveCompleteTs, sample.PeekCompleteTs) + : 0; + var overlap = Math.Max(peek, receive); + + var dominant = "article receive"; + var best = receive; + if (peek > best) + { + best = peek; + dominant = "HistoryDB Peek"; + } + + if (readyToEmit > best) + { + best = readyToEmit; + dominant = "ordered emission"; + } + + if (enqueue > best) + { + best = enqueue; + dominant = "queue admission"; + } + + if (remember > best) + { + best = remember; + dominant = "HistoryDB Remember"; + } + + if (reply > best) + { + dominant = "TX/flush (239 enqueue)"; + } + + return new StageIntervals( + msgidToPeek, + peek, + receive, + overlap, + leftover, + readyToEmit, + enqueue, + remember, + reply, + txn, + dominant); + } + + private static double Us(long start, long end) + { + if (start <= 0 || end <= 0 || end < start) + { + return 0; + } + + return Stopwatch.GetElapsedTime(start, end).TotalMicroseconds; + } + + internal readonly record struct StageIntervals( + double MsgidToPeekUs, + double PeekUs, + double ReceiveUs, + double OverlapUs, + double PeekAfterReceiveUs, + double ReadyToEmitUs, + double EnqueueUs, + double RememberUs, + double ReplyEnqueueUs, + double ServerTxnUs, + string Dominant); + } +} diff --git a/src/VectorNNTPD.ReaderService/Session/TransitPeerAuthorization.cs b/src/VectorNNTPD.ReaderService/Session/TransitPeerAuthorization.cs new file mode 100644 index 00000000..b2aef64e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Session/TransitPeerAuthorization.cs @@ -0,0 +1,180 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Session +{ + /// + /// Resolves connection-time transit/streaming peer identity from the active Transit snapshot. + /// + public interface ITransitPeerAuthorization + { + /// Gets whether any Transit peers are configured in the current snapshot. + bool IsEnabled { get; } + + /// + /// Returns the initial for . + /// + /// + /// Matching uses the precomputed IP-only ACL (literal prefixes plus already-resolved + /// AllowFrom addresses). This method never performs DNS, reverse DNS, or network I/O. + /// A unique match retains the named peer policy and grants streaming. + /// Feed commands are granted immediately only when that peer has no receive credentials. + /// When both username and password are set, feed commands wait for AUTHINFO. + /// Ambiguous multi-peer matches grant nothing. + /// Other addresses receive . + /// + NntpAuthorization Resolve(IPAddress effectiveClientAddress); + } + + /// + /// Snapshot-backed implementation of . + /// + public sealed class TransitPeerAuthorization : ITransitPeerAuthorization + { + private readonly TransitConfigurationStore _store; + private readonly ITransitDnsAddressCache _dns; + private readonly ILogger? _logger; + + /// Disabled shared instance (empty snapshot). + public static TransitPeerAuthorization Disabled { get; } = CreateStatic(TransitConfigurationSnapshot.Empty); + + /// Creates an instance from the live snapshot store (DI). + [ActivatorUtilitiesConstructor] + public TransitPeerAuthorization( + TransitConfigurationStore store, + ITransitDnsAddressCache dns, + TransitConfigurationHotReload hotReload, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(dns); + ArgumentNullException.ThrowIfNull(hotReload); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _dns = dns; + _logger = logger; + if (!store.Current.IsEmpty) + { + SessionLogMessages.TransitPeersConfigured( + logger, + store.Current.Peers.Count, + string.Join(", ", store.Current.Peers.Values.Select(static p => p.Identifier + " (" + p.PeerName + ")"))); + } + } + + private TransitPeerAuthorization( + TransitConfigurationStore store, + ITransitDnsAddressCache dns, + ILogger? logger = null) + { + _store = store; + _dns = dns; + _logger = logger; + } + + /// Creates an instance bound to an existing snapshot store (tests). + public static TransitPeerAuthorization CreateForStore( + TransitConfigurationStore store, + ITransitDnsAddressCache? dns = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(store); + return new TransitPeerAuthorization(store, dns ?? new EmptyDnsCache(), logger); + } + + /// Creates a static instance from a complete snapshot (tests). + public static TransitPeerAuthorization CreateStatic( + TransitConfigurationSnapshot snapshot, + ITransitDnsAddressCache? dns = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(snapshot); + var store = new TransitConfigurationStore(); + store.Replace(snapshot); + return new TransitPeerAuthorization(store, dns ?? new EmptyDnsCache(), logger); + } + + /// + public bool IsEnabled => !_store.Current.IsEmpty; + + /// + public NntpAuthorization Resolve(IPAddress effectiveClientAddress) + { + ArgumentNullException.ThrowIfNull(effectiveClientAddress); + var address = TrustedProxyHosts.Canonicalize(effectiveClientAddress); + var snapshot = _store.Current; + if (snapshot.IsEmpty) + { + return NntpAuthorization.Unauthenticated; + } + + List? matches = null; + TransitPeerPolicy? matched = null; + foreach (var peer in snapshot.Peers.Values) + { + if (!MatchesIpAcl(peer, address)) + { + continue; + } + + matches ??= []; + matches.Add(peer.Identifier + " (" + peer.PeerName + ")"); + matched = peer; + } + + if (matches is { Count: > 1 }) + { + if (_logger is not null) + { + SessionLogMessages.TransitPeerAmbiguous(_logger, address, string.Join(", ", matches)); + } + return NntpAuthorization.Unauthenticated; + } + + return matched is null + ? NntpAuthorization.Unauthenticated + : NntpAuthorization.ForTransitPeer(matched); + } + + /// + /// Matches against the peer's already-materialized IP ACL. + /// Does not call . + /// + private bool MatchesIpAcl(TransitPeerPolicy peer, IPAddress address) + { + foreach (var prefix in peer.LiteralPrefixes) + { + if (prefix.Contains(address)) + { + return true; + } + } + + foreach (var hostname in peer.DnsHostnames) + { + if (_dns.GetResolved(hostname).Contains(address)) + { + return true; + } + } + + return false; + } + + private sealed class EmptyDnsCache : ITransitDnsAddressCache + { + public IReadOnlySet GetResolved(string hostname) => new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteBatchId.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteBatchId.cs new file mode 100644 index 00000000..1943294f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteBatchId.cs @@ -0,0 +1,37 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Identity of one MySQL-committed consume batch used for Redis APPLY idempotency. + /// + /// + /// Generated after MySQL commits and reused for every Redis retry of that batch in this + /// process. It is not persisted. A process crash drops it and the batch is not replayed. + /// + internal static class AccountByteBatchId + { + /// HASH field prefix for an applied batch on the remaining-quota key. + public const string FieldPrefix = "b:"; + + /// + /// Maximum applied-batch marks retained per remaining key. Floor-only APPLY + /// stays correct if an older mark is evicted. + /// + public const int MaxRetainedMarks = 256; + + /// Creates a new unique batch identity. + public static string Create() => Guid.NewGuid().ToString("N"); + + /// Returns whether is a usable identity. + public static bool IsValid([NotNullWhen(true)] string? batchId) => + !string.IsNullOrWhiteSpace(batchId); + + /// HASH field name for . + public static string FieldName(string batchId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + return FieldPrefix + batchId; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteDefaults.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteDefaults.cs new file mode 100644 index 00000000..e13b480f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteDefaults.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Operational timings for account byte-quota reconciliation. + /// + /// There is no AccountBytes timer. owns the + /// single ~10-second cycle. Batching is not byte-exact. Redis is never used on + /// the NNTP write path. + /// + internal static class AccountByteDefaults + { + /// Documented batch cadence; the live scheduler is . + public static readonly TimeSpan ReconciliationPeriod = SessionStateDefaults.RenewalPeriod; + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteEngine.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteEngine.cs new file mode 100644 index 00000000..c7ef212e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteEngine.cs @@ -0,0 +1,164 @@ +using System.Globalization; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// In-process remaining-quota algorithm. Production Redis Lua is a faithful port. + /// Remaining never increases and never goes negative. + /// + /// + /// APPLY is idempotent per batch id. The remaining update is + /// min(current, mysqlRemainingAfter) (or initialize from MySQL when missing). + /// consumed is not subtracted: MySQL already subtracted that batch, and + /// another node's earlier APPLY may already have floored to a mysql_after that + /// includes this consume. Decrementing would double-count and is not retry-safe. + /// + internal sealed class AccountByteEngine + { + private readonly object _gate = new(); + private readonly Dictionary _remaining = new(StringComparer.Ordinal); + private readonly Dictionary> _applied = new(StringComparer.Ordinal); + + /// + /// Applies one consumed batch identified by . + /// Repeating the same id does not change remaining. + /// + public long Apply(string key, string batchId, long consumed, long mysqlRemainingAfter) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + if (!AccountByteBatchId.IsValid(batchId)) + { + throw new ArgumentException("Batch id is required for idempotent APPLY.", nameof(batchId)); + } + + _ = ClampNonNegative(consumed); + mysqlRemainingAfter = ClampNonNegative(mysqlRemainingAfter); + lock (_gate) + { + if (IsApplied(key, batchId)) + { + return _remaining.TryGetValue(key, out var existing) + ? ClampNonNegative(existing) + : 0; + } + + if (!_remaining.TryGetValue(key, out var current)) + { + _remaining[key] = mysqlRemainingAfter; + MarkApplied(key, batchId); + TrimApplied(key, batchId); + return mysqlRemainingAfter; + } + + current = ClampNonNegative(current); + var next = current < mysqlRemainingAfter ? current : mysqlRemainingAfter; + _remaining[key] = next; + MarkApplied(key, batchId); + TrimApplied(key, batchId); + return next; + } + } + + /// Reads remaining, or when absent. + public long Observe(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + return _remaining.TryGetValue(key, out var current) ? ClampNonNegative(current) : AccountByteKeys.Missing; + } + } + + /// Overwrites remaining without consume semantics (tests). + internal void Write(string key, long remaining) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + _remaining[key] = ClampNonNegative(remaining); + } + } + + /// Removes remaining and applied-batch marks. Returns 1 if the key existed. + internal long Delete(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + var existed = _remaining.Remove(key); + existed = _applied.Remove(key) || existed; + return existed ? 1 : 0; + } + } + + /// Returns whether the remaining key exists (tests). + internal bool Contains(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + return _remaining.ContainsKey(key); + } + } + + /// Returns whether was already applied (tests). + internal bool WasApplied(string key, string batchId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + lock (_gate) + { + return IsApplied(key, batchId); + } + } + + internal static long ClampNonNegative(long value) => value < 0 ? 0 : value; + + internal static string Format(long remaining) => + remaining.ToString(CultureInfo.InvariantCulture); + + private bool IsApplied(string key, string batchId) => + _applied.TryGetValue(key, out var batches) && batches.Contains(batchId); + + private void MarkApplied(string key, string batchId) + { + if (!_applied.TryGetValue(key, out var batches)) + { + batches = new HashSet(StringComparer.Ordinal); + _applied[key] = batches; + } + + batches.Add(batchId); + } + + private void TrimApplied(string key, string keepBatchId) + { + if (!_applied.TryGetValue(key, out var batches) + || batches.Count <= AccountByteBatchId.MaxRetainedMarks) + { + return; + } + + var extra = batches.Count - AccountByteBatchId.MaxRetainedMarks; + var doomed = new List(extra); + foreach (var id in batches) + { + if (id == keepBatchId) + { + continue; + } + + doomed.Add(id); + if (doomed.Count == extra) + { + break; + } + } + + foreach (var id in doomed) + { + batches.Remove(id); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteKeys.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteKeys.cs new file mode 100644 index 00000000..58692ada --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteKeys.cs @@ -0,0 +1,48 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Redis key material for cluster-wide remaining byte quota. + /// + /// + /// Key: nntpd:bytes:{sha256hex(accountName)} (HASH). The plaintext AUTHINFO + /// username is hashed so Redis listings do not expose account names. Field + /// remaining is the remaining byte count. Field b:{batchId} marks an + /// applied consume batch for idempotent APPLY. This is not a lease and has no key TTL. + /// + internal static class AccountByteKeys + { + /// ASCII namespace for remaining-quota keys. + public static ReadOnlySpan NamespacePrefix => "nntpd:bytes:"u8; + + /// Sentinel returned by OBSERVE when the key is missing. + public const long Missing = -1; + + /// Builds the remaining-quota Redis key for . + public static byte[] Create(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteLogMessages.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteLogMessages.cs new file mode 100644 index 00000000..6298b064 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteLogMessages.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Source-generated AccountBytes logs. EventIds 2700–2711. + internal static partial class AccountByteLogMessages + { + [LoggerMessage(EventId = 2700, Level = LogLevel.Information, Message = "Account-byte reconciliation started interval={Interval}")] + public static partial void ReconciliationStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 2701, Level = LogLevel.Information, Message = "Account-byte reconciliation stopped")] + public static partial void ReconciliationStopped(ILogger logger); + + [LoggerMessage(EventId = 2702, Level = LogLevel.Warning, Message = "Account-byte reconciliation pass failed")] + public static partial void ReconciliationPassFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2703, Level = LogLevel.Warning, Message = "Account-byte durable consume failed user={Username} bytes={Bytes}")] + public static partial void DurableConsumeFailed(ILogger logger, Exception exception, string username, long bytes); + + [LoggerMessage(EventId = 2704, Level = LogLevel.Warning, Message = "Account-byte Redis apply unavailable user={Username} consumed={Consumed} mysqlRemaining={MysqlRemaining}")] + public static partial void RedisApplyUnavailable(ILogger logger, string username, long consumed, long mysqlRemaining); + + [LoggerMessage(EventId = 2705, Level = LogLevel.Information, Message = "Account-byte quota exhausted user={Username} remaining={Remaining}")] + public static partial void QuotaExhausted(ILogger logger, string username, long remaining); + + [LoggerMessage(EventId = 2706, Level = LogLevel.Debug, Message = "Account-byte batch reconciled user={Username} consumed={Consumed} remaining={Remaining}")] + public static partial void BatchReconciled(ILogger logger, string username, long consumed, long remaining); + + [LoggerMessage(EventId = 2707, Level = LogLevel.Warning, Message = "Account-byte observe failed user={Username}")] + public static partial void ObserveFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2708, Level = LogLevel.Warning, Message = "Account-byte durable remaining query failed user={Username}")] + public static partial void DurableQueryFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2709, Level = LogLevel.Information, Message = "Account-byte final reconciliation started")] + public static partial void FinalReconciliationStarted(ILogger logger); + + [LoggerMessage(EventId = 2710, Level = LogLevel.Warning, Message = "Account-byte final reconciliation failed")] + public static partial void FinalReconciliationFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2711, Level = LogLevel.Information, Message = "Account-byte Redis state deleted user={Username} existed={Existed}")] + public static partial void ClusterStateDeleted(ILogger logger, string username, long existed); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteScripts.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteScripts.cs new file mode 100644 index 00000000..191d0dc2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteScripts.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Redis Lua for cluster remaining-quota. Semantics match . + /// The remaining HASH has no key TTL. Batch fields are marks only; they do not expire + /// the remaining value. + /// + internal static class AccountByteScripts + { + /// + /// Applies one consumed batch idempotently. Remaining is floored to + /// mysqlRemainingAfter and never increased. The same batch id is a no-op. + /// + /// + /// KEYS[1] remaining HASH. + /// ARGV: batchId, consumed, mysqlRemainingAfter. + /// consumed is accepted for the call contract but is not subtracted. + /// MySQL already subtracted the batch; decrementing Redis is not retry-safe + /// when another node has already floored to a mysql_after that includes it. + /// Returns remaining >= 0. + /// + public const string Apply = + """ + -- nntpd-bytes-apply + -- Retain at most 256 batch marks (AccountByteBatchId.MaxRetainedMarks). + local function trim_marks(key, batchField) + if redis.call('HLEN', key) <= 257 then + return + end + local fields = redis.call('HKEYS', key) + for i = 1, #fields do + local f = fields[i] + if f ~= 'remaining' and f ~= batchField then + redis.call('HDEL', key, f) + if redis.call('HLEN', key) <= 257 then + return + end + end + end + end + local key = KEYS[1] + local batchId = ARGV[1] + local mysql = tonumber(ARGV[3]) + if mysql == nil or mysql < 0 then + mysql = 0 + end + if batchId == nil or batchId == '' then + batchId = '' + end + local batchField = 'b:' .. batchId + if redis.call('EXISTS', key) == 0 then + if batchId ~= '' then + redis.call('HSET', key, 'remaining', tostring(mysql), batchField, '1') + trim_marks(key, batchField) + else + redis.call('HSET', key, 'remaining', tostring(mysql)) + end + return mysql + end + if batchId ~= '' and redis.call('HEXISTS', key, batchField) == 1 then + local cur = tonumber(redis.call('HGET', key, 'remaining')) + if cur == nil or cur < 0 then + return 0 + end + return cur + end + local current = tonumber(redis.call('HGET', key, 'remaining')) + if current == nil or current < 0 then + current = 0 + end + local next = current + if next > mysql then + next = mysql + end + if batchId ~= '' then + redis.call('HSET', key, 'remaining', tostring(next), batchField, '1') + trim_marks(key, batchField) + else + redis.call('HSET', key, 'remaining', tostring(next)) + end + return next + """; + + /// Reads remaining. Missing key returns -1. + /// KEYS[1] remaining HASH. No ARGV. Never creates or writes the key. + public const string Observe = + """ + -- nntpd-bytes-observe + if redis.call('EXISTS', KEYS[1]) == 0 then + return -1 + end + local n = tonumber(redis.call('HGET', KEYS[1], 'remaining')) + if n == nil or n < 0 then + return 0 + end + return n + """; + + /// + /// Deletes the remaining-quota HASH. Used only for explicit operator top-up + /// invalidation. Never creates or writes the key. + /// + /// KEYS[1] remaining HASH. No ARGV. Returns 1 if the key existed. + public const string Delete = + """ + -- nntpd-bytes-delete + return redis.call('DEL', KEYS[1]) + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteTracker.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteTracker.cs new file mode 100644 index 00000000..7e674fd3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/AccountByteTracker.cs @@ -0,0 +1,458 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Process-local account byte accumulator and crash-safe batch reconciler. + /// + /// + /// + /// Hot path: is an . + /// Pending and in-flight totals live only in this process. A crash loses unreconciled bytes + /// (at-most-once durable charge). That is the accepted approximation; durable quota must + /// never be charged twice for the same batch. + /// + /// + /// Reconciliation is MySQL first, then Redis APPLY with a per-batch id. After a + /// successful MySQL consume in this process, the same batch id is reused for every + /// Redis retry. Redis treats that id as exactly-once. The id is not durable. + /// Restart never replays a batch. + /// + /// + internal sealed class AccountByteTracker : IAccountByteAccountant + { + private readonly ConcurrentDictionary _ledgers = new(StringComparer.Ordinal); + private readonly IAccountByteDurableStore _durable; + private readonly IAccountByteStore _cluster; + private readonly ILogger _logger; + private readonly SemaphoreSlim _reconcileGate = new(1, 1); + + /// Initializes a new instance of the class. + public AccountByteTracker( + IAccountByteDurableStore durable, + IAccountByteStore cluster, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(durable); + ArgumentNullException.ThrowIfNull(cluster); + ArgumentNullException.ThrowIfNull(logger); + _durable = durable; + _cluster = cluster; + _logger = logger; + } + + /// + public IAccountByteSink CreateSink(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return NullAccountByteSink.Instance; + } + + return new AccountLedgerSink(GetOrAdd(accountName)); + } + + /// + public bool IsExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return false; + } + + return _ledgers.TryGetValue(accountName, out var ledger) + && Volatile.Read(ref ledger.Exhausted) == 1; + } + + /// + public void MarkExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + var ledger = GetOrAdd(accountName); + if (Interlocked.Exchange(ref ledger.Exhausted, 1) == 0) + { + AccountByteLogMessages.QuotaExhausted(_logger, accountName, remaining: 0); + } + } + + /// + public void ClearExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + if (_ledgers.TryGetValue(accountName, out var ledger)) + { + Volatile.Write(ref ledger.Exhausted, 0); + } + } + + /// + public async ValueTask DeleteAccountByteStateAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + long? deleted; + try + { + deleted = await _cluster.DeleteAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + return false; + } + + if (deleted is null) + { + AccountByteLogMessages.RedisApplyUnavailable(_logger, accountName, 0, 0); + return false; + } + + ClearExhausted(accountName); + AccountByteLogMessages.ClusterStateDeleted(_logger, accountName, deleted.Value); + return true; + } + + /// + public async ValueTask ObserveRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + long? redis = null; + try + { + redis = await _cluster.ObserveAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + } + + AccountByteConsumeResult? durable = null; + try + { + durable = await _durable.QueryRemainingAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException ex) + { + AccountByteLogMessages.DurableQueryFailed(_logger, ex, accountName); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.DurableQueryFailed(_logger, ex, accountName); + } + + if (durable is { Succeeded: false }) + { + return null; + } + + var mysqlRemaining = durable is { Succeeded: true } row + ? AccountByteEngine.ClampNonNegative(row.Remaining) + : (long?)null; + + if (redis is { } live && live != AccountByteKeys.Missing) + { + var clusterRemaining = AccountByteEngine.ClampNonNegative(live); + if (mysqlRemaining is { } durableRemaining) + { + var effective = clusterRemaining < durableRemaining ? clusterRemaining : durableRemaining; + if (clusterRemaining > durableRemaining) + { + await FloorRedisToMysqlAsync(accountName, durableRemaining, cancellationToken) + .ConfigureAwait(false); + } + + return effective; + } + + return clusterRemaining; + } + + return mysqlRemaining; + } + + /// + public async ValueTask CommitDurableAsync(CancellationToken cancellationToken = default) + { + await _reconcileGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + foreach (var pair in _ledgers) + { + cancellationToken.ThrowIfCancellationRequested(); + await CommitDurableAccountAsync(pair.Key, pair.Value, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _reconcileGate.Release(); + } + } + + /// + public async ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default) + { + await _reconcileGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + foreach (var pair in _ledgers) + { + cancellationToken.ThrowIfCancellationRequested(); + await ApplyCommittedAccountAsync(pair.Key, pair.Value, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _reconcileGate.Release(); + } + } + + /// + public async ValueTask ReconcileAsync(CancellationToken cancellationToken = default) + { + await CommitDurableAsync(cancellationToken).ConfigureAwait(false); + await ApplyCommittedAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch) + { + batch = default; + if (string.IsNullOrWhiteSpace(accountName) + || !_ledgers.TryGetValue(accountName, out var ledger) + || Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return false; + } + + var batchId = ledger.BatchId; + if (!AccountByteBatchId.IsValid(batchId)) + { + return false; + } + + batch = new AccountByteCommittedBatch( + accountName, + batchId, + ledger.LastConsumed, + ledger.LastMysqlRemaining); + return true; + } + + /// + public void CompleteApply(string accountName, long remaining) + { + if (string.IsNullOrWhiteSpace(accountName) + || !_ledgers.TryGetValue(accountName, out var ledger) + || Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return; + } + + var consumed = ledger.LastConsumed; + var mysqlRemaining = ledger.LastMysqlRemaining; + Interlocked.Exchange(ref ledger.InFlight, 0); + ledger.BatchId = null; + Volatile.Write(ref ledger.MysqlCommitted, 0); + AccountByteLogMessages.BatchReconciled(_logger, accountName, consumed, remaining); + if (remaining == 0 || mysqlRemaining == 0) + { + MarkExhausted(accountName); + } + } + + /// Returns the process-local pending total (tests). + internal long PendingBytes(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + ? Volatile.Read(ref ledger.Pending) + : 0; + + /// Returns the process-local in-flight total (tests). + internal long InFlightBytes(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + ? Volatile.Read(ref ledger.InFlight) + : 0; + + /// Returns whether MySQL already consumed the current in-flight batch (tests). + internal bool MysqlCommitted(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + && Volatile.Read(ref ledger.MysqlCommitted) == 1; + + /// Returns the in-process Redis batch id awaiting APPLY acknowledgment (tests). + internal string? CurrentBatchId(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) ? ledger.BatchId : null; + + private AccountLedger GetOrAdd(string accountName) => + _ledgers.GetOrAdd(accountName, static _ => new AccountLedger()); + + private async ValueTask FloorRedisToMysqlAsync( + string accountName, + long mysqlRemaining, + CancellationToken cancellationToken) + { + try + { + _ = await _cluster + .ApplyAsync( + accountName, + AccountByteBatchId.Create(), + consumed: 0, + mysqlRemaining, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + } + } + + private async ValueTask CommitDurableAccountAsync( + string accountName, + AccountLedger ledger, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref ledger.MysqlCommitted) == 1) + { + return; + } + + var snapshot = Interlocked.Exchange(ref ledger.Pending, 0); + if (snapshot > 0) + { + Interlocked.Add(ref ledger.InFlight, snapshot); + } + + var inFlight = Volatile.Read(ref ledger.InFlight); + if (inFlight == 0) + { + return; + } + + AccountByteConsumeResult result; + try + { + result = await _durable.ConsumeAsync(accountName, inFlight, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException ex) + { + AccountByteLogMessages.DurableConsumeFailed(_logger, ex, accountName, inFlight); + return; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.DurableConsumeFailed(_logger, ex, accountName, inFlight); + return; + } + + if (!result.Succeeded) + { + Interlocked.Exchange(ref ledger.InFlight, 0); + ledger.BatchId = null; + return; + } + + ledger.LastConsumed = result.Consumed; + ledger.LastMysqlRemaining = result.Remaining; + ledger.BatchId = AccountByteBatchId.Create(); + Volatile.Write(ref ledger.MysqlCommitted, 1); + if (result.Remaining == 0) + { + MarkExhausted(accountName); + } + } + + private async ValueTask ApplyCommittedAccountAsync( + string accountName, + AccountLedger ledger, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return; + } + + var consumed = ledger.LastConsumed; + var mysqlRemaining = ledger.LastMysqlRemaining; + var batchId = ledger.BatchId; + if (!AccountByteBatchId.IsValid(batchId)) + { + batchId = AccountByteBatchId.Create(); + ledger.BatchId = batchId; + } + + var applied = await _cluster + .ApplyAsync(accountName, batchId, consumed, mysqlRemaining, cancellationToken) + .ConfigureAwait(false); + if (applied is null) + { + AccountByteLogMessages.RedisApplyUnavailable(_logger, accountName, consumed, mysqlRemaining); + return; + } + + CompleteApply(accountName, applied.Value); + } + + private sealed class AccountLedger + { + public long Pending; + public long InFlight; + public int Exhausted; + public int MysqlCommitted; + public long LastConsumed; + public long LastMysqlRemaining; + public string? BatchId; + } + + private sealed class AccountLedgerSink(AccountLedger ledger) : IAccountByteSink + { + public void ObserveCopied(int bytes) + { + if (bytes <= 0) + { + return; + } + + Interlocked.Add(ref ledger.Pending, bytes); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteAccountant.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteAccountant.cs new file mode 100644 index 00000000..2ff30bdb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteAccountant.cs @@ -0,0 +1,170 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Process-local download-quota bookkeeping. Redis and MySQL are used by + /// AUTHINFO observe and the periodic reconciler, never on the NNTP write path. + /// + public interface IAccountByteAccountant + { + /// + /// Returns a sink that s into this + /// account's pending total. No-op when is empty. + /// + IAccountByteSink CreateSink(string accountName); + + /// Gets whether this process has observed remaining == 0 for the account. + bool IsExhausted(string accountName); + + /// + /// Reads cluster remaining (Redis if present, otherwise durable MySQL). + /// Does not create or raise a Redis key. Returns when + /// remaining cannot be determined. + /// + /// + /// Live remaining is min(Redis, MySQL) when Redis exists. A stale-high Redis + /// key is floored down to MySQL; observe never raises Redis and never uses the + /// AUTHINFO user-record cache. + /// + ValueTask ObserveRemainingAsync(string accountName, CancellationToken cancellationToken = default); + + /// Marks the account exhausted for all local sessions of this process. + void MarkExhausted(string accountName); + + /// + /// Clears the process-local exhausted flag. Used after an explicit top-up + /// invalidation or when live remaining is observed greater than zero. + /// + void ClearExhausted(string accountName); + + /// + /// Deletes cluster Redis remaining-quota state after an operator MySQL top-up. + /// The next observe uses current durable remaining. Does not increase Redis. + /// + /// + /// when the key was removed or already absent; + /// when Redis was unavailable. + /// + ValueTask DeleteAccountByteStateAsync(string accountName, CancellationToken cancellationToken = default); + + /// + /// Snapshots pending bytes and persists one account-level MySQL consume per + /// account with work. Does not APPLY Redis. Used by + /// before the combined Redis cycle. + /// + ValueTask CommitDurableAsync(CancellationToken cancellationToken = default); + + /// + /// APPLY any MySQL-committed batches that are still awaiting Redis. Combined + /// RENEW+APPLY that already called is skipped. + /// + ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default); + + /// + /// Snapshots pending, consumes MySQL, and APPLYs leftover batches. Used by + /// tests and shutdown. + /// + ValueTask ReconcileAsync(CancellationToken cancellationToken = default); + + /// Returns the MySQL-committed batch awaiting Redis APPLY, if any. + bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch); + + /// + /// Clears the in-process committed batch after a successful Redis APPLY + /// (combined EVAL or APPLY-only). + /// + void CompleteApply(string accountName, long remaining); + } + + /// One account-wide MySQL-committed consume awaiting Redis APPLY. + public readonly struct AccountByteCommittedBatch + { + /// Initializes a committed batch snapshot. + public AccountByteCommittedBatch(string accountName, string batchId, long consumed, long mysqlRemaining) + { + AccountName = accountName; + BatchId = batchId; + Consumed = consumed; + MysqlRemaining = mysqlRemaining; + } + + /// Gets the AUTHINFO account name. + public string AccountName { get; } + + /// Gets the in-process batch identity. + public string BatchId { get; } + + /// Gets bytes MySQL already subtracted for this batch. + public long Consumed { get; } + + /// Gets durable remaining after this consume. + public long MysqlRemaining { get; } + } + + /// No-op accountant for tests and unauthenticated identities. + public sealed class NullAccountByteAccountant : IAccountByteAccountant + { + /// Shared no-op instance. + public static NullAccountByteAccountant Instance { get; } = new(); + + private NullAccountByteAccountant() + { + } + + /// + public IAccountByteSink CreateSink(string accountName) => NullAccountByteSink.Instance; + + /// + public bool IsExhausted(string accountName) => false; + + /// + public ValueTask ObserveRemainingAsync(string accountName, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + /// + public void MarkExhausted(string accountName) + { + } + + /// + public void ClearExhausted(string accountName) + { + } + + /// + public ValueTask DeleteAccountByteStateAsync( + string accountName, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(true); + + /// + public ValueTask CommitDurableAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public ValueTask ReconcileAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch) + { + batch = default; + return false; + } + + /// + public void CompleteApply(string accountName, long remaining) + { + } + } + + /// Sink that ignores copied bytes. + internal sealed class NullAccountByteSink : IAccountByteSink + { + public static NullAccountByteSink Instance { get; } = new(); + + public void ObserveCopied(int bytes) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteDurableStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteDurableStore.cs new file mode 100644 index 00000000..399332e9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteDurableStore.cs @@ -0,0 +1,21 @@ +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Durable remaining-quota store (MySQL nntpusers.account_byte_limit). + internal interface IAccountByteDurableStore + { + /// + /// Atomically subtracts from remaining quota, clamping at zero. + /// + ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default); + + /// Reads remaining without mutating. Does not use the AUTHINFO user-record cache. + ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteSink.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteSink.cs new file mode 100644 index 00000000..7c7d608b --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteSink.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// + /// Hot-path observer for NNTP application bytes copied into the outbound PipeWriter. + /// + /// + /// Implementations must be allocation-free and must not perform Redis or MySQL I/O. + /// + public interface IAccountByteSink + { + /// Records copied into the output pipe for this account. + void ObserveCopied(int bytes); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteStore.cs new file mode 100644 index 00000000..72385aa8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/IAccountByteStore.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Cluster remaining-quota store. Not used on the NNTP write path. + internal interface IAccountByteStore + { + /// + /// Atomically applies one MySQL-committed batch. The same + /// is a no-op. Remaining is floored to + /// and never increased. + /// + /// Remaining after the apply, or when the store is unavailable. + ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default); + + /// + /// Reads remaining. means unavailable; + /// means the key is absent. + /// + ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default); + + /// + /// Deletes cluster remaining-quota state for an explicit MySQL top-up. + /// The next observe/apply treats the key as missing and may initialize from + /// current durable remaining. Does not raise remaining. + /// + /// + /// 1 if a key was removed, 0 if it was already absent, + /// or when the store is unavailable. + /// + ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs new file mode 100644 index 00000000..0afeb8d3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs @@ -0,0 +1,103 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// In-memory durable remaining quota for tests. Shared across trackers as one MySQL. + internal sealed class InMemoryAccountByteDurableStore : IAccountByteDurableStore + { + private readonly Lock _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + + /// When set, consume/query fail as if MySQL were down. + public bool Unavailable { get; set; } + + /// Gets how many consume operations mutated or inspected a row. + public int ConsumeCalls { get; private set; } + + /// Gets how many remaining-quota reads ran. + public int QueryCalls { get; private set; } + + /// Adds or replaces remaining quota for an account. + public void SeedByteAccount(string accountName, long remaining) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(remaining); + lock (_gate) + { + _accounts[accountName] = new AccountRow(remaining); + } + } + + /// Returns stored remaining, or when missing. + public long? Remaining(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var row) ? row.Remaining : null; + } + } + + /// + public ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + throw new NntpDbUnavailableException("MySQL byte-quota consume unavailable."); + } + + lock (_gate) + { + ConsumeCalls++; + if (!_accounts.TryGetValue(accountName, out var row)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var consumed = bytes > row.Remaining ? row.Remaining : bytes; + row.Remaining -= consumed; + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, row.Remaining, consumed)); + } + } + + /// + public ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + throw new NntpDbUnavailableException("MySQL byte-quota query unavailable."); + } + + lock (_gate) + { + QueryCalls++; + if (!_accounts.TryGetValue(accountName, out var row)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, row.Remaining, 0)); + } + } + + private sealed class AccountRow(long remaining) + { + public long Remaining { get; set; } = remaining; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs new file mode 100644 index 00000000..1366fad1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs @@ -0,0 +1,83 @@ +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Process-local cluster fake. Two trackers sharing one instance behave as two nodes. + internal sealed class InMemoryAccountByteStore : IAccountByteStore + { + private readonly AccountByteEngine _engine = new(); + + /// When set, apply/observe fail as if Redis were down. + public bool Unavailable { get; set; } + + /// Gets how many apply operations reached the engine. + public int ApplyCalls { get; private set; } + + /// Gets how many observe operations reached the engine. + public int ObserveCalls { get; private set; } + + /// + /// When greater than zero, the next apply runs the engine then returns + /// (Redis executed, response lost). + /// + public int HideNextApplyResults { get; set; } + + /// Gets the shared engine (tests). + internal AccountByteEngine Engine => _engine; + + /// + public ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + ApplyCalls++; + var remaining = _engine.Apply(EncodingKey(accountName), batchId, consumed, mysqlRemainingAfter); + if (HideNextApplyResults > 0) + { + HideNextApplyResults--; + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(remaining); + } + + /// + public ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + ObserveCalls++; + return ValueTask.FromResult(_engine.Observe(EncodingKey(accountName))); + } + + /// + public ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(_engine.Delete(EncodingKey(accountName))); + } + + internal static string EncodingKey(string accountName) => + System.Text.Encoding.UTF8.GetString(AccountByteKeys.Create(accountName)); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs new file mode 100644 index 00000000..b970ee06 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs @@ -0,0 +1,74 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Durable remaining-quota store over the existing NntpDB pool. + internal sealed class MysqlAccountByteDurableStore : IAccountByteDurableStore + { + private readonly NntpDbService _nntpDb; + + /// Initializes a new instance of the class. + public MysqlAccountByteDurableStore(NntpDbService nntpDb) + { + ArgumentNullException.ThrowIfNull(nntpDb); + _nntpDb = nntpDb; + } + + /// + public async ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.ConsumeAccountBytesAsync(accountName, bytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + throw new NntpDbUnavailableException("MySQL account byte-quota consume failed.", ex); + } + } + + /// + public async ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryAccountByteRemainingAsync(accountName, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + throw new NntpDbUnavailableException("MySQL account byte-quota query failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/RedisAccountByteStore.cs b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/RedisAccountByteStore.cs new file mode 100644 index 00000000..8638bade --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/BytesAccounting/RedisAccountByteStore.cs @@ -0,0 +1,112 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.SessionState.BytesAccounting +{ + /// Redis remaining-quota store. One EVAL per observe/apply. No key TTL. + internal sealed class RedisAccountByteStore : IAccountByteStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisAccountByteStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + return ExecuteAsync( + AccountByteScripts.Apply, + accountName, + [ + Utf8(batchId), + Utf8(AccountByteEngine.ClampNonNegative(consumed).ToString(CultureInfo.InvariantCulture)), + Utf8(AccountByteEngine.ClampNonNegative(mysqlRemainingAfter).ToString(CultureInfo.InvariantCulture)), + ], + static result => result < 0 ? 0L : result, + cancellationToken); + } + + /// + public ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default) => + ExecuteAsync( + AccountByteScripts.Observe, + accountName, + [], + static result => result, + cancellationToken); + + /// + public ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default) => + ExecuteAsync( + AccountByteScripts.Delete, + accountName, + [], + static result => result < 0 ? 0L : result, + cancellationToken); + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] values, + Func map, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return null; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return null; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + [AccountByteKeys.Create(accountName)], + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return null; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return null; + } + } + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/DistributedSessionStateTracker.cs b/src/VectorNNTPD.ReaderService/SessionState/DistributedSessionStateTracker.cs new file mode 100644 index 00000000..732d5dca --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/DistributedSessionStateTracker.cs @@ -0,0 +1,726 @@ +using System.Collections.Concurrent; +using System.Net; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Hybrid authenticated-session admission: Redis is the cluster authority for + /// both account_session_limit and account_srcip_limit. + /// + /// + /// Both limits are cluster-wide and evaluated in one atomic membership operation. + /// Limit 0 is unlimited. The source-IP local hot path is used only while + /// account_session_limit is unlimited and this node already holds a live + /// source-IP lease. A finite session limit requires every new session to take + /// the distributed path so the global session count stays exact. New admissions + /// that require Redis fail closed when membership is unavailable. + /// Owner identity is {nodeId}:{incarnation}; incarnation is a process-lifetime + /// identifier so a restart cannot mutate a previous incarnation's unexpired fields. + /// Redis TTL is crash recovery only: while this process is alive and renewal + /// succeeds, idle authenticated sessions keep consuming cluster capacity. + /// + public sealed class DistributedSessionStateTracker : ISessionStateTracker, ISessionStateLeaseManager + { + private readonly ISessionStateStore _membership; + private readonly IAccountByteAccountant? _bytes; + private readonly IAccountRateAllocator _rates; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly string _ownerId; + private readonly TimeSpan _leaseTtl; + private readonly TimeSpan _hotPathSkew; + private readonly ConcurrentDictionary _accountGates = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + private long _generations; + + /// Initializes a production tracker with default lease timings. + public DistributedSessionStateTracker( + ISessionStateStore membership, + ILogger logger, + string nodeId) + : this(membership, logger, nodeId, TimeProvider.System, SessionStateDefaults.LeaseTtl, SessionStateDefaults.HotPathSkew, incarnation: null, bytes: null, rates: null) + { + } + + /// Initializes a tracker with explicit timing and incarnation (tests). + internal DistributedSessionStateTracker( + ISessionStateStore membership, + ILogger logger, + string nodeId, + TimeProvider timeProvider, + TimeSpan? leaseTtl = null, + TimeSpan? hotPathSkew = null, + string? incarnation = null, + IAccountByteAccountant? bytes = null, + IAccountRateAllocator? rates = null) + { + ArgumentNullException.ThrowIfNull(membership); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentNullException.ThrowIfNull(timeProvider); + _membership = membership; + _bytes = bytes; + _rates = rates ?? NullAccountRateAllocator.Instance; + _logger = logger; + _nodeId = nodeId; + _ownerId = string.Concat(nodeId, ":", string.IsNullOrWhiteSpace(incarnation) ? Guid.NewGuid().ToString("N") : incarnation); + _timeProvider = timeProvider; + _leaseTtl = leaseTtl ?? SessionStateDefaults.LeaseTtl; + _hotPathSkew = hotPathSkew ?? SessionStateDefaults.HotPathSkew; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(_leaseTtl, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThan(_hotPathSkew, TimeSpan.Zero); + } + + /// Gets the configured node identity. + internal string NodeId => _nodeId; + + /// Gets {nodeId}:{incarnation} written into Redis ownership fields. + internal string OwnerId => _ownerId; + + /// Gets how many admits were satisfied from a live local source-IP lease. + internal int LocalHotPathAdmits { get; private set; } + + /// Gets how many admits required a distributed membership call. + internal int DistributedAdmits { get; private set; } + + /// Gets how many admits were rejected by the distinct-IP limit. + internal int DistributedSourceRejects { get; private set; } + + /// Gets how many admits were rejected by the cluster session limit. + internal int DistributedSessionRejects { get; private set; } + + /// Gets how many admits were rejected by the distinct-IP limit. + internal int DistributedRejects => DistributedSourceRejects; + + /// Gets how many admits failed closed because membership was unavailable. + internal int UnavailableRejects { get; private set; } + + /// Gets how many admitted sessions this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default) => + TryAdmitAsync(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit, rateLimitBps: 0, cancellationToken); + + /// + public async ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(sourceAddress); + + var ip = SourceAddressIdentity.Format(sourceAddress); + var accountGate = _accountGates.GetOrAdd(accountName, static _ => new SemaphoreSlim(1, 1)); + await accountGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (TryAdmitLocalHotPath(accountName, sessionId, ip, sessionLimit, srcIpLimit, rateLimitBps, out var local)) + { + return local; + } + + if (sessionLimit <= 0 && srcIpLimit <= 0 && rateLimitBps <= 0) + { + return SessionAdmissionResult.Success; + } + + var trackSessions = sessionLimit > 0 || rateLimitBps > 0; + long sessionGeneration; + long sourceGeneration; + lock (_gate) + { + _accounts.TryGetValue(accountName, out var account); + sessionGeneration = trackSessions + ? (account is { SessionGeneration: not 0 } ? account.SessionGeneration : NextGeneration()) + : 0; + sourceGeneration = srcIpLimit > 0 + && account is not null + && account.Ips.TryGetValue(ip, out var owner) + && owner is { Distributed: true, Generation: not 0 } + ? owner.Generation + : (srcIpLimit > 0 ? NextGeneration() : 0); + } + + var now = _timeProvider.GetUtcNow(); + SessionStateAdmitResult membership; + try + { + membership = await _membership.TryAdmitAsync( + accountName, + ip, + _ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + _leaseTtl, + cancellationToken, + trackSessions).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // TRY_ADMIT is atomic. Cancellation after a successful EVAL (or a + // store that throws OCE after writing) must still decrement, or + // this process has no local session that can later release. + await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!membership.Accepted) + { + if (membership.Status == SessionStateAdmitStatus.Unavailable) + { + UnavailableRejects++; + SessionStateLogMessages.SessionStateUnavailable(_logger, accountName, ip, srcIpLimit, _nodeId); + return SessionAdmissionResult.Unavailable; + } + + if (membership.Status == SessionStateAdmitStatus.RejectedSessionLimit) + { + DistributedSessionRejects++; + SessionStateLogMessages.SessionAdmissionRejected(_logger, accountName, ip, sessionLimit, _nodeId); + return SessionAdmissionResult.SessionLimitExceeded; + } + + DistributedSourceRejects++; + SessionStateLogMessages.SourceAddressRejected(_logger, accountName, ip, srcIpLimit, _nodeId); + return SessionAdmissionResult.SourceAddressLimitExceeded; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + CommitDistributedAdmit( + accountName, + sessionId, + ip, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + trackSessions); + } + catch (OperationCanceledException) + { + await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + DistributedAdmits++; + if (rateLimitBps > 0) + { + _rates.ObserveClusterSessionCount(accountName, membership.SessionTotal); + } + + SessionStateLogMessages.SessionAdmittedDistributed( + _logger, + accountName, + ip, + srcIpLimit, + _nodeId, + membership.Status.ToString()); + return SessionAdmissionResult.Success; + } + finally + { + accountGate.Release(); + } + } + + /// + public async ValueTask ReleaseAsync( + string accountName, + string sessionId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + + var accountGate = _accountGates.GetOrAdd(accountName, static _ => new SemaphoreSlim(1, 1)); + await accountGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + string? ip = null; + long sessionGeneration = 0; + long sourceGeneration = 0; + var releaseMembership = false; + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account) + || !account.Sessions.Remove(sessionId, out var session)) + { + return; + } + + ReleaseCalls++; + + ip = session.Ip; + sessionGeneration = account.SessionGeneration; + var distributedSession = account.DistributedSession; + if (account.Ips.TryGetValue(ip, out var owner)) + { + owner.Count--; + sourceGeneration = owner.Generation; + if (owner.Count <= 0) + { + account.Ips.Remove(ip); + releaseMembership = owner.Distributed || distributedSession; + } + else + { + releaseMembership = distributedSession; + } + } + else + { + releaseMembership = distributedSession; + } + + if (account.Sessions.Count == 0) + { + _accounts.Remove(accountName); + } + } + + if (releaseMembership && ip is not null) + { + var remaining = await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + cancellationToken) + .ConfigureAwait(false); + // Redis unavailable maps to 0, the same integer as "last cluster session + // released". Observing 0 while local R sessions remain treats remotes as + // gone and can raise joiners from blocked to the full account rate. + if (remaining > 0) + { + _rates.ObserveClusterSessionCount(accountName, remaining); + } + + SessionStateLogMessages.SessionStateReleased(_logger, accountName, ip, _nodeId); + } + } + finally + { + accountGate.Release(); + } + } + + /// + public async ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + List<(string Account, long SessionGeneration, List<(string Ip, long Generation)> Sources)> owners; + lock (_gate) + { + owners = []; + foreach (var (accountName, account) in _accounts) + { + List<(string Ip, long Generation)> sources = []; + foreach (var (ip, owner) in account.Ips) + { + if (owner is { Distributed: true, Count: > 0 }) + { + sources.Add((ip, owner.Generation)); + } + } + + if (!account.DistributedSession && sources.Count == 0) + { + continue; + } + + owners.Add(( + accountName, + account.DistributedSession ? account.SessionGeneration : 0, + sources)); + } + } + + var now = _timeProvider.GetUtcNow(); + foreach (var (accountName, sessionGeneration, sources) in owners) + { + cancellationToken.ThrowIfCancellationRequested(); + SessionStateRenewResult status; + try + { + status = await RenewAccountAsync( + accountName, + sessionGeneration, + sources, + now, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateLeaseRenewFailed( + _logger, + ex, + accountName, + sources.Count > 0 ? sources[0].Ip : "-", + _nodeId); + continue; + } + + if (status.Status == SessionStateRenewStatus.Renewed) + { + if (status.SessionTotal > 0) + { + _rates.ObserveClusterSessionCount(accountName, status.SessionTotal); + } + + foreach (var (ip, _) in sources) + { + MarkLeaseRenewed(accountName, ip, now); + SessionStateLogMessages.SessionStateLeaseRenewed(_logger, accountName, ip, _nodeId); + } + + continue; + } + + if (status.Status == SessionStateRenewStatus.Unavailable) + { + SessionStateLogMessages.SessionStateLeaseRenewUnavailable( + _logger, + accountName, + sources.Count > 0 ? sources[0].Ip : "-", + _nodeId); + continue; + } + + foreach (var (ip, _) in sources) + { + InvalidateHotPath(accountName, ip); + SessionStateLogMessages.SessionStateLeaseLost(_logger, accountName, ip, _nodeId); + } + } + } + + private async ValueTask RenewAccountAsync( + string accountName, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + CancellationToken cancellationToken) + { + if (_bytes is not null + && _bytes.TryGetCommittedBatch(accountName, out var batch)) + { + var combined = await _membership.RenewAndApplyAsync( + accountName, + _ownerId, + sessionGeneration, + sources, + now, + _leaseTtl, + batch.BatchId, + batch.Consumed, + batch.MysqlRemaining, + cancellationToken).ConfigureAwait(false); + if (combined.Remaining is { } remaining) + { + _bytes.CompleteApply(accountName, remaining); + } + + return new SessionStateRenewResult(combined.Renew); + } + + return await _membership.RenewAsync( + accountName, + _ownerId, + sessionGeneration, + sources, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + + /// + public async ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) + { + List accounts; + lock (_gate) + { + accounts = []; + foreach (var (accountName, account) in _accounts) + { + if (account.DistributedSession || account.Ips.Values.Any(static owner => owner.Distributed)) + { + accounts.Add(accountName); + } + } + } + + foreach (var accountName in accounts) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await _membership.ReleaseOwnerAsync(accountName, _ownerId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed(_logger, ex, accountName, _ownerId); + } + } + } + + /// Returns the local authenticated session count for . + internal int GetLocalSessionCount(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + } + } + + /// Returns the local reference count for an account/source pair. + internal int GetLocalIpCount(string accountName, string normalizedSourceIp) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(normalizedSourceIp, out var owner) + ? owner.Count + : 0; + } + } + + private bool TryAdmitLocalHotPath( + string accountName, + string sessionId, + string ip, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + out SessionAdmissionResult result) + { + lock (_gate) + { + _accounts.TryGetValue(accountName, out var account); + if (account is not null && account.Sessions.ContainsKey(sessionId)) + { + result = SessionAdmissionResult.Success; + return true; + } + + if (sessionLimit <= 0 && srcIpLimit <= 0 && rateLimitBps <= 0) + { + account ??= GetOrCreateAccount(accountName); + account.Sessions[sessionId] = new SessionAdmission(ip); + if (!account.Ips.TryGetValue(ip, out var unlimited)) + { + unlimited = new IpOwner(); + account.Ips[ip] = unlimited; + } + + unlimited.Count++; + result = SessionAdmissionResult.Success; + return true; + } + + // A finite session limit or positive rate share consumes a cluster-wide + // slot; the source-IP hot path must not skip Redis. + if (sessionLimit > 0 || rateLimitBps > 0) + { + result = SessionAdmissionResult.Success; + return false; + } + + var now = _timeProvider.GetUtcNow(); + if (account is not null + && account.Ips.TryGetValue(ip, out var owner) + && owner.Distributed + && owner.HotPathValid + && owner.LeaseExpiresAt - _hotPathSkew > now) + { + owner.Count++; + owner.SrcIpLimit = srcIpLimit; + account.Sessions[sessionId] = new SessionAdmission(ip); + LocalHotPathAdmits++; + SessionStateLogMessages.SessionAdmittedLocal(_logger, accountName, ip, srcIpLimit, _nodeId); + result = SessionAdmissionResult.Success; + return true; + } + + if (account is not null && account.Sessions.Count == 0 && account.Ips.Count == 0) + { + _accounts.Remove(accountName); + } + + result = SessionAdmissionResult.Success; + return false; + } + } + + private void CommitDistributedAdmit( + string accountName, + string sessionId, + string ip, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + bool trackSessions) + { + lock (_gate) + { + var account = GetOrCreateAccount(accountName); + if (account.Sessions.ContainsKey(sessionId)) + { + return; + } + + if (sessionLimit > 0 || trackSessions) + { + account.DistributedSession = true; + account.SessionGeneration = sessionGeneration; + } + + if (!account.Ips.TryGetValue(ip, out var owner)) + { + owner = new IpOwner(); + account.Ips[ip] = owner; + } + + owner.Count++; + if (srcIpLimit > 0) + { + owner.Generation = sourceGeneration; + owner.SrcIpLimit = srcIpLimit; + owner.Distributed = true; + owner.HotPathValid = sessionLimit <= 0; + owner.LeaseExpiresAt = now + _leaseTtl; + } + + account.Sessions[sessionId] = new SessionAdmission(ip); + } + } + + private AccountAdmission GetOrCreateAccount(string accountName) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + account = new AccountAdmission(); + _accounts[accountName] = account; + } + + return account; + } + + private void MarkLeaseRenewed(string accountName, string ip, DateTimeOffset now) + { + lock (_gate) + { + if (_accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(ip, out var owner) + && owner.Distributed) + { + owner.LeaseExpiresAt = now + _leaseTtl; + owner.HotPathValid = !account.DistributedSession; + } + } + } + + private void InvalidateHotPath(string accountName, string ip) + { + lock (_gate) + { + if (_accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(ip, out var owner)) + { + owner.HotPathValid = false; + owner.LeaseExpiresAt = DateTimeOffset.MinValue; + } + } + } + + private long NextGeneration() => Interlocked.Increment(ref _generations); + + private sealed class AccountAdmission + { + public Dictionary Sessions { get; } = new(StringComparer.Ordinal); + + public Dictionary Ips { get; } = new(StringComparer.Ordinal); + + public bool DistributedSession { get; set; } + + public long SessionGeneration { get; set; } + } + + private sealed class SessionAdmission + { + public SessionAdmission(string ip) + { + Ip = ip; + } + + public string Ip { get; } + } + + private sealed class IpOwner + { + public int Count { get; set; } + + public long Generation { get; set; } + + public int SrcIpLimit { get; set; } + + public bool Distributed { get; set; } + + public bool HotPathValid { get; set; } + + public DateTimeOffset LeaseExpiresAt { get; set; } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/ISessionStateLeaseManager.cs b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateLeaseManager.cs new file mode 100644 index 00000000..4d3f1420 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateLeaseManager.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Process-local lease renewal and shutdown release for cluster session-state ownership. + /// + /// + /// Renewal proves this node is still responsible for its authenticated sessions. + /// It is independent of NNTP traffic. One renew operation per account extends + /// session and source-IP ownership together, or extends none of them. + /// The background is the sole scheduler; this + /// abstraction owns the actual ownership snapshot and Redis operations. When a + /// MySQL-committed byte batch exists for an owned account, renewal and APPLY + /// share one Redis EVAL. A successful renew also returns the cluster session + /// total so local rate caps can be updated without a second scheduler. + /// + public interface ISessionStateLeaseManager + { + /// + /// Refreshes this incarnation's session and source-IP leases for accounts that + /// still have local authenticated sessions. Does not recreate missing ownership. + /// + ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default); + + /// Releases leftover node/incarnation ownership without touching the database. + ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/ISessionStateStore.cs b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateStore.cs new file mode 100644 index 00000000..787bcbbb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateStore.cs @@ -0,0 +1,77 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Cluster-wide account session and source-IP membership. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Admit, release, and renew each use one atomic operation. + /// + public interface ISessionStateStore + { + /// + /// Atomically admits one session and this owner's source-IP membership. + /// + ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false); + + /// + /// Atomically decrements this owner's session count and source-IP count. + /// Returns the remaining cluster session total. Unavailable membership + /// returns 0; callers must not treat that as a live total. + /// + ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's session and source-IP leases. + /// All requested fields are extended or none are. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's session/source leases and APPLYs one + /// MySQL-committed byte batch for the same account. APPLY runs even when + /// renewal is lost. + /// is when this store cannot apply byte state. + /// + ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default); + + /// Deletes leftover ownership for this process incarnation. + ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/ISessionStateTracker.cs b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateTracker.cs new file mode 100644 index 00000000..9614d4bc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/ISessionStateTracker.cs @@ -0,0 +1,47 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Atomically admits and releases concurrent authenticated reader sessions per account. + /// + /// + /// session_limit 0 and srcip_limit 0 are unlimited. + /// session_limit is the cluster-wide cap on authenticated sessions. + /// srcip_limit is a cluster-wide cap on distinct source IPs that currently + /// have at least one admitted session, not a per-IP session cap. Production uses + /// : both limits are enforced + /// atomically through Redis ownership leases. + /// + public interface ISessionStateTracker + { + /// + /// Attempts to admit for . + /// Re-admitting the same session id is idempotent. + /// + ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default); + + /// + /// Admits a session and, when (bits per second) + /// is greater than zero, tracks the cluster session count for rate allocation. + /// + ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) => + TryAdmitAsync(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit, cancellationToken); + + /// Releases a previously admitted session. Idempotent when the session is unknown. + ValueTask ReleaseAsync(string accountName, string sessionId, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateStore.cs b/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateStore.cs new file mode 100644 index 00000000..e6fa0185 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateStore.cs @@ -0,0 +1,256 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Process-local cluster fake for session + source-IP membership. Two trackers + /// sharing one instance behave as two NNTP nodes. Ordinary unit tests must not + /// require live Redis. + /// + internal sealed class InMemorySessionStateStore : ISessionStateStore + { + private readonly SessionStateEngine _engine = new(); + + /// When set, admit/release/renew fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Optional gate that blocks the next admit until completed or cancelled. + public TaskCompletionSource? BlockAdmit { get; set; } + + /// Optional gate that blocks the next release until completed or cancelled. + public TaskCompletionSource? BlockRelease { get; set; } + + /// Optional gate that blocks the next renew until completed or cancelled. + public TaskCompletionSource? BlockRenew { get; set; } + + /// Set when is entered, before waits. + public TaskCompletionSource? NotifyRenewStarted { get; set; } + + /// When set, throws for this account so other accounts can still renew. + public string? FaultRenewAccount { get; set; } + + /// Gets how many membership renew operations were attempted. + public int RenewCalls { get; private set; } + + /// Gets the shared admission engine (tests). + internal SessionStateEngine Engine => _engine; + + /// In-memory source hash identity for . + internal static string SourceKey(string accountName) => "src:" + accountName; + + /// In-memory session hash identity for . + internal static string SessionKey(string accountName) => "sess:" + accountName; + + /// Returns whether currently holds . + internal bool HasSourceOwner(string accountName, string normalizedSourceIp, string ownerId, long nowUnixMs) => + _engine.HasSourceOwner(SourceKey(accountName), normalizedSourceIp, ownerId, nowUnixMs); + + /// Returns the cluster-wide authenticated session count for . + internal int ActiveSessionCount(string accountName, long nowUnixMs) => + _engine.ActiveSessionCount(SessionKey(accountName), nowUnixMs); + + /// Returns this owner's unexpired session count. + internal int OwnerSessionCount(string accountName, string ownerId, long nowUnixMs) => + _engine.OwnerSessionCount(SessionKey(accountName), ownerId, nowUnixMs); + + /// Returns distinct unexpired source IPs for . + internal IReadOnlyCollection ActiveSourceIps(string accountName, long nowUnixMs) => + _engine.ActiveSourceIps(SourceKey(accountName), nowUnixMs); + + /// Returns the stored session-ownership expiry, or when absent. + internal long? SessionExpiry(string accountName, string ownerId) => + _engine.TryGetOwnership(SessionKey(accountName), ownerId, out var expiry, out _, out _) + ? expiry + : null; + + /// Returns the stored source-ownership expiry, or when absent. + internal long? SourceExpiry(string accountName, string normalizedSourceIp, string ownerId) => + _engine.TryGetOwnership( + SourceKey(accountName), + SessionStateKeys.SourceField(normalizedSourceIp, ownerId), + out var expiry, + out _, + out _) + ? expiry + : null; + + /// Changes a source field's generation so a later renew is all-or-nothing rejected. + internal void BreakSourceGeneration(string accountName, string normalizedSourceIp, string ownerId) + { + var field = SessionStateKeys.SourceField(normalizedSourceIp, ownerId); + if (!_engine.TryGetOwnership(SourceKey(accountName), field, out var expiry, out var generation, out var count)) + { + throw new InvalidOperationException("Source ownership is missing."); + } + + _engine.WriteOwnership(SourceKey(accountName), field, expiry, generation + 1, count); + } + + /// + public async ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (BlockAdmit is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return new SessionStateAdmitResult(SessionStateAdmitStatus.Unavailable); + } + + var code = _engine.TryAdmit( + SourceKey(accountName), + SessionKey(accountName), + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + sessionGeneration, + sourceGeneration, + trackSessions); + var status = SessionStateEngine.UnpackAdmitStatus(code); + var total = SessionStateEngine.UnpackAdmitSessionTotal(code); + return status switch + { + SessionStateEngine.AcceptedExisting => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedExisting, total), + SessionStateEngine.AcceptedNew => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedNew, total), + SessionStateEngine.RejectedSessionLimit => + new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSessionLimit), + _ => new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSourceLimit), + }; + } + + /// + public async ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (BlockRelease is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return 0; + } + + return (int)_engine.Release( + SourceKey(accountName), + SessionKey(accountName), + normalizedSourceIp, + ownerId, + sessionGeneration, + sourceGeneration); + } + + /// + public async ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentNullException.ThrowIfNull(sources); + if (FaultRenewAccount is { } fault + && string.Equals(fault, accountName, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Injected renewal fault."); + } + + NotifyRenewStarted?.TrySetResult(); + if (BlockRenew is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + RenewCalls++; + if (Unavailable) + { + return new SessionStateRenewResult(SessionStateRenewStatus.Unavailable); + } + + var result = _engine.Renew( + SourceKey(accountName), + SessionKey(accountName), + ownerId, + sessionGeneration, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + sources); + return result == 0 + ? new SessionStateRenewResult(SessionStateRenewStatus.Lost) + : new SessionStateRenewResult(SessionStateRenewStatus.Renewed, (int)result); + } + + /// + public async ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + _ = batchId; + _ = consumed; + _ = mysqlRemainingAfter; + var status = await RenewAsync( + accountName, + ownerId, + sessionGeneration, + sources, + now, + leaseTtl, + cancellationToken).ConfigureAwait(false); + return new SessionStateRenewAndApplyResult(status.Status, remaining: null); + } + + /// + public ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (!Unavailable) + { + _ = _engine.ReleaseOwner(SourceKey(accountName), SessionKey(accountName), ownerId); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateTracker.cs b/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateTracker.cs new file mode 100644 index 00000000..31034792 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/InMemorySessionStateTracker.cs @@ -0,0 +1,175 @@ +using System.Net; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Process-local atomic admission tracker used by unit tests. Production cluster + /// session and source-IP admission uses . + /// + public sealed class InMemorySessionStateTracker : ISessionStateTracker + { + private readonly object _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + private readonly IAccountRateAllocator _rates; + + /// Initializes a process-local tracker. + public InMemorySessionStateTracker(IAccountRateAllocator? rates = null) + { + _rates = rates ?? NullAccountRateAllocator.Instance; + } + + /// Gets how many admitted sessions this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask( + TryAdmit(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit)); + } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = TryAdmit(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit); + if (result == SessionAdmissionResult.Success && rateLimitBps > 0) + { + _rates.ObserveClusterSessionCount(accountName, GetLocalSessionCount(accountName)); + } + + return new ValueTask(result); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + string sessionId, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Release(accountName, sessionId); + return ValueTask.CompletedTask; + } + + /// Synchronous admit used by existing in-process tests. + public SessionAdmissionResult TryAdmit( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(sourceAddress); + + var ip = SourceAddressIdentity.Format(sourceAddress); + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + account = new AccountAdmission(); + _accounts[accountName] = account; + } + + if (account.Sessions.TryGetValue(sessionId, out var existingIp)) + { + _ = existingIp; + return SessionAdmissionResult.Success; + } + + if (sessionLimit > 0 && account.Sessions.Count >= sessionLimit) + { + return SessionAdmissionResult.SessionLimitExceeded; + } + + if (srcIpLimit > 0 + && !account.IpCounts.ContainsKey(ip) + && account.IpCounts.Count >= srcIpLimit) + { + return SessionAdmissionResult.SourceAddressLimitExceeded; + } + + account.Sessions[sessionId] = ip; + account.IpCounts[ip] = account.IpCounts.GetValueOrDefault(ip) + 1; + return SessionAdmissionResult.Success; + } + } + + /// + public void Release(string accountName, string sessionId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + if (!account.Sessions.Remove(sessionId, out var ip)) + { + return; + } + + ReleaseCalls++; + + if (account.IpCounts.TryGetValue(ip, out var count)) + { + if (count <= 1) + { + account.IpCounts.Remove(ip); + } + else + { + account.IpCounts[ip] = count - 1; + } + } + + if (account.Sessions.Count == 0) + { + _accounts.Remove(accountName); + _rates.ObserveClusterSessionCount(accountName, 0); + } + else + { + _rates.ObserveClusterSessionCount(accountName, account.Sessions.Count); + } + } + } + + private int GetLocalSessionCount(string accountName) + { + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + } + } + + /// Formats the admitted source identity. IPv4-mapped IPv6 becomes IPv4 text. + internal static string FormatSourceAddress(IPAddress address) => SourceAddressIdentity.Format(address); + + private sealed class AccountAdmission + { + public Dictionary Sessions { get; } = new(StringComparer.Ordinal); + public Dictionary IpCounts { get; } = new(StringComparer.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateAllocator.cs b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateAllocator.cs new file mode 100644 index 00000000..a244b7f3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateAllocator.cs @@ -0,0 +1,239 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.ReaderService.SessionState.RateLimiting +{ + /// + /// Process-local map of account sessions to outbound caps. Cluster session + /// counts come from SessionState admit/release/renew; the write path never + /// consults this type. + /// + /// + /// A cross-node join would overshoot if the new session took an equal share + /// while remotes still held floor(rate / previousCount). The unused + /// remainder accountBytes % remotes is safe only when every remote is + /// that previous equal-share group. A second joining node sees remotes that + /// include the first joiner and would take another remainder while the + /// established node still holds nearly the whole aggregate. While remotes + /// exist, new sessions are blocked. Established local sessions may only drop + /// to a budget that assumes previously split remotes still hold this node's + /// last per-session cap and newly observed remotes hold 0. Equal shares + /// resume only when this node owns every remaining session. + /// + internal sealed class AccountRateAllocator : IAccountRateAllocator + { + private readonly ConcurrentDictionary _clusterCounts = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _accounts = new(StringComparer.Ordinal); + + /// Initializes a process-local allocator. + public AccountRateAllocator(TimeProvider? timeProvider = null, TimeSpan? equalSplitHold = null) + { + _ = timeProvider; + _ = equalSplitHold; + } + + /// + public void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(cap); + if (rateBps <= 0) + { + cap.UpdateMaxSendBytesPerSecond(0); + return; + } + + var account = _accounts.GetOrAdd(accountName, static _ => new AccountRates()); + lock (account.Gate) + { + account.RateBps = rateBps; + account.Sessions[sessionId] = cap; + if (_clusterCounts.TryGetValue(accountName, out var observed) && observed > 0) + { + account.ClusterCount = observed; + } + + Apply(account); + } + } + + /// + public void Unregister(string accountName, string sessionId) + { + if (string.IsNullOrWhiteSpace(accountName) + || string.IsNullOrWhiteSpace(sessionId) + || !_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + lock (account.Gate) + { + account.Sessions.TryRemove(sessionId, out _); + if (account.Sessions.IsEmpty) + { + _accounts.TryRemove(accountName, out _); + return; + } + + Apply(account); + } + } + + /// + public void ObserveClusterSessionCount(string accountName, int clusterSessionCount) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + var count = clusterSessionCount < 0 ? 0 : clusterSessionCount; + _clusterCounts[accountName] = count; + if (!_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + lock (account.Gate) + { + account.ClusterCount = count; + Apply(account); + } + } + + /// + public long? GetSessionCap(string accountName, string sessionId) + { + if (string.IsNullOrWhiteSpace(accountName) + || string.IsNullOrWhiteSpace(sessionId) + || !_accounts.TryGetValue(accountName, out var account)) + { + return null; + } + + lock (account.Gate) + { + return account.Sessions.TryGetValue(sessionId, out var cap) + ? cap.MaxSendBytesPerSecond + : null; + } + } + + /// Returns the last observed cluster count (tests). + internal int ClusterCount(string accountName) => + _clusterCounts.TryGetValue(accountName, out var count) ? count : 0; + + /// Returns how many local sessions are registered (tests). + internal int LocalSessionCount(string accountName) => + _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + + private void Apply(AccountRates account) + { + var rate = account.RateBps; + if (rate <= 0) + { + return; + } + + var count = account.ClusterCount; + if (count <= 0) + { + count = account.Sessions.Count; + } + + var local = account.Sessions.Count; + if (local == 0) + { + return; + } + + var remotes = count - local; + var target = AccountRateFormula.EqualShareCap(rate, count); + if (remotes <= 0) + { + foreach (var pair in account.Sessions) + { + pair.Value.UpdateMaxSendBytesPerSecond(target); + } + + account.LastCount = count; + account.LastLocal = local; + return; + } + + var accountBytes = AccountRateFormula.AccountBytesPerSecond(rate); + long remoteHold; + if (account.LastCount <= 0) + { + var remoteShare = AccountRateFormula.PerSessionBytesPerSecond(rate, remotes); + remoteHold = checked(remoteShare * remotes); + } + else + { + var oldRemotes = account.LastCount - account.LastLocal; + if (oldRemotes < 0) + { + oldRemotes = 0; + } + + var oldShare = 0L; + // codeql[cs/linq/missed-select] Allocation-free max scan; Select would materialize caps. + foreach (var pair in account.Sessions) + { + var cap = pair.Value.MaxSendBytesPerSecond; + if (cap > oldShare) + { + oldShare = cap; + } + } + + if (oldShare <= 0) + { + oldShare = AccountRateFormula.PerSessionBytesPerSecond(rate, account.LastCount); + } + + remoteHold = checked(oldShare * oldRemotes); + } + + var localBudget = accountBytes - remoteHold; + var perLocal = localBudget > 0 ? localBudget / local : 0; + foreach (var pair in account.Sessions) + { + var current = pair.Value.MaxSendBytesPerSecond; + if (current <= 0) + { + pair.Value.UpdateMaxSendBytesPerSecond(AccountRateFormula.BlockedBytesPerSecond); + continue; + } + + var safe = target; + if (perLocal < safe) + { + safe = perLocal; + } + + if (current < safe) + { + safe = current; + } + + pair.Value.UpdateMaxSendBytesPerSecond( + safe > 0 ? safe : AccountRateFormula.BlockedBytesPerSecond); + } + + account.LastCount = count; + account.LastLocal = local; + } + + private sealed class AccountRates + { + public object Gate { get; } = new(); + public int RateBps; + public int ClusterCount; + public int LastCount; + public int LastLocal; + public ConcurrentDictionary Sessions { get; } = new(StringComparer.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateFormula.cs b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateFormula.cs new file mode 100644 index 00000000..501fdc78 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/AccountRateFormula.cs @@ -0,0 +1,107 @@ +namespace VectorNNTPD.ReaderService.SessionState.RateLimiting +{ + /// + /// Converts account_rate_limit (bits per second) to integer bytes/sec and + /// divides that aggregate by the cluster-wide active session count. + /// + /// + /// account_rate_limit = 0 is unlimited (existing policy). Conversion is + /// floor(bps / 8). Division uses integer floor so + /// sum(per-session caps) <= account bytes/sec. + /// The denominator is the live active session count, never + /// account_session_limit. A positive configured rate that floors to + /// 0 bytes/sec is , not unlimited. + /// + public static class AccountRateFormula + { + /// Bits in one byte. Database account_rate_limit is bits/sec. + public const int BitsPerByte = 8; + + /// + /// Account-wide cap in bytes/sec: floor(rateBps / 8). + /// <= 0 is unlimited and returns 0. + /// A positive rate below 8 bps returns 0 here; callers that apply a + /// limiter cap must use so that is blocked. + /// + public static long AccountBytesPerSecond(int rateBps) + { + if (rateBps <= 0) + { + return 0; + } + + // int.MaxValue / 8 fits in int and long; no Mbps multiplier remains. + return (long)rateBps / BitsPerByte; + } + + /// + /// Per-session cap in bytes/sec: floor(accountBytesPerSec / activeSessions). + /// Returns 0 (unlimited / no allocation) when the account rate is unlimited + /// or there are no active sessions. A positive rate that floors to 0 is + /// not unlimited; use on the write path. + /// + public static long PerSessionBytesPerSecond(int rateBps, int activeSessions) + { + if (rateBps <= 0 || activeSessions <= 0) + { + return 0; + } + + return AccountBytesPerSecond(rateBps) / activeSessions; + } + + /// + /// Equal-share cap for a positive rate. A floor of 0 is + /// so the limiter cannot treat it as unlimited. + /// + public static long EqualShareCap(int rateBps, int activeSessions) + { + if (rateBps <= 0) + { + return 0; + } + + var perSession = PerSessionBytesPerSecond(rateBps, activeSessions); + return perSession > 0 ? perSession : BlockedBytesPerSecond; + } + + /// + /// Returns whether perSession * activeSessions stays at or under the account cap. + /// + public static bool AggregateDoesNotExceed(int rateBps, int activeSessions) + { + if (rateBps <= 0 || activeSessions <= 0) + { + return true; + } + + var perSession = PerSessionBytesPerSecond(rateBps, activeSessions); + return perSession * activeSessions <= AccountBytesPerSecond(rateBps); + } + + /// + /// Bytes/sec a newly admitted session may take while remotes may still hold + /// floor(account / previousSessions). This is the unused remainder + /// after that previous equal split. 0 means no residual; callers must + /// not treat it as unlimited. + /// + public static long ConservativeJoinBytesPerSecond(int rateBps, int previousActiveSessions) + { + if (rateBps <= 0 || previousActiveSessions <= 0) + { + return 0; + } + + return AccountBytesPerSecond(rateBps) % previousActiveSessions; + } + + /// + /// Negative sentinel: block writes until the cap is replaced. Distinct from + /// 0, which remains unlimited. + /// + public const long BlockedBytesPerSecond = -1; + + /// Counts a stored cap toward a cluster aggregate. Blocked contributes 0; unlimited is 0. + public static long AllocatedBytesPerSecond(long cap) => cap > 0 ? cap : 0; + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IAccountRateAllocator.cs b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IAccountRateAllocator.cs new file mode 100644 index 00000000..7db3715e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IAccountRateAllocator.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.ReaderService.SessionState.RateLimiting +{ + /// + /// Process-local account rate allocation. SessionState observes the cluster + /// session count; this type updates every registered local session cap. + /// + public interface IAccountRateAllocator + { + /// + /// Registers a connected session. is + /// account_rate_limit in bits per second, not Mbps. Applies the last + /// observed cluster count, or one session when none has been observed yet. + /// + void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps); + + /// Drops a session. Remaining local sessions keep the last observed count. + void Unregister(string accountName, string sessionId); + + /// + /// Applies the cluster session total to local caps. Existing local sessions + /// that already hold a positive equal-share cap may only drop to a share + /// that fits accountBytes − remotes × floor(accountBytes / remotes). + /// A newly admitted session on a node that does not own every session is + /// blocked. Equal shares resume when this node owns every remaining session. + /// + void ObserveClusterSessionCount(string accountName, int clusterSessionCount); + + /// Returns the current cap for a registered session (tests). + long? GetSessionCap(string accountName, string sessionId); + } + + /// No-op allocator for hosts and tests that do not attach rate limiters. + public sealed class NullAccountRateAllocator : IAccountRateAllocator + { + /// Shared no-op instance. + public static NullAccountRateAllocator Instance { get; } = new(); + + private NullAccountRateAllocator() + { + } + + /// + public void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps) + { + } + + /// + public void Unregister(string accountName, string sessionId) + { + } + + /// + public void ObserveClusterSessionCount(string accountName, int clusterSessionCount) + { + } + + /// + public long? GetSessionCap(string accountName, string sessionId) => null; + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IOutboundRateCap.cs b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IOutboundRateCap.cs new file mode 100644 index 00000000..71b9efff --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/IOutboundRateCap.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.ReaderService.SessionState.RateLimiting +{ + /// Dynamically adjustable outbound bytes-per-second cap. + public interface IOutboundRateCap + { + /// + /// Gets the current cap. 0 is unlimited. < 0 blocks writes + /// until a later update (not unlimited). + /// + long MaxSendBytesPerSecond { get; } + + /// Replaces the cap. Existing and future writes observe the new value. + void UpdateMaxSendBytesPerSecond(long bytesPerSecond); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/OutboundRateLimiter.cs b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/OutboundRateLimiter.cs new file mode 100644 index 00000000..d979ef8d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RateLimiting/OutboundRateLimiter.cs @@ -0,0 +1,199 @@ +namespace VectorNNTPD.ReaderService.SessionState.RateLimiting +{ + /// + /// Stream wrapper that throttles writes to a dynamically adjustable bytes/sec cap. + /// Reads, flushes, and cap == 0 are passthrough. cap < 0 blocks + /// until the cap is updated. + /// + /// + /// Adapted from Vector.NNTP DynamicSendRateLimitedStream: one-second write window, + /// 25 ms backoff, live during writes. + /// Writes are serialized. Cancellation aborts the backoff. Dispose does not wait + /// for an in-flight write; the write token must be cancelled by the connection. + /// + public sealed class OutboundRateLimiter : Stream, IOutboundRateCap + { + private readonly Stream _inner; + private readonly bool _leaveInnerOpen; + private readonly TimeProvider _time; + private readonly SemaphoreSlim _writeGate = new(1, 1); + private long _maxSendBytesPerSecond; + private long _windowStartMs; + private long _bytesInWindow; + private int _disposed; + + /// Initializes a limiter over . + public OutboundRateLimiter( + Stream inner, + long initialMaxSendBytesPerSecond, + bool leaveInnerOpen, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + _leaveInnerOpen = leaveInnerOpen; + _time = timeProvider ?? TimeProvider.System; + _maxSendBytesPerSecond = initialMaxSendBytesPerSecond; + _windowStartMs = NowMs(); + } + + /// + public override bool CanRead => _inner.CanRead; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => _inner.CanWrite; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public long MaxSendBytesPerSecond => Interlocked.Read(ref _maxSendBytesPerSecond); + + /// + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => + Interlocked.Exchange(ref _maxSendBytesPerSecond, bytesPerSecond); + + /// + public override void Flush() => _inner.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _inner.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + _inner.ReadAsync(buffer, cancellationToken); + + /// + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + /// + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var cap = Interlocked.Read(ref _maxSendBytesPerSecond); + if (buffer.IsEmpty || cap == 0) + { + await _inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + return; + } + + await _writeGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var offset = 0; + while (offset < buffer.Length) + { + cap = Interlocked.Read(ref _maxSendBytesPerSecond); + if (cap == 0) + { + await _inner.WriteAsync(buffer[offset..], cancellationToken).ConfigureAwait(false); + return; + } + + if (cap < 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(25), _time, cancellationToken) + .ConfigureAwait(false); + continue; + } + + var sliceBytes = buffer.Length - offset; + if (sliceBytes > cap) + { + sliceBytes = cap > int.MaxValue ? int.MaxValue : (int)cap; + } + + await ThrottleAsync(sliceBytes, cap, cancellationToken).ConfigureAwait(false); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + await _inner.WriteAsync(buffer.Slice(offset, sliceBytes), cancellationToken) + .ConfigureAwait(false); + _bytesInWindow += sliceBytes; + offset += sliceBytes; + } + } + finally + { + _writeGate.Release(); + } + } + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writeGate.Dispose(); + if (!_leaveInnerOpen) + { + _inner.Dispose(); + } + } + + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writeGate.Dispose(); + if (!_leaveInnerOpen) + { + await _inner.DisposeAsync().ConfigureAwait(false); + } + } + + await base.DisposeAsync().ConfigureAwait(false); + } + + private async Task ThrottleAsync(int nextBytes, long cap, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var now = NowMs(); + if (now - _windowStartMs >= 1000) + { + _windowStartMs = now; + _bytesInWindow = 0; + } + + if (_bytesInWindow + nextBytes <= cap) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(25), _time, cancellationToken).ConfigureAwait(false); + } + } + + private long NowMs() => _time.GetUtcNow().ToUnixTimeMilliseconds(); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/RedisSessionStateStore.cs b/src/VectorNNTPD.ReaderService/SessionState/RedisSessionStateStore.cs new file mode 100644 index 00000000..8b7393ec --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/RedisSessionStateStore.cs @@ -0,0 +1,256 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Redis-backed cluster session + source-IP membership using one atomic Lua EVAL + /// on the shared connection. + /// + internal sealed class RedisSessionStateStore : ISessionStateStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisSessionStateStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) => + ExecuteAsync( + SessionStateScripts.TryAdmit, + accountName, + MembershipKeys(accountName), + [ + Utf8(normalizedSourceIp), + Utf8(ownerId), + Utf8(sessionLimit.ToString(CultureInfo.InvariantCulture)), + Utf8(srcIpLimit.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(sourceGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(trackSessions ? "1" : "0"), + ], + result => + { + var status = SessionStateEngine.UnpackAdmitStatus(result); + var total = SessionStateEngine.UnpackAdmitSessionTotal(result); + return status switch + { + SessionStateEngine.AcceptedExisting => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedExisting, total), + SessionStateEngine.AcceptedNew => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedNew, total), + SessionStateEngine.RejectedSessionLimit => + new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSessionLimit), + _ => new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSourceLimit), + }; + }, + new SessionStateAdmitResult(SessionStateAdmitStatus.Unavailable), + cancellationToken); + + /// + public ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) => + ExecuteAsync( + SessionStateScripts.Release, + accountName, + MembershipKeys(accountName), + [ + Utf8(normalizedSourceIp), + Utf8(ownerId), + Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(sourceGeneration.ToString(CultureInfo.InvariantCulture)), + ], + static result => result < 0 ? 0 : (int)result, + fallback: 0, + cancellationToken); + + /// + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + var values = new ReadOnlyMemory[5 + (sources.Count * 2)]; + values[0] = Utf8(ownerId); + values[1] = Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)); + values[2] = Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)); + values[3] = Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)); + values[4] = Utf8(sources.Count.ToString(CultureInfo.InvariantCulture)); + for (var i = 0; i < sources.Count; i++) + { + values[5 + (i * 2)] = Utf8(sources[i].Ip); + values[6 + (i * 2)] = Utf8(sources[i].Generation.ToString(CultureInfo.InvariantCulture)); + } + + return ExecuteAsync( + SessionStateScripts.Renew, + accountName, + MembershipKeys(accountName), + values, + result => result == 0 + ? new SessionStateRenewResult(SessionStateRenewStatus.Lost) + : new SessionStateRenewResult(SessionStateRenewStatus.Renewed, (int)result), + new SessionStateRenewResult(SessionStateRenewStatus.Unavailable), + cancellationToken); + } + + /// + public ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + var values = new ReadOnlyMemory[8 + (sources.Count * 2)]; + values[0] = Utf8(ownerId); + values[1] = Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)); + values[2] = Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)); + values[3] = Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)); + values[4] = Utf8(sources.Count.ToString(CultureInfo.InvariantCulture)); + for (var i = 0; i < sources.Count; i++) + { + values[5 + (i * 2)] = Utf8(sources[i].Ip); + values[6 + (i * 2)] = Utf8(sources[i].Generation.ToString(CultureInfo.InvariantCulture)); + } + + values[5 + (sources.Count * 2)] = Utf8(batchId); + values[6 + (sources.Count * 2)] = Utf8( + AccountByteEngine.ClampNonNegative(consumed).ToString(CultureInfo.InvariantCulture)); + values[7 + (sources.Count * 2)] = Utf8( + AccountByteEngine.ClampNonNegative(mysqlRemainingAfter) + .ToString(CultureInfo.InvariantCulture)); + + return ExecuteAsync( + SessionStateScripts.RenewAndApply, + accountName, + CombinedKeys(accountName), + values, + packed => + { + var (status, remaining) = SessionStateBytePack.Decode(packed); + return new SessionStateRenewAndApplyResult(status, remaining); + }, + new SessionStateRenewAndApplyResult(SessionStateRenewStatus.Unavailable, remaining: null), + cancellationToken); + } + + /// + public async ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + SessionStateScripts.ReleaseOwner, + accountName, + MembershipKeys(accountName), + [Utf8(ownerId)], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + keys, + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + } + + private static ReadOnlyMemory[] MembershipKeys(string accountName) => + [SessionStateKeys.CreateSource(accountName), SessionStateKeys.CreateSession(accountName)]; + + private static ReadOnlyMemory[] CombinedKeys(string accountName) => + [ + SessionStateKeys.CreateSource(accountName), + SessionStateKeys.CreateSession(accountName), + AccountByteKeys.Create(accountName), + ]; + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionAdmissionResult.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionAdmissionResult.cs new file mode 100644 index 00000000..b9850a7c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionAdmissionResult.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// Outcome of authenticated-session admission. + public enum SessionAdmissionResult + { + /// The session was admitted, or admission was skipped because both limits are 0. + Success, + + /// account_session_limit would be exceeded. Wire: 481 Too many sessions. + SessionLimitExceeded, + + /// + /// A new distinct source address would exceed account_srcip_limit. + /// Wire: 481 Too many source addresses. + /// + SourceAddressLimitExceeded, + + /// + /// Cluster membership could not be established. New admission that requires Redis + /// fails closed. Wire: 503 Temporary authentication failure. + /// + Unavailable, + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateAdmitResult.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateAdmitResult.cs new file mode 100644 index 00000000..18ea0268 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateAdmitResult.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// Cluster membership outcome for one combined session/source-IP admit attempt. + public enum SessionStateAdmitStatus + { + /// Admitted; the source IP was already active. + AcceptedExisting, + + /// Admitted; a new distinct source IP was established. + AcceptedNew, + + /// Rejected: cluster account_session_limit would be exceeded. + RejectedSessionLimit, + + /// Rejected: cluster account_srcip_limit would be exceeded. + RejectedSourceLimit, + + /// Redis (or the membership store) was unavailable. Fail closed. + Unavailable, + } + + /// Result of a cluster session/source-IP admit attempt. + public readonly struct SessionStateAdmitResult + { + /// Initializes a membership admit result. + public SessionStateAdmitResult(SessionStateAdmitStatus status, int sessionTotal = 0) + { + Status = status; + SessionTotal = sessionTotal < 0 ? 0 : sessionTotal; + } + + /// Gets the membership outcome. + public SessionStateAdmitStatus Status { get; } + + /// + /// Cluster-wide unexpired session count after this attempt (0 when rejected + /// or unavailable). + /// + public int SessionTotal { get; } + + /// Gets whether membership was granted. + public bool Accepted => + Status is SessionStateAdmitStatus.AcceptedExisting or SessionStateAdmitStatus.AcceptedNew; + } + + /// Outcome of a node ownership lease renewal. + public enum SessionStateRenewStatus + { + /// This owner's requested leases were refreshed. + Renewed, + + /// At least one requested ownership field was gone or mismatched. + Lost, + + /// The membership store could not be reached. + Unavailable, + } + + /// Renewal outcome plus the cluster session total used for rate allocation. + public readonly struct SessionStateRenewResult + { + /// Initializes a renewal result. + public SessionStateRenewResult(SessionStateRenewStatus status, int sessionTotal = 0) + { + Status = status; + SessionTotal = sessionTotal < 0 ? 0 : sessionTotal; + } + + /// Gets the renewal outcome. + public SessionStateRenewStatus Status { get; } + + /// Unexpired cluster session count when is Renewed. + public int SessionTotal { get; } + + /// Converts a status-only result (session total 0). + public static implicit operator SessionStateRenewResult(SessionStateRenewStatus status) => + new(status); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateBytePack.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateBytePack.cs new file mode 100644 index 00000000..c6724b96 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateBytePack.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Encodes combined RENEW+APPLY into one Redis integer. Remaining is always + /// non-negative; a lost renewal is -remaining - 1. + /// + internal static class SessionStateBytePack + { + /// Packs renew status and remaining for a combined EVAL return. + public static long Encode(bool renewed, long remaining) + { + if (remaining < 0) + { + remaining = 0; + } + + return renewed ? remaining : -remaining - 1; + } + + /// Unpacks a combined EVAL integer into renew status and remaining. + public static (SessionStateRenewStatus Status, long Remaining) Decode(long packed) + { + if (packed >= 0) + { + return (SessionStateRenewStatus.Renewed, packed); + } + + var remaining = -packed - 1; + return (SessionStateRenewStatus.Lost, remaining < 0 ? 0 : remaining); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateDefaults.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateDefaults.cs new file mode 100644 index 00000000..23fff6af --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateDefaults.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// Lease and hot-path constants for cluster-wide session and source-IP admission. + /// + /// The Redis TTL is crash/failure recovery only. It is not an idle-session + /// timeout. A healthy node renews every 10 seconds while authenticated sessions + /// exist, so ownership stays alive indefinitely even when clients issue no + /// commands. A 30-second lease recovers crashed-node ownership quickly without + /// treating a brief Redis blip as membership loss. Two missed heartbeats expire + /// the lease. The 2-second hot-path skew refuses a local fast-path decision that + /// would outlive the distributed lease. + /// + internal static class SessionStateDefaults + { + /// Distributed ownership TTL written into Redis on admit and renew. + public static readonly TimeSpan LeaseTtl = TimeSpan.FromSeconds(30); + + /// + /// Interval between process-wide lease renewal passes. + /// uses the same 10-second period. + /// + public static readonly TimeSpan RenewalPeriod = TimeSpan.FromSeconds(10); + + /// + /// Local hot-path requires the known lease to remain valid for at least this long. + /// + public static readonly TimeSpan HotPathSkew = TimeSpan.FromSeconds(2); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateEngine.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateEngine.cs new file mode 100644 index 00000000..5c3c17ec --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateEngine.cs @@ -0,0 +1,521 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Atomic cluster session + source-IP membership algorithm. The Redis Lua scripts + /// are a faithful port of these operations. + /// + /// + /// Two hashes per account, mutated in one locked/EVAL step: + /// source fields {ip}\x1f{ownerId} and session fields {ownerId}, + /// values {expiryUnixMs}|{generation}|{count}. Session limit is checked + /// before source-IP limit. Rejection makes no state changes. The session count is + /// the sum of unexpired per-owner counts; the source count is the number of + /// distinct unexpired IPs. + /// + internal sealed class SessionStateEngine + { + /// Accepted; the source IP was already active. + public const long AcceptedExisting = 3; + + /// Accepted; a new distinct source IP was established. + public const long AcceptedNew = 2; + + /// Rejected: cluster session limit would be exceeded. + public const long RejectedSessionLimit = 1; + + /// Rejected: cluster source-address limit would be exceeded. + public const long RejectedSourceLimit = 0; + + /// Packs an accepted admit status with the cluster session total. + public static long PackAdmit(long status, int sessionTotal) => + status <= RejectedSessionLimit + ? status + : status | ((long)(sessionTotal < 0 ? 0 : sessionTotal) << 2); + + /// Reads the raw admit status from a packed EVAL integer. + public static long UnpackAdmitStatus(long packed) => packed <= RejectedSessionLimit ? packed : packed & 3; + + /// Reads the cluster session total from a packed accepted EVAL integer. + public static int UnpackAdmitSessionTotal(long packed) => + packed <= RejectedSessionLimit ? 0 : (int)(packed >> 2); + + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// Returns the unpacked admit status (tests that ignore the packed total). + internal long TryAdmitStatus( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long nowUnixMs, + long leaseMs, + long sessionGeneration, + long sourceGeneration, + bool trackSessions = false) => + UnpackAdmitStatus( + TryAdmit( + sourceKey, + sessionKey, + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + nowUnixMs, + leaseMs, + sessionGeneration, + sourceGeneration, + trackSessions)); + + /// + /// Atomically admits one session and this owner's source-IP membership. + /// + public long TryAdmit( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long nowUnixMs, + long leaseMs, + long sessionGeneration, + long sourceGeneration, + bool trackSessions = false) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + var sources = GetOrCreateHash(sourceKey); + var sessions = GetOrCreateHash(sessionKey); + PruneExpired(sources, nowUnixMs); + PruneExpired(sessions, nowUnixMs); + + var sessionTotal = SumCounts(sessions); + if (sessionLimit > 0 && sessionTotal >= sessionLimit) + { + RemoveIfEmpty(sourceKey, sources); + RemoveIfEmpty(sessionKey, sessions); + return RejectedSessionLimit; + } + + var activeIps = DistinctActiveIps(sources); + if (srcIpLimit > 0 && !activeIps.Contains(normalizedSourceIp) && activeIps.Count >= srcIpLimit) + { + RemoveIfEmpty(sourceKey, sources); + RemoveIfEmpty(sessionKey, sessions); + return RejectedSourceLimit; + } + + var expiry = nowUnixMs + leaseMs; + if (sessionLimit > 0 || trackSessions) + { + IncrementOwned(sessions, ownerId, expiry, sessionGeneration); + } + + if (srcIpLimit > 0) + { + IncrementOwned( + sources, + SessionStateKeys.SourceField(normalizedSourceIp, ownerId), + expiry, + sourceGeneration); + } + + var status = activeIps.Contains(normalizedSourceIp) ? AcceptedExisting : AcceptedNew; + return PackAdmit(status, SumCounts(sessions)); + } + } + + /// + /// Decrements this owner's session count and source-IP count when generations match. + /// + public long Release( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + DecrementOwned(sessions, ownerId, sessionGeneration); + RemoveIfEmpty(sessionKey, sessions); + } + + if (_hashes.TryGetValue(sourceKey, out var sources)) + { + DecrementOwned(sources, SessionStateKeys.SourceField(normalizedSourceIp, ownerId), sourceGeneration); + RemoveIfEmpty(sourceKey, sources); + } + + return _hashes.TryGetValue(sessionKey, out var remaining) + ? SumCounts(remaining) + : 0; + } + } + + /// Deletes every field owned by (process shutdown). + public long ReleaseOwner(string sourceKey, string sessionKey, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + sessions.Remove(ownerId); + RemoveIfEmpty(sessionKey, sessions); + } + + if (_hashes.TryGetValue(sourceKey, out var sources)) + { + List? owned = null; + // codeql[cs/linq/missed-where] Lock-held key collection without LINQ allocation. + foreach (var field in sources.Keys) + { + if (SessionStateKeys.TrySplitField(field, out _, out var fieldOwner) + && string.Equals(fieldOwner, ownerId, StringComparison.Ordinal)) + { + owned ??= []; + owned.Add(field); + } + } + + if (owned is not null) + { + foreach (var field in owned) + { + sources.Remove(field); + } + } + + RemoveIfEmpty(sourceKey, sources); + } + + return 1; + } + } + + /// + /// Refreshes this owner's unexpired matching session and source leases. + /// Returns 1 when every requested field was renewed; 0 when any was lost. + /// + public long Renew( + string sourceKey, + string sessionKey, + string ownerId, + long sessionGeneration, + long nowUnixMs, + long leaseMs, + IReadOnlyList<(string Ip, long Generation)> sources) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + ArgumentNullException.ThrowIfNull(sources); + + lock (_gate) + { + if (sessionGeneration != 0 + && !CanRenewField(sessionKey, ownerId, sessionGeneration, nowUnixMs, out _)) + { + return 0; + } + + foreach (var (ip, generation) in sources) + { + if (!CanRenewField( + sourceKey, + SessionStateKeys.SourceField(ip, ownerId), + generation, + nowUnixMs, + out _)) + { + return 0; + } + } + + if (sessionGeneration != 0) + { + RenewField(sessionKey, ownerId, sessionGeneration, nowUnixMs, leaseMs); + } + + foreach (var (ip, generation) in sources) + { + RenewField( + sourceKey, + SessionStateKeys.SourceField(ip, ownerId), + generation, + nowUnixMs, + leaseMs); + } + + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + PruneExpired(sessions, nowUnixMs); + RemoveIfEmpty(sessionKey, sessions); + var total = SumCounts(sessions); + return total < 1 ? 1 : total; + } + + return 1; + } + } + + /// Returns the cluster-wide authenticated session count. + internal int ActiveSessionCount(string sessionKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + lock (_gate) + { + if (!_hashes.TryGetValue(sessionKey, out var sessions)) + { + return 0; + } + + PruneExpired(sessions, nowUnixMs); + RemoveIfEmpty(sessionKey, sessions); + return SumCounts(sessions); + } + } + + /// Returns distinct unexpired source IPs. + internal IReadOnlyCollection ActiveSourceIps(string sourceKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + lock (_gate) + { + if (!_hashes.TryGetValue(sourceKey, out var sources)) + { + return []; + } + + PruneExpired(sources, nowUnixMs); + RemoveIfEmpty(sourceKey, sources); + return DistinctActiveIps(sources); + } + } + + /// Returns this owner's unexpired session count. + internal int OwnerSessionCount(string sessionKey, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(sessionKey, out var sessions) + || !sessions.TryGetValue(ownerId, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out _, out var count) + || expiry <= nowUnixMs) + { + return 0; + } + + return count; + } + } + + /// Reads a HASH field without mutating it (tests). + internal bool TryGetOwnership( + string key, + string field, + out long expiryUnixMs, + out long generation, + out int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + if (_hashes.TryGetValue(key, out var hash) + && hash.TryGetValue(field, out var current) + && SessionStateKeys.TrySplitValue(current, out expiryUnixMs, out generation, out count)) + { + return true; + } + } + + expiryUnixMs = 0; + generation = 0; + count = 0; + return false; + } + + /// Overwrites a HASH field without admit/release semantics (tests). + internal void WriteOwnership(string key, string field, long expiryUnixMs, long generation, int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + GetOrCreateHash(key)[field] = SessionStateKeys.Value(expiryUnixMs, generation, count); + } + } + + /// Returns whether this owner currently holds . + internal bool HasSourceOwner(string sourceKey, string normalizedSourceIp, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + var field = SessionStateKeys.SourceField(normalizedSourceIp, ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(sourceKey, out var sources) + || !sources.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out _, out var count)) + { + return false; + } + + return expiry > nowUnixMs && count > 0; + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private bool CanRenewField(string key, string field, long generation, long nowUnixMs, out int count) + { + count = 0; + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out count) + || storedGeneration != generation + || expiry <= nowUnixMs) + { + return false; + } + + return true; + } + + private void RenewField(string key, string field, long generation, long nowUnixMs, long leaseMs) + { + if (!CanRenewField(key, field, generation, nowUnixMs, out var count) + || !_hashes.TryGetValue(key, out var hash)) + { + return; + } + + hash[field] = SessionStateKeys.Value(nowUnixMs + leaseMs, generation, count); + } + + private static void IncrementOwned(Dictionary hash, string field, long expiry, long generation) + { + // Same generation: this owner still holds the field. A different generation + // is a new local epoch (local state was wiped) and must replace stale count. + if (hash.TryGetValue(field, out var current) + && SessionStateKeys.TrySplitValue(current, out _, out var storedGeneration, out var count) + && storedGeneration == generation) + { + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count + 1); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, generation, 1); + } + + private static void DecrementOwned(Dictionary hash, string field, long generation) + { + if (!hash.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out var count) + || storedGeneration != generation) + { + return; + } + + if (count <= 1) + { + hash.Remove(field); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count - 1); + } + + private static void PruneExpired(Dictionary hash, long nowUnixMs) + { + List? expired = null; + foreach (var (field, value) in hash) + { + if (!SessionStateKeys.TrySplitValue(value, out var expiry, out _, out _) || expiry <= nowUnixMs) + { + expired ??= []; + expired.Add(field); + } + } + + if (expired is null) + { + return; + } + + foreach (var field in expired) + { + hash.Remove(field); + } + } + + private static int SumCounts(Dictionary hash) + { + var total = 0; + // codeql[cs/linq/missed-where] TrySplitValue out-params; lock-held sum without LINQ allocation. + foreach (var value in hash.Values) + { + if (SessionStateKeys.TrySplitValue(value, out _, out _, out var count)) + { + total += count; + } + } + + return total; + } + + private static HashSet DistinctActiveIps(Dictionary hash) + { + var active = new HashSet(StringComparer.Ordinal); + // codeql[cs/linq/missed-where] TrySplitField out-params; lock-held scan without LINQ allocation. + foreach (var field in hash.Keys) + { + if (SessionStateKeys.TrySplitField(field, out var ip, out _)) + { + active.Add(ip); + } + } + + return active; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateKeys.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateKeys.cs new file mode 100644 index 00000000..68ed2edb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateKeys.cs @@ -0,0 +1,120 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Redis key material for cluster-wide account session and source-IP membership. + /// + /// + /// Keys: nntpd:srcip:{sha256hex(account)} and nntpd:sess:{sha256hex(account)}. + /// Account names are hashed so Redis key listings do not expose usernames. + /// Source field: {normalizedIp}\x1f{ownerId}. + /// Session field: {ownerId}. + /// Value: {expiryUnixMs}|{generation}|{count}. + /// ownerId is {nodeId}:{incarnation} so a process restart cannot + /// overwrite a previous incarnation's unexpired ownership. + /// + internal static class SessionStateKeys + { + /// ASCII namespace for source-IP ownership hashes. + public static ReadOnlySpan SourceNamespacePrefix => "nntpd:srcip:"u8; + + /// ASCII namespace for session ownership hashes. + public static ReadOnlySpan SessionNamespacePrefix => "nntpd:sess:"u8; + + /// Separates the normalized IP from the owner id in a source HASH field. + public const char FieldSeparator = '\u001f'; + + /// Separates expiry, generation, and count in a HASH value. + public const char ValueSeparator = '|'; + + /// Builds the source-IP Redis HASH key for . + public static byte[] CreateSource(string accountName) => Create(SourceNamespacePrefix, accountName); + + /// Builds the session Redis HASH key for . + public static byte[] CreateSession(string accountName) => Create(SessionNamespacePrefix, accountName); + + /// Builds the HASH field for a node-owned source IP. + public static string SourceField(string normalizedSourceIp, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + return string.Concat(normalizedSourceIp, FieldSeparator, ownerId); + } + + /// Formats a lease value as expiryMs|generation|count. + public static string Value(long expiryUnixMs, long generation, int count) => + string.Concat( + expiryUnixMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + count.ToString(CultureInfo.InvariantCulture)); + + /// Parses a source HASH field into source IP and owner id. + public static bool TrySplitField(string field, out string ip, out string ownerId) + { + ArgumentNullException.ThrowIfNull(field); + var separator = field.IndexOf(FieldSeparator, StringComparison.Ordinal); + if (separator <= 0 || separator >= field.Length - 1) + { + ip = string.Empty; + ownerId = string.Empty; + return false; + } + + ip = field[..separator]; + ownerId = field[(separator + 1)..]; + return true; + } + + /// Parses a HASH value into expiry, generation, and count. + public static bool TrySplitValue(string value, out long expiryUnixMs, out long generation, out int count) + { + ArgumentNullException.ThrowIfNull(value); + expiryUnixMs = 0; + generation = 0; + count = 0; + var first = value.IndexOf(ValueSeparator, StringComparison.Ordinal); + if (first <= 0 || first >= value.Length - 1) + { + return false; + } + + var second = value.IndexOf(ValueSeparator, first + 1); + if (second <= first + 1 || second >= value.Length - 1) + { + return false; + } + + return long.TryParse(value.AsSpan(0, first), out expiryUnixMs) + && long.TryParse(value.AsSpan(first + 1, second - first - 1), out generation) + && int.TryParse(value.AsSpan(second + 1), out count); + } + + private static byte[] Create(ReadOnlySpan prefix, string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateLogMessages.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateLogMessages.cs new file mode 100644 index 00000000..889de524 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// Source-generated session-state diagnostics. Never includes secrets. + internal static partial class SessionStateLogMessages + { + [LoggerMessage(EventId = 2310, Level = LogLevel.Debug, Message = "Session-state admission accepted locally user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SessionAdmittedLocal(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2311, Level = LogLevel.Debug, Message = "Session-state admission accepted via cluster membership user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId} path={Path}")] + public static partial void SessionAdmittedDistributed(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId, string path); + + [LoggerMessage(EventId = 2312, Level = LogLevel.Information, Message = "Session-state source-address admission rejected user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SourceAddressRejected(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2323, Level = LogLevel.Information, Message = "Session admission rejected user={Username} ip={SourceIp} limit={SessionLimit} node={NodeId}")] + public static partial void SessionAdmissionRejected(ILogger logger, string username, string sourceIp, int sessionLimit, string nodeId); + + [LoggerMessage(EventId = 2313, Level = LogLevel.Warning, Message = "Session-state admission failed closed; cluster membership unavailable user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SessionStateUnavailable(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2314, Level = LogLevel.Debug, Message = "Session-state node ownership released user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateReleased(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2315, Level = LogLevel.Debug, Message = "Session-state lease renewed user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewed(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2316, Level = LogLevel.Warning, Message = "Session-state lease renewal failed user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewFailed(ILogger logger, Exception exception, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2317, Level = LogLevel.Warning, Message = "Session-state lease renewal skipped; membership unavailable user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewUnavailable(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2318, Level = LogLevel.Warning, Message = "Session-state lease lost; local hot path invalidated user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseLost(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2319, Level = LogLevel.Warning, Message = "Session-state release failed user={Username} ip={SourceIp}")] + public static partial void SessionStateReleaseFailed(ILogger logger, Exception exception, string username, string sourceIp); + + [LoggerMessage(EventId = 2320, Level = LogLevel.Information, Message = "Session-state lease renewal started interval={Interval}")] + public static partial void SessionStateRenewalStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 2321, Level = LogLevel.Information, Message = "Session-state lease renewal stopped")] + public static partial void SessionStateRenewalStopped(ILogger logger); + + [LoggerMessage(EventId = 2322, Level = LogLevel.Warning, Message = "Session-state lease renewal pass failed")] + public static partial void SessionStateRenewalPassFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateRenewAndApplyResult.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateRenewAndApplyResult.cs new file mode 100644 index 00000000..5f53424e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateRenewAndApplyResult.cs @@ -0,0 +1,22 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// Outcome of one atomic session-renewal plus byte-quota APPLY. + public readonly struct SessionStateRenewAndApplyResult + { + /// Initializes a combined renew/apply result. + public SessionStateRenewAndApplyResult(SessionStateRenewStatus renew, long? remaining) + { + Renew = renew; + Remaining = remaining; + } + + /// Gets the session/source lease renewal outcome. + public SessionStateRenewStatus Renew { get; } + + /// + /// Remaining after APPLY, or when this store did not + /// apply byte state (in-memory renew-only path). + /// + public long? Remaining { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateScripts.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateScripts.cs new file mode 100644 index 00000000..0acd429e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateScripts.cs @@ -0,0 +1,481 @@ +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Redis Lua scripts for atomic cluster session + source-IP admission. + /// Semantics match . + /// + internal static class SessionStateScripts + { + /// + /// Atomically evaluates both limits and, on accept, increments session and + /// source-IP ownership. Rejection writes no fields. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: ip, owner, sessionLimit, srcIpLimit, nowMs, leaseMs, sessionGen, sourceGen, + /// trackSessions (1 to increment the session hash when sessionLimit is 0). + /// Accept returns status + (sessionTotal * 4) (3 existing source, 2 new source). + /// Reject remains 1 session-limit, 0 source-limit. Session limit is checked first. + /// + public const string TryAdmit = + """ + -- nntpd-admit-try + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local ip = ARGV[1] + local owner = ARGV[2] + local sessionLimit = tonumber(ARGV[3]) + local srcLimit = tonumber(ARGV[4]) + local now = tonumber(ARGV[5]) + local lease = tonumber(ARGV[6]) + local sessionGen = ARGV[7] + local sourceGen = ARGV[8] + local srcField = ip .. '\31' .. owner + local expiry = now + lease + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function prune(key) + local entries = redis.call('HGETALL', key) + local expired = {} + for i = 1, #entries, 2 do + local exp = parse_value(entries[i + 1]) + if exp == nil or exp <= now then + expired[#expired + 1] = entries[i] + end + end + if #expired > 0 then + redis.call('HDEL', key, unpack(expired)) + end + end + + local function increment(key, field, gen) + local cur = redis.call('HGET', key, field) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen and storedGen == gen then + redis.call('HSET', key, field, tostring(expiry) .. '|' .. storedGen .. '|' .. tostring((count or 0) + 1)) + return + end + end + redis.call('HSET', key, field, tostring(expiry) .. '|' .. gen .. '|1') + end + + prune(srcKey) + prune(sessKey) + if redis.call('HLEN', srcKey) == 0 then + redis.call('DEL', srcKey) + end + if redis.call('HLEN', sessKey) == 0 then + redis.call('DEL', sessKey) + end + + local sessionTotal = 0 + local sessEntries = redis.call('HGETALL', sessKey) + for i = 1, #sessEntries, 2 do + local _, _, count = parse_value(sessEntries[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + if sessionLimit ~= nil and sessionLimit > 0 and sessionTotal >= sessionLimit then + return 1 + end + + local active = {} + local srcEntries = redis.call('HGETALL', srcKey) + for i = 1, #srcEntries, 2 do + local f = srcEntries[i] + local sep = string.find(f, '\31', 1, true) + local fip = sep and string.sub(f, 1, sep - 1) or f + active[fip] = true + end + + if srcLimit ~= nil and srcLimit > 0 and not active[ip] then + local distinct = 0 + for _ in pairs(active) do + distinct = distinct + 1 + end + if distinct >= srcLimit then + return 0 + end + end + + local trackSessions = tonumber(ARGV[9]) == 1 + if (sessionLimit ~= nil and sessionLimit > 0) or trackSessions then + increment(sessKey, owner, sessionGen) + end + if srcLimit ~= nil and srcLimit > 0 then + increment(srcKey, srcField, sourceGen) + end + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local _, _, count = parse_value(counted[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + if active[ip] then + return 3 + (sessionTotal * 4) + end + return 2 + (sessionTotal * 4) + """; + + /// + /// Decrements this owner's session and source-IP counts when generations match. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: ip, owner, sessionGen, sourceGen. Returns the remaining cluster session total. + /// + public const string Release = + """ + -- nntpd-admit-release + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local ip = ARGV[1] + local owner = ARGV[2] + local sessionGen = ARGV[3] + local sourceGen = ARGV[4] + local srcField = ip .. '\31' .. owner + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function decrement(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen then + return + end + if count == nil or count <= 1 then + redis.call('HDEL', key, field) + else + redis.call('HSET', key, field, tostring(exp) .. '|' .. storedGen .. '|' .. tostring(count - 1)) + end + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + decrement(sessKey, owner, sessionGen) + decrement(srcKey, srcField, sourceGen) + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local _, _, count = parse_value(counted[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + return sessionTotal + """; + + /// + /// Refreshes this owner's unexpired matching session and source leases. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: owner, sessionGen, nowMs, leaseMs, ipCount, ip1, gen1, ... + /// Returns 0 when any requested field was lost. On success returns the + /// unexpired cluster session total (minimum 1 so a source-only renew is not Lost). + /// A sessionGen of 0 skips the session field. + /// + public const string Renew = + """ + -- nntpd-admit-renew + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local owner = ARGV[1] + local sessionGen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local ipCount = tonumber(ARGV[5]) or 0 + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function can_renew(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return false + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return false + end + return true, count + end + + local function renew_field(key, field, gen, count) + redis.call('HSET', key, field, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + end + + if sessionGen ~= '0' then + local ok = can_renew(sessKey, owner, sessionGen) + if not ok then + return 0 + end + end + + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok = can_renew(srcKey, ip .. '\31' .. owner, gen) + if not ok then + return 0 + end + i = i + 1 + end + + if sessionGen ~= '0' then + local ok, count = can_renew(sessKey, owner, sessionGen) + if ok then + renew_field(sessKey, owner, sessionGen, count) + end + end + + i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok, count = can_renew(srcKey, ip .. '\31' .. owner, gen) + if ok then + renew_field(srcKey, ip .. '\31' .. owner, gen, count) + end + i = i + 1 + end + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local exp, _, count = parse_value(counted[i + 1]) + if exp ~= nil and exp > now then + sessionTotal = sessionTotal + (count or 0) + end + end + if sessionTotal < 1 then + sessionTotal = 1 + end + return sessionTotal + """; + + /// + /// Renews this owner's unexpired matching leases, then APPLYs one byte batch. + /// APPLY always runs; a lost renewal does not skip quota reconciliation. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash, KEYS[3] remaining-quota HASH. + /// ARGV: owner, sessionGen, nowMs, leaseMs, ipCount, ip1, gen1, ..., + /// batchId, consumed, mysqlRemainingAfter. + /// Returns packed : remaining when renewed, + /// -remaining - 1 when lost. APPLY is floor-only and idempotent. + /// + public const string RenewAndApply = + """ + -- nntpd-admit-renew-and-apply + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local bytesKey = KEYS[3] + local owner = ARGV[1] + local sessionGen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local ipCount = tonumber(ARGV[5]) or 0 + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function can_renew(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return false + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return false + end + return true, count + end + + local function renew_field(key, field, gen, count) + redis.call('HSET', key, field, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + end + + local function trim_marks(key, batchField) + if redis.call('HLEN', key) <= 257 then + return + end + local fields = redis.call('HKEYS', key) + for i = 1, #fields do + local f = fields[i] + if f ~= 'remaining' and f ~= batchField then + redis.call('HDEL', key, f) + if redis.call('HLEN', key) <= 257 then + return + end + end + end + end + + local function apply_bytes() + local batchId = ARGV[6 + (ipCount * 2)] + local mysql = tonumber(ARGV[8 + (ipCount * 2)]) + if mysql == nil or mysql < 0 then + mysql = 0 + end + if batchId == nil or batchId == '' then + batchId = '' + end + local batchField = 'b:' .. batchId + if redis.call('EXISTS', bytesKey) == 0 then + if batchId ~= '' then + redis.call('HSET', bytesKey, 'remaining', tostring(mysql), batchField, '1') + trim_marks(bytesKey, batchField) + else + redis.call('HSET', bytesKey, 'remaining', tostring(mysql)) + end + return mysql + end + if batchId ~= '' and redis.call('HEXISTS', bytesKey, batchField) == 1 then + local cur = tonumber(redis.call('HGET', bytesKey, 'remaining')) + if cur == nil or cur < 0 then + return 0 + end + return cur + end + local current = tonumber(redis.call('HGET', bytesKey, 'remaining')) + if current == nil or current < 0 then + current = 0 + end + local next = current + if next > mysql then + next = mysql + end + if batchId ~= '' then + redis.call('HSET', bytesKey, 'remaining', tostring(next), batchField, '1') + trim_marks(bytesKey, batchField) + else + redis.call('HSET', bytesKey, 'remaining', tostring(next)) + end + return next + end + + local renewed = 1 + if sessionGen ~= '0' then + local ok = can_renew(sessKey, owner, sessionGen) + if not ok then + renewed = 0 + end + end + + if renewed == 1 then + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok = can_renew(srcKey, ip .. '\31' .. owner, gen) + if not ok then + renewed = 0 + break + end + i = i + 1 + end + end + + if renewed == 1 then + if sessionGen ~= '0' then + local ok, count = can_renew(sessKey, owner, sessionGen) + if ok then + renew_field(sessKey, owner, sessionGen, count) + end + end + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok, count = can_renew(srcKey, ip .. '\31' .. owner, gen) + if ok then + renew_field(srcKey, ip .. '\31' .. owner, gen, count) + end + i = i + 1 + end + end + + local remaining = apply_bytes() + if remaining == nil or remaining < 0 then + remaining = 0 + end + if renewed == 1 then + return remaining + end + return -remaining - 1 + """; + + /// Deletes every field owned by this process incarnation. + /// KEYS[1] source hash, KEYS[2] session hash. ARGV: owner. Always returns 1. + public const string ReleaseOwner = + """ + -- nntpd-admit-release-owner + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local owner = ARGV[1] + redis.call('HDEL', sessKey, owner) + if redis.call('HLEN', sessKey) == 0 then + redis.call('DEL', sessKey) + end + local entries = redis.call('HGETALL', srcKey) + local owned = {} + for i = 1, #entries, 2 do + local f = entries[i] + local sep = string.find(f, '\31', 1, true) + local fieldOwner = sep and string.sub(f, sep + 1) or '' + if fieldOwner == owner then + owned[#owned + 1] = f + end + end + if #owned > 0 then + redis.call('HDEL', srcKey, unpack(owned)) + end + if redis.call('HLEN', srcKey) == 0 then + redis.call('DEL', srcKey) + end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SessionStateService.cs b/src/VectorNNTPD.ReaderService/SessionState/SessionStateService.cs new file mode 100644 index 00000000..0bcf7e2a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SessionStateService.cs @@ -0,0 +1,177 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Sole periodic scheduler for SessionState: lease renewal, source-address + /// ownership, byte-quota reconciliation, and rate observation. + /// + /// + /// This service is a scheduler. It does not accept connections, authenticate clients, + /// or keep its own session registry. Every renewal interval it commits account-wide + /// MySQL remaining-quota consumes, then asks + /// to renew currently active local ownership. When an account has both distributed + /// ownership and a committed byte batch, renewal and Redis APPLY share one EVAL. + /// Accounts with pending bytes but no SessionState ownership receive APPLY-only. + /// Each account with distributed ownership is one Redis operation covering session + /// count, every locally owned source IP, and byte state when present. Renewal is + /// node liveness, not client activity: idle authenticated sessions keep consuming + /// cluster capacity while this process can renew. stops the + /// loop, performs a bounded two-pass final byte reconcile, then releases leftover incarnation + /// ownership. Listeners are registered after this service so they stop first and + /// sessions can finalize before renewal ends. Process crash is detected by lease + /// expiry after renewals stop. Unreconciled process-memory pending bytes are lost. + /// + public sealed class SessionStateService : IApplicationService + { + private readonly ISessionStateLeaseManager _leases; + private readonly IAccountByteAccountant _bytes; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public SessionStateService( + ISessionStateLeaseManager leases, + IAccountByteAccountant bytes, + ILogger logger) + : this(leases, bytes, logger, TimeProvider.System, SessionStateDefaults.RenewalPeriod) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal SessionStateService( + ISessionStateLeaseManager leases, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + : this(leases, NullAccountByteAccountant.Instance, logger, timeProvider, interval) + { + } + + /// Initializes a new instance with byte accounting and an explicit interval (tests). + internal SessionStateService( + ISessionStateLeaseManager leases, + IAccountByteAccountant bytes, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(leases); + ArgumentNullException.ThrowIfNull(bytes); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _leases = leases; + _bytes = bytes; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "SessionState"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + AccountByteLogMessages.FinalReconciliationStarted(_logger); + try + { + await _bytes.ReconcileAsync(cancellationToken).ConfigureAwait(false); + // Pending observed while a MySQL-committed batch was still awaiting APPLY + // stays in Pending. A second pass consumes it after the first APPLY completes. + await _bytes.ReconcileAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.FinalReconciliationFailed(_logger, ex); + } + + try + { + await _leases.ReleaseAllOwnershipAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + SessionStateLogMessages.SessionStateRenewalStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + try + { + await ReconcileOnceAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateRenewalPassFailed(_logger, ex); + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + SessionStateLogMessages.SessionStateRenewalStopped(_logger); + } + + private async Task ReconcileOnceAsync(CancellationToken cancellationToken) + { + await _bytes.CommitDurableAsync(cancellationToken).ConfigureAwait(false); + await _leases.RenewLeasesAsync(cancellationToken).ConfigureAwait(false); + await _bytes.ApplyCommittedAsync(cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/SessionState/SourceAddressIdentity.cs b/src/VectorNNTPD.ReaderService/SessionState/SourceAddressIdentity.cs new file mode 100644 index 00000000..6e488a41 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/SessionState/SourceAddressIdentity.cs @@ -0,0 +1,36 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.SessionState +{ + /// + /// Canonical source-IP identity used by session and cluster source-IP admission. + /// + /// + /// IPv4-mapped IPv6 addresses are converted to IPv4 so ::ffff:192.0.2.10 + /// and 192.0.2.10 are one identity. Other IPv4 and IPv6 addresses are + /// independent; equivalent textual forms of the same address normalize through + /// . + /// + internal static class SourceAddressIdentity + { + /// Returns the canonical textual identity for . + public static string Format(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return Normalize(address).ToString(); + } + + /// + /// Network-order address bytes after the same IPv4-mapped normalization as + /// . IPv4 is 4 bytes; IPv6 is 16 bytes. + /// + public static byte[] ToNetworkBytes(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return Normalize(address).GetAddressBytes(); + } + + private static IPAddress Normalize(IPAddress address) => + address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/ArticlePlacement.cs b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacement.cs new file mode 100644 index 00000000..c91e4aaf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacement.cs @@ -0,0 +1,145 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// Observable result of one single-copy placement attempt. + public enum ArticlePlacementKind : byte + { + /// Journal accept completed. Matches ArticleAcceptOutcome.Accepted (1). + Accepted = 1, + + /// Identical article already represented. Matches ArticleAcceptOutcome.Duplicate (2). + Duplicate = 2, + + /// Conflicting ArtHash or ArtSize. Matches ArticleAcceptOutcome.Conflict (3). + Conflict = 3, + + /// Journal pressure rejection. Matches ArticleAcceptOutcome.RejectedPressure (4). + RejectedPressure = 4, + + /// Canonical or integrity rejection. Matches ArticleAcceptOutcome.RejectedInvalid (5). + RejectedInvalid = 5, + + /// Capacity rejection. Matches ArticleAcceptOutcome.RejectedCapacity (6). + RejectedCapacity = 6, + + /// No eligible registry entry. No dial. + NoActiveServer = 7, + + /// Connect, TLS, protocol FAIL, or I/O failure before END was fully written. + TransportFailure = 8, + + /// Caller cancelled before the server could accept. + Cancelled = 9, + + /// END was written and RESULT was not observed. The journal may already hold the article. + AcknowledgementNotObserved = 10, + } + + /// One finished placement attempt. + /// Classification. + /// Transport or protocol failure name. Never article bytes. + public readonly record struct ArticlePlacementResult(ArticlePlacementKind Kind, string? Failure = null) + { + /// Gets whether the StorageServer accepted responsibility or already held the same article. + public bool Succeeded => Kind is ArticlePlacementKind.Accepted or ArticlePlacementKind.Duplicate; + + /// Maps a RESULT outcome byte. Unknown bytes are transport failures. + public static ArticlePlacementResult FromOutcomeByte(byte outcome) => outcome switch + { + (byte)ArticlePlacementKind.Accepted => new(ArticlePlacementKind.Accepted), + (byte)ArticlePlacementKind.Duplicate => new(ArticlePlacementKind.Duplicate), + (byte)ArticlePlacementKind.Conflict => new(ArticlePlacementKind.Conflict), + (byte)ArticlePlacementKind.RejectedPressure => new(ArticlePlacementKind.RejectedPressure), + (byte)ArticlePlacementKind.RejectedInvalid => new(ArticlePlacementKind.RejectedInvalid), + (byte)ArticlePlacementKind.RejectedCapacity => new(ArticlePlacementKind.RejectedCapacity), + _ => new(ArticlePlacementKind.TransportFailure, "result-outcome"), + }; + } + + /// Selects one placement target from an already-active registry snapshot. + public static class StorageServerPlacementSelector + { + /// + /// Selects the eligible entry with the greatest , + /// then the lowest , then ordinal FQDN. + /// + public static bool TrySelect(IReadOnlyList active, out StorageServerFleetEntry selected) + { + ArgumentNullException.ThrowIfNull(active); + selected = default; + var found = false; + foreach (var entry in active) + { + if (entry.VatpPort is not (>= 1 and <= 65535)) + { + continue; + } + + if (!found || Prefer(entry, selected)) + { + selected = entry; + found = true; + } + } + + return found; + } + + private static bool Prefer(in StorageServerFleetEntry candidate, in StorageServerFleetEntry current) + { + if (candidate.AvailableBytes != current.AvailableBytes) + { + return candidate.AvailableBytes > current.AvailableBytes; + } + + if (candidate.ServerId != current.ServerId) + { + return candidate.ServerId < current.ServerId; + } + + return string.CompareOrdinal(candidate.Fqdn, current.Fqdn) < 0; + } + } + + /// Builds VATP META from a CanonicalV1 record. + public static class ArticlePlacementMeta + { + private static readonly NntpArticleHeaderName[] DateHeaders = + [ + NntpArticleHeaderName.Date, + NntpArticleHeaderName.InjectionDate, + NntpArticleHeaderName.NntpPostingDate, + NntpArticleHeaderName.Posted, + NntpArticleHeaderName.XDate, + NntpArticleHeaderName.DeliveryDate, + ]; + + /// + /// Recovers the winning Date-family header by locating fields that match the record, + /// then builds META with . + /// + public static bool TryFromRecord(in ArticleRecord record, out ArticleCanonicalTransferMeta meta) + { + meta = default; + if (record.ParseStatus != ArticleParseStatus.CanonicalV1 || record.ArtSize <= 0) + { + return false; + } + + var artData = record.ArtData.Span; + foreach (var name in DateHeaders) + { + if (ArticleFieldTable.Locate(artData, name).Equals(record.Fields)) + { + meta = ArticleCanonicalTransferMeta.FromRecord(in record, name); + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementClient.cs b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementClient.cs new file mode 100644 index 00000000..619917eb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementClient.cs @@ -0,0 +1,673 @@ +using System.Buffers; +using System.Globalization; +using System.Net.Security; +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Storage +{ + using VectorNNTPD.Common.Core; + /// + /// Long-lived STORE client. One TLS connection per placement target. Does not use . + /// + /// + /// Connections stay open while they are healthy. Each connection has one frame writer and at most + /// outstanding STORE streams. A lost RESULT after END is + /// and is not retried. Placement itself + /// stays disabled in the spool writer. + /// + public sealed class ArticlePlacementClient : IArticlePlacementClient, IAsyncDisposable + { + private static readonly TimeSpan InitialBackoff = TimeSpan.FromMilliseconds(200); + private static readonly TimeSpan MaximumBackoff = TimeSpan.FromSeconds(5); + + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ArticleTransferLimits _limits; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly Dictionary _connections = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary> _dialing = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _nextDial = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _backoff = new(StringComparer.OrdinalIgnoreCase); + private int _disposed; + private int _connects; + private int _dials; + + /// Optional certificate callback for tests only. + internal RemoteCertificateValidationCallback? ServerCertificateValidationCallback { get; set; } + + /// Optional TCP connect host override for tests. TLS TargetHost remains the advertised FQDN. + internal string? TestTcpConnectHost { get; set; } + + /// Gets how many TCP connections this client has established. + internal int ConnectCount => Volatile.Read(ref _connects); + + /// Gets how many dial attempts have started, including failures. + internal int DialAttempts => Volatile.Read(ref _dials); + + /// Gets the outstanding STORE limit applied to each connection. + internal int MaxOutstandingStores { get; } + + /// Sums wait and RESULT counters across connections still owned by this client. + internal PlacementTransportCounters Snapshot() + { + lock (_gate) + { + var counters = default(PlacementTransportCounters); + foreach (var connection in _connections.Values) + { + counters = counters.Add(connection); + } + + return counters; + } + } + + /// Initializes a placement client with two outstanding STOREs and the production VATP limits. + public ArticlePlacementClient(ILogger logger) + : this(logger, ArticleIngestionOptions.DefaultMaxOutstandingStoresPerConnection) + { + } + + /// Initializes a placement client with an explicit outstanding-STORE limit. + public ArticlePlacementClient(ILogger logger, int maxOutstandingStores) + : this(logger, new VatpClientOptions(), ArticleTransferLimits.Default, maxOutstandingStores, TimeProvider.System) + { + } + + internal ArticlePlacementClient( + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + int maxOutstandingStores = ArticleIngestionOptions.DefaultMaxOutstandingStoresPerConnection, + TimeProvider? time = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(limits); + if (maxOutstandingStores is not (1 or 2)) + { + throw new ArgumentOutOfRangeException( + nameof(maxOutstandingStores), + maxOutstandingStores, + "Outstanding STORE operations per connection must be 1 or 2."); + } + + _logger = logger; + _options = options; + _limits = limits; + _time = time ?? TimeProvider.System; + MaxOutstandingStores = maxOutstandingStores; + } + + /// + public async ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref _disposed) != 0) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "shutdown"); + } + + if (target.VatpPort is not int port || port is < 1 or > 65535) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "vatp-port"); + } + + if (!ArticlePlacementMeta.TryFromRecord(in record, out var meta)) + { + return new ArticlePlacementResult(ArticlePlacementKind.RejectedInvalid, "meta"); + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + var leased = await LeaseAsync(target.Fqdn, port, cancellationToken).ConfigureAwait(false); + if (leased.Failure is { } failure) + { + return failure; + } + + var connection = leased.Connection + ?? throw new InvalidOperationException("Placement lease returned neither a connection nor a failure."); + var result = await connection.PlaceAsync(record, meta, cancellationToken).ConfigureAwait(false); + if (connection.IsDead) + { + Forget(target.Fqdn, port, connection); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or TimeoutException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + return result; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "timeout"); + } + catch (OperationCanceledException) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, DescribeTransportFailure(ex)); + } + } + + /// Closes every placement connection and rejects later calls. + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + ArticlePlacementConnection[] connections; + lock (_gate) + { + connections = _connections.Values.ToArray(); + _connections.Clear(); + } + + var outstanding = 0; + foreach (var connection in connections) + { + outstanding += connection.ActiveStores; + await connection.DisposeAsync().ConfigureAwait(false); + } + + ArticlePlacementTransportLogMessages.Shutdown(_logger, outstanding); + } + + private async Task<(ArticlePlacementConnection? Connection, ArticlePlacementResult? Failure)> LeaseAsync( + string fqdn, + int port, + CancellationToken cancellationToken) + { + var key = fqdn + ":" + port.ToString(CultureInfo.InvariantCulture); + Task? pending = null; + var owner = false; + ArticlePlacementConnection? dead = null; + ArticlePlacementResult? deferred = null; + lock (_gate) + { + if (_disposed != 0) + { + return (null, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "shutdown")); + } + + if (_connections.TryGetValue(key, out var live) && !live.IsDead) + { + ArticlePlacementTransportLogMessages.Reused(_logger, fqdn, port, live.ActiveStores); + return (live, null); + } + + if (_connections.TryGetValue(key, out dead)) + { + _connections.Remove(key); + } + + if (_dialing.TryGetValue(key, out pending)) + { + // Dial already in flight; await the captured pending task below. + } + else if (_nextDial.TryGetValue(key, out var next) && _time.GetUtcNow() < next) + { + var wait = next - _time.GetUtcNow(); + ArticlePlacementTransportLogMessages.ReconnectDeferred( + _logger, fqdn, port, (long)wait.TotalMilliseconds, "backoff"); + deferred = new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "backoff"); + } + else + { + pending = DialAsync(fqdn, port, key); + _dialing[key] = pending; + owner = true; + } + } + + if (dead is not null) + { + try + { + await dead.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or TimeoutException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + if (deferred is not null) + { + return (null, deferred); + } + + try + { + var connection = await pending!.WaitAsync(cancellationToken).ConfigureAwait(false); + return (connection, null); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return (null, new ArticlePlacementResult(ArticlePlacementKind.Cancelled)); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is SocketException or IOException or TimeoutException or OperationCanceledException) + { + if (owner) + { + NoteFailure(key); + } + + var failure = DescribeTransportFailure(ex); + ArticlePlacementTransportLogMessages.ConnectionFailed(_logger, fqdn, port, failure); + return (null, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, failure)); + } + finally + { + if (owner) + { + _ = ReleaseDialAsync(key, pending!); + } + } + } + + private async Task ReleaseDialAsync(string key, Task pending) + { + try + { + await pending.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + lock (_gate) + { + if (_dialing.TryGetValue(key, out var current) && ReferenceEquals(current, pending)) + { + _dialing.Remove(key); + } + } + } + + private async Task DialAsync(string fqdn, int port, string key) + { + Interlocked.Increment(ref _dials); + var connection = await ArticlePlacementConnection.EstablishAsync( + fqdn, + port, + TestTcpConnectHost ?? fqdn, + _logger, + _options, + _limits, + MaxOutstandingStores, + ServerCertificateValidationCallback, + CancellationToken.None) + .ConfigureAwait(false); + Interlocked.Increment(ref _connects); + lock (_gate) + { + _backoff[key] = InitialBackoff; + _nextDial.Remove(key); + if (_disposed != 0) + { + _ = connection.DisposeAsync(); + throw new IOException("shutdown"); + } + + _connections[key] = connection; + } + + return connection; + } + + private void NoteFailure(string key) + { + lock (_gate) + { + var current = _backoff.TryGetValue(key, out var backoff) ? backoff : InitialBackoff; + _nextDial[key] = _time.GetUtcNow().Add(current); + var doubled = TimeSpan.FromTicks(Math.Min(current.Ticks * 2, MaximumBackoff.Ticks)); + _backoff[key] = doubled; + } + } + + private void Forget(string fqdn, int port, ArticlePlacementConnection connection) + { + var key = fqdn + ":" + port.ToString(CultureInfo.InvariantCulture); + lock (_gate) + { + if (_connections.TryGetValue(key, out var current) && ReferenceEquals(current, connection)) + { + _connections.Remove(key); + } + } + } + + private static string DescribeTransportFailure(Exception ex) + { + if (ex is TimeoutException) + { + return "timeout"; + } + + if (ex is SocketException socket) + { + return string.Create( + CultureInfo.InvariantCulture, + $"SocketException: SocketErrorCode={socket.SocketErrorCode} HResult={socket.HResult} Message={socket.Message}"); + } + + return ex.GetType().Name; + } + } + + /// Wait and RESULT totals for connections still held by one placement client. + internal readonly record struct PlacementTransportCounters( + long SlotWaitTicks, + long CreditWaitTicks, + long ResultCount, + long ResultTicks, + long ResultMaxTicks, + int PeakActive, + long DataFramesWritten) + { + /// Adds one connection's counters. + internal PlacementTransportCounters Add(ArticlePlacementConnection connection) => + new( + SlotWaitTicks + connection.SlotWaitTicks, + CreditWaitTicks + connection.CreditWaitTicks, + ResultCount + connection.ResultCount, + ResultTicks + connection.ResultTicks, + Math.Max(ResultMaxTicks, connection.ResultMaxTicks), + Math.Max(PeakActive, connection.PeakActive), + DataFramesWritten + connection.DataFramesWritten); + } + + /// STORE/META/DATA/END sender over an already connected stream. + internal static class VatpPlacementTransfer + { + public static async Task PlaceAsync( + Stream stream, + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + ArticleTransferLimits limits, + TimeSpan ioTimeout, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + var reader = new FrameReader(stream, ioTimeout); + var endWritten = false; + try + { + cancellationToken.ThrowIfCancellationRequested(); + var maxFrame = limits.DefaultMaxFramePayload; + await WriteAsync(stream, VatpFrameEncoder.EncodeHello(maxFrame), cancellationToken).ConfigureAwait(false); + var hello = await reader.ReadAsync(maxFrame, cancellationToken).ConfigureAwait(false); + if (hello is null) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "hello-closed"); + } + + if (hello.Value.Header.Type != VatpFrameType.Hello + || !VatpHello.TryDecode(hello.Value.Payload, out var decodedHello, out _)) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "hello"); + } + + maxFrame = Math.Min(decodedHello.MaxFramePayload, VatpProtocol.DefaultMaxFramePayload); + const uint streamId = 1; + var id = new byte[ArticleId.Length]; + record.ArtId.CopyTo(id); + await WriteAsync(stream, VatpFrameEncoder.EncodeStore(streamId, id), cancellationToken).ConfigureAwait(false); + var metaBytes = VatpMetaCodec.Encode(in meta); + await WriteAsync(stream, VatpFrameEncoder.EncodeMeta(streamId, metaBytes), cancellationToken).ConfigureAwait(false); + + var window = new ArticleTransferWindow(0, limits.MaxStreamCreditBytes); + var offset = 0; + var artData = record.ArtData; + while (offset < record.ArtSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var length = ArticleTransferReadyRing.ComputeDataPayloadLength( + record.ArtSize - offset, + window.Credit, + maxFrame); + if (length <= 0) + { + var paced = await ReadControlAsync(reader, maxFrame, endWritten: false, cancellationToken) + .ConfigureAwait(false); + if (paced.Terminal is { } terminal) + { + return terminal; + } + + if (paced.AddedCredit > 0) + { + _ = window.Add(paced.AddedCredit); + } + + continue; + } + + if (!window.TryConsume(length)) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window"); + } + + var fin = offset + length == record.ArtSize; + await WriteAsync( + stream, + VatpFrameEncoder.EncodeData(streamId, artData.Slice(offset, length), fin), + cancellationToken) + .ConfigureAwait(false); + offset += length; + } + + await WriteAsync(stream, VatpFrameEncoder.EncodeEnd(streamId), cancellationToken).ConfigureAwait(false); + endWritten = true; + while (true) + { + var final = await ReadControlAsync(reader, maxFrame, endWritten: true, cancellationToken) + .ConfigureAwait(false); + if (final.Terminal is { } terminal) + { + return terminal; + } + + if (final.AddedCredit > 0) + { + _ = window.Add(final.AddedCredit); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (!endWritten) + { + try + { + await WriteAsync(stream, VatpFrameEncoder.EncodeCancel(1), CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + + return new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or TimeoutException or ObjectDisposedException) + { + return endWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, ex.GetType().Name); + } + } + + private readonly record struct ControlRead(ArticlePlacementResult? Terminal, uint AddedCredit); + + private static async Task ReadControlAsync( + FrameReader reader, + uint maxFrame, + bool endWritten, + CancellationToken cancellationToken) + { + var frame = await reader.ReadAsync(maxFrame, cancellationToken).ConfigureAwait(false); + if (frame is null) + { + return new ControlRead( + endWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"), + 0); + } + + switch (frame.Value.Header.Type) + { + case VatpFrameType.Window: + return VatpControlPayload.TryDecodeWindow(Copy(frame.Value.Payload), out var add) + ? new ControlRead(null, add) + : new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window"), 0); + case VatpFrameType.Result: + if (!VatpResultPayload.TryDecode(frame.Value.Payload, out var outcome, out _)) + { + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "result"), 0); + } + + return new ControlRead(ArticlePlacementResult.FromOutcomeByte(outcome), 0); + case VatpFrameType.Fail: + if (!VatpControlPayload.TryDecodeFail(Copy(frame.Value.Payload), out var error, out _)) + { + return new ControlRead( + new ArticlePlacementResult( + endWritten + ? ArticlePlacementKind.AcknowledgementNotObserved + : ArticlePlacementKind.TransportFailure, + "fail"), + 0); + } + + if (error == VatpErrorCode.Cancelled) + { + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.Cancelled), 0); + } + + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, error.ToString()), 0); + default: + return new ControlRead( + new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, frame.Value.Header.Type.ToString()), + 0); + } + } + + private static byte[] Copy(ReadOnlySequence payload) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return bytes; + } + + private static async Task WriteAsync( + Stream stream, + VatpFrameEncoder.EncodedFrame frame, + CancellationToken cancellationToken) + { + if (!frame.Header.IsEmpty) + { + await stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (!frame.Payload.IsEmpty) + { + await stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + + private sealed class FrameReader + { + private readonly Stream _stream; + private readonly TimeSpan _ioTimeout; + private readonly byte[] _buffer = new byte[64 * 1024]; + private readonly List _pending = []; + + public FrameReader(Stream stream, TimeSpan ioTimeout) + { + _stream = stream; + _ioTimeout = ioTimeout; + } + + public async Task ReadAsync(uint maxFramePayload, CancellationToken cancellationToken) + { + while (true) + { + var parsed = VatpFrameParser.ParseOneFrame(new ReadOnlySequence(_pending.ToArray()), maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + throw new IOException(parsed.Error.ToString()); + } + + if (parsed.Status == VatpFrameParseStatus.Success && parsed.Frame is { } frame) + { + _pending.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + return CopyFrame(frame); + } + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(_ioTimeout); + int read; + try + { + read = await _stream.ReadAsync(_buffer, timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("placement-io"); + } + + if (read == 0) + { + return null; + } + + for (var i = 0; i < read; i++) + { + _pending.Add(_buffer[i]); + } + } + } + + private static VatpParsedFrame CopyFrame(VatpParsedFrame frame) + { + var payload = new byte[frame.Payload.Length]; + frame.Payload.CopyTo(payload); + return new VatpParsedFrame(frame.Header, new ReadOnlySequence(payload)); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementConnection.cs b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementConnection.cs new file mode 100644 index 00000000..4230da58 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementConnection.cs @@ -0,0 +1,918 @@ +using System.Buffers; +using System.Net.Security; +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// + /// One TLS connection to one placement target. One writer. At most + /// STORE streams. + /// + internal sealed class ArticlePlacementConnection : IAsyncDisposable + { + private readonly string _fqdn; + private readonly int _port; + private readonly TcpClient _tcp; + private readonly SslStream _ssl; + private readonly ILogger _logger; + private readonly ArticleTransferLimits _limits; + private readonly TimeSpan _ioTimeout; + private readonly SemaphoreSlim _slots; + private readonly SemaphoreSlim _write = new(1, 1); + private readonly object _tableGate = new(); + private readonly Dictionary _operations = []; + private readonly CancellationTokenSource _lifetime = new(); + private readonly FrameBuffer _frames = new(); + private Task? _reader; + private int _dead; + private int _disposed; + private int _active; + private int _peakActive; + private uint _nextStreamId = 1; + private uint _maxFramePayload; + private long _slotWaitTicks; + private long _creditWaitTicks; + private long _resultTicks; + private long _resultCount; + private long _resultMaxTicks; + private long _dataFramesWritten; + private int _storesAwaitingInitialCredit; + private readonly object _readTimeoutGate = new(); + private CancellationTokenSource? _activeReadTimeout; + + private ArticlePlacementConnection( + string fqdn, + int port, + TcpClient tcp, + SslStream ssl, + ILogger logger, + ArticleTransferLimits limits, + TimeSpan ioTimeout, + int maxOutstanding, + uint maxFramePayload) + { + _fqdn = fqdn; + _port = port; + _tcp = tcp; + _ssl = ssl; + _logger = logger; + _limits = limits; + _ioTimeout = ioTimeout; + MaxOutstanding = maxOutstanding; + _maxFramePayload = maxFramePayload; + _slots = new SemaphoreSlim(maxOutstanding, maxOutstanding); + } + + public int MaxOutstanding { get; } + + /// Gets the highest number of STORE operations that were in flight together. + internal int PeakActive => Volatile.Read(ref _peakActive); + + /// Gets ticks spent waiting for a free STORE slot. + internal long SlotWaitTicks => Volatile.Read(ref _slotWaitTicks); + + /// Gets ticks spent waiting for WINDOW credit. + internal long CreditWaitTicks => Volatile.Read(ref _creditWaitTicks); + + /// Gets how many RESULT waits were recorded. + internal long ResultCount => Volatile.Read(ref _resultCount); + + /// Gets the total ticks spent waiting for RESULT after END. + internal long ResultTicks => Volatile.Read(ref _resultTicks); + + /// Gets the longest RESULT wait in ticks. + internal long ResultMaxTicks => Volatile.Read(ref _resultMaxTicks); + + /// Gets DATA frames this connection has written. Credit is zero until the server sends WINDOW. + internal long DataFramesWritten => Volatile.Read(ref _dataFramesWritten); + + public bool IsDead => Volatile.Read(ref _dead) != 0; + + public int ActiveStores => Volatile.Read(ref _active); + + public static async Task EstablishAsync( + string fqdn, + int port, + string connectHost, + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + int maxOutstanding, + RemoteCertificateValidationCallback? certificateValidation, + CancellationToken cancellationToken) + { + var tcp = new TcpClient { NoDelay = true }; + try + { + tcp.SendBufferSize = 1 << 20; + tcp.ReceiveBufferSize = 1 << 20; + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(options.ConnectTimeout); + try + { + await tcp.ConnectAsync(connectHost, port, connectCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + if (ex.InnerException is SocketException) + { + throw ex.InnerException; + } + + throw new TimeoutException("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + SslStream ssl; + try + { + ssl = await VatpTlsClient.AuthenticateAsClientAsync( + tcp.GetStream(), + fqdn, + options.TlsHandshakeTimeout, + certificateValidation, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("timeout"); + } + var connection = new ArticlePlacementConnection( + fqdn, + port, + tcp, + ssl, + logger, + limits, + options.IoTimeout, + maxOutstanding, + limits.DefaultMaxFramePayload); + try + { + await connection.ExchangeHelloAsync(cancellationToken).ConfigureAwait(false); + connection._reader = Task.Run(connection.ReadLoopAsync); + ArticlePlacementTransportLogMessages.Established(logger, fqdn, port, maxOutstanding); + return connection; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public async Task PlaceAsync( + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + CancellationToken cancellationToken) + { + if (IsDead || Volatile.Read(ref _disposed) != 0) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + } + + var slotWait = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + await _slots.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Add(ref _slotWaitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - slotWait); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + catch (ObjectDisposedException) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + + try + { + using (var operation = Begin(record)) + { + try + { + var now = Interlocked.Increment(ref _active); + int previousPeak; + do + { + previousPeak = Volatile.Read(ref _peakActive); + if (now <= previousPeak) + { + break; + } + } + while (Interlocked.CompareExchange(ref _peakActive, now, previousPeak) != previousPeak); + + return await SendAsync(operation, record, meta, cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _active); + lock (_tableGate) + { + _operations.Remove(operation.StreamId); + } + } + } + } + finally + { + try + { + _slots.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + var outstanding = ActiveStores; + MarkDead(); + try + { + await _lifetime.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + try + { + if (_reader is not null) + { + await _reader.WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + } + } + catch (TimeoutException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + FaultOutstanding(shutdown: true); + ArticlePlacementTransportLogMessages.Closed(_logger, _fqdn, _port, outstanding); + await _ssl.DisposeAsync().ConfigureAwait(false); + _tcp.Dispose(); + _slots.Dispose(); + _write.Dispose(); + _lifetime.Dispose(); + } + + private StoreOperation Begin(ArticleRecord record) + { + lock (_tableGate) + { + if (_nextStreamId == 0) + { + MarkDead(); + throw new IOException("stream-id-space"); + } + + var streamId = _nextStreamId++; + var operation = new StoreOperation(streamId, record.ArtId, _limits); + _operations.Add(streamId, operation); + return operation; + } + } + + private async Task SendAsync( + StoreOperation operation, + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetime.Token); + try + { + var idBytes = new byte[ArticleId.Length]; + record.ArtId.CopyTo(idBytes); + await WriteFrameAsync(VatpFrameEncoder.EncodeStore(operation.StreamId, idBytes), linked.Token) + .ConfigureAwait(false); + operation.Started = true; + var metaBytes = VatpMetaCodec.Encode(in meta); + operation.AwaitingInitialCredit = 1; + Interlocked.Increment(ref _storesAwaitingInitialCredit); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(operation.StreamId, metaBytes), linked.Token) + .ConfigureAwait(false); + DisarmReadTimeout(); + + var artData = record.ArtData; + var offset = 0; + while (offset < record.ArtSize) + { + linked.Token.ThrowIfCancellationRequested(); + int length; + lock (operation.Gate) + { + length = ArticleTransferReadyRing.ComputeDataPayloadLength( + record.ArtSize - offset, + operation.Window.Credit, + _maxFramePayload); + if (length > 0 && !operation.Window.TryConsume(length)) + { + length = 0; + } + } + + if (length <= 0) + { + var creditWait = System.Diagnostics.Stopwatch.GetTimestamp(); + await operation.Credit.WaitAsync(linked.Token).ConfigureAwait(false); + Interlocked.Add(ref _creditWaitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - creditWait); + if (operation.IsCompleted) + { + return operation.Result; + } + + continue; + } + + var fin = offset + length == record.ArtSize; + await WriteFrameAsync( + VatpFrameEncoder.EncodeData(operation.StreamId, artData.Slice(offset, length), fin), + linked.Token) + .ConfigureAwait(false); + Interlocked.Increment(ref _dataFramesWritten); + offset += length; + } + + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(operation.StreamId), linked.Token).ConfigureAwait(false); + operation.EndWritten = true; + var resultWait = System.Diagnostics.Stopwatch.GetTimestamp(); + var result = await operation.Completion.Task.WaitAsync(_ioTimeout, linked.Token).ConfigureAwait(false); + NoteResultWait(System.Diagnostics.Stopwatch.GetTimestamp() - resultWait); + LogOutcome(operation, result); + return result; + } + catch (TimeoutException) + { + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "timeout"); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + catch (OperationCanceledException) when (operation.IsCompleted) + { + LogOutcome(operation, operation.Result); + return operation.Result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (operation.EndWritten) + { + var ambiguous = new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved); + Finish(operation, ambiguous); + LogOutcome(operation, ambiguous); + return ambiguous; + } + + await TryWriteCancelAsync(operation).ConfigureAwait(false); + var cancelled = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + Finish(operation, cancelled); + LogOutcome(operation, cancelled); + return cancelled; + } + catch (OperationCanceledException) + { + var result = _disposed != 0 + ? operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException) + { + MarkDead(); + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, ex.GetType().Name); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + } + + private async Task ExchangeHelloAsync(CancellationToken cancellationToken) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeHello(_maxFramePayload), cancellationToken).ConfigureAwait(false); + var frame = await ReadFrameAsync(cancellationToken).ConfigureAwait(false); + if (frame is null + || frame.Value.Header.Type != VatpFrameType.Hello + || !VatpHello.TryDecode(frame.Value.Payload, out var hello, out _)) + { + throw new IOException("hello"); + } + + _maxFramePayload = Math.Min(hello.MaxFramePayload, VatpProtocol.DefaultMaxFramePayload); + } + + private async Task ReadLoopAsync() + { + try + { + while (!IsDead && Volatile.Read(ref _disposed) == 0) + { + var frame = await ReadFrameAsync(_lifetime.Token).ConfigureAwait(false); + if (frame is null) + { + MarkDead(); + FaultOutstanding(shutdown: false); + return; + } + + Dispatch(frame.Value); + } + } + catch (OperationCanceledException) + { + FaultOutstanding(shutdown: Volatile.Read(ref _disposed) != 0); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException or TimeoutException) + { + MarkDead(); + ArticlePlacementTransportLogMessages.ConnectionFailed(_logger, _fqdn, _port, ex.GetType().Name); + FaultOutstanding(shutdown: false); + } + } + + private void Dispatch(VatpParsedFrame frame) + { + if (frame.Header.Type == VatpFrameType.Hello) + { + return; + } + + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + MarkDead(); + FaultOutstanding(shutdown: false); + return; + } + + StoreOperation? operation; + lock (_tableGate) + { + _operations.TryGetValue(frame.Header.StreamId, out operation); + } + + if (operation is null) + { + return; + } + + switch (frame.Header.Type) + { + case VatpFrameType.Window: + if (!VatpControlPayload.TryDecodeWindow(Copy(frame.Payload), out var add)) + { + Finish(operation, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window")); + break; + } + + var gainedInitialCredit = false; + lock (operation.Gate) + { + gainedInitialCredit = operation.AwaitingInitialCredit == 1 && add > 0; + _ = operation.Window.Add(add); + } + + if (gainedInitialCredit && Interlocked.Exchange(ref operation.AwaitingInitialCredit, 0) == 1) + { + Interlocked.Decrement(ref _storesAwaitingInitialCredit); + } + + try + { + operation.Credit.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + break; + case VatpFrameType.Result: + if (!VatpResultPayload.TryDecode(frame.Payload, out var outcome, out _)) + { + Finish(operation, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "result")); + break; + } + + Finish(operation, ArticlePlacementResult.FromOutcomeByte(outcome)); + break; + case VatpFrameType.Fail: + if (!VatpControlPayload.TryDecodeFail(Copy(frame.Payload), out var error, out _)) + { + Finish(operation, new ArticlePlacementResult( + operation.EndWritten + ? ArticlePlacementKind.AcknowledgementNotObserved + : ArticlePlacementKind.TransportFailure, + "fail")); + break; + } + + Finish(operation, error == VatpErrorCode.Cancelled + ? new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, error.ToString())); + break; + default: + Finish(operation, new ArticlePlacementResult( + ArticlePlacementKind.TransportFailure, + frame.Header.Type.ToString())); + break; + } + } + + private void FaultOutstanding(bool shutdown) + { + StoreOperation[] operations; + lock (_tableGate) + { + operations = _operations.Values.ToArray(); + } + + foreach (var operation in operations) + { + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : shutdown + ? new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + Finish(operation, result); + } + } + + private async Task TryWriteCancelAsync(StoreOperation operation) + { + if (IsDead) + { + return; + } + + try + { + await WriteFrameAsync(VatpFrameEncoder.EncodeCancel(operation.StreamId), CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException or OperationCanceledException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task WriteFrameAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + await _write.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + try + { + if (!frame.Header.IsEmpty) + { + await _ssl.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (!frame.Payload.IsEmpty) + { + await _ssl.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + MarkDead(); + throw; + } + } + finally + { + try + { + _write.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private void NoteResultWait(long ticks) + { + Interlocked.Increment(ref _resultCount); + Interlocked.Add(ref _resultTicks, ticks); + long previous; + do + { + previous = Volatile.Read(ref _resultMaxTicks); + if (ticks <= previous) + { + return; + } + } + while (Interlocked.CompareExchange(ref _resultMaxTicks, ticks, previous) != previous); + } + + private async Task ReadFrameAsync(CancellationToken cancellationToken) + { + while (true) + { + if (_frames.TryTake(_maxFramePayload, out var frame)) + { + return frame; + } + + using (var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + { + lock (_readTimeoutGate) + { + _activeReadTimeout = timeout; + if (Volatile.Read(ref _storesAwaitingInitialCredit) == 0) + { + timeout.CancelAfter(_ioTimeout); + } + } + + int read; + try + { + read = await _frames.ReadFromAsync(_ssl, timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("placement-io"); + } + finally + { + lock (_readTimeoutGate) + { + if (ReferenceEquals(_activeReadTimeout, timeout)) + { + _activeReadTimeout = null; + } + } + } + + if (read == 0) + { + return null; + } + } + } + } + + /// Completes one STORE and drops its initial-credit wait from the read-timeout count. + private void Finish(StoreOperation operation, ArticlePlacementResult result) + { + if (operation.TryFinish(result)) + { + Interlocked.Decrement(ref _storesAwaitingInitialCredit); + } + } + + /// + /// Disarms the outstanding read while at least one STORE is waiting for its first WINDOW. + /// The next read arms the timeout again once that count returns to zero. + /// + private void DisarmReadTimeout() + { + lock (_readTimeoutGate) + { + if (Volatile.Read(ref _storesAwaitingInitialCredit) == 0) + { + return; + } + + var timeout = _activeReadTimeout; + if (timeout is null) + { + return; + } + + try + { + timeout.CancelAfter(Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private void LogOutcome(StoreOperation operation, ArticlePlacementResult result) + { + if (result.Kind == ArticlePlacementKind.AcknowledgementNotObserved) + { + ArticlePlacementTransportLogMessages.StoreAmbiguous(_logger, _fqdn, _port, operation.StreamId); + return; + } + + if (result.Succeeded) + { + ArticlePlacementTransportLogMessages.StoreCompleted( + _logger, _fqdn, _port, operation.StreamId, result.Kind.ToString()); + return; + } + + ArticlePlacementTransportLogMessages.StoreRejected( + _logger, _fqdn, _port, operation.StreamId, result.Kind.ToString(), result.Failure); + } + + private void MarkDead() => Volatile.Write(ref _dead, 1); + + private static byte[] Copy(ReadOnlySequence payload) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return bytes; + } + + private sealed class StoreOperation : IDisposable + { + private int _completed; + + public StoreOperation(uint streamId, ArticleId articleId, ArticleTransferLimits limits) + { + StreamId = streamId; + ArticleId = articleId; + Window = new ArticleTransferWindow(0, limits.MaxStreamCreditBytes); + Credit = new SemaphoreSlim(0); + Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public uint StreamId { get; } + + public ArticleId ArticleId { get; } + + public object Gate { get; } = new(); + + // codeql[cs/missed-readonly-modifier] Mutable struct; readonly would copy on Add/TryConsume and break window credit. + public ArticleTransferWindow Window; + + public SemaphoreSlim Credit { get; } + + public TaskCompletionSource Completion { get; } + + public bool Started { get; set; } + + private volatile bool _endWritten; + + public bool EndWritten + { + get => _endWritten; + set => _endWritten = value; + } + + public bool IsCompleted => Volatile.Read(ref _completed) != 0; + + public ArticlePlacementResult Result { get; private set; } + + /// 1 after META until the first WINDOW or a terminal result, then 0. + public int AwaitingInitialCredit; + + /// + /// Completes the operation once. + /// Returns when this call clears a pending initial-credit wait. + /// + public bool TryFinish(ArticlePlacementResult result) + { + var releasedCreditWait = Interlocked.Exchange(ref AwaitingInitialCredit, 0) == 1; + if (Interlocked.Exchange(ref _completed, 1) != 0) + { + return releasedCreditWait; + } + + Result = result; + Completion.TrySetResult(result); + try + { + Credit.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + return releasedCreditWait; + } + + public void Dispose() => Credit.Dispose(); + } + + /// Accumulates socket bytes and yields one parsed frame at a time. + private sealed class FrameBuffer + { + private byte[] _buffer = new byte[64 * 1024]; + private int _start; + private int _end; + + public bool TryTake(uint maxFramePayload, out VatpParsedFrame frame) + { + frame = default; + if (_end <= _start) + { + return false; + } + + var parsed = VatpFrameParser.ParseOneFrame( + new ReadOnlySequence(_buffer, _start, _end - _start), + maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + throw new IOException(parsed.Error.ToString()); + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } parsedFrame) + { + return false; + } + + var payload = new byte[parsedFrame.Payload.Length]; + parsedFrame.Payload.CopyTo(payload); + frame = new VatpParsedFrame(parsedFrame.Header, new ReadOnlySequence(payload)); + _start += checked((int)parsed.ConsumedBytes); + if (_start == _end) + { + _start = 0; + _end = 0; + } + + return true; + } + + public async ValueTask ReadFromAsync(Stream stream, CancellationToken cancellationToken) + { + Compact(); + if (_end == _buffer.Length) + { + Array.Resize(ref _buffer, _buffer.Length * 2); + } + + var read = await stream.ReadAsync(_buffer.AsMemory(_end), cancellationToken).ConfigureAwait(false); + _end += read; + return read; + } + + private void Compact() + { + if (_start == 0) + { + return; + } + + if (_start == _end) + { + _start = 0; + _end = 0; + return; + } + + Buffer.BlockCopy(_buffer, _start, _buffer, 0, _end - _start); + _end -= _start; + _start = 0; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementTransportLogMessages.cs b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementTransportLogMessages.cs new file mode 100644 index 00000000..e349ff12 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/ArticlePlacementTransportLogMessages.cs @@ -0,0 +1,62 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// Source-generated logs for the long-lived placement connection. Not per DATA frame. + internal static partial class ArticlePlacementTransportLogMessages + { + [LoggerMessage( + EventId = 2640, + Level = LogLevel.Information, + Message = "Placement connection established fqdn={Fqdn} vatpPort={VatpPort} outstandingLimit={OutstandingLimit}")] + public static partial void Established(ILogger logger, string Fqdn, int VatpPort, int OutstandingLimit); + + [LoggerMessage( + EventId = 2641, + Level = LogLevel.Debug, + Message = "Placement connection reused fqdn={Fqdn} vatpPort={VatpPort} activeStores={ActiveStores}")] + public static partial void Reused(ILogger logger, string Fqdn, int VatpPort, int ActiveStores); + + [LoggerMessage( + EventId = 2642, + Level = LogLevel.Warning, + Message = "Placement reconnect deferred fqdn={Fqdn} vatpPort={VatpPort} backoffMs={BackoffMs} failure={Failure}")] + public static partial void ReconnectDeferred(ILogger logger, string Fqdn, int VatpPort, long BackoffMs, string Failure); + + [LoggerMessage( + EventId = 2643, + Level = LogLevel.Warning, + Message = "Placement connection failed fqdn={Fqdn} vatpPort={VatpPort} failure={Failure}")] + public static partial void ConnectionFailed(ILogger logger, string Fqdn, int VatpPort, string Failure); + + [LoggerMessage( + EventId = 2644, + Level = LogLevel.Information, + Message = "Placement connection closed fqdn={Fqdn} vatpPort={VatpPort} outstanding={Outstanding}")] + public static partial void Closed(ILogger logger, string Fqdn, int VatpPort, int Outstanding); + + [LoggerMessage( + EventId = 2645, + Level = LogLevel.Debug, + Message = "Placement STORE completed fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId} outcome={Outcome}")] + public static partial void StoreCompleted(ILogger logger, string Fqdn, int VatpPort, uint StreamId, string Outcome); + + [LoggerMessage( + EventId = 2646, + Level = LogLevel.Warning, + Message = "Placement STORE rejected fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId} outcome={Outcome} failure={Failure}")] + public static partial void StoreRejected(ILogger logger, string Fqdn, int VatpPort, uint StreamId, string Outcome, string? Failure); + + [LoggerMessage( + EventId = 2647, + Level = LogLevel.Warning, + Message = "Placement STORE acknowledgement not observed fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId}")] + public static partial void StoreAmbiguous(ILogger logger, string Fqdn, int VatpPort, uint StreamId); + + [LoggerMessage( + EventId = 2648, + Level = LogLevel.Information, + Message = "Placement transport shutdown outstanding={Outstanding}")] + public static partial void Shutdown(ILogger logger, int Outstanding); + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/IArticlePlacementClient.cs b/src/VectorNNTPD.ReaderService/Storage/IArticlePlacementClient.cs new file mode 100644 index 00000000..693ab0c9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/IArticlePlacementClient.cs @@ -0,0 +1,17 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// + /// Sends one CanonicalV1 article to one StorageServer. + /// The client keeps one TLS connection per target and does not dial for every article. + /// + public interface IArticlePlacementClient + { + /// Places on . One attempt. + ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupContracts.cs b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupContracts.cs new file mode 100644 index 00000000..c283a965 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupContracts.cs @@ -0,0 +1,348 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// One validated StorageServer presence response retained for a single lookup. + /// Responding StorageServer id. + /// Responding StorageServer FQDN. + /// TLS VATP listen port for retrieval. + internal readonly record struct StorageArticleCandidate(int ServerId, string Fqdn, int VatpPort); + + /// + /// Later positives for one in-flight lookup. Completes when an alternate arrives or the + /// original lookup window closes. Does not start another lookup. + /// + internal interface IStorageArticleAlternateSource + { + /// Completes when the lookup window closes. + Task WhenClosed { get; } + + /// + /// Returns the second retained candidate, or when the original + /// lookup window closes without one. + /// + ValueTask WaitForAlternateAsync(CancellationToken cancellationToken); + } + + /// Outcome of a StorageServer fleet article-presence lookup. + public enum StorageArticleLookupOutcome + { + /// At least one StorageServer responded positively. + Found = 0, + + /// No positive response arrived before the lookup timeout. + NotFound = 1, + } + + /// Result of a StorageServer fleet article-presence lookup. + /// Classified outcome. + /// Logical lookup identity. + /// Queried article identity. + /// Winning StorageServer id when found. + /// Winning StorageServer FQDN when found. + /// TLS VATP listen port when found. + /// Detail when not found. + public sealed record StorageArticleLookupResult( + StorageArticleLookupOutcome Outcome, + Guid RequestId, + ArticleId ArticleId, + int? ServerId, + string? Fqdn, + int? VatpPort, + string? Error) + { + /// + /// Later candidates for this lookup, present only while the original lookup window can + /// still accept them. Null when the lookup did not find a first candidate. + /// + internal IStorageArticleAlternateSource? Alternates { get; init; } + + /// Builds a not-found result. + public static StorageArticleLookupResult NotFound(Guid requestId, ArticleId articleId, string error) => + new(StorageArticleLookupOutcome.NotFound, requestId, articleId, null, null, null, error); + } + + /// NNTPD client for StorageServer fleet article-presence lookups. + public interface IStorageArticleLookupClient + { + /// + /// Asks all StorageServers "who has this article?" once. The task completes when the + /// first valid positive arrives or the lookup timeout elapses. The registration stays + /// until that same timeout so one later positive can still be retained. + /// + Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken); + } + + /// + /// Correlates StorageServer lookup responses by AMQP CorrelationId. + /// Matching positives are retained until the lookup window closes. Duplicates and + /// unknowns are ignored. The registration is not removed on the first positive. + /// + internal sealed class StorageArticleLookupResponseRouter + { + private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); + + /// Gets outstanding correlation count (tests). + internal int OutstandingCount => _pending.Count; + + /// Registers a lookup awaiting responses. + internal bool TryRegister(string correlationId, StorageArticleLookupOperation operation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + ArgumentNullException.ThrowIfNull(operation); + if (operation.IsCompleted) + { + return false; + } + + if (!_pending.TryAdd(correlationId, new PendingLookup(operation))) + { + throw new InvalidOperationException($"Duplicate storage lookup correlation '{correlationId}'."); + } + + if (operation.IsCompleted) + { + _pending.TryRemove(correlationId, out _); + return false; + } + + return true; + } + + /// Removes a registration. + internal void Unregister(string correlationId) + { + if (!string.IsNullOrWhiteSpace(correlationId)) + { + _pending.TryRemove(correlationId, out _); + } + } + + /// Cancels all outstanding lookups. + internal void CancelAll() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _pending) + { + if (_pending.TryRemove(pair.Key, out var pending)) + { + pending.Operation.TrySetCanceled(); + } + } + } + + /// + /// Dispatches a parsed positive response. Returns when the + /// response was retained as a candidate. Does not remove the registration. + /// + internal bool TryComplete(string? correlationId, StorageArticleLookupResponse response) + { + ArgumentNullException.ThrowIfNull(response); + if (string.IsNullOrWhiteSpace(correlationId)) + { + return false; + } + + if (!_pending.TryGetValue(correlationId, out var pending)) + { + return false; + } + + if (pending.Operation.RequestId != response.RequestId) + { + return false; + } + + if (pending.Operation.ArticleId != response.ArticleId) + { + return false; + } + + return pending.Operation.TryAccept(response); + } + + private sealed record PendingLookup(StorageArticleLookupOperation Operation); + } + + /// + /// One in-flight StorageServer fleet lookup. Retains at most two distinct candidates in + /// arrival order and wakes waiters on the first without closing the lookup window. + /// + internal sealed class StorageArticleLookupOperation : IStorageArticleAlternateSource + { + private const int MaxRetainedCandidates = 2; + private readonly object _gate = new(); + private readonly List _candidates = new(MaxRetainedCandidates); + private readonly HashSet _serverIds = []; + private readonly HashSet _endpoints = new(StringComparer.OrdinalIgnoreCase); + private readonly TaskCompletionSource _firstReady = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _alternateOrClosed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _closed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private bool _windowClosed; + + /// Initializes a lookup operation. + internal StorageArticleLookupOperation(Guid requestId, ArticleId articleId) + { + RequestId = requestId; + ArticleId = articleId; + } + + /// Logical request identity. + internal Guid RequestId { get; } + + /// Queried article identity. + internal ArticleId ArticleId { get; } + + /// Completes when the first candidate is retained or the window closes without one. + internal Task FirstReady => _firstReady.Task; + + /// + public Task WhenClosed => _closed.Task; + + /// Whether the window has been closed. + internal bool IsCompleted + { + get + { + lock (_gate) + { + return _windowClosed; + } + } + } + + /// Reads the first retained candidate. + internal bool TryGetFirst(out StorageArticleCandidate candidate) + { + lock (_gate) + { + if (_candidates.Count > 0) + { + candidate = _candidates[0]; + return true; + } + } + + candidate = default; + return false; + } + + /// + /// Retains when it is a new valid candidate. + /// Invalid, duplicate, and over-cap responses are ignored. + /// + internal bool TryAccept(StorageArticleLookupResponse response) + { + if (!TryValidate(response, out var candidate, out var endpointKey)) + { + return false; + } + + lock (_gate) + { + if (_windowClosed || _candidates.Count >= MaxRetainedCandidates) + { + return false; + } + + if (_serverIds.Contains(candidate.ServerId) || _endpoints.Contains(endpointKey)) + { + return false; + } + + _serverIds.Add(candidate.ServerId); + _endpoints.Add(endpointKey); + _candidates.Add(candidate); + if (_candidates.Count == 1) + { + _firstReady.TrySetResult(true); + } + else + { + _alternateOrClosed.TrySetResult(true); + } + + return true; + } + } + + /// Closes the window so no further candidate can be retained. + internal void CloseWindow() + { + lock (_gate) + { + if (_windowClosed) + { + return; + } + + _windowClosed = true; + } + + _firstReady.TrySetResult(false); + _alternateOrClosed.TrySetResult(false); + _closed.TrySetResult(); + } + + /// Cancels waiters. Used when the lookup service is stopping. + internal bool TrySetCanceled() + { + CloseWindow(); + return true; + } + + /// + public async ValueTask WaitForAlternateAsync(CancellationToken cancellationToken) + { + lock (_gate) + { + if (_candidates.Count >= 2) + { + return _candidates[1]; + } + + if (_windowClosed) + { + return null; + } + } + + await _alternateOrClosed.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_gate) + { + return _candidates.Count >= 2 ? _candidates[1] : null; + } + } + + private bool TryValidate( + StorageArticleLookupResponse response, + out StorageArticleCandidate candidate, + out string endpointKey) + { + candidate = default; + endpointKey = string.Empty; + if (!ServerIdRules.IsInRange(response.ServerId)) + { + return false; + } + + if (response.ArticleId != ArticleId + || !VatpEndpointFields.IsCanonicalFqdn(response.Fqdn) + || !VatpEndpointFields.IsCanonicalPort(response.VatpPort)) + { + return false; + } + + candidate = new StorageArticleCandidate(response.ServerId, response.Fqdn, response.VatpPort); + endpointKey = response.Fqdn + ":" + response.VatpPort.ToString(System.Globalization.CultureInfo.InvariantCulture); + return true; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupLogMessages.cs b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupLogMessages.cs new file mode 100644 index 00000000..9e20a4e3 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupLogMessages.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.ReaderService.Storage +{ + /// Source-generated StorageServer article-lookup log messages. + internal static partial class StorageArticleLookupLogMessages + { + [LoggerMessage( + EventId = 2870, + Level = LogLevel.Information, + Message = "Storage article lookup ready (replyTo={ReplyTo}, generation={Generation})")] + public static partial void Ready(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2871, + Level = LogLevel.Information, + Message = "Storage article lookup session replaced (replyTo={ReplyTo}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2872, + Level = LogLevel.Error, + Message = "Storage article lookup session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2873, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for storage article lookup")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2874, + Level = LogLevel.Debug, + Message = "Storage article lookup published (requestId={RequestId}, correlationId={CorrelationId}, articleId={ArticleId}, generation={Generation})")] + public static partial void Published( + ILogger logger, + Guid RequestId, + string CorrelationId, + string ArticleId, + long Generation); + + [LoggerMessage( + EventId = 2875, + Level = LogLevel.Debug, + Message = "Storage article lookup response ignored (correlationId={CorrelationId}, reason={Reason})")] + public static partial void ResponseIgnored(ILogger logger, string CorrelationId, string Reason); + + [LoggerMessage( + EventId = 2876, + Level = LogLevel.Error, + Message = "Storage article lookup delivery handling failed")] + public static partial void DeliveryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2877, + Level = LogLevel.Information, + Message = "Storage article lookup stopped")] + public static partial void Stopped(ILogger logger); + + [LoggerMessage( + EventId = 2878, + Level = LogLevel.Debug, + Message = "Storage article lookup skipped because the registry has no StorageServers (articleId={ArticleId})")] + public static partial void SkippedEmptyRegistry(ILogger logger, string ArticleId); + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupService.cs b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupService.cs new file mode 100644 index 00000000..ea436d8d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageArticleLookupService.cs @@ -0,0 +1,480 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.RabbitMq; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// + /// NNTPD-owned StorageServer fleet article-presence lookup: publish to cache.requests, + /// consume replies on an ephemeral reply queue, and return the first valid positive immediately. + /// + /// + /// The registration stays until the existing lookup timeout so one later distinct positive + /// can be retained. Independent of ArticleWork RPC. Negative responses are not expected. + /// When has no entries, no cache.requests + /// publication is made and the existing miss result is returned so ARTICLE/HEAD/BODY/STAT + /// continue to ArticleWork. An empty registry is not an article-not-found result. + /// A registry that still contains any entry, including a stale or draining one, uses the + /// existing publication path. + /// + internal sealed class StorageArticleLookupService : IApplicationService, IStorageArticleLookupClient + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly IStorageServerRegistry? _storageServers; + private readonly StorageArticleLookupResponseRouter _router = new(); + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly object _sessionGate = new(); + private readonly Guid _instanceId = Guid.NewGuid(); + private CancellationTokenSource? _shutdownCts; + private LookupSession? _session; + private int _started; + + /// Initializes a new storage article lookup service. + public StorageArticleLookupService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + ILogger logger, + TimeProvider? timeProvider = null, + IStorageServerRegistry? storageServers = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _storageServers = storageServers; + } + + /// + public string Name => "StorageArticleLookup"; + + /// + public Task? Execution => null; + + /// Gets the current reply queue name when attached (tests). + internal string? CurrentReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo; + } + } + } + + /// Gets outstanding correlations (tests). + internal int OutstandingCorrelations => _router.OutstandingCount; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + /// + public async Task LookupAsync( + ArticleId articleId, + CancellationToken cancellationToken) + { + if (_storageServers is not null && _storageServers.Snapshot().Count == 0) + { + StorageArticleLookupLogMessages.SkippedEmptyRegistry(_logger, articleId.ToLowerHexString()); + return StorageArticleLookupResult.NotFound( + Guid.NewGuid(), + articleId, + "Storage article lookup timed out with no positive response."); + } + + if (Volatile.Read(ref _started) == 0 || _shutdownCts is null) + { + throw new InvalidOperationException("Storage article lookup is not started."); + } + + var requestId = Guid.NewGuid(); + var correlationId = Guid.NewGuid().ToString("D"); + var operation = new StorageArticleLookupOperation(requestId, articleId); + // codeql[cs/missed-using-statement] Linked CTS ownership transfers to ObserveLookupWindowAsync when the lookup window starts; disposed there or in the !windowStarted finally below. A using around LookupAsync would cancel the background window early. + var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token); + bool registered; + try + { + // codeql[cs/dispose-not-called-on-throw] linked is disposed in catch below, on !registered, or by ObserveLookupWindowAsync after ownership transfer; method-scoped using would cancel the window early. + registered = _router.TryRegister(correlationId, operation); + } + catch + { + linked.Dispose(); + throw; + } + + if (!registered) + { + // codeql[cs/dispose-not-called-on-throw] Explicit dispose on registration failure; throw path disposed in catch. Ownership otherwise transfers to ObserveLookupWindowAsync. + linked.Dispose(); + return StorageArticleLookupResult.NotFound(requestId, articleId, "Lookup could not be registered."); + } + + var windowStarted = false; + try + { + await PublishRequestAsync(requestId, correlationId, articleId, linked.Token).ConfigureAwait(false); + windowStarted = true; + _ = ObserveLookupWindowAsync(operation, correlationId, linked); + await operation.FirstReady.WaitAsync(linked.Token).ConfigureAwait(false); + if (operation.TryGetFirst(out var first)) + { + return new StorageArticleLookupResult( + StorageArticleLookupOutcome.Found, + requestId, + articleId, + first.ServerId, + first.Fqdn, + first.VatpPort, + Error: null) + { + Alternates = operation, + }; + } + + return StorageArticleLookupResult.NotFound( + requestId, + articleId, + "Storage article lookup timed out with no positive response."); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + operation.CloseWindow(); + throw; + } + catch (OperationCanceledException) + { + operation.CloseWindow(); + return StorageArticleLookupResult.NotFound( + requestId, + articleId, + "Storage article lookup was canceled."); + } + finally + { + if (!windowStarted) + { + _router.Unregister(correlationId); + linked.Dispose(); + } + } + } + + private async Task ObserveLookupWindowAsync( + StorageArticleLookupOperation operation, + string correlationId, + CancellationTokenSource linked) + { + using (linked) + { + try + { + await Task.Delay(CacheFleetTopology.LookupTimeout, _timeProvider, linked.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + finally + { + operation.CloseWindow(); + _router.Unregister(correlationId); + } + } + } + + /// Completes a lookup from a parsed response body (tests). + internal bool TryDispatchResponse(string correlationId, StorageArticleLookupResponse response) => + _router.TryComplete(correlationId, response); + + private async Task PublishRequestAsync( + Guid requestId, + string correlationId, + ArticleId articleId, + CancellationToken cancellationToken) + { + LookupSession session; + lock (_sessionGate) + { + session = _session ?? throw new InvalidOperationException("Storage article lookup session is not ready."); + } + + if (!_rabbitMq.TryGetCurrent(out var handle) + || !handle.IsCurrent + || handle.Generation != session.Generation) + { + throw new InvalidOperationException( + $"Storage article lookup publication failed: connection generation {session.Generation} is not current."); + } + + var body = StorageArticleLookupWireProtocol.SerializeRequestV1( + new StorageArticleLookupRequest( + StorageArticleLookupWireProtocol.CurrentVersion, + requestId, + articleId)); + + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + lock (_sessionGate) + { + if (!ReferenceEquals(_session, session)) + { + throw new InvalidOperationException("Storage article lookup session was replaced."); + } + } + + await session.PublishChannel + .PublishAsync( + CacheRequestsTopology.ExchangeName, + routingKey: string.Empty, + correlationId, + requestId.ToString("D"), + session.ReplyTo, + StorageArticleLookupWireProtocol.JsonContentType, + CacheRequestsTopology.ExpirationMilliseconds, + body, + cancellationToken) + .ConfigureAwait(false); + + StorageArticleLookupLogMessages.Published( + _logger, + requestId, + correlationId, + articleId.ToLowerHexString(), + session.Generation); + } + finally + { + _publishGate.Release(); + } + } + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + StorageArticleLookupLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for storage article lookup."); + } + + var generation = handle.Generation; + IRabbitMqRpcChannel? publishChannel = null; + IRabbitMqRpcChannel? consumeChannel = null; + try + { + publishChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + consumeChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + var replyTo = CreateReplyQueueName(); + await consumeChannel.QueueDeclareAsync( + replyTo, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + _ = await consumeChannel + .ConsumeAsync(replyTo, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new LookupSession(generation, replyTo, publishChannel, consumeChannel); + LookupSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + StorageArticleLookupLogMessages.SessionReplaced(_logger, replyTo, generation); + } + else + { + StorageArticleLookupLogMessages.Ready(_logger, replyTo, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + StorageArticleLookupLogMessages.SessionAttachFailed(_logger, ex, generation); + if (publishChannel is not null) + { + await publishChannel.DisposeAsync().ConfigureAwait(false); + } + + if (consumeChannel is not null) + { + await consumeChannel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqRpcDelivery delivery) + { + try + { + if (!StorageArticleLookupWireProtocol.TryParseResponseV1( + delivery.Body.Span, + out var response, + out var reason) + || response is null) + { + StorageArticleLookupLogMessages.ResponseIgnored(_logger, delivery.CorrelationId ?? string.Empty, reason); + return; + } + + if (!_router.TryComplete(delivery.CorrelationId, response)) + { + StorageArticleLookupLogMessages.ResponseIgnored( + _logger, + delivery.CorrelationId ?? string.Empty, + "Unknown, duplicate, or mismatched correlation."); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageArticleLookupLogMessages.DeliveryFailed(_logger, ex); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageArticleLookupLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + _router.CancelAll(); + + LookupSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + StorageArticleLookupLogMessages.Stopped(_logger); + } + + private string CreateReplyQueueName() + { + var serverId = _nntpdOptions.Value.ServerId ?? 0; + return RabbitMqTopologyNames.Normalize($"nntpd.{serverId:00}.{_instanceId:N}.cache.lookup"); + } + + private sealed class LookupSession : IAsyncDisposable + { + internal LookupSession( + long generation, + string replyTo, + IRabbitMqRpcChannel publishChannel, + IRabbitMqRpcChannel consumeChannel) + { + Generation = generation; + ReplyTo = replyTo; + PublishChannel = publishChannel; + ConsumeChannel = consumeChannel; + } + + internal long Generation { get; } + + internal string ReplyTo { get; } + + internal IRabbitMqRpcChannel PublishChannel { get; } + + internal IRabbitMqRpcChannel ConsumeChannel { get; } + + public async ValueTask DisposeAsync() + { + await PublishChannel.DisposeAsync().ConfigureAwait(false); + await ConsumeChannel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetConsumerService.cs b/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetConsumerService.cs new file mode 100644 index 00000000..0a5c45d9 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetConsumerService.cs @@ -0,0 +1,365 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.RabbitMq; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// + /// Consumes StorageServer advertisements from cache.broadcast into + /// . + /// + /// + /// Declares an ephemeral per-instance queue cache.<nntpd-fqdn> (non-durable, + /// exclusive, auto-delete) bound to the fanout exchange so every NNTPD receives every + /// advertisement without competing consumers. + /// + internal sealed class StorageServerFleetConsumerService : IApplicationService + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly IStorageServerRegistry _registry; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly object _sessionGate = new(); + private CancellationTokenSource? _shutdownCts; + private ConsumeSession? _session; + private int _started; + + /// Initializes a new fleet advertisement consumer. + public StorageServerFleetConsumerService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IStorageServerRegistry registry, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(registry); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _registry = registry; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public string Name => "StorageServerFleetConsumer"; + + /// + public Task? Execution => null; + + /// Gets the current instance queue name when attached (tests). + internal string? CurrentQueueName + { + get + { + lock (_sessionGate) + { + return _session?.QueueName; + } + } + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + StorageServerFleetLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for StorageServer fleet consume."); + } + + var fqdn = _nntpdOptions.Value.Fqdn; + if (string.IsNullOrWhiteSpace(fqdn)) + { + throw new InvalidOperationException("StorageServer fleet consume requires the generated NNTPD FQDN."); + } + + var queueName = CacheBroadcastTopology.BuildInstanceQueueName(fqdn); + var generation = handle.Generation; + IRabbitMqManualAckChannel? channel = null; + try + { + channel = await handle.Connection + .CreateManualAckChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + await channel.ExchangeDeclareAsync( + CacheBroadcastTopology.ExchangeName, + CacheBroadcastTopology.ExchangeTypeName, + CacheBroadcastTopology.ExchangeDurable, + CacheBroadcastTopology.ExchangeAutoDelete, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + await channel.QueueDeclareAsync( + queueName, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + await channel.QueueBindAsync( + queueName, + CacheBroadcastTopology.ExchangeName, + routingKey: string.Empty, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + var consumerTag = await channel + .BasicConsumeAsync(queueName, prefetchCount: 64, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new ConsumeSession(generation, queueName, consumerTag, channel); + ConsumeSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + StorageServerFleetLogMessages.SessionReplaced(_logger, queueName, generation); + } + else + { + StorageServerFleetLogMessages.Ready(_logger, queueName, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + StorageServerFleetLogMessages.SessionAttachFailed(_logger, ex, generation); + if (channel is not null) + { + await channel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqManualAckDelivery delivery) + { + ConsumeSession? session; + lock (_sessionGate) + { + session = _session; + } + + if (session is null || delivery.Generation != session.Generation) + { + return; + } + + try + { + if (!string.Equals( + delivery.ContentType, + StorageServerAdvertisementWireProtocol.JsonContentType, + StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(delivery.ContentType)) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, "Unexpected content type."); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + var receivedAt = _timeProvider.GetUtcNow(); + if (StorageServerLifecycleWireProtocol.TryParseV1( + delivery.Body.Span, + out var announcement, + out var lifecycleReason, + out var lifecycleRecognized)) + { + if (announcement is null) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, lifecycleReason); + } + else + { + _registry.ApplyLifecycle(announcement, receivedAt); + } + + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + if (lifecycleRecognized) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, lifecycleReason); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + if (!StorageServerAdvertisementWireProtocol.TryParseV1( + delivery.Body.Span, + out var advertisement, + out var reason) + || advertisement is null) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, reason); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + _registry.ApplyAdvertisement(advertisement, receivedAt); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageServerFleetLogMessages.DeliveryFailed(_logger, ex); + try + { + await session.Channel + .BasicNackAsync(delivery.DeliveryTag, requeue: false, CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception nackEx) + { + StorageServerFleetLogMessages.DeliveryFailed(_logger, nackEx); + } + } + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageServerFleetLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + ConsumeSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + StorageServerFleetLogMessages.Stopped(_logger); + } + + private sealed class ConsumeSession : IAsyncDisposable + { + internal ConsumeSession( + long generation, + string queueName, + string consumerTag, + IRabbitMqManualAckChannel channel) + { + Generation = generation; + QueueName = queueName; + ConsumerTag = consumerTag; + Channel = channel; + } + + internal long Generation { get; } + + internal string QueueName { get; } + + internal string ConsumerTag { get; } + + internal IRabbitMqManualAckChannel Channel { get; } + + public async ValueTask DisposeAsync() + { + try + { + if (Channel.IsOpen) + { + await Channel.BasicCancelAsync(ConsumerTag, CancellationToken.None).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Channel may already be closed by connection loss. + } + + await Channel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetLogMessages.cs b/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetLogMessages.cs new file mode 100644 index 00000000..9c3c7375 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageServerFleetLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.ReaderService.Storage +{ + /// Source-generated StorageServer fleet consumer log messages. + internal static partial class StorageServerFleetLogMessages + { + [LoggerMessage( + EventId = 2860, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer ready (queue={Queue}, generation={Generation})")] + public static partial void Ready(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2861, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer session replaced (queue={Queue}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2862, + Level = LogLevel.Error, + Message = "StorageServer fleet consumer session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2863, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for StorageServer fleet consume")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2864, + Level = LogLevel.Warning, + Message = "Rejected StorageServer advertisement payload ({Reason})")] + public static partial void PayloadRejected(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 2865, + Level = LogLevel.Error, + Message = "StorageServer fleet delivery handling failed")] + public static partial void DeliveryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2866, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.ReaderService/Storage/StorageServerRegistry.cs b/src/VectorNNTPD.ReaderService/Storage/StorageServerRegistry.cs new file mode 100644 index 00000000..845f4b34 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Storage/StorageServerRegistry.cs @@ -0,0 +1,267 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.ReaderService.Storage +{ + /// Liveness classification for a known StorageServer. + public enum StorageServerFleetState + { + /// LastSeen is within the cache-fleet liveness window. + Active = 0, + + /// LastSeen is older than the cache-fleet liveness window. + Stale = 1, + } + + /// + /// One in-memory StorageServer fleet entry maintained from cache broadcast advertisements. + /// + /// Numeric StorageServer identity. + /// StorageServer FQDN (registry key). + /// Latest advertised total storage bytes. + /// Latest advertised used storage bytes. + /// Latest advertised available storage bytes. + /// UTC time when the latest accepted advertisement or Draining announcement was applied. + /// + /// Advertised TLS VATP port. Null entries stay in the liveness view and are not placement targets. + /// + /// + /// When , the entry is excluded from + /// even while is inside the liveness window. Liveness expiry does not clear it. + /// + /// + /// Newest advertisement or Draining timestamp accepted for this FQDN. An older timestamp is not applied. + /// This orders one advertisement stream. It is not a process identity. + /// + public readonly record struct StorageServerFleetEntry( + int ServerId, + string Fqdn, + long TotalBytes, + long UsedBytes, + long AvailableBytes, + DateTimeOffset LastSeen, + int? VatpPort = null, + bool IsDraining = false, + DateTimeOffset NewestMessageTimestamp = default) + { + /// + /// Classifies this entry relative to using + /// . + /// + public StorageServerFleetState GetState(DateTimeOffset utcNow) => + utcNow - LastSeen <= CacheFleetTopology.LivenessWindow + ? StorageServerFleetState.Active + : StorageServerFleetState.Stale; + + /// Returns whether this entry is active at . + public bool IsActive(DateTimeOffset utcNow) => GetState(utcNow) == StorageServerFleetState.Active; + } + + /// + /// In-memory authoritative view of currently known StorageServers for future placement decisions. + /// + /// + /// Hot-path lookups must use this registry only. Do not call RabbitMQ, disk, or network from + /// selection logic merely to learn which StorageServers are alive. + /// A periodic advertisement is the readiness, liveness, capacity, and VATP endpoint signal. + /// Draining removes the FQDN from immediately and does not change + /// capacity bytes. There is no separate Ready announcement and no process epoch. + /// + public interface IStorageServerRegistry + { + /// + /// Applies a parsed advertisement, creating or updating the FQDN-keyed entry and refreshing + /// when the advertisement is accepted. + /// + /// Validated advertisement payload. + /// UTC receive time used as LastSeen when the advertisement is accepted. + /// + /// An advertisement older than is ignored. + /// While the entry is draining, only a strictly newer advertisement is accepted, and that advertisement + /// clears draining. A restarted process whose timestamps are not strictly newer than the accepted + /// Draining timestamp cannot be distinguished from a delayed advertisement and stays out of + /// . Receive time is not compared with the message timestamp. + /// + void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc); + + /// Attempts to read the entry for . + bool TryGet(string fqdn, out StorageServerFleetEntry entry); + + /// Returns a snapshot of all known entries (active and stale). + IReadOnlyList Snapshot(); + + /// + /// Returns entries that are inside the liveness window and not draining at . + /// + IReadOnlyList GetActive(DateTimeOffset utcNow); + + /// + /// Applies a Draining announcement for the announcement FQDN. + /// + /// Validated Draining announcement. + /// UTC receive time used as LastSeen when the announcement is accepted. + /// + /// Draining removes the FQDN from immediately. Capacity bytes already recorded + /// for the FQDN are kept. A Draining timestamp older than + /// is ignored, so a delayed Draining + /// announcement does not override a newer advertisement. Liveness expiry does not clear Draining. + /// + void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc); + } + + /// Thread-safe in-memory StorageServer fleet registry. + public sealed class StorageServerRegistry : IStorageServerRegistry + { + private readonly ConcurrentDictionary _entries = + new(StringComparer.OrdinalIgnoreCase); + + /// + public void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc) + { + ArgumentNullException.ThrowIfNull(advertisement); + ArgumentException.ThrowIfNullOrWhiteSpace(advertisement.Fqdn); + + var key = advertisement.Fqdn.Trim(); + var receivedAt = receivedAtUtc.ToUniversalTime(); + _entries.AddOrUpdate( + key, + _ => EntryFromAdvertisement(advertisement, key, receivedAt), + (_, existing) => MergeAdvertisement(existing, advertisement, key, receivedAt)); + } + + /// + public void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc) + { + ArgumentNullException.ThrowIfNull(announcement); + ArgumentException.ThrowIfNullOrWhiteSpace(announcement.Fqdn); + if (announcement.State != StorageServerLifecycleState.Draining) + { + return; + } + + var key = announcement.Fqdn.Trim(); + var receivedAt = receivedAtUtc.ToUniversalTime(); + _entries.AddOrUpdate( + key, + _ => EntryFromDraining(announcement, key, receivedAt), + (_, existing) => MergeDraining(existing, announcement, key, receivedAt)); + } + + private static StorageServerFleetEntry EntryFromAdvertisement( + StorageServerAdvertisement advertisement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = advertisement.Timestamp.ToUniversalTime(); + return new StorageServerFleetEntry( + advertisement.ServerId, + key, + advertisement.TotalBytes, + advertisement.UsedBytes, + advertisement.AvailableBytes, + receivedAt, + advertisement.VatpPort, + IsDraining: false, + timestamp); + } + + private static StorageServerFleetEntry EntryFromDraining( + StorageServerLifecycleAnnouncement announcement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = announcement.Timestamp.ToUniversalTime(); + return new StorageServerFleetEntry( + announcement.ServerId, + key, + TotalBytes: 0, + UsedBytes: 0, + AvailableBytes: 0, + receivedAt, + announcement.VatpPort, + IsDraining: true, + timestamp); + } + + private static StorageServerFleetEntry MergeAdvertisement( + StorageServerFleetEntry existing, + StorageServerAdvertisement advertisement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = advertisement.Timestamp.ToUniversalTime(); + if (timestamp < existing.NewestMessageTimestamp) + { + return existing; + } + + if (existing.IsDraining && timestamp <= existing.NewestMessageTimestamp) + { + return existing; + } + + return new StorageServerFleetEntry( + advertisement.ServerId, + key, + advertisement.TotalBytes, + advertisement.UsedBytes, + advertisement.AvailableBytes, + receivedAt, + advertisement.VatpPort, + IsDraining: false, + timestamp); + } + + private static StorageServerFleetEntry MergeDraining( + StorageServerFleetEntry existing, + StorageServerLifecycleAnnouncement announcement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = announcement.Timestamp.ToUniversalTime(); + if (timestamp < existing.NewestMessageTimestamp) + { + return existing; + } + + return existing with + { + ServerId = announcement.ServerId, + Fqdn = key, + LastSeen = receivedAt, + VatpPort = announcement.VatpPort, + IsDraining = true, + NewestMessageTimestamp = timestamp, + }; + } + + /// + public bool TryGet(string fqdn, out StorageServerFleetEntry entry) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + return _entries.TryGetValue(fqdn.Trim(), out entry); + } + + /// + public IReadOnlyList Snapshot() => [.. _entries.Values]; + + /// + public IReadOnlyList GetActive(DateTimeOffset utcNow) + { + var now = utcNow.ToUniversalTime(); + List? active = null; + foreach (var entry in _entries.Values) + { + if (!entry.IsActive(now) || entry.IsDraining) + { + continue; + } + + active ??= []; + active.Add(entry); + } + + return active is null ? Array.Empty() : active; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryLogMessages.cs b/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryLogMessages.cs new file mode 100644 index 00000000..3133f67f --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryLogMessages.cs @@ -0,0 +1,151 @@ +namespace VectorNNTPD.ReaderService.Telemetry +{ + /// Source-generated standard application telemetry messages. + internal static partial class ApplicationTelemetryLogMessages + { + [LoggerMessage( + EventId = 2400, + Level = LogLevel.Information, + Message = "HistoryDb lookups={Lookups} hits={Hits} misses={Misses} errors={Errors} wait_ms={WaitMs}")] + public static partial void HistoryDb( + ILogger logger, + long Lookups, + long Hits, + long Misses, + long Errors, + long WaitMs); + + [LoggerMessage( + EventId = 2401, + Level = LogLevel.Information, + Message = "TransitIngressQueue articles={Articles} bytes={Bytes} peak_articles={PeakArticles} peak_bytes={PeakBytes} waiting={Waiting} limit={Limit} admit_fail={AdmissionFailures} admission_wait_ms={AdmissionWaitMs}")] + public static partial void TransitIngressQueue( + ILogger logger, + int Articles, + long Bytes, + int PeakArticles, + long PeakBytes, + int Waiting, + long Limit, + long AdmissionFailures, + long AdmissionWaitMs); + + [LoggerMessage( + EventId = 2402, + Level = LogLevel.Information, + Message = "ActiveSessions active={Active} established={Established} idle={Idle} receiving={Receiving} waiting_history={WaitingHistory} waiting_queue={WaitingQueue} waiting_window={WaitingWindow} completing={Completing}")] + public static partial void ActiveSessions( + ILogger logger, + int Active, + int Established, + int Idle, + int Receiving, + int WaitingHistory, + int WaitingQueue, + int WaitingWindow, + int Completing); + + [LoggerMessage( + EventId = 2403, + Level = LogLevel.Information, + Message = "{PeerName} active={Active}/{MaxIncomingConnections} peak={Peak} accepted={Accepted} transmitted={Transmitted} rejected={Rejected} articles={Articles} bytes={Bytes} avg_article={AverageArticleBytes} avg_mbps={AverageMbps:F2} checks={Checks}")] + public static partial void TransitPeer( + ILogger logger, + string PeerName, + int Active, + int MaxIncomingConnections, + int Peak, + long Accepted, + long Transmitted, + long Rejected, + long Articles, + long Bytes, + long AverageArticleBytes, + double AverageMbps, + long Checks); + + [LoggerMessage( + EventId = 2404, + Level = LogLevel.Information, + Message = "IngestionWorker items={Items} arts_s={ArticlesPerSec:F2} busy_pct={BusyPct:F1} idle_ms={IdleMs} busy_ms={BusyMs} dq_p95_ms={DequeueP95Ms} to_pub_p95_ms={ToPublishP95Ms} encode_p95_ms={EncodeP95Ms} news_p95_ms={NewsP95Ms} item_avg_ms={ItemAvgMs} item_p95_ms={ItemP95Ms}")] + public static partial void IngestionWorker( + ILogger logger, + long Items, + double ArticlesPerSec, + double BusyPct, + long IdleMs, + long BusyMs, + long DequeueP95Ms, + long ToPublishP95Ms, + long EncodeP95Ms, + long NewsP95Ms, + double ItemAvgMs, + long ItemP95Ms); + + [LoggerMessage( + EventId = 2405, + Level = LogLevel.Information, + Message = "OverviewDbHandoff gate_p95_ms={GateP95Ms} pub_p95_ms={PublishP95Ms} confirm_avg_ms={ConfirmAvgMs} confirm_p95_ms={ConfirmP95Ms} handoff_avg_ms={HandoffAvgMs} handoff_p95_ms={HandoffP95Ms} confirm_busy_pct={ConfirmBusyPct:F1} fail={ConfirmFailures} timeout={ConfirmTimeouts} in_flight={InFlightPublishes} in_flight_max={MaxInFlightPublishes}")] + public static partial void OverviewDbHandoff( + ILogger logger, + long GateP95Ms, + long PublishP95Ms, + double ConfirmAvgMs, + long ConfirmP95Ms, + double HandoffAvgMs, + long HandoffP95Ms, + double ConfirmBusyPct, + long ConfirmFailures, + long ConfirmTimeouts, + int InFlightPublishes, + int MaxInFlightPublishes); + + [LoggerMessage( + EventId = 2406, + Level = LogLevel.Information, + Message = "TakeThisPipeline occupied={Occupied} occupied_max={OccupiedMax} occupied_full_sessions={OccupiedFullSessions} occupied_full_waits={OccupiedFullWaits} processing={Processing} processing_max={ProcessingMax} emit_gate_p95_ms={EmitGateP95Ms} enqueue_avg_ms={EnqueueAvgMs} enqueue_p95_ms={EnqueueP95Ms} budget_wait_p95_ms={BudgetWaitP95Ms} budget_waits={BudgetWaits} budget_exhausted={BudgetExhausted}")] + public static partial void TakeThisPipeline( + ILogger logger, + int Occupied, + int OccupiedMax, + int OccupiedFullSessions, + long OccupiedFullWaits, + int Processing, + int ProcessingMax, + long EmitGateP95Ms, + double EnqueueAvgMs, + long EnqueueP95Ms, + long BudgetWaitP95Ms, + long BudgetWaits, + int BudgetExhausted); + + [LoggerMessage( + EventId = 2407, + Level = LogLevel.Information, + Message = "IngestionWorkerPool workers={Workers} min={MinWorkers} max={MaxWorkers} scale_up={ScaleUps} scale_down={ScaleDowns} queue_util={QueueUtilisation:F3} waiting={WaitingProducers}")] + public static partial void IngestionWorkerPool( + ILogger logger, + int Workers, + int MinWorkers, + int MaxWorkers, + long ScaleUps, + long ScaleDowns, + double QueueUtilisation, + int WaitingProducers); + + [LoggerMessage( + EventId = 2408, + Level = LogLevel.Information, + Message = "OverviewDbWorkQueue articles={Articles} bytes={Bytes} waiting={Waiting} publishers={Publishers} pub_min={MinPublishers} pub_max={MaxPublishers} scale_up={ScaleUps} scale_down={ScaleDowns}")] + public static partial void OverviewDbWorkQueue( + ILogger logger, + int Articles, + long Bytes, + int Waiting, + int Publishers, + int MinPublishers, + int MaxPublishers, + long ScaleUps, + long ScaleDowns); + } +} diff --git a/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryService.cs b/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryService.cs new file mode 100644 index 00000000..760aea39 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Telemetry/ApplicationTelemetryService.cs @@ -0,0 +1,410 @@ +using System.Diagnostics; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Telemetry +{ + /// + /// Always-on one-minute application telemetry. Independent of FeedDiagnostics. + /// + public sealed class ApplicationTelemetryService : IApplicationService, IAsyncDisposable + { + /// Production emit period. + public static readonly TimeSpan Period = TimeSpan.FromMinutes(1); + + private readonly IArticleIngestionQueue _queue; + private readonly INntpSessionCensus _census; + private readonly IHistoryLookupMetrics? _history; + private readonly TransitConfigurationStore _transit; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly IAsyncInterval? _injectedInterval; + private readonly TimeSpan _period; + private readonly CancellationTokenSource _runCts = new(); + private readonly object _emitGate = new(); + private Task? _execution; + private int _disposed; + private int _emitBusy; + private long _priorLookups; + private long _priorHits; + private long _priorMisses; + private long _priorErrors; + private long _priorWaitTicks; + private long _priorAdmissionFailures; + private long _priorAdmissionWaitTicks; + private readonly Dictionary _peerPriors = new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + public ApplicationTelemetryService( + IArticleIngestionQueue queue, + INntpSessionCensus census, + ILogger logger, + IHistoryLookupMetrics? history = null, + TimeProvider? timeProvider = null, + TransitConfigurationStore? transit = null, + ITransitPeerMetrics? peerMetrics = null, + IngestionPipelineMetrics? pipelineMetrics = null) + : this(queue, census, logger, history, timeProvider, interval: null, period: null, transit, peerMetrics, pipelineMetrics) + { + } + + /// Initializes a new instance with an injectable interval (tests). + internal ApplicationTelemetryService( + IArticleIngestionQueue queue, + INntpSessionCensus census, + ILogger logger, + IHistoryLookupMetrics? history, + TimeProvider? timeProvider, + IAsyncInterval? interval, + TimeSpan? period, + TransitConfigurationStore? transit = null, + ITransitPeerMetrics? peerMetrics = null, + IngestionPipelineMetrics? pipelineMetrics = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(census); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _census = census; + _history = history; + _transit = transit ?? new TransitConfigurationStore(); + _peerMetrics = peerMetrics; + _pipeline = pipelineMetrics; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _injectedInterval = interval; + _period = period ?? Period; + } + + /// + public string Name => "ApplicationTelemetry"; + + /// + public Task? Execution => _execution; + + /// Gets how many snapshots have been emitted (tests). + internal int EmitCount { get; private set; } + + /// Gets whether an emit is currently running (tests). + internal bool IsEmitBusy => Volatile.Read(ref _emitBusy) != 0; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _execution = RunAsync(_runCts.Token); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is null) + { + return; + } + + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var owned = _injectedInterval is null; + var interval = _injectedInterval ?? new PeriodicTimerInterval(_period, _timeProvider); + try + { + while (!cancellationToken.IsCancellationRequested) + { + var more = await interval.WaitNextAsync(cancellationToken).ConfigureAwait(false); + if (!more) + { + break; + } + + Emit(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + finally + { + if (owned) + { + await interval.DisposeAsync().ConfigureAwait(false); + } + } + } + + /// Emits one snapshot (tests). Production uses the interval loop only. + internal void Emit() + { + lock (_emitGate) + { + Interlocked.Exchange(ref _emitBusy, 1); + try + { + EmitCore(); + EmitCount++; + } + finally + { + Interlocked.Exchange(ref _emitBusy, 0); + } + } + } + + private void EmitCore() + { + var history = _history?.Capture() ?? default; + var lookups = NonNegativeDelta(history.Lookups, ref _priorLookups); + var hits = NonNegativeDelta(history.Hits, ref _priorHits); + var misses = NonNegativeDelta(history.Misses, ref _priorMisses); + var errors = NonNegativeDelta(history.Errors, ref _priorErrors); + var waitMs = TicksToMilliseconds(NonNegativeDelta(history.WaitTicks, ref _priorWaitTicks)); + ApplicationTelemetryLogMessages.HistoryDb(_logger, lookups, hits, misses, errors, waitMs); + + var admitFail = NonNegativeDelta(_queue.AdmissionFailureCount, ref _priorAdmissionFailures); + var admitWaitMs = TicksToMilliseconds( + NonNegativeDelta(_queue.AdmissionWaitTicks, ref _priorAdmissionWaitTicks)); + ApplicationTelemetryLogMessages.TransitIngressQueue( + _logger, + _queue.Count, + _queue.QueuedBytes, + _queue.PeakCount, + _queue.PeakQueuedBytes, + _queue.WaitingProducerCount, + _queue.MemoryLimitBytes, + admitFail, + admitWaitMs); + + var sessions = _census.Capture(); + ApplicationTelemetryLogMessages.ActiveSessions( + _logger, + sessions.Active, + sessions.Established, + sessions.Idle, + sessions.Receiving, + sessions.WaitingHistory, + sessions.WaitingQueue, + sessions.WaitingWindow, + sessions.Completing); + + EmitPipeline(sessions); + EmitConfiguredPeers(); + } + + private void EmitPipeline(NntpSessionCensusSnapshot sessions) + { + if (_pipeline is null) + { + return; + } + + var snapshot = _pipeline.CaptureInterval(); + var busyMs = IngestionPipelineSnapshot.TicksToMilliseconds(snapshot.BusyTicks); + var idleMs = IngestionPipelineSnapshot.TicksToMilliseconds(snapshot.IdleTicks); + ApplicationTelemetryLogMessages.IngestionWorker( + _logger, + snapshot.WorkerItems, + snapshot.ArticlesPerSecond(_period), + snapshot.BusyPercent, + idleMs, + busyMs, + snapshot.DequeueWait.P95Ms, + snapshot.ToPublishStart.P95Ms, + snapshot.Encode.P95Ms, + snapshot.News.P95Ms, + snapshot.WorkerItem.AvgMs, + snapshot.WorkerItem.P95Ms); + ApplicationTelemetryLogMessages.OverviewDbHandoff( + _logger, + snapshot.GateWait.P95Ms, + snapshot.Publish.P95Ms, + snapshot.Confirm.AvgMs, + snapshot.Confirm.P95Ms, + snapshot.Handoff.AvgMs, + snapshot.Handoff.P95Ms, + snapshot.ConfirmBusyPercent, + snapshot.ConfirmFailures, + snapshot.ConfirmTimeouts, + snapshot.InFlightPublishes, + snapshot.MaxInFlightPublishes); + + var utilisation = _queue.MemoryLimitBytes <= 0 + ? 0d + : Math.Clamp(_queue.QueuedBytes / (double)_queue.MemoryLimitBytes, 0d, 1d); + ApplicationTelemetryLogMessages.IngestionWorkerPool( + _logger, + snapshot.CurrentWorkers, + snapshot.MinWorkers, + snapshot.MaxWorkers, + snapshot.ScaleUps, + snapshot.ScaleDowns, + utilisation, + _queue.WaitingProducerCount); + + var overviewStage = _pipeline.CaptureOverviewDbStage(); + ApplicationTelemetryLogMessages.OverviewDbWorkQueue( + _logger, + overviewStage.WorkQueueCount, + overviewStage.WorkQueueBytes, + overviewStage.WorkQueueWaiting, + overviewStage.PublisherWorkers, + overviewStage.MinPublisherWorkers, + overviewStage.MaxPublisherWorkers, + overviewStage.PublisherScaleUps, + overviewStage.PublisherScaleDowns); + + var budgetExhausted = _queue.WaitingProducerCount > 0 + || _queue.QueuedBytes >= _queue.MemoryLimitBytes + ? 1 + : 0; + var occupiedMax = Math.Max(sessions.TakeThisOccupiedMax, snapshot.MaxTakeThisOccupied); + var processingMax = Math.Max(sessions.TakeThisProcessingMax, snapshot.MaxTakeThisProcessing); + ApplicationTelemetryLogMessages.TakeThisPipeline( + _logger, + sessions.TakeThisOccupied, + occupiedMax, + sessions.TakeThisFullSessions, + snapshot.OccupiedFullWaits, + sessions.TakeThisProcessing, + processingMax, + snapshot.EmitGateWait.P95Ms, + snapshot.TakeThisEnqueue.AvgMs, + snapshot.TakeThisEnqueue.P95Ms, + snapshot.BudgetWait.P95Ms, + snapshot.BudgetWaits, + budgetExhausted); + } + + private void EmitConfiguredPeers() + { + if (_peerMetrics is null) + { + return; + } + + var peers = _transit.Current.Peers; + if (peers.Count == 0) + { + return; + } + + foreach (var identifier in peers.Keys.OrderBy(static id => id, StringComparer.Ordinal)) + { + var policy = peers[identifier]; + var snapshot = _peerMetrics.Capture(identifier); + if (!_peerPriors.TryGetValue(identifier, out var prior)) + { + prior = new PeerPrior(); + _peerPriors[identifier] = prior; + } + + var accepted = NonNegativeDelta(snapshot.Accepted, ref prior.Accepted); + var rejected = NonNegativeDelta(snapshot.Rejected, ref prior.Rejected); + var articles = NonNegativeDelta(snapshot.ArticlesReceived, ref prior.Articles); + var bytes = NonNegativeDelta(snapshot.ArticleBytes, ref prior.Bytes); + var checks = NonNegativeDelta(snapshot.Checks, ref prior.Checks); + var transmitted = NonNegativeDelta(snapshot.Transmitted, ref prior.Transmitted); + var average = articles > 0 ? bytes / articles : 0; + ApplicationTelemetryLogMessages.TransitPeer( + _logger, + identifier, + snapshot.Active, + policy.MaxIncomingConnections, + snapshot.Peak, + accepted, + transmitted, + rejected, + articles, + bytes, + average, + AverageMbps(bytes, _period), + checks); + } + } + + /// + /// Decimal megabits/sec from interval article bytes over the telemetry period. + /// + internal static double AverageMbps(long bytes, TimeSpan interval) + { + if (bytes <= 0) + { + return 0; + } + + var seconds = interval.TotalSeconds; + if (seconds <= 0) + { + return 0; + } + + return bytes * 8d / seconds / 1_000_000d; + } + + private static long NonNegativeDelta(long current, ref long prior) + { + var previous = prior; + prior = current; + return current < previous ? 0 : current - previous; + } + + private static long TicksToMilliseconds(long ticks) + { + if (ticks <= 0) + { + return 0; + } + + return (long)Stopwatch.GetElapsedTime(0, ticks).TotalMilliseconds; + } + + private sealed class PeerPrior + { + public long Accepted; + public long Rejected; + public long Articles; + public long Bytes; + public long Checks; + public long Transmitted; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Telemetry/IAsyncInterval.cs b/src/VectorNNTPD.ReaderService/Telemetry/IAsyncInterval.cs new file mode 100644 index 00000000..107696de --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Telemetry/IAsyncInterval.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.ReaderService.Telemetry +{ + /// Wait-for-next-period seam used by application telemetry. + internal interface IAsyncInterval : IAsyncDisposable + { + /// + /// Waits until the next period. Returns when the interval is completed. + /// + ValueTask WaitNextAsync(CancellationToken cancellationToken); + } + + /// implementation of . + internal sealed class PeriodicTimerInterval : IAsyncInterval + { + private readonly PeriodicTimer _timer; + + /// Initializes a period timer using . + public PeriodicTimerInterval(TimeSpan period, TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(timeProvider); + _timer = new PeriodicTimer(period, timeProvider); + } + + /// + public ValueTask WaitNextAsync(CancellationToken cancellationToken) => + _timer.WaitForNextTickAsync(cancellationToken); + + /// + public ValueTask DisposeAsync() + { + _timer.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/ITransitDnsAddressCache.cs b/src/VectorNNTPD.ReaderService/Transit/ITransitDnsAddressCache.cs new file mode 100644 index 00000000..8212e3a1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/ITransitDnsAddressCache.cs @@ -0,0 +1,36 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Thread-safe cache of DNS-resolved AllowFrom addresses. + /// + /// + /// A refresh atomically replaces the address set for one hostname. Transient failures + /// keep the previous set. Readers never observe a partial hostname update. + /// + public interface ITransitDnsAddressCache + { + /// Minimum interval between DNS refreshes for one hostname. + static readonly TimeSpan MinimumRefreshInterval = TimeSpan.FromSeconds(60); + + /// Gets the current resolved address set for . + /// + /// In-memory only. Must not perform DNS, reverse DNS, or network I/O. + /// Connection-time identification calls this after refresh has materialized addresses. + /// + IReadOnlySet GetResolved(string hostname); + + /// Resolves every hostname referenced by . + Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken); + + /// Refreshes hostnames whose TTL/minimum interval has elapsed. + Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken); + + /// + /// Returns the delay until the next hostname is due, or + /// when no hostnames are configured. + /// + TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/ITransitDnsResolver.cs b/src/VectorNNTPD.ReaderService/Transit/ITransitDnsResolver.cs new file mode 100644 index 00000000..553ed037 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/ITransitDnsResolver.cs @@ -0,0 +1,65 @@ +using System.Net; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// Resolves Transit AllowFrom hostnames to A/AAAA addresses with optional DNS TTL. + public interface ITransitDnsResolver + { + /// Resolves asynchronously. + Task ResolveAsync(string hostname, CancellationToken cancellationToken); + } + + /// Classifies a DNS lookup outcome for AllowFrom refresh. + public enum TransitDnsOutcome + { + /// One or more addresses were returned. + Success = 0, + + /// + /// Authoritative empty result (NXDOMAIN or NOERROR with no A/AAAA). + /// The cached address set may be cleared. + /// + Empty = 1, + + /// + /// Temporary failure (timeout, SERVFAIL, transport error). + /// The last valid address set must be retained. + /// + TransientFailure = 2, + } + + /// Result of one hostname lookup. + public sealed class TransitDnsResolveResult + { + /// Creates a successful result with addresses and optional TTL. + public static TransitDnsResolveResult Success(IReadOnlyList addresses, TimeSpan? timeToLive) => + new(TransitDnsOutcome.Success, addresses, timeToLive); + + /// Creates an authoritative empty result. + public static TransitDnsResolveResult Empty(TimeSpan? timeToLive = null) => + new(TransitDnsOutcome.Empty, Array.Empty(), timeToLive); + + /// Creates a transient failure that must preserve the last address set. + public static TransitDnsResolveResult TransientFailure() => + new(TransitDnsOutcome.TransientFailure, Array.Empty(), timeToLive: null); + + private TransitDnsResolveResult(TransitDnsOutcome outcome, IReadOnlyList addresses, TimeSpan? timeToLive) + { + Outcome = outcome; + Addresses = addresses; + TimeToLive = timeToLive; + } + + /// Gets the lookup outcome. + public TransitDnsOutcome Outcome { get; } + + /// Gets resolved addresses (empty unless ). + public IReadOnlyList Addresses { get; } + + /// + /// Gets the DNS TTL when the resolver provided one; otherwise . + /// A missing TTL uses the 60-second minimum refresh interval as the fallback. + /// + public TimeSpan? TimeToLive { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/ITransitInboundConnectionLimiter.cs b/src/VectorNNTPD.ReaderService/Transit/ITransitInboundConnectionLimiter.cs new file mode 100644 index 00000000..c615becb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/ITransitInboundConnectionLimiter.cs @@ -0,0 +1,100 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Admits inbound connections associated with a named Transit peer. + /// + /// + /// Listeners remain Transit-facing. Redis is the admission authority for named + /// peers. A process-local count is diagnostics only. + /// + public interface ITransitInboundConnectionLimiter + { + /// + /// Attempts to consume one inbound slot for using the + /// current snapshot's MaxIncomingConnections. + /// + /// + /// An outcome that is admitted when a cluster slot was taken (or no slot is + /// required). Rejected when the peer is unknown, closed, at its cluster + /// limit, or Redis is unavailable. + /// + ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default); + } + + /// Outcome of a Transit inbound slot acquire. + public readonly struct TransitInboundAdmitResult + { + /// Initializes an inbound admit outcome. + public TransitInboundAdmitResult(bool admitted, TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + Admitted = admitted; + Lease = lease; + } + + /// Gets whether the connection may proceed. + public bool Admitted { get; } + + /// Gets the lease to dispose when the connection ends. Never null. + public TransitInboundConnectionLease Lease { get; } + + /// Rejected without a held slot. + public static TransitInboundAdmitResult Rejected { get; } = + new(false, TransitInboundConnectionLease.None); + + /// Admitted without a held slot (non-transit or disabled limiter). + public static TransitInboundAdmitResult Uncounted { get; } = + new(true, TransitInboundConnectionLease.None); + + /// Admitted with a slot that must be released exactly once. + public static TransitInboundAdmitResult Accept(TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + return new TransitInboundAdmitResult(true, lease); + } + } + + /// Releases one inbound Transit peer slot when disposed. One-shot. + public sealed class TransitInboundConnectionLease : IAsyncDisposable, IDisposable + { + private readonly Func? _release; + private int _released; + + internal TransitInboundConnectionLease(Func release) + { + ArgumentNullException.ThrowIfNull(release); + _release = release; + } + + private TransitInboundConnectionLease() + { + } + + /// Empty lease (no slot was taken). + public static TransitInboundConnectionLease None { get; } = new(); + + /// Gets whether this lease holds a slot that has not yet been released. + public bool IsHeld => _release is not null && Volatile.Read(ref _released) == 0; + + /// + public async ValueTask DisposeAsync() + { + if (_release is null) + { + return; + } + + if (Interlocked.Exchange(ref _released, 1) != 0) + { + return; + } + + await _release(CancellationToken.None).ConfigureAwait(false); + } + + /// + public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/ITransitPeerMetrics.cs b/src/VectorNNTPD.ReaderService/Transit/ITransitPeerMetrics.cs new file mode 100644 index 00000000..efbcf8e1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/ITransitPeerMetrics.cs @@ -0,0 +1,98 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// Lifetime counters for one configured Transit peer. + public readonly struct TransitPeerMetricsSnapshot + { + /// Initializes a peer snapshot. + public TransitPeerMetricsSnapshot( + string peerId, + int active, + int peak, + long accepted, + long rejected, + long articlesReceived, + long articleBytes, + long checks, + long transmitted) + { + PeerId = peerId; + Active = active; + Peak = peak; + Accepted = accepted; + Rejected = rejected; + ArticlesReceived = articlesReceived; + ArticleBytes = articleBytes; + Checks = checks; + Transmitted = transmitted; + } + + /// Gets the Transit identifier (configuration dictionary key). + public string PeerId { get; } + + /// Gets current established sessions belonging to this peer. + public int Active { get; } + + /// Gets the lifetime high-water of simultaneous established sessions. + public int Peak { get; } + + /// Gets lifetime accepted inbound connections. + public long Accepted { get; } + + /// Gets lifetime rejected inbound connections. + public long Rejected { get; } + + /// Gets lifetime framed articles received from this peer. + public long ArticlesReceived { get; } + + /// Gets lifetime framed article payload bytes received from this peer. + public long ArticleBytes { get; } + + /// Gets lifetime CHECK operations for this peer. + public long Checks { get; } + + /// + /// Gets lifetime articles successfully transmitted to this peer. + /// Remains 0 until outbound delivery accounting increments it. + /// + public long Transmitted { get; } + } + + /// + /// Authoritative process-wide Transit peer counters. Independent of FeedDiagnostics. + /// + public interface ITransitPeerMetrics + { + /// Records a successful inbound admission for a named peer. + void RecordAccepted(string peerId); + + /// + /// Retracts a prior when the session is reclassified + /// as a non-transit reader after AUTHINFO. + /// + void RetractAccepted(string peerId); + + /// Records an inbound admission rejection for a named peer. + void RecordRejected(string peerId); + + /// Records that an established session for entered RunAsync. + void RecordSessionRegistered(string peerId); + + /// Records that an established session for left RunAsync. + void RecordSessionUnregistered(string peerId); + + /// Records one framed article received from . + void RecordArticleReceived(string peerId, int bytes); + + /// Records one CHECK for . + void RecordCheck(string peerId); + + /// + /// Records articles successfully transmitted to . + /// Call only from the outbound delivery path after a successful send. + /// + void RecordTransmitted(string peerId, long articles = 1); + + /// Captures lifetime counters for (zeros when unseen). + TransitPeerMetricsSnapshot Capture(string peerId); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPattern.cs b/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPattern.cs new file mode 100644 index 00000000..6d834384 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPattern.cs @@ -0,0 +1,457 @@ +using System.Text; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Compiled newsfeeds(5) subscription expression using INN uwildmat_poison semantics. + /// + /// + /// + /// The configuration value is a single comma-separated string, not a JSON array and not a + /// .NET glob or regular expression. Grammar and matching follow INN + /// libinn-uwildmat / newsfeeds(5): + /// + /// + /// Comma separates patterns; \, is a literal comma. + /// Each pattern is matched against the complete newsgroup name (anchored). + /// The rightmost matching pattern decides the result. + /// Leading ! excludes; leading @ poisons. + /// * any sequence, ? one Unicode scalar, [...] / [^...] sets, \ escapes. + /// + /// + /// Empty segments, a dangling escape, or an unclosed character set fail validation. + /// + /// + public sealed class NewsfeedsPattern + { + private readonly IReadOnlyList _segments; + + private NewsfeedsPattern(string expression, IReadOnlyList segments) + { + Expression = expression; + _segments = segments; + } + + /// Gets the original configuration expression. + public string Expression { get; } + + /// Parses and validates a newsfeeds-style expression. + public static bool TryParse(string? expression, out NewsfeedsPattern? pattern, out string? error) + { + pattern = null; + error = null; + if (expression is null) + { + error = "Patterns must not be null."; + return false; + } + + if (!TrySplit(expression, out var rawSegments, out error)) + { + return false; + } + + if (rawSegments.Count == 0) + { + error = "Patterns must contain at least one wildmat expression."; + return false; + } + + var segments = new List(rawSegments.Count); + for (var i = 0; i < rawSegments.Count; i++) + { + var raw = rawSegments[i]; + if (raw.Length == 0) + { + error = "Patterns contains an empty expression (leading, trailing, or doubled comma)."; + return false; + } + + var kind = NewsfeedsPatternDecision.Match; + var start = 0; + if (raw[0] == '!') + { + kind = NewsfeedsPatternDecision.Exclude; + start = 1; + } + else if (raw[0] == '@') + { + kind = NewsfeedsPatternDecision.Poison; + start = 1; + } + + if (start >= raw.Length) + { + error = "Patterns contains an empty wildmat after '!' or '@'."; + return false; + } + + var wildmat = raw[start..]; + if (!TryValidateWildmat(wildmat, out error)) + { + return false; + } + + segments.Add(new Segment(kind, wildmat)); + } + + pattern = new NewsfeedsPattern(expression, segments); + return true; + } + + /// + /// Returns whether is subscribed (inclusion, not exclude/poison). + /// + public bool MatchesNewsgroup(string newsgroup) => + Evaluate(newsgroup) == NewsfeedsPatternDecision.Match; + + /// Evaluates the expression against a single newsgroup name. + public NewsfeedsPatternDecision Evaluate(string newsgroup) + { + ArgumentNullException.ThrowIfNull(newsgroup); + var decision = NewsfeedsPatternDecision.NoMatch; + // codeql[cs/linq/missed-where] Hot-path wildmat evaluation without LINQ allocation. + foreach (var segment in _segments) + { + if (Wildmat.IsMatch(newsgroup, segment.Wildmat)) + { + decision = segment.Kind; + } + } + + return decision; + } + + /// + /// Evaluates poison/subscription across an article's newsgroups. + /// + /// + /// Any poison group poisons the article even when another cross-posted group would match. + /// Otherwise the article is subscribed if any group matches. + /// + public NewsfeedsPatternDecision EvaluateArticle(IEnumerable newsgroups) + { + ArgumentNullException.ThrowIfNull(newsgroups); + var subscribed = false; + // codeql[cs/linq/missed-select] Early-exit poison check; Select enumerator allocation on article path. + foreach (var group in newsgroups) + { + var decision = Evaluate(group); + if (decision == NewsfeedsPatternDecision.Poison) + { + return NewsfeedsPatternDecision.Poison; + } + + if (decision == NewsfeedsPatternDecision.Match) + { + subscribed = true; + } + } + + return subscribed ? NewsfeedsPatternDecision.Match : NewsfeedsPatternDecision.NoMatch; + } + + private static bool TrySplit(string expression, out List segments, out string? error) + { + segments = []; + error = null; + var current = new StringBuilder(); + var escaped = false; + foreach (var ch in expression) + { + if (escaped) + { + current.Append(ch); + escaped = false; + continue; + } + + if (ch == '\\') + { + current.Append(ch); + escaped = true; + continue; + } + + if (ch == ',') + { + segments.Add(current.ToString()); + current.Clear(); + continue; + } + + current.Append(ch); + } + + if (escaped) + { + error = "Patterns ends with a dangling backslash escape."; + return false; + } + + segments.Add(current.ToString()); + return true; + } + + private static bool TryValidateWildmat(string wildmat, out string? error) + { + error = null; + var escaped = false; + for (var i = 0; i < wildmat.Length; i++) + { + var ch = wildmat[i]; + if (escaped) + { + escaped = false; + continue; + } + + if (ch == '\\') + { + escaped = true; + continue; + } + + if (ch != '[') + { + continue; + } + + if (!TryConsumeCharacterSet(wildmat, ref i, out error)) + { + return false; + } + } + + if (escaped) + { + error = "Patterns contains a dangling backslash escape."; + return false; + } + + return true; + } + + private static bool TryConsumeCharacterSet(string wildmat, ref int index, out string? error) + { + error = null; + var i = index + 1; + if (i >= wildmat.Length) + { + error = "Patterns contains an unclosed character set."; + return false; + } + + if (wildmat[i] == '^') + { + i++; + } + + if (i < wildmat.Length && wildmat[i] == ']') + { + i++; + } + + while (i < wildmat.Length && wildmat[i] != ']') + { + i++; + } + + if (i >= wildmat.Length) + { + error = "Patterns contains an unclosed character set."; + return false; + } + + index = i; + return true; + } + + private readonly struct Segment + { + public Segment(NewsfeedsPatternDecision kind, string wildmat) + { + Kind = kind; + Wildmat = wildmat; + } + + public NewsfeedsPatternDecision Kind { get; } + + public string Wildmat { get; } + } + + /// INN uwildmat single-pattern matcher (no comma / ! / @ handling). + internal static class Wildmat + { + public static bool IsMatch(string text, string pattern) => + Match(text.AsSpan(), pattern.AsSpan()); + + private static bool Match(ReadOnlySpan text, ReadOnlySpan pattern) + { + while (!pattern.IsEmpty) + { + if (pattern[0] == '*') + { + while (pattern.Length > 1 && pattern[1] == '*') + { + pattern = pattern[1..]; + } + + pattern = pattern[1..]; + if (pattern.IsEmpty) + { + return true; + } + + var remaining = text; + while (true) + { + if (Match(remaining, pattern)) + { + return true; + } + + if (remaining.IsEmpty) + { + return false; + } + + remaining = ConsumeRune(remaining); + } + } + + if (text.IsEmpty) + { + return false; + } + + if (pattern[0] == '\\') + { + if (pattern.Length < 2) + { + return false; + } + + if (!TryConsumeLiteral(ref text, pattern[1])) + { + return false; + } + + pattern = pattern[2..]; + continue; + } + + if (pattern[0] == '?') + { + text = ConsumeRune(text); + pattern = pattern[1..]; + continue; + } + + if (pattern[0] == '[') + { + if (!TryMatchCharacterSet(ref text, ref pattern)) + { + return false; + } + + continue; + } + + if (!TryConsumeLiteral(ref text, pattern[0])) + { + return false; + } + + pattern = pattern[1..]; + } + + return text.IsEmpty; + } + + private static bool TryMatchCharacterSet(ref ReadOnlySpan text, ref ReadOnlySpan pattern) + { + if (text.IsEmpty) + { + return false; + } + + var body = pattern[1..]; + var negated = false; + if (!body.IsEmpty && body[0] == '^') + { + negated = true; + body = body[1..]; + } + + if (body.IsEmpty) + { + return false; + } + + Rune.DecodeFromUtf16(text, out var candidate, out var candidateLen); + var matched = false; + var i = 0; + if (body[0] == ']') + { + matched = candidate.Value == ']'; + i = 1; + } + + while (i < body.Length && body[i] != ']') + { + if (i + 2 < body.Length && body[i + 1] == '-' && body[i + 2] != ']') + { + var start = body[i]; + var end = body[i + 2]; + if (candidate.Value >= start && candidate.Value <= end) + { + matched = true; + } + + i += 3; + continue; + } + + if (candidate.Value == body[i]) + { + matched = true; + } + + i++; + } + + if (i >= body.Length || body[i] != ']') + { + return false; + } + + if (matched == negated) + { + return false; + } + + text = text[candidateLen..]; + pattern = body[(i + 1)..]; + return true; + } + + private static bool TryConsumeLiteral(ref ReadOnlySpan text, char expected) + { + if (text.IsEmpty || text[0] != expected) + { + return false; + } + + text = text[1..]; + return true; + } + + private static ReadOnlySpan ConsumeRune(ReadOnlySpan text) + { + Rune.DecodeFromUtf16(text, out _, out var len); + return text[len..]; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPatternDecision.cs b/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPatternDecision.cs new file mode 100644 index 00000000..82ab65cf --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/NewsfeedsPatternDecision.cs @@ -0,0 +1,20 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Result of evaluating a newsfeeds(5) / uwildmat_poison subscription expression. + /// + public enum NewsfeedsPatternDecision + { + /// No pattern matched the newsgroup. + NoMatch = 0, + + /// The rightmost matching pattern is an inclusion (subscribed). + Match = 1, + + /// The rightmost matching pattern is an exclusion (!). + Exclude = 2, + + /// The rightmost matching pattern is poison (@). + Poison = 3, + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueCompiler.cs b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueCompiler.cs new file mode 100644 index 00000000..0d370e15 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueCompiler.cs @@ -0,0 +1,217 @@ +using System.Globalization; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// Compiles one Transit catalogue record into the immutable snapshot. + internal static class TransitCatalogueCompiler + { + /// Maximum article-type mask. Matches . + public const int UnrestrictedArticleTypes = 65535; + + /// Compiles or throws when a plane is not a valid snapshot. + public static TransitConfigurationSnapshot Compile(TransitCatalogueRecord record) + { + ArgumentNullException.ThrowIfNull(record); + if (record.PublicationId < 1) + { + throw new InvalidOperationException("Transit publication_id must be at least 1."); + } + + var peers = new Dictionary(record.Peers.Count, StringComparer.Ordinal); + foreach (var peer in record.Peers) + { + if (!peers.TryAdd(peer.Identifier, CompilePeer(peer))) + { + throw new InvalidOperationException( + "Transit publication contains a duplicate peer identifier '" + peer.Identifier + "'."); + } + } + + return new TransitConfigurationSnapshot(peers, record.WantTrash, record.LogTrash, record.PublicationId); + } + + private static TransitPeerPolicy CompilePeer(TransitCataloguePeerRecord peer) + { + ValidateLimit(peer.Identifier, "max_inbound", peer.MaxInbound); + ValidateLimit(peer.Identifier, "max_outbound", peer.MaxOutbound); + ValidateSize(peer.Identifier, "receive max_article_bytes", peer.ReceiveMaxArticleBytes); + ValidateSendSize(peer.Identifier, peer.SendMaxArticleBytes); + var receiveTypes = ParseArticleTypes(peer.Identifier, "receive", peer.ReceiveArticleTypes); + var sendTypes = ParseArticleTypes(peer.Identifier, "send", peer.SendArticleTypes); + var receivePatterns = ParsePatterns(peer.Identifier, "receive", peer.ReceivePatterns); + var sendPatterns = ParsePatterns(peer.Identifier, "send", peer.SendPatterns); + ValidateCredentials(peer.Identifier, "receive", peer.ReceiveUsername, peer.ReceivePassword); + ValidateCredentials(peer.Identifier, "send", peer.SendUsername, peer.SendPassword); + if (!Enum.TryParse(peer.SslMode, ignoreCase: false, out var ssl) || !Enum.IsDefined(ssl)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' ssl_mode is not None, Tls, or StartTls."); + } + + if (string.IsNullOrWhiteSpace(peer.PeerName)) + { + throw new InvalidOperationException("Transit peer '" + peer.Identifier + "' peer_name is empty."); + } + + var literals = new List(); + var hostnames = new List(); + for (var i = 0; i < peer.AllowFrom.Count; i++) + { + if (!TransitAllowFromEntry.TryParse(peer.AllowFrom[i], out var entry, out var error)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' AllowFrom[" + i.ToString(CultureInfo.InvariantCulture) + + "] is invalid: " + (error ?? "unparsed.")); + } + + switch (entry) + { + case TransitAllowFromEntry.Literal literal: + literals.Add(literal.Prefix); + break; + case TransitAllowFromEntry.Hostname hostname: + if (!hostnames.Contains(hostname.DnsName, StringComparer.OrdinalIgnoreCase)) + { + hostnames.Add(hostname.DnsName); + } + + break; + default: + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' AllowFrom entry was not parsed."); + } + } + + var connectTo = new List(peer.Endpoints.Count); + var seenEndpoints = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < peer.Endpoints.Count; i++) + { + var endpoint = peer.Endpoints[i]; + var key = endpoint.Host + "\n" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!seenEndpoints.Add(key)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' has a duplicate Send endpoint."); + } + + var text = endpoint.Host.Contains(':', StringComparison.Ordinal) + ? "[" + endpoint.Host + "]:" + endpoint.Port.ToString(CultureInfo.InvariantCulture) + : endpoint.Host + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!TransitConnectEndpoint.TryParse(text, out var parsed, out var error) || parsed is null) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' Send endpoint is invalid: " + (error ?? "unparsed.")); + } + + connectTo.Add(parsed); + } + + var exclusions = new List(peer.PathExclusions.Count); + var seenExclusions = new HashSet(StringComparer.Ordinal); + foreach (var token in peer.PathExclusions) + { + if (string.IsNullOrWhiteSpace(token) || !seenExclusions.Add(token)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' has a duplicate or empty Path exclusion."); + } + + exclusions.Add(token); + } + + return new TransitPeerPolicy( + peer.Identifier, + peer.PeerName, + peer.MaxInbound, + peer.MaxOutbound, + literals, + hostnames, + connectTo, + peer.ReceiveUsername, + peer.ReceivePassword, + ssl, + sendPatterns, + peer.DeferOnDuplicate, + peer.PathToken, + peer.ReceiveMaxArticleBytes, + sendTypes, + receivePatterns, + receiveTypes, + peer.SendMaxArticleBytes, + peer.SendUsername, + peer.SendPassword, + exclusions); + } + + private static void ValidateLimit(string identifier, string column, int value) + { + if (value < 0 || value > TransitPeerPolicy.MaxConnectionLimit) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + column + " is outside 0–" + + TransitPeerPolicy.MaxConnectionLimit.ToString(CultureInfo.InvariantCulture) + "."); + } + } + + private static void ValidateSize(string identifier, string column, long value) + { + if (value < 1 || value > int.MaxValue) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + column + " is outside 1–2147483647."); + } + } + + /// + /// Accepts SQL NULL as unlimited. An explicit send ceiling uses the same positive range as receive. + /// + private static void ValidateSendSize(string identifier, long? value) + { + if (value is null) + { + return; + } + + ValidateSize(identifier, "send max_article_bytes", value.Value); + } + + private static TransitMessageTypes ParseArticleTypes(string identifier, string plane, int mask) + { + if (mask < 0 || mask > UnrestrictedArticleTypes) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " article_types is outside 0–65535."); + } + + return (TransitMessageTypes)mask; + } + + private static NewsfeedsPattern ParsePatterns(string identifier, string plane, string expression) + { + NewsfeedsPattern? pattern = null; + string? error = null; + if (string.IsNullOrWhiteSpace(expression) + || !NewsfeedsPattern.TryParse(expression, out pattern, out error) + || pattern is null) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " patterns are invalid: " + + (error ?? "empty.")); + } + + return pattern; + } + + private static void ValidateCredentials(string identifier, string plane, string username, string password) + { + var userSet = username.Length > 0; + var passwordSet = password.Length > 0; + if (userSet != passwordSet) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " credentials must both be set or both be blank."); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueRecord.cs b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueRecord.cs new file mode 100644 index 00000000..f020c8b4 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueRecord.cs @@ -0,0 +1,195 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// One published Transit catalogue row set, before pattern and endpoint compilation. + /// + public sealed class TransitCatalogueRecord + { + /// Empty published catalogue used by offline hosts. No peers; junk logging stays on. + public static TransitCatalogueRecord Empty { get; } = new(1, 1, 1, 1, true, true, []); + + /// Initializes a catalogue row set. + public TransitCatalogueRecord( + long publicationId, + long receiveRevision, + long sendRevision, + long globalRevision, + bool wantTrash, + bool logTrash, + IReadOnlyList peers) + { + ArgumentNullException.ThrowIfNull(peers); + PublicationId = publicationId; + ReceiveRevision = receiveRevision; + SendRevision = sendRevision; + GlobalRevision = globalRevision; + WantTrash = wantTrash; + LogTrash = logTrash; + Peers = peers; + } + + /// Gets nntptransitpublication.publication_id. + public long PublicationId { get; } + + /// Gets the receive revision named by the publication. + public long ReceiveRevision { get; } + + /// Gets the send revision named by the publication. + public long SendRevision { get; } + + /// Gets the global revision named by the publication. + public long GlobalRevision { get; } + + /// Gets the site-wide want-trash flag. + public bool WantTrash { get; } + + /// Gets the site-wide log-trash flag. + public bool LogTrash { get; } + + /// Gets every peer in the publication. Receive and send identifiers are the same set. + public IReadOnlyList Peers { get; } + } + + /// One peer's identity plus both planes, still in catalogue column form. + public sealed class TransitCataloguePeerRecord + { + /// Initializes one peer row. + public TransitCataloguePeerRecord( + string identifier, + string peerName, + int maxInbound, + string receiveUsername, + string receivePassword, + bool deferOnDuplicate, + long receiveMaxArticleBytes, + int receiveArticleTypes, + string receivePatterns, + IReadOnlyList allowFrom, + int maxOutbound, + string sslMode, + string sendUsername, + string sendPassword, + long? sendMaxArticleBytes, + int sendArticleTypes, + string sendPatterns, + string pathToken, + IReadOnlyList endpoints, + IReadOnlyList pathExclusions) + { + ArgumentNullException.ThrowIfNull(identifier); + ArgumentNullException.ThrowIfNull(peerName); + ArgumentNullException.ThrowIfNull(receiveUsername); + ArgumentNullException.ThrowIfNull(receivePassword); + ArgumentNullException.ThrowIfNull(receivePatterns); + ArgumentNullException.ThrowIfNull(allowFrom); + ArgumentNullException.ThrowIfNull(sslMode); + ArgumentNullException.ThrowIfNull(sendUsername); + ArgumentNullException.ThrowIfNull(sendPassword); + ArgumentNullException.ThrowIfNull(sendPatterns); + ArgumentNullException.ThrowIfNull(pathToken); + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentNullException.ThrowIfNull(pathExclusions); + Identifier = identifier; + PeerName = peerName; + MaxInbound = maxInbound; + ReceiveUsername = receiveUsername; + ReceivePassword = receivePassword; + DeferOnDuplicate = deferOnDuplicate; + ReceiveMaxArticleBytes = receiveMaxArticleBytes; + ReceiveArticleTypes = receiveArticleTypes; + ReceivePatterns = receivePatterns; + AllowFrom = allowFrom; + MaxOutbound = maxOutbound; + SslMode = sslMode; + SendUsername = sendUsername; + SendPassword = sendPassword; + SendMaxArticleBytes = sendMaxArticleBytes; + SendArticleTypes = sendArticleTypes; + SendPatterns = sendPatterns; + PathToken = pathToken; + Endpoints = endpoints; + PathExclusions = pathExclusions; + } + + /// Gets the peer identifier. + public string Identifier { get; } + + /// Gets the display name. + public string PeerName { get; } + + /// Gets max_inbound. 0 closes receive. + public int MaxInbound { get; } + + /// Gets the receive username. + public string ReceiveUsername { get; } + + /// Gets the receive password. Never log this value. + public string ReceivePassword { get; } + + /// Gets whether duplicate offers are deferred. + public bool DeferOnDuplicate { get; } + + /// Gets the receive article size limit. + public long ReceiveMaxArticleBytes { get; } + + /// Gets the receive article-type mask. + public int ReceiveArticleTypes { get; } + + /// Gets the receive newsfeeds expression. + public string ReceivePatterns { get; } + + /// Gets AllowFrom entries in ordinal order. + public IReadOnlyList AllowFrom { get; } + + /// Gets max_outbound. 0 closes send. + public int MaxOutbound { get; } + + /// Gets ssl_mode (None, Tls, or StartTls). + public string SslMode { get; } + + /// Gets the send username. + public string SendUsername { get; } + + /// Gets the send password. Never log this value. + public string SendPassword { get; } + + /// + /// Gets the send article size limit in bytes. + /// is unlimited. A value is an explicit ceiling and is not filled from the receive limit. + /// + public long? SendMaxArticleBytes { get; } + + /// Gets the send article-type mask. + public int SendArticleTypes { get; } + + /// Gets the send newsfeeds expression. + public string SendPatterns { get; } + + /// Gets the Path token. + public string PathToken { get; } + + /// Gets ConnectTo endpoints in ordinal order. + public IReadOnlyList Endpoints { get; } + + /// Gets extra Path exclusions in ordinal order. + public IReadOnlyList PathExclusions { get; } + } + + /// One send endpoint stored as an unbracketed host and an explicit port. + public sealed class TransitCatalogueEndpointRecord + { + /// Initializes an endpoint row. + public TransitCatalogueEndpointRecord(string host, int port) + { + ArgumentNullException.ThrowIfNull(host); + Host = host; + Port = port; + } + + /// Gets the unbracketed host. + public string Host { get; } + + /// Gets the TCP port. + public int Port { get; } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueService.cs b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueService.cs new file mode 100644 index 00000000..5ab476eb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitCatalogueService.cs @@ -0,0 +1,219 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Loads the published Transit catalogue from NntpDB and replaces the process snapshot. + /// + /// + /// Initial load failure prevents RUNNING. A failed refresh keeps the last snapshot. + /// The same publication_id does not replace the snapshot. Send configuration is stored + /// for a later feeder and does not open outbound connections. Passwords are not logged. + /// + internal sealed class TransitCatalogueService : IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly NntpDbService _nntpDb; + private readonly TransitConfigurationStore _store; + private readonly IOptionsMonitor _nntpdOptions; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private IDisposable? _optionsSubscription; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a production catalogue service. + public TransitCatalogueService( + NntpDbService nntpDb, + TransitConfigurationStore store, + IOptionsMonitor nntpdOptions, + ILogger logger) + : this(nntpDb, store, nntpdOptions, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a service with an explicit clock and interval (tests). + internal TransitCatalogueService( + NntpDbService nntpDb, + TransitConfigurationStore store, + IOptionsMonitor nntpdOptions, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _store = store; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "TransitCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + _optionsSubscription = _nntpdOptions.OnChange(OnOptionsChanged); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + TransitLogMessages.CatalogueInitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is not null) + { + try + { + await _execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _optionsSubscription?.Dispose(); + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitLogMessages.CatalogueRefreshFailed(_logger, ex); + } + finally + { + Interlocked.Exchange(ref _refreshing, 0); + } + } + + private async ValueTask LoadAsync(CancellationToken cancellationToken) + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var record = await connection.QueryTransitCatalogueAsync(cancellationToken).ConfigureAwait(false); + if (record is null) + { + throw new InvalidOperationException( + "nntptransitcurrent is missing policy_id = 1. NNTPD will not invent a local Transit catalogue."); + } + + return TransitCatalogueCompiler.Compile(record); + } + + private void Publish(TransitConfigurationSnapshot snapshot) + { + ApplyGlobalFlags(_nntpdOptions.CurrentValue, snapshot); + if (_store.Current.PublicationId == snapshot.PublicationId && _store.Current.PublicationId != 0) + { + return; + } + + _store.Replace(snapshot); + TransitLogMessages.CataloguePublished(_logger, snapshot.PublicationId, snapshot.Peers.Count); + } + + private void OnOptionsChanged(NntpdOptions options, string? name) => + ApplyGlobalFlags(options, _store.Current); + + private static void ApplyGlobalFlags(NntpdOptions options, TransitConfigurationSnapshot snapshot) + { + if (snapshot.PublicationId == 0) + { + return; + } + + options.Transit.WantTrash = snapshot.WantTrash; + options.Transit.LogTrash = snapshot.LogTrash; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationHotReload.cs b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationHotReload.cs new file mode 100644 index 00000000..3c849852 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationHotReload.cs @@ -0,0 +1,73 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Applies updates to . + /// + /// + /// An empty top-level Transit section does not replace the store. The published + /// MySQL catalogue is authoritative, and a JSON reload must not wipe it. Non-empty + /// in-memory sections still replace the snapshot so tests can exercise admission. + /// Invalid non-empty reloads are ignored so the last valid snapshot remains. + /// + public sealed class TransitConfigurationHotReload : IDisposable + { + private readonly IDisposable? _subscription; + + /// Initializes hot reload from the options monitor into the snapshot store. + public TransitConfigurationHotReload( + IOptionsMonitor monitor, + TransitConfigurationStore store, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(monitor); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(logger); + + Apply(monitor.CurrentValue, store, logger, isReload: false); + _subscription = monitor.OnChange(options => Apply(options, store, logger, isReload: true)); + } + + /// + public void Dispose() => _subscription?.Dispose(); + + private static void Apply( + TransitPeersOptions options, + TransitConfigurationStore store, + ILogger logger, + bool isReload) + { + if (options.Count == 0) + { + return; + } + + var result = new TransitPeersOptionsValidator().Validate(Options.DefaultName, options); + if (result.Failed) + { + var first = result.Failures?.FirstOrDefault() ?? result.FailureMessage ?? "invalid Transit configuration"; + if (isReload) + { + TransitLogMessages.InvalidReloadIgnored(logger, first); + } + else + { + throw new OptionsValidationException( + Options.DefaultName, + typeof(TransitPeersOptions), + result.Failures); + } + + return; + } + + store.Replace(TransitConfigurationSnapshot.Create(options)); + if (isReload) + { + TransitLogMessages.ConfigurationReloaded(logger, options.Count); + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationSnapshot.cs b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationSnapshot.cs new file mode 100644 index 00000000..bf4f24cb --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationSnapshot.cs @@ -0,0 +1,161 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Immutable, fully validated Transit peer configuration snapshot. + /// + /// + /// Readers observe either the previous complete snapshot or this complete snapshot. + /// DNS-resolved addresses live in so hostname + /// refreshes do not rebuild unrelated peer policy. + /// + public sealed class TransitConfigurationSnapshot + { + /// Empty snapshot (no peers). Global junk flags stay at their process defaults. + public static TransitConfigurationSnapshot Empty { get; } = new(new Dictionary(StringComparer.Ordinal)); + + private static readonly NewsfeedsPattern OptionsReceivePatterns = ParseOptionsReceivePatterns(); + + internal TransitConfigurationSnapshot( + IReadOnlyDictionary peers, + bool wantTrash = true, + bool logTrash = true, + long publicationId = 0) + { + Peers = peers; + WantTrash = wantTrash; + LogTrash = logTrash; + PublicationId = publicationId; + } + + /// Gets peers keyed by configured identifier (ordinal, case-sensitive). + public IReadOnlyDictionary Peers { get; } + + /// Gets the published nntptransitpublication.publication_id, or 0 for an in-memory snapshot. + public long PublicationId { get; } + + /// Gets the site-wide want-trash flag from the published global revision. + public bool WantTrash { get; } + + /// Gets the site-wide log-trash flag from the published global revision. + public bool LogTrash { get; } + + /// Gets whether any peers are configured. + public bool IsEmpty => Peers.Count == 0; + + /// + /// Builds a snapshot from already-validated in-memory options. + /// + /// + /// has one size, one pattern, and one message-type list. + /// Those map to Receive size, Send patterns, and Send article types. Receive patterns + /// are * and Receive article types are unrestricted. The MySQL catalogue compiler + /// sets the two planes independently. + /// + public static TransitConfigurationSnapshot Create(TransitPeersOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var peers = new Dictionary(options.Count, StringComparer.Ordinal); + foreach (var (identifier, peer) in options) + { + peers[identifier] = CreatePeer(identifier, peer); + } + + return new TransitConfigurationSnapshot(peers); + } + + internal static TransitPeerPolicy CreatePeer(string identifier, TransitPeerOptions options) + { + var literals = new List(); + var hostnames = new List(); + foreach (var raw in options.AllowFrom ?? []) + { + if (!TransitAllowFromEntry.TryParse(raw, out var entry, out _)) + { + continue; + } + + switch (entry) + { + case TransitAllowFromEntry.Literal literal: + literals.Add(literal.Prefix); + break; + case TransitAllowFromEntry.Hostname hostname: + if (!hostnames.Contains(hostname.DnsName, StringComparer.OrdinalIgnoreCase)) + { + hostnames.Add(hostname.DnsName); + } + + break; + } + } + + var connectTo = new List(); + // codeql[cs/linq/missed-where] TryParse out-params; foreach preserves parse-and-collect clarity. + foreach (var raw in options.ConnectTo ?? []) + { + if (TransitConnectEndpoint.TryParse(raw, out var endpoint, out _) && endpoint is not null) + { + connectTo.Add(endpoint); + } + } + + if (!NewsfeedsPattern.TryParse(options.Patterns, out var patterns, out _) || patterns is null) + { + throw new InvalidOperationException("Patterns must be validated before snapshot creation."); + } + + if (!TransitSslParser.TryParse(options.Ssl, out var ssl, out _)) + { + throw new InvalidOperationException("Ssl must be validated before snapshot creation."); + } + + var username = (options.Username ?? string.Empty).Trim(); + var password = options.Password ?? string.Empty; + if (!TransitMessageTypesParser.TryParse(options.MessageTypes, out var messageTypes, out _)) + { + throw new InvalidOperationException("MessageTypes must be validated before snapshot creation."); + } + + var peerName = options.PeerName ?? string.Empty; + if (string.IsNullOrWhiteSpace(peerName)) + { + throw new InvalidOperationException("PeerName must be validated before snapshot creation."); + } + + return new TransitPeerPolicy( + identifier, + peerName, + options.MaxIncomingConnections!.Value, + options.MaxOutgoingConnections!.Value, + literals, + hostnames, + connectTo, + username, + password, + ssl, + patterns, + options.DeferOnDuplicate, + options.PathToken ?? string.Empty, + options.MaxSize, + messageTypes, + OptionsReceivePatterns, + TransitMessageTypes.All, + options.MaxSize, + string.Empty, + string.Empty, + []); + } + + private static NewsfeedsPattern ParseOptionsReceivePatterns() + { + if (!NewsfeedsPattern.TryParse("*", out var pattern, out _) || pattern is null) + { + throw new InvalidOperationException("Receive pattern '*' must parse."); + } + + return pattern; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationStore.cs b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationStore.cs new file mode 100644 index 00000000..c4275a94 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitConfigurationStore.cs @@ -0,0 +1,56 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Holds the active immutable Transit configuration snapshot. + /// + /// + /// Replacement is atomic. Readers use without taking a lock. + /// A listener registered via observes complete snapshots only. + /// + public sealed class TransitConfigurationStore + { + private TransitConfigurationSnapshot _current = TransitConfigurationSnapshot.Empty; + private event Action? Changed; + + /// Gets the current complete snapshot. + public TransitConfigurationSnapshot Current => Volatile.Read(ref _current!); + + /// Replaces the active snapshot with a fully built instance. + public void Replace(TransitConfigurationSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Volatile.Write(ref _current, snapshot); + Changed?.Invoke(snapshot); + } + + /// Subscribes to complete snapshot replacements. + public IDisposable Subscribe(Action listener) + { + ArgumentNullException.ThrowIfNull(listener); + Changed += listener; + return new Subscription(this, listener); + } + + private sealed class Subscription : IDisposable + { + private TransitConfigurationStore? _store; + private Action? _listener; + + public Subscription(TransitConfigurationStore store, Action listener) + { + _store = store; + _listener = listener; + } + + public void Dispose() + { + var store = Interlocked.Exchange(ref _store, null); + var listener = Interlocked.Exchange(ref _listener, null); + if (store is not null && listener is not null) + { + store.Changed -= listener; + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitConnectionAdmission.cs b/src/VectorNNTPD.ReaderService/Transit/TransitConnectionAdmission.cs new file mode 100644 index 00000000..4cb6cb00 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitConnectionAdmission.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Admits an inbound connection against the identified Transit peer's MaxIncomingConnections. + /// + internal static class TransitConnectionAdmission + { + /// + /// Consumes a cluster slot when the session is associated with a Transit peer. + /// Non-transit clients are admitted without a slot and never create peer state. + /// + public static async ValueTask TryAdmitAsync( + ITransitInboundConnectionLimiter limiter, + NntpSession session, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(limiter); + ArgumentNullException.ThrowIfNull(session); + var peerName = session.Authorization.TransitPeerName; + if (peerName is null) + { + return TransitInboundAdmitResult.Uncounted; + } + + return await limiter.TryAcquireAsync(peerName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Writes RFC 3977/4644 400 Service temporarily unavailable and completes. + /// + public static async ValueTask WriteUnavailableAsync( + INntpConnection connection, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(connection); + await using var writer = new NntpResponseWriter(connection.Output); + await writer + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitDnsAddressCache.cs b/src/VectorNNTPD.ReaderService/Transit/TransitDnsAddressCache.cs new file mode 100644 index 00000000..15876935 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitDnsAddressCache.cs @@ -0,0 +1,242 @@ +using System.Collections.Concurrent; +using System.Net; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Per-hostname DNS cache with TTL-based refresh and a 60-second minimum interval. + /// + public sealed class TransitDnsAddressCache : ITransitDnsAddressCache + { + private static readonly IReadOnlySet EmptySet = new HashSet(); + + private readonly ITransitDnsResolver _resolver; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _minimumRefreshInterval; + private readonly ConcurrentDictionary _hosts = + new(StringComparer.OrdinalIgnoreCase); + + /// Initializes the cache with the production 60-second minimum refresh interval. + public TransitDnsAddressCache( + ITransitDnsResolver resolver, + ILogger logger, + TimeProvider? timeProvider = null) + : this(resolver, logger, ITransitDnsAddressCache.MinimumRefreshInterval, timeProvider) + { + } + + /// Test constructor with an injectable minimum refresh interval. + internal TransitDnsAddressCache( + ITransitDnsResolver resolver, + ILogger logger, + TimeSpan minimumRefreshInterval, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(resolver); + ArgumentNullException.ThrowIfNull(logger); + _resolver = resolver; + _logger = logger; + _minimumRefreshInterval = minimumRefreshInterval; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + /// + /// Dictionary lookup only. Does not call , + /// Dns.GetHostEntry, Dns.GetHostAddresses, or perform network I/O. + /// + public IReadOnlySet GetResolved(string hostname) + { + ArgumentException.ThrowIfNullOrWhiteSpace(hostname); + return _hosts.TryGetValue(hostname, out var state) ? state.Addresses : EmptySet; + } + + /// + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + RefreshCoreAsync(snapshot, dueOnly: false, cancellationToken); + + /// + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + RefreshCoreAsync(snapshot, dueOnly: true, cancellationToken); + + /// + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + var names = CollectHostnames(snapshot); + if (names.Count == 0) + { + return Timeout.InfiniteTimeSpan; + } + + var now = _timeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var name in names.Keys) + { + if (!_hosts.TryGetValue(name, out var state)) + { + return TimeSpan.Zero; + } + + if (state.NextRefreshUtc < next) + { + next = state.NextRefreshUtc; + } + } + + var delay = next - now; + return delay < TimeSpan.Zero ? TimeSpan.Zero : delay; + } + + private async Task RefreshCoreAsync( + TransitConfigurationSnapshot snapshot, + bool dueOnly, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(snapshot); + var hosts = CollectHostnames(snapshot); + PruneUnused(hosts.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase)); + var now = _timeProvider.GetUtcNow(); + foreach (var (name, peerNames) in hosts) + { + cancellationToken.ThrowIfCancellationRequested(); + if (dueOnly + && _hosts.TryGetValue(name, out var existing) + && existing.NextRefreshUtc > now) + { + continue; + } + + await RefreshHostnameAsync(snapshot, name, peerNames, cancellationToken).ConfigureAwait(false); + } + } + + private async Task RefreshHostnameAsync( + TransitConfigurationSnapshot snapshot, + string hostname, + IReadOnlyList peerNames, + CancellationToken cancellationToken) + { + TransitDnsResolveResult result; + string? failureReason = null; + try + { + result = await _resolver.ResolveAsync(hostname, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + failureReason = ex.Message; + result = TransitDnsResolveResult.TransientFailure(); + } + + var now = _timeProvider.GetUtcNow(); + var next = now + RefreshInterval(result.TimeToLive); + switch (result.Outcome) + { + case TransitDnsOutcome.Success: + var set = Deduplicate(result.Addresses); + _hosts[hostname] = new HostState(set, next); + TransitLogMessages.DnsResolved(_logger, hostname, set.Count, next); + break; + case TransitDnsOutcome.Empty: + _hosts[hostname] = new HostState(new HashSet(), next); + LogDnsFailure( + snapshot, + peerNames, + hostname, + failureReason ?? "NXDOMAIN or NOERROR with no A/AAAA records"); + break; + default: + _hosts[hostname] = _hosts.TryGetValue(hostname, out var previous) + ? new HostState(previous.Addresses, next) + : new HostState(new HashSet(), next); + + LogDnsFailure(snapshot, peerNames, hostname, failureReason ?? "transient DNS failure"); + break; + } + } + + private void LogDnsFailure( + TransitConfigurationSnapshot snapshot, + IReadOnlyList peerIds, + string hostname, + string reason) + { + foreach (var peerId in peerIds) + { + var display = snapshot.Peers.TryGetValue(peerId, out var peer) ? peer.PeerName : peerId; + TransitLogMessages.DnsResolutionFailed(_logger, peerId, display, hostname, reason); + } + } + + private TimeSpan RefreshInterval(TimeSpan? ttl) + { + if (ttl is not { } value || value <= TimeSpan.Zero) + { + return _minimumRefreshInterval; + } + + return value > _minimumRefreshInterval ? value : _minimumRefreshInterval; + } + + private static Dictionary> CollectHostnames(TransitConfigurationSnapshot snapshot) + { + var names = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var peer in snapshot.Peers.Values) + { + foreach (var hostname in peer.DnsHostnames) + { + if (!names.TryGetValue(hostname, out var peers)) + { + peers = []; + names[hostname] = peers; + } + + if (!peers.Contains(peer.Identifier, StringComparer.Ordinal)) + { + peers.Add(peer.Identifier); + } + } + } + + return names; + } + + private void PruneUnused(HashSet live) + { + // codeql[cs/linq/missed-where] ConcurrentDictionary Keys with TryRemove side effects. + foreach (var key in _hosts.Keys) + { + if (!live.Contains(key)) + { + _hosts.TryRemove(key, out _); + } + } + } + + private static IReadOnlySet Deduplicate(IReadOnlyList addresses) + { + var set = new HashSet(); + foreach (var address in addresses) + { + set.Add(TrustedProxyHosts.Canonicalize(address)); + } + + return set; + } + + private sealed class HostState(IReadOnlySet addresses, DateTimeOffset nextRefreshUtc) + { + public IReadOnlySet Addresses { get; } = addresses; + + public DateTimeOffset NextRefreshUtc { get; } = nextRefreshUtc; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitDnsClientResolver.cs b/src/VectorNNTPD.ReaderService/Transit/TransitDnsClientResolver.cs new file mode 100644 index 00000000..0725c40a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitDnsClientResolver.cs @@ -0,0 +1,136 @@ +using System.Net; +using DnsClient; +using DnsClient.Protocol; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Resolves AllowFrom hostnames with DnsClient so A/AAAA record TTLs are available. + /// + /// + /// + /// Each lookup queries A and AAAA. DnsClient exposes TimeToLive on answer records; + /// the refresh interval uses the minimum TTL among returned records, floored at 60 seconds + /// by . + /// + /// + /// If a successful answer somehow omitted TTL information, + /// is and the cache falls back to the 60-second minimum. This resolver + /// does not invent a fixed polling interval in place of record TTLs. + /// + /// + public sealed class TransitDnsClientResolver : ITransitDnsResolver + { + private readonly ILookupClient _client; + + /// Initializes a resolver with the system DNS lookup client. + public TransitDnsClientResolver() + : this(new LookupClient()) + { + } + + /// Initializes a resolver with an injected lookup client (tests). + public TransitDnsClientResolver(ILookupClient client) + { + ArgumentNullException.ThrowIfNull(client); + _client = client; + } + + /// + public async Task ResolveAsync(string hostname, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(hostname); + var name = hostname.Trim().TrimEnd('.'); + + IDnsQueryResponse a; + IDnsQueryResponse aaaa; + try + { + a = await _client.QueryAsync(name, QueryType.A, cancellationToken: cancellationToken) + .ConfigureAwait(false); + aaaa = await _client.QueryAsync(name, QueryType.AAAA, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + return TransitDnsResolveResult.TransientFailure(); + } + + var aTransient = IsTransient(a); + var aaaaTransient = IsTransient(aaaa); + if (aTransient && aaaaTransient) + { + return TransitDnsResolveResult.TransientFailure(); + } + + var addresses = new List(); + var ttlSeconds = int.MaxValue; + var sawTtl = false; + Collect(a, addresses, ref ttlSeconds, ref sawTtl); + Collect(aaaa, addresses, ref ttlSeconds, ref sawTtl); + + if (addresses.Count == 0) + { + if (aTransient || aaaaTransient) + { + return TransitDnsResolveResult.TransientFailure(); + } + + return TransitDnsResolveResult.Empty(sawTtl ? TimeSpan.FromSeconds(ttlSeconds) : null); + } + + return TransitDnsResolveResult.Success( + addresses, + sawTtl ? TimeSpan.FromSeconds(ttlSeconds) : null); + } + + private static bool IsTransient(IDnsQueryResponse response) + { + if (!response.HasError) + { + return false; + } + + return response.Header.ResponseCode is not DnsHeaderResponseCode.NotExistentDomain; + } + + private static void Collect( + IDnsQueryResponse response, + List addresses, + ref int ttlSeconds, + ref bool sawTtl) + { + if (response.HasError) + { + return; + } + + foreach (var record in response.Answers) + { + IPAddress? address = record switch + { + ARecord a => a.Address, + AaaaRecord aaaa => aaaa.Address, + _ => null, + }; + if (address is null) + { + continue; + } + + addresses.Add(TrustedProxyHosts.Canonicalize(address)); + if (record.TimeToLive > 0) + { + sawTtl = true; + ttlSeconds = Math.Min(ttlSeconds, record.TimeToLive); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitDnsRefreshService.cs b/src/VectorNNTPD.ReaderService/Transit/TransitDnsRefreshService.cs new file mode 100644 index 00000000..f999b522 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitDnsRefreshService.cs @@ -0,0 +1,139 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Session; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Refreshes Transit AllowFrom DNS names according to each hostname's TTL (minimum 60 seconds). + /// + /// + /// Does not poll VectorNNTPD.ReaderService.json. Configuration reloads arrive through + /// (IOptionsMonitor). The wait interval is the + /// next hostname due time, not a fixed poll. + /// + public sealed class TransitDnsRefreshService : IApplicationService, IAsyncDisposable + { + private readonly TransitConfigurationStore _store; + private readonly ITransitDnsAddressCache _cache; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private IDisposable? _subscription; + private TaskCompletionSource _wakeup = NewWakeup(); + private Task? _execution; + private int _disposed; + + /// Initializes a new instance of the class. + public TransitDnsRefreshService( + TransitConfigurationStore store, + ITransitDnsAddressCache cache, + ITransitPeerAuthorization authorization, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(cache); + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _cache = cache; + _logger = logger; + } + + /// + public string Name => "TransitDnsRefresh"; + + /// + public Task? Execution => _execution; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + await _cache.RefreshAllAsync(_store.Current, cancellationToken).ConfigureAwait(false); + _subscription = _store.Subscribe(_ => _wakeup.TrySetResult()); + _execution = RunAsync(_runCts.Token); + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _subscription?.Dispose(); + _subscription = null; + await _runCts.CancelAsync().ConfigureAwait(false); + _wakeup.TrySetResult(); + if (_execution is not null) + { + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _subscription?.Dispose(); + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var delay = _cache.GetDelayUntilNextRefresh(_store.Current); + var wakeup = _wakeup; + try + { + if (delay == Timeout.InfiniteTimeSpan) + { + await wakeup.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + else if (delay > TimeSpan.Zero) + { + var delayTask = Task.Delay(delay, cancellationToken); + await Task.WhenAny(delayTask, wakeup.Task).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + Interlocked.Exchange(ref _wakeup, NewWakeup()); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await _cache.RefreshDueAsync(_store.Current, cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitLogMessages.RefreshLoopTerminated(_logger, ex); + throw; + } + } + + private static TaskCompletionSource NewWakeup() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitInboundConnectionLimiter.cs b/src/VectorNNTPD.ReaderService/Transit/TransitInboundConnectionLimiter.cs new file mode 100644 index 00000000..c131a453 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitInboundConnectionLimiter.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Transit-facing inbound limiter. Redis is authoritative for named peers; + /// the process-local count is diagnostics and lifecycle bookkeeping only. + /// + /// + /// Lowering MaxIncomingConnections on reload does not disconnect existing holders. + /// Removing a peer rejects new acquires for that name; existing leases still release. + /// 0 admits no new inbound connections. Redis unavailability fails closed. + /// + public sealed class TransitInboundConnectionLimiter : ITransitInboundConnectionLimiter + { + private readonly TransitConfigurationStore _store; + private readonly ITransitPeerStateTracker _tracker; + + /// Shared no-op limiter for tests that do not exercise inbound limits. + public static ITransitInboundConnectionLimiter Disabled { get; } = new DisabledLimiter(); + + /// Initializes a limiter bound to the active Transit snapshot and cluster tracker. + public TransitInboundConnectionLimiter(TransitConfigurationStore store, ITransitPeerStateTracker tracker) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(tracker); + _store = store; + _tracker = tracker; + } + + /// Gets the current local owned count for (tests/diagnostics). + public int GetCount(string peerName) => _tracker.GetLocalCount(peerName); + + /// + public async ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + if (!_store.Current.Peers.TryGetValue(peerName, out var peer)) + { + return TransitInboundAdmitResult.Rejected; + } + + var max = peer.MaxIncomingConnections; + var admitted = await _tracker + .TryAdmitAsync(peerName, max, cancellationToken) + .ConfigureAwait(false); + if (!admitted.Accepted) + { + return TransitInboundAdmitResult.Rejected; + } + + var generation = admitted.Generation; + return TransitInboundAdmitResult.Accept( + new TransitInboundConnectionLease(ct => _tracker.ReleaseAsync(peerName, generation, ct))); + } + + private sealed class DisabledLimiter : ITransitInboundConnectionLimiter + { + public ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(TransitInboundAdmitResult.Uncounted); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitLogMessages.cs b/src/VectorNNTPD.ReaderService/Transit/TransitLogMessages.cs new file mode 100644 index 00000000..d9bf47aa --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitLogMessages.cs @@ -0,0 +1,59 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// Source-generated Transit configuration and DNS log messages. + internal static partial class TransitLogMessages + { + [LoggerMessage( + EventId = 1702, + Level = LogLevel.Error, + Message = "Ignoring invalid Transit configuration reload; last valid snapshot remains active. {Failure}")] + public static partial void InvalidReloadIgnored(ILogger logger, string Failure); + + [LoggerMessage( + EventId = 1703, + Level = LogLevel.Information, + Message = "Transit peer configuration reloaded ({Count} peer(s))")] + public static partial void ConfigurationReloaded(ILogger logger, int Count); + + [LoggerMessage( + EventId = 1704, + Level = LogLevel.Information, + Message = "Transit AllowFrom DNS {Hostname} resolved to {Count} address(es); next refresh at {NextRefreshUtc}")] + public static partial void DnsResolved(ILogger logger, string Hostname, int Count, DateTimeOffset NextRefreshUtc); + + [LoggerMessage( + EventId = 1705, + Level = LogLevel.Warning, + Message = "Transit peer {PeerId} ({PeerName}) AllowFrom DNS resolution failed for {Hostname}: {Reason}")] + public static partial void DnsResolutionFailed( + ILogger logger, + string PeerId, + string PeerName, + string Hostname, + string Reason); + + [LoggerMessage( + EventId = 1706, + Level = LogLevel.Error, + Message = "Transit AllowFrom DNS refresh loop terminated unexpectedly")] + public static partial void RefreshLoopTerminated(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2270, + Level = LogLevel.Error, + Message = "Transit catalogue initial load failed")] + public static partial void CatalogueInitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2271, + Level = LogLevel.Warning, + Message = "Transit catalogue refresh failed; last published snapshot remains active")] + public static partial void CatalogueRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2272, + Level = LogLevel.Information, + Message = "Transit catalogue published {PublicationId} ({PeerCount} peer(s))")] + public static partial void CataloguePublished(ILogger logger, long PublicationId, int PeerCount); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerMetrics.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerMetrics.cs new file mode 100644 index 00000000..e69dded7 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerMetrics.cs @@ -0,0 +1,155 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Always-on Transit peer counters keyed by configured identifier. + /// + public sealed class TransitPeerMetrics : ITransitPeerMetrics + { + private readonly ConcurrentDictionary _peers = new(StringComparer.Ordinal); + + /// + public void RecordAccepted(string peerId) => Get(peerId).IncrementAccepted(); + + /// + public void RetractAccepted(string peerId) => Get(peerId).DecrementAccepted(); + + /// + public void RecordRejected(string peerId) => Get(peerId).IncrementRejected(); + + /// + public void RecordSessionRegistered(string peerId) => Get(peerId).RegisterSession(); + + /// + public void RecordSessionUnregistered(string peerId) => Get(peerId).UnregisterSession(); + + /// + public void RecordArticleReceived(string peerId, int bytes) => Get(peerId).AddArticle(bytes); + + /// + public void RecordCheck(string peerId) => Get(peerId).IncrementChecks(); + + /// + public void RecordTransmitted(string peerId, long articles = 1) => Get(peerId).AddTransmitted(articles); + + /// + public TransitPeerMetricsSnapshot Capture(string peerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerId); + return _peers.TryGetValue(peerId, out var counters) + ? counters.Capture(peerId) + : new TransitPeerMetricsSnapshot(peerId, 0, 0, 0, 0, 0, 0, 0, 0); + } + + private Counters Get(string peerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerId); + return _peers.GetOrAdd(peerId, static _ => new Counters()); + } + + private sealed class Counters + { + private int _active; + private int _peak; + private long _accepted; + private long _rejected; + private long _articlesReceived; + private long _articleBytes; + private long _checks; + private long _transmitted; + + public void IncrementAccepted() => Interlocked.Increment(ref _accepted); + + public void DecrementAccepted() + { + while (true) + { + var current = Volatile.Read(ref _accepted); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref _accepted, current - 1, current) == current) + { + return; + } + } + } + + public void IncrementRejected() => Interlocked.Increment(ref _rejected); + + public void IncrementChecks() => Interlocked.Increment(ref _checks); + + public void RegisterSession() + { + var next = Interlocked.Increment(ref _active); + UpdatePeak(next); + } + + public void UnregisterSession() + { + while (true) + { + var current = Volatile.Read(ref _active); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref _active, current - 1, current) == current) + { + return; + } + } + } + + public void AddArticle(int bytes) + { + Interlocked.Increment(ref _articlesReceived); + if (bytes > 0) + { + Interlocked.Add(ref _articleBytes, bytes); + } + } + + public void AddTransmitted(long articles) + { + if (articles > 0) + { + Interlocked.Add(ref _transmitted, articles); + } + } + + public TransitPeerMetricsSnapshot Capture(string peerId) => + new( + peerId, + Math.Max(0, Volatile.Read(ref _active)), + Volatile.Read(ref _peak), + Volatile.Read(ref _accepted), + Volatile.Read(ref _rejected), + Volatile.Read(ref _articlesReceived), + Volatile.Read(ref _articleBytes), + Volatile.Read(ref _checks), + Volatile.Read(ref _transmitted)); + + private void UpdatePeak(int candidate) + { + while (true) + { + var current = Volatile.Read(ref _peak); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref _peak, candidate, current) == current) + { + return; + } + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerPolicy.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerPolicy.cs new file mode 100644 index 00000000..f083dd2d --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerPolicy.cs @@ -0,0 +1,183 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Immutable policy snapshot for one named Transit peer. + /// + /// + /// Do not log . Use in diagnostics. + /// + public sealed class TransitPeerPolicy + { + /// Maximum accepted inbound or outbound connection limit. + public const int MaxConnectionLimit = 4096; + + internal TransitPeerPolicy( + string identifier, + string peerName, + int maxIncomingConnections, + int maxOutgoingConnections, + IReadOnlyList literalPrefixes, + IReadOnlyList dnsHostnames, + IReadOnlyList connectTo, + string username, + string password, + TransitSslMode ssl, + NewsfeedsPattern patterns, + bool deferOnDuplicate, + string pathToken, + long maxSize, + TransitMessageTypes messageTypes, + NewsfeedsPattern receivePatterns, + TransitMessageTypes receiveArticleTypes, + long? sendMaxArticleBytes, + string sendUsername, + string sendPassword, + IReadOnlyList pathExclusions) + { + ArgumentNullException.ThrowIfNull(receivePatterns); + ArgumentNullException.ThrowIfNull(sendUsername); + ArgumentNullException.ThrowIfNull(sendPassword); + ArgumentNullException.ThrowIfNull(pathExclusions); + Identifier = identifier; + PeerName = peerName; + MaxIncomingConnections = maxIncomingConnections; + MaxOutgoingConnections = maxOutgoingConnections; + LiteralPrefixes = literalPrefixes; + DnsHostnames = dnsHostnames; + ConnectTo = connectTo; + Username = username; + Password = password; + Ssl = ssl; + Patterns = patterns; + DeferOnDuplicate = deferOnDuplicate; + PathToken = pathToken; + MaxSize = maxSize; + MessageTypes = messageTypes; + ReceivePatterns = receivePatterns; + ReceiveArticleTypes = receiveArticleTypes; + SendMaxArticleBytes = sendMaxArticleBytes; + SendUsername = sendUsername; + SendPassword = sendPassword; + PathExclusions = pathExclusions; + } + + /// + /// Gets the stable protocol/machine identifier (Transit dictionary key). + /// + /// Exact configured string. Not derived from . + public string Identifier { get; } + + /// + /// Gets the human-readable administrative display name. + /// + /// Exact configured PeerName. Not a protocol argument. + public string PeerName { get; } + + /// Gets the inbound connection limit for this peer. + public int MaxIncomingConnections { get; } + + /// + /// Gets the outbound connection limit for this peer. + /// 0 closes send. Outbound feeding is not executed from this value. + /// + public int MaxOutgoingConnections { get; } + + /// Gets literal AllowFrom prefixes (addresses and CIDRs). + public IReadOnlyList LiteralPrefixes { get; } + + /// Gets AllowFrom DNS hostnames (resolved separately). + public IReadOnlyList DnsHostnames { get; } + + /// Gets parsed ConnectTo endpoints (not connected). + public IReadOnlyList ConnectTo { get; } + + /// Gets the Receive AUTHINFO username, or empty when receive credentials are not configured. + public string Username { get; } + + /// + /// Gets the peer AUTHINFO password, or empty when credentials are not configured. + /// Never write this value to logs. + /// + public string Password { get; } + + /// Gets a value indicating whether both username and password are configured. + public bool HasPeerCredentials => Username.Length > 0 && Password.Length > 0; + + /// Gets the canonical TLS mode. + public TransitSslMode Ssl { get; } + + /// Gets the Send newsfeeds expression. Inbound acceptance does not apply it. + public NewsfeedsPattern Patterns { get; } + + /// + /// Gets whether a History hit on CHECK and IHAVE is deferred + /// (431/436) instead of rejected (438/435). + /// TAKETHIS does not read this value. + /// + public bool DeferOnDuplicate { get; } + + /// + /// Gets the exact configured Path-header token (not consumed by routing yet). + /// + public string PathToken { get; } + + /// + /// Gets the Receive article size limit in bytes. + /// Inbound acceptance uses the smaller of this value and the published global maxartsize. + /// A non-positive value, or a value above , adds no peer ceiling. + /// + public long MaxSize { get; } + + /// Gets the Send article-type mask. Inbound acceptance does not apply it. + public TransitMessageTypes MessageTypes { get; } + + /// + /// Gets the Receive newsfeeds expression. + /// TAKETHIS and IHAVE accept an article only when returns match. + /// + public NewsfeedsPattern ReceivePatterns { get; } + + /// + /// Gets the Receive article-type mask. 65535 is unrestricted. + /// TAKETHIS and IHAVE require every flag to be present, via . + /// + public TransitMessageTypes ReceiveArticleTypes { get; } + + /// + /// Gets the Send article size limit in bytes. Independent of . + /// is unlimited. A value is an explicit ceiling. Outbound feeding does not apply it. + /// + public long? SendMaxArticleBytes { get; } + + /// Gets the Send username, or empty when send credentials are not configured. + public string SendUsername { get; } + + /// + /// Gets the Send password, or empty when send credentials are not configured. + /// Never write this value to logs. + /// + public string SendPassword { get; } + + /// Gets a value indicating whether both send username and password are configured. + public bool HasSendCredentials => SendUsername.Length > 0 && SendPassword.Length > 0; + + /// Gets additional Path hops that suppress sending to this peer. + public IReadOnlyList PathExclusions { get; } + + /// + public override string ToString() => Identifier; + + /// Returns whether and match this peer. + public bool CredentialsMatch(string username, string password) + { + ArgumentNullException.ThrowIfNull(username); + ArgumentNullException.ThrowIfNull(password); + return HasPeerCredentials + && string.Equals(Username, username, StringComparison.Ordinal) + && string.Equals(Password, password, StringComparison.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs new file mode 100644 index 00000000..3d4e384a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs @@ -0,0 +1,373 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Hybrid Transit inbound admission: Redis is the cluster authority for + /// MaxIncomingConnections. Local counts are bookkeeping only. + /// + /// + /// Identity is . Source IP is never + /// used as a Redis key. MaxIncomingConnections == 0 is closed, not + /// unlimited. Every named-peer admit requires a successful distributed TRY_ADMIT. + /// Owner identity is {nodeId}:{incarnation}. Redis TTL is crash recovery + /// only: while this process is alive and renewal succeeds, idle Transit + /// connections keep consuming cluster capacity. + /// + public sealed class DistributedTransitPeerStateTracker : ITransitPeerStateTracker, ITransitPeerStateLeaseManager + { + private readonly ITransitPeerStateStore _membership; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly string _ownerId; + private readonly TimeSpan _leaseTtl; + private readonly ConcurrentDictionary _identifierGates = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private readonly Dictionary _peers = new(StringComparer.Ordinal); + private readonly HashSet _ownedIdentifiers = new(StringComparer.Ordinal); + private long _generations; + private bool _accepting = true; + + /// Initializes a production tracker with default lease timings. + public DistributedTransitPeerStateTracker( + ITransitPeerStateStore membership, + ILogger logger, + string nodeId) + : this(membership, logger, nodeId, TimeProvider.System, TransitPeerStateDefaults.LeaseTtl) + { + } + + /// Initializes a tracker with explicit timing and incarnation (tests). + internal DistributedTransitPeerStateTracker( + ITransitPeerStateStore membership, + ILogger logger, + string nodeId, + TimeProvider timeProvider, + TimeSpan? leaseTtl = null, + string? incarnation = null) + { + ArgumentNullException.ThrowIfNull(membership); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentNullException.ThrowIfNull(timeProvider); + _membership = membership; + _logger = logger; + _nodeId = nodeId; + _ownerId = string.Concat(nodeId, ":", string.IsNullOrWhiteSpace(incarnation) ? Guid.NewGuid().ToString("N") : incarnation); + _timeProvider = timeProvider; + _leaseTtl = leaseTtl ?? TransitPeerStateDefaults.LeaseTtl; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(_leaseTtl, TimeSpan.Zero); + } + + /// Gets the configured node identity. + internal string NodeId => _nodeId; + + /// Gets {nodeId}:{incarnation} written into Redis ownership fields. + internal string OwnerId => _ownerId; + + /// Gets how many admits required a distributed membership call that succeeded. + internal int DistributedAdmits { get; private set; } + + /// Gets how many admits were rejected by the cluster inbound limit or a closed peer. + internal int DistributedRejects { get; private set; } + + /// Gets how many admits failed closed because membership was unavailable. + internal int UnavailableRejects { get; private set; } + + /// Gets how many admitted connections this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public async ValueTask TryAdmitAsync( + string identifier, + int maxIncoming, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + + var identifierGate = _identifierGates.GetOrAdd(identifier, static _ => new SemaphoreSlim(1, 1)); + await identifierGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + long generation; + lock (_gate) + { + if (!_accepting) + { + DistributedRejects++; + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected); + } + + generation = _peers.TryGetValue(identifier, out var peer) && peer.Generation != 0 + ? peer.Generation + : NextGeneration(); + } + + var now = _timeProvider.GetUtcNow(); + TransitPeerStateAdmitResult membership; + try + { + membership = await _membership.TryAdmitAsync( + identifier, + _ownerId, + maxIncoming, + generation, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // TRY_ADMIT is atomic. Cancellation after a successful EVAL (or a + // store that throws OCE after writing) must still decrement, or + // this process has no local connection that can later release. + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + if (!membership.Accepted) + { + if (membership.Status == TransitPeerStateAdmitStatus.Unavailable) + { + UnavailableRejects++; + TransitPeerStateLogMessages.TransitPeerUnavailable(_logger, identifier, maxIncoming, _nodeId); + return membership; + } + + DistributedRejects++; + TransitPeerStateLogMessages.TransitPeerRejected(_logger, identifier, maxIncoming, _nodeId); + return membership; + } + + CommitDistributedAdmit(identifier, generation, now); + } + catch (OperationCanceledException) + { + if (membership.Accepted) + { + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + CancellationToken.None) + .ConfigureAwait(false); + } + + throw; + } + + DistributedAdmits++; + TransitPeerStateLogMessages.TransitPeerAdmitted(_logger, identifier, maxIncoming, _nodeId); + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation); + } + finally + { + identifierGate.Release(); + } + } + + /// + public async ValueTask ReleaseAsync( + string identifier, + long generation, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + + var identifierGate = _identifierGates.GetOrAdd(identifier, static _ => new SemaphoreSlim(1, 1)); + await identifierGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var releaseMembership = false; + lock (_gate) + { + if (!_peers.TryGetValue(identifier, out var peer) || peer.LocalCount <= 0) + { + return; + } + + ReleaseCalls++; + peer.LocalCount--; + releaseMembership = true; + if (peer.LocalCount <= 0) + { + _peers.Remove(identifier); + } + } + + if (releaseMembership) + { + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + cancellationToken) + .ConfigureAwait(false); + TransitPeerStateLogMessages.TransitPeerReleased(_logger, identifier, _nodeId); + } + } + finally + { + identifierGate.Release(); + } + } + + /// + public async ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + List<(string Identifier, long Generation)> owners; + lock (_gate) + { + owners = []; + foreach (var (identifier, peer) in _peers) + { + if (peer.LocalCount > 0 && peer.Generation != 0) + { + owners.Add((identifier, peer.Generation)); + } + } + } + + var now = _timeProvider.GetUtcNow(); + foreach (var (identifier, generation) in owners) + { + cancellationToken.ThrowIfCancellationRequested(); + TransitPeerStateRenewStatus status; + try + { + status = await _membership.RenewAsync( + identifier, + _ownerId, + generation, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerLeaseRenewFailed(_logger, ex, identifier, _nodeId); + continue; + } + + if (status == TransitPeerStateRenewStatus.Renewed) + { + MarkLeaseRenewed(identifier, now); + TransitPeerStateLogMessages.TransitPeerLeaseRenewed(_logger, identifier, _nodeId); + continue; + } + + if (status == TransitPeerStateRenewStatus.Unavailable) + { + TransitPeerStateLogMessages.TransitPeerLeaseRenewUnavailable(_logger, identifier, _nodeId); + continue; + } + + TransitPeerStateLogMessages.TransitPeerLeaseLost(_logger, identifier, _nodeId); + } + } + + /// + public async ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) + { + List identifiers; + lock (_gate) + { + _accepting = false; + identifiers = [.. _ownedIdentifiers]; + } + + foreach (var identifier in identifiers) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await _membership.ReleaseOwnerAsync(identifier, _ownerId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerReleaseOwnerFailed(_logger, ex, identifier, _ownerId); + } + } + } + + /// + public int GetLocalCount(string identifier) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + lock (_gate) + { + return _peers.TryGetValue(identifier, out var peer) ? peer.LocalCount : 0; + } + } + + /// Returns whether this tracker still accepts new admissions (tests). + internal bool IsAccepting + { + get + { + lock (_gate) + { + return _accepting; + } + } + } + + private void CommitDistributedAdmit(string identifier, long generation, DateTimeOffset now) + { + lock (_gate) + { + if (!_peers.TryGetValue(identifier, out var peer)) + { + peer = new PeerOwnership(); + _peers[identifier] = peer; + } + + peer.Generation = generation; + peer.LocalCount++; + peer.LeaseExpiresAt = now + _leaseTtl; + _ownedIdentifiers.Add(identifier); + } + } + + private void MarkLeaseRenewed(string identifier, DateTimeOffset now) + { + lock (_gate) + { + if (_peers.TryGetValue(identifier, out var peer)) + { + peer.LeaseExpiresAt = now + _leaseTtl; + } + } + } + + private long NextGeneration() => Interlocked.Increment(ref _generations); + + private sealed class PeerOwnership + { + public int LocalCount { get; set; } + + public long Generation { get; set; } + + public DateTimeOffset LeaseExpiresAt { get; set; } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs new file mode 100644 index 00000000..c83686da --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Process-local lease renewal and shutdown release for Transit inbound ownership. + /// + /// + /// Renewal proves this node is still responsible for its admitted Transit + /// connections. It is independent of NNTP traffic, article transfers, CHECK, + /// TAKETHIS, commands, bytes transferred, and idle state. + /// is the scheduler; this abstraction + /// owns the ownership snapshot and Redis operations. + /// + public interface ITransitPeerStateLeaseManager + { + /// + /// Refreshes this incarnation's inbound leases for peers that still have + /// local admitted connections. Does not recreate missing ownership. + /// + ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default); + + /// + /// Releases leftover node/incarnation ownership. After this call, new + /// admissions on this tracker fail closed so shutdown cannot race ReleaseOwner + /// with a later TRY_ADMIT. + /// + ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateStore.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateStore.cs new file mode 100644 index 00000000..74c32785 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateStore.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Cluster-wide Transit inbound-connection membership. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Admit, release, and renew each use one atomic operation. + /// + public interface ITransitPeerStateStore + { + /// + /// Atomically admits one inbound connection for this owner when the cluster + /// count is below . + /// + ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// + /// Atomically decrements this owner's inbound count when the generation matches. + /// + ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's inbound lease. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// Deletes leftover ownership for this process incarnation on this peer. + ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateTracker.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateTracker.cs new file mode 100644 index 00000000..a5a0e9b1 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/ITransitPeerStateTracker.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Admits and releases inbound Transit connections against cluster-wide + /// MaxIncomingConnections for a . + /// + /// + /// Redis is authoritative. A local owned-count is diagnostics and lifecycle + /// bookkeeping only; it never permits admission without distributed authorization. + /// Source IP is not an identity here. MaxIncomingConnections == 0 is closed. + /// + public interface ITransitPeerStateTracker + { + /// + /// Attempts to admit one inbound connection for . + /// + ValueTask TryAdmitAsync( + string identifier, + int maxIncoming, + CancellationToken cancellationToken = default); + + /// + /// Releases one previously admitted inbound connection for this identifier + /// and generation. Idempotent when this process has no local ownership. + /// + ValueTask ReleaseAsync( + string identifier, + long generation, + CancellationToken cancellationToken = default); + + /// Returns the process-local owned count for . + int GetLocalCount(string identifier); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs new file mode 100644 index 00000000..ff21d865 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs @@ -0,0 +1,187 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Process-local cluster fake for Transit inbound membership. Two trackers + /// sharing one instance behave as two NNTP nodes. Ordinary unit tests must not + /// require live Redis. + /// + internal sealed class InMemoryTransitPeerStateStore : ITransitPeerStateStore + { + private readonly TransitPeerStateEngine _engine = new(); + + /// When set, admit/release/renew fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Optional gate that blocks the next admit until completed or cancelled. + public TaskCompletionSource? BlockAdmit { get; set; } + + /// Optional gate that blocks the next release until completed or cancelled. + public TaskCompletionSource? BlockRelease { get; set; } + + /// Optional gate that blocks the next renew until completed or cancelled. + public TaskCompletionSource? BlockRenew { get; set; } + + /// Set when is entered, before waits. + public TaskCompletionSource? NotifyRenewStarted { get; set; } + + /// When set, throws after a successful engine write. + public Exception? ThrowAfterAdmit { get; set; } + + /// When set, throws for this identifier so others can still renew. + public string? FaultRenewIdentifier { get; set; } + + /// Gets how many membership renew operations were attempted. + /// + /// After reaches the engine, this increments only once that + /// renew has returned, so a non-zero count means the renewed expiry is already visible. + /// The unavailable path increments without an engine write. + /// + public int RenewCalls { get; private set; } + + /// Gets how many admit operations reached the engine. + public int AdmitCalls { get; private set; } + + /// Gets how many release operations reached the engine. + public int ReleaseCalls { get; private set; } + + /// Gets the shared admission engine (tests). + internal TransitPeerStateEngine Engine => _engine; + + /// In-memory hash identity for . + internal static string ConnectionKey(string identifier) => "tconn:" + identifier; + + /// Returns the cluster-wide unexpired inbound count. + internal int ActiveCount(string identifier, long nowUnixMs) => + _engine.ActiveCount(ConnectionKey(identifier), nowUnixMs); + + /// Returns this owner's unexpired inbound count. + internal int OwnerCount(string identifier, string ownerId, long nowUnixMs) => + _engine.OwnerCount(ConnectionKey(identifier), ownerId, nowUnixMs); + + /// Returns the stored ownership expiry, or when absent. + internal long? Expiry(string identifier, string ownerId) => + _engine.TryGetOwnership(ConnectionKey(identifier), ownerId, out var expiry, out _, out _) + ? expiry + : null; + + /// + public async ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (BlockAdmit is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Unavailable); + } + + AdmitCalls++; + var code = _engine.TryAdmit( + ConnectionKey(identifier), + ownerId, + maxIncoming, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + generation); + if (ThrowAfterAdmit is { } fault) + { + throw fault; + } + + return code == TransitPeerStateEngine.Accepted + ? new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation) + : new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected); + } + + /// + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (BlockRelease is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return; + } + + ReleaseCalls++; + _ = _engine.Release(ConnectionKey(identifier), ownerId, generation); + } + + /// + public async ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (FaultRenewIdentifier is { } fault + && string.Equals(fault, identifier, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Injected renewal fault."); + } + + NotifyRenewStarted?.TrySetResult(); + if (BlockRenew is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + RenewCalls++; + return TransitPeerStateRenewStatus.Unavailable; + } + + var renewed = _engine.Renew( + ConnectionKey(identifier), + ownerId, + generation, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds) == 1; + RenewCalls++; + return renewed + ? TransitPeerStateRenewStatus.Renewed + : TransitPeerStateRenewStatus.Lost; + } + + /// + public ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + cancellationToken.ThrowIfCancellationRequested(); + if (!Unavailable) + { + _ = _engine.ReleaseOwner(ConnectionKey(identifier), ownerId); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs new file mode 100644 index 00000000..0b0fd38c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs @@ -0,0 +1,159 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Redis-backed cluster Transit inbound membership using one atomic Lua EVAL + /// on the shared connection. + /// + internal sealed class RedisTransitPeerStateStore : ITransitPeerStateStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisTransitPeerStateStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + ExecuteAsync( + TransitPeerStateScripts.TryAdmit, + identifier, + [ + Utf8(ownerId), + Utf8(maxIncoming.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + ], + result => result == TransitPeerStateEngine.Accepted + ? new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation) + : new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected), + new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Unavailable), + cancellationToken); + + /// + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + TransitPeerStateScripts.Release, + identifier, + [ + Utf8(ownerId), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + ], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + /// + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + ExecuteAsync( + TransitPeerStateScripts.Renew, + identifier, + [ + Utf8(ownerId), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + ], + result => result == 1 + ? TransitPeerStateRenewStatus.Renewed + : TransitPeerStateRenewStatus.Lost, + TransitPeerStateRenewStatus.Unavailable, + cancellationToken); + + /// + public async ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + TransitPeerStateScripts.ReleaseOwner, + identifier, + [Utf8(ownerId)], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + private async ValueTask ExecuteAsync( + string script, + string identifier, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + [TransitPeerStateKeys.Create(identifier)], + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + } + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs new file mode 100644 index 00000000..c0b516ff --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs @@ -0,0 +1,54 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// Cluster membership outcome for one Transit inbound admit attempt. + public enum TransitPeerStateAdmitStatus + { + /// Admitted; this owner now holds one additional inbound connection. + Accepted, + + /// + /// Rejected: MaxIncomingConnections is 0 (closed) or the cluster + /// already holds at least that many unexpired inbound connections. + /// + Rejected, + + /// Redis (or the membership store) was unavailable. Fail closed. + Unavailable, + } + + /// Result of a cluster Transit inbound admit attempt. + public readonly struct TransitPeerStateAdmitResult + { + /// Initializes a Transit membership admit result. + public TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus status, long generation = 0) + { + Status = status; + Generation = generation; + } + + /// Gets the membership outcome. + public TransitPeerStateAdmitStatus Status { get; } + + /// + /// Gets the generation that must be supplied to . + /// Zero when admission was not granted. + /// + public long Generation { get; } + + /// Gets whether membership was granted. + public bool Accepted => Status == TransitPeerStateAdmitStatus.Accepted; + } + + /// Outcome of a Transit node ownership lease renewal. + public enum TransitPeerStateRenewStatus + { + /// This owner's requested lease was refreshed. + Renewed, + + /// The requested ownership field was gone, expired, or generation-mismatched. + Lost, + + /// The membership store could not be reached. + Unavailable, + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateDefaults.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateDefaults.cs new file mode 100644 index 00000000..5929d57a --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateDefaults.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// Lease timings for cluster-wide Transit inbound connection ownership. + /// + /// These match the established SessionState operational values (30s lease, 10s + /// renewal, 2s hot-path skew) without introducing Transit-specific configuration. + /// Redis expiry is crash/failure recovery only. It is not a Transit idle timeout. + /// A healthy node renews every 10 seconds while it still owns inbound connections, + /// independently of NNTP traffic. Transit has no local admit hot path: every named + /// peer requires distributed authorization. MaxIncomingConnections == 0 + /// means closed, not unlimited. + /// + internal static class TransitPeerStateDefaults + { + /// Distributed ownership TTL written into Redis on admit and renew. + public static readonly TimeSpan LeaseTtl = TimeSpan.FromSeconds(30); + + /// + /// Interval between process-wide lease renewal passes. + /// uses the same 10-second period. + /// + public static readonly TimeSpan RenewalPeriod = TimeSpan.FromSeconds(10); + + /// + /// SessionState hot-path skew. Transit does not admit without Redis; the + /// constant is retained so operational timing stays aligned. + /// + public static readonly TimeSpan HotPathSkew = TimeSpan.FromSeconds(2); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateEngine.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateEngine.cs new file mode 100644 index 00000000..275f9cd5 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateEngine.cs @@ -0,0 +1,326 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Atomic cluster Transit inbound-connection algorithm. The Redis Lua scripts + /// are a faithful port of these operations. + /// + /// + /// One HASH per Transit Identifier. Field {ownerId}, value + /// {expiryUnixMs}|{generation}|{count}. The cluster count is the sum of + /// unexpired per-owner counts. max <= 0 rejects without incrementing. + /// Rejection after prune writes no ownership. Source IP is not part of this state. + /// + internal sealed class TransitPeerStateEngine + { + /// Accepted; ownership was created or incremented. + public const long Accepted = 1; + + /// Rejected: closed (max <= 0) or cluster count already at the limit. + public const long Rejected = 0; + + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// + /// Atomically admits one inbound connection for this owner when the cluster + /// count is below . + /// + public long TryAdmit( + string connectionKey, + string ownerId, + int maxIncoming, + long nowUnixMs, + long leaseMs, + long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + var owners = GetOrCreateHash(connectionKey); + PruneExpired(owners, nowUnixMs); + if (maxIncoming <= 0) + { + RemoveIfEmpty(connectionKey, owners); + return Rejected; + } + + if (SumCounts(owners) >= maxIncoming) + { + RemoveIfEmpty(connectionKey, owners); + return Rejected; + } + + IncrementOwned(owners, ownerId, nowUnixMs + leaseMs, generation); + return Accepted; + } + } + + /// + /// Decrements this owner's count when the generation matches. Deletes the + /// field at zero and the hash when empty. + /// + public long Release(string connectionKey, string ownerId, long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(connectionKey, out var owners)) + { + DecrementOwned(owners, ownerId, generation); + RemoveIfEmpty(connectionKey, owners); + } + + return 1; + } + } + + /// + /// Refreshes this owner's unexpired matching lease. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + public long Renew( + string connectionKey, + string ownerId, + long generation, + long nowUnixMs, + long leaseMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (!CanRenewField(connectionKey, ownerId, generation, nowUnixMs, out _)) + { + return 0; + } + + RenewField(connectionKey, ownerId, generation, nowUnixMs, leaseMs); + return 1; + } + } + + /// Deletes only this owner's field. Other owners and peers are untouched. + public long ReleaseOwner(string connectionKey, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(connectionKey, out var owners)) + { + owners.Remove(ownerId); + RemoveIfEmpty(connectionKey, owners); + } + + return 1; + } + } + + /// Returns the cluster-wide unexpired inbound count for this peer. + internal int ActiveCount(string connectionKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + lock (_gate) + { + if (!_hashes.TryGetValue(connectionKey, out var owners)) + { + return 0; + } + + PruneExpired(owners, nowUnixMs); + RemoveIfEmpty(connectionKey, owners); + return SumCounts(owners); + } + } + + /// Returns this owner's unexpired inbound count. + internal int OwnerCount(string connectionKey, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(connectionKey, out var owners) + || !owners.TryGetValue(ownerId, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out _, out var count) + || expiry <= nowUnixMs) + { + return 0; + } + + return count; + } + } + + /// Reads a HASH field without mutating it (tests). + internal bool TryGetOwnership( + string key, + string field, + out long expiryUnixMs, + out long generation, + out int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + if (_hashes.TryGetValue(key, out var hash) + && hash.TryGetValue(field, out var current) + && TransitPeerStateKeys.TrySplitValue(current, out expiryUnixMs, out generation, out count)) + { + return true; + } + } + + expiryUnixMs = 0; + generation = 0; + count = 0; + return false; + } + + /// Overwrites a HASH field without admit/release semantics (tests). + internal void WriteOwnership(string key, string field, long expiryUnixMs, long generation, int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + GetOrCreateHash(key)[field] = TransitPeerStateKeys.Value(expiryUnixMs, generation, count); + } + } + + /// Returns field names currently stored for (tests). + internal IReadOnlyCollection OwnerFields(string connectionKey) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + lock (_gate) + { + return _hashes.TryGetValue(connectionKey, out var owners) + ? [.. owners.Keys] + : []; + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private bool CanRenewField(string key, string field, long generation, long nowUnixMs, out int count) + { + count = 0; + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out count) + || storedGeneration != generation + || expiry <= nowUnixMs) + { + return false; + } + + return true; + } + + private void RenewField(string key, string field, long generation, long nowUnixMs, long leaseMs) + { + if (!CanRenewField(key, field, generation, nowUnixMs, out var count) + || !_hashes.TryGetValue(key, out var hash)) + { + return; + } + + hash[field] = TransitPeerStateKeys.Value(nowUnixMs + leaseMs, generation, count); + } + + private static void IncrementOwned(Dictionary hash, string field, long expiry, long generation) + { + // Same generation: this owner still holds the field. A different generation + // is a new local epoch (local state was wiped) and must replace stale count. + if (hash.TryGetValue(field, out var current) + && TransitPeerStateKeys.TrySplitValue(current, out _, out var storedGeneration, out var count) + && storedGeneration == generation) + { + hash[field] = TransitPeerStateKeys.Value(expiry, storedGeneration, count + 1); + return; + } + + hash[field] = TransitPeerStateKeys.Value(expiry, generation, 1); + } + + private static void DecrementOwned(Dictionary hash, string field, long generation) + { + if (!hash.TryGetValue(field, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out var count) + || storedGeneration != generation) + { + return; + } + + if (count <= 1) + { + hash.Remove(field); + return; + } + + hash[field] = TransitPeerStateKeys.Value(expiry, storedGeneration, count - 1); + } + + private static void PruneExpired(Dictionary hash, long nowUnixMs) + { + List? expired = null; + foreach (var (field, value) in hash) + { + if (!TransitPeerStateKeys.TrySplitValue(value, out var expiry, out _, out _) || expiry <= nowUnixMs) + { + expired ??= []; + expired.Add(field); + } + } + + if (expired is null) + { + return; + } + + foreach (var field in expired) + { + hash.Remove(field); + } + } + + private static int SumCounts(Dictionary hash) + { + var total = 0; + // codeql[cs/linq/missed-where] TrySplitValue out-params; lock-held sum without LINQ allocation. + foreach (var value in hash.Values) + { + if (TransitPeerStateKeys.TrySplitValue(value, out _, out _, out var count)) + { + total += count; + } + } + + return total; + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateKeys.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateKeys.cs new file mode 100644 index 00000000..fff3445c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateKeys.cs @@ -0,0 +1,84 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Redis key material for cluster-wide Transit inbound connection ownership. + /// + /// + /// Key: nntpd:tconn:{sha256hex(identifier)}. + /// Field: {ownerId} where ownerId is {nodeId}:{incarnation}. + /// Value: {expiryUnixMs}|{generation}|{count}. + /// The Transit dictionary key () is hashed + /// so Redis key listings do not expose peer names. Source IP is never part of this key. + /// Ownership uses value expiry, not a Redis key TTL. + /// + internal static class TransitPeerStateKeys + { + /// ASCII namespace for Transit inbound-connection ownership hashes. + public static ReadOnlySpan NamespacePrefix => "nntpd:tconn:"u8; + + /// Separates expiry, generation, and count in a HASH value. + public const char ValueSeparator = '|'; + + /// Builds the Transit inbound-connection Redis HASH key for . + public static byte[] Create(string identifier) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(identifier); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(identifier, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(identifier), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + + /// Formats a lease value as expiryMs|generation|count. + public static string Value(long expiryUnixMs, long generation, int count) => + string.Concat( + expiryUnixMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + count.ToString(CultureInfo.InvariantCulture)); + + /// Parses a HASH value into expiry, generation, and count. + public static bool TrySplitValue(string value, out long expiryUnixMs, out long generation, out int count) + { + ArgumentNullException.ThrowIfNull(value); + expiryUnixMs = 0; + generation = 0; + count = 0; + var first = value.IndexOf(ValueSeparator, StringComparison.Ordinal); + if (first <= 0 || first >= value.Length - 1) + { + return false; + } + + var second = value.IndexOf(ValueSeparator, first + 1); + if (second <= first + 1 || second >= value.Length - 1) + { + return false; + } + + return long.TryParse(value.AsSpan(0, first), out expiryUnixMs) + && long.TryParse(value.AsSpan(first + 1, second - first - 1), out generation) + && int.TryParse(value.AsSpan(second + 1), out count); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs new file mode 100644 index 00000000..3ee68607 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// Source-generated Transit peer-state diagnostics. Never includes secrets. + internal static partial class TransitPeerStateLogMessages + { + [LoggerMessage(EventId = 1710, Level = LogLevel.Debug, Message = "Transit peer-state admission accepted identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerAdmitted(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1711, Level = LogLevel.Information, Message = "Transit peer-state admission rejected identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerRejected(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1712, Level = LogLevel.Warning, Message = "Transit peer-state admission failed closed; cluster membership unavailable identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerUnavailable(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1713, Level = LogLevel.Debug, Message = "Transit peer-state ownership released identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerReleased(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1714, Level = LogLevel.Debug, Message = "Transit peer-state lease renewed identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewed(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1715, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal failed identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewFailed(ILogger logger, Exception exception, string identifier, string nodeId); + + [LoggerMessage(EventId = 1716, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal skipped; membership unavailable identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewUnavailable(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1717, Level = LogLevel.Warning, Message = "Transit peer-state lease lost; ownership not extended identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseLost(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1718, Level = LogLevel.Warning, Message = "Transit peer-state release-owner failed identifier={Identifier} owner={OwnerId}")] + public static partial void TransitPeerReleaseOwnerFailed(ILogger logger, Exception exception, string identifier, string ownerId); + + [LoggerMessage(EventId = 1719, Level = LogLevel.Information, Message = "Transit peer-state lease renewal started interval={Interval}")] + public static partial void TransitPeerRenewalStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 1720, Level = LogLevel.Information, Message = "Transit peer-state lease renewal stopped")] + public static partial void TransitPeerRenewalStopped(ILogger logger); + + [LoggerMessage(EventId = 1721, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal pass failed")] + public static partial void TransitPeerRenewalPassFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateScripts.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateScripts.cs new file mode 100644 index 00000000..285a30fc --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateScripts.cs @@ -0,0 +1,190 @@ +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Redis Lua scripts for atomic cluster Transit inbound-connection admission. + /// Semantics match . + /// + internal static class TransitPeerStateScripts + { + /// + /// Atomically evaluates the inbound limit and, on accept, increments this + /// owner's count. Rejection writes no ownership increment. + /// + /// + /// KEYS[1] tconn hash. + /// ARGV: owner, maxIncoming, nowMs, leaseMs, generation. + /// Returns 1 accepted, 0 rejected (closed or at limit). + /// Expired fields are pruned before the limit is summed. + /// max <= 0 rejects without incrementing. + /// + public const string TryAdmit = + """ + -- nntpd-tconn-try + local key = KEYS[1] + local owner = ARGV[1] + local maxIncoming = tonumber(ARGV[2]) + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local gen = ARGV[5] + local expiry = now + lease + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function prune(hashKey) + local entries = redis.call('HGETALL', hashKey) + local expired = {} + for i = 1, #entries, 2 do + local exp = parse_value(entries[i + 1]) + if exp == nil or exp <= now then + expired[#expired + 1] = entries[i] + end + end + if #expired > 0 then + redis.call('HDEL', hashKey, unpack(expired)) + end + end + + local function increment(hashKey, field, requestedGen) + local cur = redis.call('HGET', hashKey, field) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen and storedGen == requestedGen then + redis.call('HSET', hashKey, field, tostring(expiry) .. '|' .. storedGen .. '|' .. tostring((count or 0) + 1)) + return + end + end + redis.call('HSET', hashKey, field, tostring(expiry) .. '|' .. requestedGen .. '|1') + end + + prune(key) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + + if maxIncoming == nil or maxIncoming <= 0 then + return 0 + end + + local total = 0 + local entries = redis.call('HGETALL', key) + for i = 1, #entries, 2 do + local _, _, count = parse_value(entries[i + 1]) + total = total + (count or 0) + end + if total >= maxIncoming then + return 0 + end + + increment(key, owner, gen) + return 1 + """; + + /// + /// Decrements this owner's count when the generation matches. + /// + /// + /// KEYS[1] tconn hash. ARGV: owner, generation. Always returns 1. + /// A mismatched generation is a no-op. The field is deleted at count 0; + /// the hash is deleted when empty. + /// + public const string Release = + """ + -- nntpd-tconn-release + local key = KEYS[1] + local owner = ARGV[1] + local gen = ARGV[2] + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local cur = redis.call('HGET', key, owner) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen == gen then + if count == nil or count <= 1 then + redis.call('HDEL', key, owner) + else + redis.call('HSET', key, owner, tostring(exp) .. '|' .. storedGen .. '|' .. tostring(count - 1)) + end + end + end + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + return 1 + """; + + /// + /// Refreshes this owner's unexpired matching lease. Missing ownership is not recreated. + /// + /// + /// KEYS[1] tconn hash. ARGV: owner, generation, nowMs, leaseMs. + /// Returns 1 when the field was renewed, 0 when it was missing, expired, or mismatched. + /// + public const string Renew = + """ + -- nntpd-tconn-renew + local key = KEYS[1] + local owner = ARGV[1] + local gen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local cur = redis.call('HGET', key, owner) + if not cur then + return 0 + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return 0 + end + redis.call('HSET', key, owner, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + return 1 + """; + + /// Deletes only this owner's field on this peer hash. + /// KEYS[1] tconn hash. ARGV: owner. Always returns 1. + public const string ReleaseOwner = + """ + -- nntpd-tconn-release-owner + local key = KEYS[1] + local owner = ARGV[1] + redis.call('HDEL', key, owner) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateService.cs b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateService.cs new file mode 100644 index 00000000..91d00970 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitPeerState/TransitPeerStateService.cs @@ -0,0 +1,130 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Periodic distributed Transit inbound-ownership maintenance. + /// + /// + /// This service is a scheduler. It does not accept connections or keep its own + /// connection registry. Every renewal interval it asks + /// to renew currently active local + /// ownership. Renewal is process/node liveness, not connection activity: idle + /// Transit connections continue to consume cluster capacity while this process + /// can renew. stops the loop and then releases leftover + /// incarnation ownership. Listeners are registered after this service so they + /// stop first and admitted connections can finalize before renewal ends. + /// Process crash is detected by lease expiry after renewals stop. + /// + public sealed class TransitPeerStateService : IApplicationService + { + private readonly ITransitPeerStateLeaseManager _leases; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public TransitPeerStateService( + ITransitPeerStateLeaseManager leases, + ILogger logger) + : this(leases, logger, TimeProvider.System, TransitPeerStateDefaults.RenewalPeriod) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal TransitPeerStateService( + ITransitPeerStateLeaseManager leases, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(leases); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _leases = leases; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "TransitPeerState"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + try + { + await _leases.ReleaseAllOwnershipAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + TransitPeerStateLogMessages.TransitPeerRenewalStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + try + { + await _leases.RenewLeasesAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerRenewalPassFailed(_logger, ex); + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + TransitPeerStateLogMessages.TransitPeerRenewalStopped(_logger); + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transit/TransitReceivePolicy.cs b/src/VectorNNTPD.ReaderService/Transit/TransitReceivePolicy.cs new file mode 100644 index 00000000..57971b2e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transit/TransitReceivePolicy.cs @@ -0,0 +1,138 @@ +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Transit +{ + /// + /// Applies one connection's captured Receive peer policy to an already classified article. + /// + /// + /// The peer object is the session authorization captured at identification. + /// Send , , + /// and are not read. + /// CHECK has no article body and does not call this type. + /// + internal static class TransitReceivePolicy + { + /// TAKETHIS/IHAVE detail when is outside the receive mask. + internal const string ArticleTypeRejectDetail = "rejected article type"; + + /// TAKETHIS/IHAVE detail when does not subscribe the article. + internal const string NewsgroupPatternRejectDetail = "rejected newsgroup pattern"; + + /// + /// Returns the receive ceiling for one article operation. + /// + /// + /// Published nntpsharedconfig.maxartsize, or the caller fallback when no snapshot is on the session. + /// This value is always the hard ceiling. + /// + /// Peer captured for this connection, or when the session is not a named peer. + /// + /// when is absent or its + /// is non-positive or above + /// (no peer ceiling). Otherwise the smaller of the two. + /// + internal static int EffectiveMaxArticleBytes(int globalOrFallback, TransitPeerPolicy? peer) + { + if (peer is null) + { + return globalOrFallback; + } + + var peerMax = peer.MaxSize; + if (peerMax <= 0 || peerMax > int.MaxValue) + { + return globalOrFallback; + } + + var peerCeiling = (int)peerMax; + return peerCeiling < globalOrFallback ? peerCeiling : globalOrFallback; + } + + /// + /// Returns whether the captured receive mask or newsgroup expression rejects . + /// + /// Peer captured for this connection. does not reject. + /// Classified CanonicalV1 record. Classification is not repeated here. + /// + /// or when this method returns . + /// + /// when the article must be rejected. + /// + /// Article types use : 65535 allows every combination, + /// and every other mask requires each flag on to be present. + /// Newsgroups use on the article's Newsgroups header. + /// Only is accepted. Exclude, poison, and no-match reject. + /// + internal static bool TryReject(TransitPeerPolicy? peer, in ArticleRecord record, out string rejectDetail) + { + rejectDetail = string.Empty; + if (peer is null) + { + return false; + } + + var allowed = (ArticleType)peer.ReceiveArticleTypes; + if (!ArticleTypeCapabilities.Allows(allowed, record.ArtType)) + { + rejectDetail = ArticleTypeRejectDetail; + return true; + } + + if (peer.ReceivePatterns.EvaluateArticle(SplitNewsgroups(record.Newsgroups)) != NewsfeedsPatternDecision.Match) + { + rejectDetail = NewsgroupPatternRejectDetail; + return true; + } + + return false; + } + + /// + /// Splits a Newsgroups header into trimmed group tokens for . + /// + /// Header value bytes from the canonical record. + /// Group names. Empty and whitespace-only tokens are omitted. + /// + /// The string conversion exists only because takes strings. + /// Comma separation and surrounding SP/HTAB trimming follow the header, not a second wildmat implementation. + /// + private static List SplitNewsgroups(ReadOnlySpan newsgroups) + { + var groups = new List(4); + var start = 0; + for (var i = 0; i <= newsgroups.Length; i++) + { + if (i < newsgroups.Length && newsgroups[i] != (byte)',') + { + continue; + } + + var token = newsgroups[start..i]; + start = i + 1; + var tokenStart = 0; + var tokenEnd = token.Length; + while (tokenStart < tokenEnd && IsSpace(token[tokenStart])) + { + tokenStart++; + } + + while (tokenEnd > tokenStart && IsSpace(token[tokenEnd - 1])) + { + tokenEnd--; + } + + if (tokenEnd > tokenStart) + { + groups.Add(Encoding.UTF8.GetString(token[tokenStart..tokenEnd])); + } + } + + return groups; + } + + /// Returns whether is SP or HTAB. + private static bool IsSpace(byte value) => value is (byte)' ' or (byte)'\t'; + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/IVatpArticleClient.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/IVatpArticleClient.cs new file mode 100644 index 00000000..0229b54c --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/IVatpArticleClient.cs @@ -0,0 +1,23 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Fetches canonical articles from BackFiller or StorageServer listeners over VATP. + public interface IVatpArticleClient + { + /// + /// Fetches one article from the explicit VATP endpoint. + /// + /// Lowercase dotted DNS name of the listener. + /// TLS listen port in the range 1–65535. + /// Correlation id from ArticleWork or the storage lookup. + /// Expected ArticleId for OPEN binding. + /// Cancellation; sends CANCEL when the connection is usable. + Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpArticleClient.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpArticleClient.cs new file mode 100644 index 00000000..ae138879 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpArticleClient.cs @@ -0,0 +1,70 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Pool-backed VATP article fetch orchestration. + internal sealed class VatpArticleClient : IVatpArticleClient + { + private readonly VatpConnectionPool _pool; + private readonly ILogger _logger; + + public VatpArticleClient(VatpConnectionPool pool, ILogger logger) + { + _pool = pool; + _logger = logger; + } + + /// + public async Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + if (requestId == Guid.Empty) + { + return VatpFetchResult.ProtocolFailure("RequestId must not be empty.", requestId, articleId); + } + + if (!VatpEndpointFields.IsCanonicalFqdn(fqdn)) + { + return VatpFetchResult.ProtocolFailure("VATP endpoint requires a lowercase dotted DNS fqdn.", requestId, articleId); + } + + if (!VatpEndpointFields.IsCanonicalPort(vatpPort)) + { + return VatpFetchResult.ProtocolFailure("VATP endpoint requires a port in the range 1–65535.", requestId, articleId); + } + + var (connection, acquireFailure) = await _pool.AcquireAsync(fqdn, vatpPort, cancellationToken) + .ConfigureAwait(false); + if (acquireFailure is { } failure) + { + return VatpFetchResult.ConnectionFailure(failure.Error, requestId, articleId); + } + + if (connection is null) + { + return VatpFetchResult.ConnectionFailure("Failed to acquire VATP connection.", requestId, articleId); + } + + try + { + return await connection.FetchArticleAsync(requestId, articleId, cancellationToken).ConfigureAwait(false); + } + finally + { + if (connection.IsDead) + { + _pool.RemoveDead(connection); + } + else + { + _pool.Release(connection); + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientLogMessages.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientLogMessages.cs new file mode 100644 index 00000000..01350105 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientLogMessages.cs @@ -0,0 +1,89 @@ +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Source-generated VATP client log messages. + internal static partial class VatpClientLogMessages + { + [LoggerMessage( + EventId = 2800, + Level = LogLevel.Debug, + Message = "VATP connecting to {Host}:{Port}.")] + public static partial void Connecting(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2801, + Level = LogLevel.Debug, + Message = "VATP TLS established for {Host}:{Port}.")] + public static partial void TlsEstablished(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2802, + Level = LogLevel.Debug, + Message = "VATP HELLO complete on {Host}:{Port} maxFramePayload={MaxFramePayload}.")] + public static partial void HelloComplete(ILogger logger, string Host, int Port, uint MaxFramePayload); + + [LoggerMessage( + EventId = 2803, + Level = LogLevel.Debug, + Message = "VATP OPEN streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void OpenSent( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2804, + Level = LogLevel.Debug, + Message = "VATP META streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void MetaReceived( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2805, + Level = LogLevel.Debug, + Message = "VATP transfer complete streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void TransferComplete( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2806, + Level = LogLevel.Debug, + Message = "VATP transfer failed streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex} errorCode={ErrorCode}.")] + public static partial void TransferFailed( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex, + VatpErrorCode ErrorCode); + + [LoggerMessage( + EventId = 2807, + Level = LogLevel.Debug, + Message = "VATP CANCEL streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void CancelSent( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2808, + Level = LogLevel.Debug, + Message = "VATP connection closed {Host}:{Port}.")] + public static partial void ConnectionClosed(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2809, + Level = LogLevel.Debug, + Message = "VATP protocol violation on {Host}:{Port}: {Detail}.")] + public static partial void ProtocolViolation(ILogger logger, string Host, int Port, string Detail); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientOptions.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientOptions.cs new file mode 100644 index 00000000..b2f8090e --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpClientOptions.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Internal VATP client resource and timeout defaults. + internal sealed class VatpClientOptions + { + /// Maximum TLS connections per cache endpoint. + public int MaxConnectionsPerEndpoint { get; init; } = 4; + + /// TCP connect timeout. + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(15); + + /// TLS handshake timeout. + public TimeSpan TlsHandshakeTimeout { get; init; } = TimeSpan.FromSeconds(15); + + /// + /// Per-read I/O timeout while a connection is active. + /// It is not armed while a STORE on the connection is still waiting for its first WINDOW. + /// + public TimeSpan IoTimeout { get; init; } = TimeSpan.FromMinutes(5); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnection.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnection.cs new file mode 100644 index 00000000..896ab3e2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnection.cs @@ -0,0 +1,842 @@ +using System.Buffers; +using System.Net.Security; +using System.Net.Sockets; +using System.Threading.Channels; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// One multiplexed TLS+VATP connection to a cache listener. + internal sealed class VatpConnection : IAsyncDisposable + { + private readonly string _host; + private readonly int _port; + private readonly Stream _stream; + private readonly TcpClient _tcp; + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ArticleTransferLimits _limits; + private readonly ArticleTransferStreamTable _streamTable; + private readonly Dictionary _pending = new(); + private readonly object _gate = new(); + private readonly SemaphoreSlim _writeLock = new(1, 1); + private readonly Channel _outbound; + private uint _nextStreamId = 1; + private uint _maxFramePayload = VatpProtocol.DefaultMaxFramePayload; + private Task? _runTask; + private CancellationTokenSource? _runCts; + private int _disposed; + private int _dead; + + private VatpConnection( + string host, + int port, + TcpClient tcp, + Stream stream, + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + uint maxFramePayload) + { + _host = host; + _port = port; + _tcp = tcp; + _stream = stream; + _logger = logger; + _options = options; + _limits = limits; + _streamTable = new ArticleTransferStreamTable(limits); + _maxFramePayload = maxFramePayload; + _outbound = Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + } + + /// Gets whether the connection is dead and must be removed from the pool. + public bool IsDead => Volatile.Read(ref _dead) != 0; + + /// Gets whether another stream can be opened on this connection. + public bool HasStreamCapacity + { + get + { + lock (_gate) + { + return !IsDead && _streamTable.Count < _limits.MaxStreamsPerConnection; + } + } + } + + /// Endpoint host used for TLS and logging. + public string Host => _host; + + /// Endpoint port. + public int Port => _port; + + internal static async Task ConnectAsync( + string host, + int port, + ILogger logger, + VatpClientOptions options, + RemoteCertificateValidationCallback? serverCertificateValidationCallback, + CancellationToken cancellationToken, + string? tcpConnectHost = null) + { + VatpClientLogMessages.Connecting(logger, host, port); + var tcp = new TcpClient(); + try + { + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(options.ConnectTimeout); + await tcp.ConnectAsync(tcpConnectHost ?? host, port, connectCts.Token).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + tcp.Dispose(); + throw new VatpConnectionException("TCP connect failed.", ex); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + throw new VatpConnectionException("TCP connect timed out."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + + Stream transport; + try + { + var ssl = await VatpTlsClient.AuthenticateAsClientAsync( + tcp.GetStream(), + host, + options.TlsHandshakeTimeout, + serverCertificateValidationCallback, + cancellationToken).ConfigureAwait(false); + transport = ssl; + VatpClientLogMessages.TlsEstablished(logger, host, port); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + tcp.Dispose(); + throw new VatpConnectionException("TLS handshake failed.", ex); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + throw new VatpConnectionException("TLS handshake timed out."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + + var limits = ArticleTransferLimits.Default; + var connection = new VatpConnection( + host, + port, + tcp, + transport, + logger, + options, + limits, + VatpProtocol.DefaultMaxFramePayload); + + try + { + var maxPayload = await connection.PerformHelloExchangeAsync(cancellationToken).ConfigureAwait(false); + connection._maxFramePayload = maxPayload; + VatpClientLogMessages.HelloComplete(logger, host, port, maxPayload); + connection.StartRunLoop(); + return connection; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + internal async Task FetchArticleAsync( + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (IsDead) + { + return VatpFetchResult.ConnectionFailure("VATP connection is no longer usable.", requestId, articleId); + } + + uint streamId; + PendingStream pending; + lock (_gate) + { + if (_streamTable.Count >= _limits.MaxStreamsPerConnection) + { + return VatpFetchResult.ConnectionFailure("VATP connection stream table is full.", requestId, articleId); + } + + streamId = AllocateStreamIdLocked(); + var openResult = _streamTable.TryOpen(streamId, requestId, articleId, out var receiveStream); + if (!openResult.Success || receiveStream is null) + { + RecycleStreamIdLocked(streamId); + return VatpFetchResult.ProtocolFailure( + $"Failed to open local stream: {openResult.Error}", + requestId, + articleId); + } + + pending = new PendingStream(receiveStream, articleId); + _pending[streamId] = pending; + } + + var articleHex = articleId.ToLowerHexString(); + VatpClientLogMessages.OpenSent(_logger, streamId, requestId, articleHex); + + Span artBytes = stackalloc byte[ArticleId.Length]; + articleId.CopyTo(artBytes); + await EnqueueFrameAsync( + VatpFrameEncoder.EncodeOpen(streamId, requestId, artBytes), + cancellationToken).ConfigureAwait(false); + + using var cancelReg = cancellationToken.Register(static state => + { + var tuple = ((VatpConnection Connection, uint StreamId))state!; + _ = tuple.Connection.TrySendCancelAsync(tuple.StreamId); + }, (this, streamId)); + + try + { + return await pending.Completion.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return VatpFetchResult.Cancelled(requestId, articleId, pending.ReceiveStream.ReceivedBytes); + } + finally + { + lock (_gate) + { + _pending.Remove(streamId); + _streamTable.TryRemove(streamId); + RecycleStreamIdLocked(streamId); + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + MarkDead("Connection disposed."); + try + { + _runCts?.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + _outbound.Writer.TryComplete(); + if (_runTask is not null) + { + try + { + await _runTask.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + await _stream.DisposeAsync().ConfigureAwait(false); + _tcp.Dispose(); + _writeLock.Dispose(); + VatpClientLogMessages.ConnectionClosed(_logger, _host, _port); + } + + private async Task PerformHelloExchangeAsync(CancellationToken cancellationToken) + { + await WriteFrameDirectAsync( + VatpFrameEncoder.EncodeHello(_limits.DefaultMaxFramePayload), + cancellationToken).ConfigureAwait(false); + + var frame = await ReadOneFrameAsync(cancellationToken).ConfigureAwait(false); + if (frame.Header.Type != VatpFrameType.Hello + || frame.Header.StreamId != VatpProtocol.ConnectionStreamId) + { + throw new VatpConnectionException("Expected server HELLO."); + } + + Span helloBytes = stackalloc byte[VatpProtocol.HelloPayloadLength]; + CopyPayload(frame.Payload, helloBytes); + if (!VatpHello.TryDecode(helloBytes, out var hello, out var error)) + { + throw new VatpConnectionException($"Invalid server HELLO: {error}"); + } + + return Math.Min(hello.MaxFramePayload, _limits.DefaultMaxFramePayload); + } + + private void StartRunLoop() + { + _runCts = new CancellationTokenSource(); + _runTask = RunAsync(_runCts.Token); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var writer = RunWriterAsync(cancellationToken); + try + { + await RunReadLoopAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + MarkDead(ex.Message); + } + finally + { + _outbound.Writer.TryComplete(); + try + { + await writer.ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + FailAllPending(VatpFetchResult.ConnectionFailure("VATP connection closed.")); + } + } + + private async Task RunWriterAsync(CancellationToken cancellationToken) + { + await foreach (var frame in _outbound.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await WriteFrameDirectAsync(frame, cancellationToken).ConfigureAwait(false); + } + } + + private async Task RunReadLoopAsync(CancellationToken cancellationToken) + { + // Socket reads are independent of VATP framing: one ReadAsync may yield a partial + // frame, one frame, or many frames. Only the incomplete tail is retained, and that + // tail is bounded to one negotiated maximum frame. + var readBuffer = ArrayPool.Shared.Rent(32 * 1024); + var parseBuffer = ArrayPool.Shared.Rent(VatpProtocol.HeaderLengthBytes + (int)_maxFramePayload); + var buffered = 0; + try + { + while (!cancellationToken.IsCancellationRequested && !IsDead) + { + var maxIncomplete = VatpProtocol.HeaderLengthBytes + (int)_maxFramePayload; + if (buffered > maxIncomplete) + { + MarkDead("Read buffer exceeded negotiated frame size."); + return; + } + + int bytesRead; + try + { + using var ioCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ioCts.CancelAfter(_options.IoTimeout); + bytesRead = await _stream.ReadAsync(readBuffer.AsMemory(0, readBuffer.Length), ioCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + MarkDead("Read timed out."); + return; + } + + if (bytesRead == 0) + { + MarkDead("Peer closed connection."); + return; + } + + EnsureParseCapacity(ref parseBuffer, buffered + bytesRead); + readBuffer.AsSpan(0, bytesRead).CopyTo(parseBuffer.AsSpan(buffered)); + buffered += bytesRead; + + var consumed = 0; + while (consumed < buffered) + { + var candidate = new ReadOnlySequence(parseBuffer, consumed, buffered - consumed); + var parsed = VatpFrameParser.ParseOneFrame(in candidate, _maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, parsed.Error.ToString()); + MarkDead($"Invalid frame: {parsed.Error}"); + return; + } + + consumed += checked((int)parsed.ConsumedBytes); + if (!await HandleFrameAsync(parsed.Frame!.Value, cancellationToken).ConfigureAwait(false)) + { + return; + } + } + + if (consumed > 0) + { + parseBuffer.AsSpan(consumed, buffered - consumed).CopyTo(parseBuffer); + buffered -= consumed; + } + + // After dispatching every complete frame, only an incomplete frame may remain. + if (buffered > maxIncomplete) + { + MarkDead("Read buffer exceeded negotiated frame size."); + return; + } + } + } + finally + { + ArrayPool.Shared.Return(readBuffer); + ArrayPool.Shared.Return(parseBuffer); + } + } + + private async Task HandleFrameAsync(VatpParsedFrame frame, CancellationToken cancellationToken) + { + if (frame.Header.Type == VatpFrameType.Fail) + { + return HandleFailFrame(frame); + } + + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, "Unexpected connection-level frame."); + MarkDead("Unexpected connection-level frame."); + return false; + } + + PendingStream? pending; + ArticleTransferReceiveStream? receive; + lock (_gate) + { + if (!_pending.TryGetValue(frame.Header.StreamId, out pending)) + { + // Late META/DATA/END after local stream cleanup (e.g. fetch already + // completed while WINDOW/FAIL races drained) must not kill the connection. + VatpClientLogMessages.ProtocolViolation( + _logger, + _host, + _port, + $"Ignoring frame for inactive StreamId {frame.Header.StreamId}."); + return !IsDead; + } + + receive = pending.ReceiveStream; + } + + switch (frame.Header.Type) + { + case VatpFrameType.Meta: + VatpClientLogMessages.MetaReceived( + _logger, + frame.Header.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString()); + if (!receive.TryAcceptMeta(frame.Payload).Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "META rejected.")); + } + + break; + case VatpFrameType.Data: + { + // Receive credit is consumed on accept. Replenish locally and advertise + // the same amount via WINDOW so ArtSize may exceed InitialStreamWindowBytes + // without treating the initial window as an article-size ceiling. + var apply = receive.TryAcceptData(frame.Payload, frame.Header.HasFin); + if (!apply.Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "DATA rejected.")); + break; + } + + var payloadLength = checked((uint)frame.Payload.Length); + // Pair outbound WINDOW with local credit restore. Skip both when the + // stream was cancelled between accept and replenish (no cross-stream credit). + if (payloadLength > 0 && receive.TryAcceptWindow(payloadLength).Success) + { + await EnqueueFrameAsync( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, payloadLength), + cancellationToken).ConfigureAwait(false); + } + + if (receive.IsTerminal) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "DATA rejected.")); + } + + break; + } + + case VatpFrameType.End: + if (!receive.TryAcceptEnd().Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "END rejected.")); + break; + } + + if (receive.TryTakeRecord(out var record)) + { + VatpClientLogMessages.TransferComplete( + _logger, + frame.Header.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString()); + CompletePending( + frame.Header.StreamId, + VatpFetchResult.FromSuccess( + record, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes)); + } + else + { + CompletePending( + frame.Header.StreamId, + VatpFetchResult.IncompleteOrMalformed( + "END received without consumable record.", + receive.Failure, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes)); + } + + break; + default: + VatpClientLogMessages.ProtocolViolation( + _logger, + _host, + _port, + $"Unexpected frame type {frame.Header.Type}."); + MarkDead("Unexpected frame type."); + return false; + } + + return !IsDead; + } + + private bool HandleFailFrame(VatpParsedFrame frame) + { + Span payload = stackalloc byte[(int)Math.Min(frame.Payload.Length, VatpProtocol.FailMaxPayloadLength)]; + CopyPayload(frame.Payload, payload); + if (!VatpControlPayload.TryDecodeFail(payload, out var errorCode, out var reason)) + { + MarkDead("Invalid FAIL payload."); + return false; + } + + var reasonText = reason.IsEmpty ? null : System.Text.Encoding.ASCII.GetString(reason); + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + MarkDead(reasonText ?? errorCode.ToString()); + return false; + } + + // Server may emit UnknownStream for WINDOW/CANCEL that arrive after it already + // removed a finished stream (common when ArtSize fits in the initial window). + // That race must not fail an in-flight or just-completed transfer, nor poison siblings. + if (errorCode == VatpErrorCode.UnknownStream) + { + return !IsDead; + } + + lock (_gate) + { + if (_pending.TryGetValue(frame.Header.StreamId, out var pending)) + { + VatpClientLogMessages.TransferFailed( + _logger, + frame.Header.StreamId, + pending.ReceiveStream.RequestId, + pending.ArticleId.ToLowerHexString(), + errorCode); + CompletePendingLocked( + frame.Header.StreamId, + VatpFetchResult.RemoteFailure( + reasonText, + errorCode, + pending.ReceiveStream.RequestId, + pending.ArticleId, + pending.ReceiveStream.ReceivedBytes)); + } + } + + return !IsDead; + } + + private void CompletePending(uint streamId, VatpFetchResult result) + { + lock (_gate) + { + CompletePendingLocked(streamId, result); + } + } + + private void CompletePendingLocked(uint streamId, VatpFetchResult result) + { + if (_pending.TryGetValue(streamId, out var pending)) + { + pending.Completion.TrySetResult(result); + } + } + + private VatpFetchResult ToFetchFailure(ArticleTransferReceiveStream receive, string detail) + { + VatpClientLogMessages.TransferFailed( + _logger, + receive.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString(), + receive.Failure); + if (receive.Failure == VatpErrorCode.Cancelled) + { + return VatpFetchResult.Cancelled( + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes); + } + + return VatpFetchResult.IncompleteOrMalformed( + detail, + receive.Failure, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes); + } + + private void FailAllPending(VatpFetchResult result) + { + lock (_gate) + { + foreach (var (streamId, pending) in _pending.ToArray()) + { + pending.Completion.TrySetResult( + VatpFetchResult.ConnectionFailure( + result.Error, + pending.ReceiveStream.RequestId, + pending.ArticleId, + pending.ReceiveStream.ReceivedBytes)); + _pending.Remove(streamId); + _streamTable.TryRemove(streamId); + RecycleStreamIdLocked(streamId); + } + } + } + + private void MarkDead(string reason) + { + if (Interlocked.Exchange(ref _dead, 1) == 1) + { + return; + } + + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, reason); + FailAllPending(VatpFetchResult.ConnectionFailure(reason)); + } + + private async Task TrySendCancelAsync(uint streamId) + { + PendingStream? pending; + lock (_gate) + { + if (!_pending.TryGetValue(streamId, out pending) || pending.ReceiveStream.IsTerminal) + { + return; + } + + _ = pending.ReceiveStream.TryCancel(); + } + + VatpClientLogMessages.CancelSent( + _logger, + streamId, + pending!.ReceiveStream.RequestId, + pending.ArticleId.ToLowerHexString()); + + try + { + await EnqueueFrameAsync(VatpFrameEncoder.EncodeCancel(streamId), CancellationToken.None) + .ConfigureAwait(false); + } + catch (ChannelClosedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task EnqueueFrameAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + if (IsDead) + { + throw new VatpConnectionException("Connection is dead."); + } + + await _outbound.Writer.WriteAsync(frame, cancellationToken).ConfigureAwait(false); + } + + private async Task WriteFrameDirectAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (frame.Header.Length > 0) + { + await _stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (frame.Payload.Length > 0) + { + await _stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _writeLock.Release(); + } + } + + private async Task ReadOneFrameAsync(CancellationToken cancellationToken) + { + var header = new byte[VatpProtocol.HeaderLengthBytes]; + await ReadExactAsync(header, cancellationToken).ConfigureAwait(false); + var parsedHeader = VatpFrameHeader.ReadFrom(header); + var payload = new byte[parsedHeader.PayloadLength]; + if (payload.Length > 0) + { + await ReadExactAsync(payload, cancellationToken).ConfigureAwait(false); + } + + return new VatpParsedFrame(parsedHeader, new ReadOnlySequence(payload)); + } + + private async Task ReadExactAsync(Memory buffer, CancellationToken cancellationToken) + { + var total = 0; + while (total < buffer.Length) + { + using var ioCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ioCts.CancelAfter(_options.IoTimeout); + var read = await _stream.ReadAsync(buffer[total..], ioCts.Token).ConfigureAwait(false); + if (read == 0) + { + throw new VatpConnectionException("Unexpected end of stream."); + } + + total += read; + } + } + + private uint AllocateStreamIdLocked() + { + // Do not reuse StreamIds on a live connection. Late WINDOW/END/FAIL for a just- + // finished stream must not be applied to a successor transfer that recycled the id. + // Concurrent capacity remains bounded by MaxStreamsPerConnection via the stream table. + if (_nextStreamId == 0) + { + _nextStreamId = 1; + } + + return _nextStreamId++; + } + + private void RecycleStreamIdLocked(uint streamId) + { + // Intentionally unused for wire StreamId values; retained as a call-site hook so + // stream-table removal stays paired with allocation cleanup. + _ = streamId; + } + + private static void EnsureParseCapacity(ref byte[] parseBuffer, int required) + { + if (required <= parseBuffer.Length) + { + return; + } + + var bigger = ArrayPool.Shared.Rent(Math.Max(parseBuffer.Length * 2, required)); + parseBuffer.AsSpan(0, Math.Min(parseBuffer.Length, required)).CopyTo(bigger); + ArrayPool.Shared.Return(parseBuffer); + parseBuffer = bigger; + } + + private static void CopyPayload(in ReadOnlySequence payload, Span destination) + { + if (payload.Length > destination.Length) + { + throw new InvalidOperationException("Payload buffer too small."); + } + + payload.CopyTo(destination); + } + + private sealed class PendingStream(ArticleTransferReceiveStream receiveStream, ArticleId articleId) + { + public ArticleTransferReceiveStream ReceiveStream { get; } = receiveStream; + + public ArticleId ArticleId { get; } = articleId; + + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + /// Connection setup failure surfaced to the pool. + internal sealed class VatpConnectionException : Exception + { + public VatpConnectionException(string message) + : base(message) + { + } + + public VatpConnectionException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnectionPool.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnectionPool.cs new file mode 100644 index 00000000..18ca79f2 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpConnectionPool.cs @@ -0,0 +1,246 @@ +using System.Collections.Concurrent; +using System.Net.Security; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Pools multiplexed VATP connections keyed by cache endpoint. + internal sealed class VatpConnectionPool : IAsyncDisposable + { + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ConcurrentDictionary _endpoints = new(); + private int _disposed; + + /// Optional certificate callback for tests only. + internal RemoteCertificateValidationCallback? ServerCertificateValidationCallback { get; set; } + + /// Optional TCP connect host override for tests (TLS TargetHost remains the URI host). + internal string? TestTcpConnectHost { get; set; } + + public VatpConnectionPool(ILogger logger) + : this(logger, new VatpClientOptions()) + { + } + + internal VatpConnectionPool(ILogger logger, VatpClientOptions options) + { + _logger = logger; + _options = options; + } + + internal async Task<(VatpConnection? Connection, VatpFetchResult? Failure)> AcquireAsync( + string host, + int port, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var key = new EndpointKey(host, port); + var pool = _endpoints.GetOrAdd(key, static _ => new EndpointPool()); + + var existing = pool.TryAcquireExisting(); + if (existing is not null) + { + return (existing, null); + } + + if (pool.ConnectionCount >= _options.MaxConnectionsPerEndpoint) + { + return pool.ConnectionCount == 0 + ? (null, VatpFetchResult.ConnectionFailure("No VATP connections available for endpoint.")) + : (null, VatpFetchResult.ConnectionFailure( + "All VATP connections to the endpoint are at stream capacity.")); + } + + await pool.CreateGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + existing = pool.TryAcquireExisting(); + if (existing is not null) + { + return (existing, null); + } + + if (pool.ConnectionCount >= _options.MaxConnectionsPerEndpoint) + { + return ( + null, + VatpFetchResult.ConnectionFailure( + "All VATP connections to the endpoint are at stream capacity.")); + } + + try + { + var created = await VatpConnection.ConnectAsync( + host, + port, + _logger, + _options, + ServerCertificateValidationCallback, + cancellationToken, + TestTcpConnectHost).ConfigureAwait(false); + pool.Add(created); + return (created, null); + } + catch (VatpConnectionException ex) + { + return (null, VatpFetchResult.ConnectionFailure(ex.Message)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return (null, VatpFetchResult.ConnectionFailure(ex.Message)); + } + } + finally + { + pool.CreateGate.Release(); + } + } + + internal void Release(VatpConnection connection) + { + if (connection.IsDead) + { + RemoveDead(connection); + } + } + + internal void RemoveDead(VatpConnection connection) + { + var key = new EndpointKey(connection.Host, connection.Port); + if (_endpoints.TryGetValue(key, out var pool)) + { + pool.Remove(connection); + } + + _ = connection.DisposeAsync(); + } + + internal int GetEndpointConnectionCount(string host, int port) + { + var key = new EndpointKey(host, port); + return _endpoints.TryGetValue(key, out var pool) ? pool.ConnectionCount : 0; + } + + internal int GetEndpointAliveConnectionCount(string host, int port) + { + var key = new EndpointKey(host, port); + return _endpoints.TryGetValue(key, out var pool) ? pool.AliveConnectionCount : 0; + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + foreach (var pool in _endpoints.Values) + { + await pool.DisposeAsync().ConfigureAwait(false); + } + + _endpoints.Clear(); + } + + private readonly record struct EndpointKey(string Host, int Port); + + private sealed class EndpointPool + { + private readonly List _connections = []; + private readonly object _gate = new(); + + internal SemaphoreSlim CreateGate { get; } = new(1, 1); + + public int ConnectionCount + { + get + { + lock (_gate) + { + return _connections.Count; + } + } + } + + public int AliveConnectionCount + { + get + { + lock (_gate) + { + PurgeDeadLocked(); + return _connections.Count; + } + } + } + + public void Add(VatpConnection connection) + { + lock (_gate) + { + _connections.Add(connection); + } + } + + public void Remove(VatpConnection connection) + { + lock (_gate) + { + _connections.Remove(connection); + } + } + + public VatpConnection? TryAcquireExisting() + { + lock (_gate) + { + PurgeDeadLocked(); + foreach (var candidate in _connections) + { + if (candidate.HasStreamCapacity) + { + return candidate; + } + } + } + + return null; + } + + public async ValueTask DisposeAsync() + { + VatpConnection[] snapshot; + lock (_gate) + { + snapshot = _connections.ToArray(); + _connections.Clear(); + } + + CreateGate.Dispose(); + foreach (var connection in snapshot) + { + await connection.DisposeAsync().ConfigureAwait(false); + } + } + + private void PurgeDeadLocked() + { + for (var i = _connections.Count - 1; i >= 0; i--) + { + if (_connections[i].IsDead) + { + var dead = _connections[i]; + _connections.RemoveAt(i); + _ = dead.DisposeAsync(); + } + } + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchKind.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchKind.cs new file mode 100644 index 00000000..555c6ba8 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchKind.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Outcome classification for a VATP article fetch. + public enum VatpFetchKind + { + /// Canonical article received and validated. + Success, + + /// Remote peer rejected or failed the transfer (FAIL frame, open rejected). + RemoteTransferFailure, + + /// Local protocol violation or unexpected framing. + ProtocolFailure, + + /// Connect, TLS, or transport I/O failure. + ConnectionFailure, + + /// Operation cancelled by caller. + Cancelled, + + /// Article identity could not be used for OPEN. + InvalidArticleId, + + /// Transfer ended without a consumable canonical record. + IncompleteOrMalformedArticle, + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchResult.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchResult.cs new file mode 100644 index 00000000..12086d98 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpFetchResult.cs @@ -0,0 +1,116 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// Result of . + public readonly record struct VatpFetchResult + { + private VatpFetchResult( + VatpFetchKind kind, + ArticleRecord record, + string? error, + VatpErrorCode? errorCode, + Guid? requestId, + ArticleId? articleId, + int acceptedDataBytes) + { + if (acceptedDataBytes < 0) + { + throw new ArgumentOutOfRangeException(nameof(acceptedDataBytes)); + } + + Kind = kind; + Record = record; + Error = error; + ErrorCode = errorCode; + RequestId = requestId; + ArticleId = articleId; + AcceptedDataBytes = acceptedDataBytes; + } + + /// Gets the outcome kind. + public VatpFetchKind Kind { get; } + + /// Gets the article when is . + public ArticleRecord Record { get; } + + /// Gets optional diagnostic text (never article payload). + public string? Error { get; } + + /// Gets the VATP error code when supplied by the peer or state machine. + public VatpErrorCode? ErrorCode { get; } + + /// Gets the OPEN RequestId when known. + public Guid? RequestId { get; } + + /// Gets the OPEN ArticleId when known. + public ArticleId? ArticleId { get; } + + /// + /// Gets the number of DATA payload bytes copied into the receive stream before this result was built. + /// + /// + /// Zero when no receive stream existed or no DATA payload had been copied. + /// When a stream exists, this is that stream's copied DATA count at completion. + /// + public int AcceptedDataBytes { get; } + + /// Creates a success result. + public static VatpFetchResult FromSuccess( + ArticleRecord record, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new(VatpFetchKind.Success, record, null, null, requestId, articleId, acceptedDataBytes); + + /// Creates a remote transfer failure. + public static VatpFetchResult RemoteFailure( + string? error, + VatpErrorCode? errorCode, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new(VatpFetchKind.RemoteTransferFailure, default, error, errorCode, requestId, articleId, acceptedDataBytes); + + /// Creates a protocol failure. + public static VatpFetchResult ProtocolFailure( + string? error, + Guid? requestId = null, + ArticleId? articleId = null, + int acceptedDataBytes = 0) => + new(VatpFetchKind.ProtocolFailure, default, error, null, requestId, articleId, acceptedDataBytes); + + /// Creates a connection failure. + public static VatpFetchResult ConnectionFailure( + string? error, + Guid? requestId = null, + ArticleId? articleId = null, + int acceptedDataBytes = 0) => + new(VatpFetchKind.ConnectionFailure, default, error, null, requestId, articleId, acceptedDataBytes); + + /// Creates a cancelled result. + public static VatpFetchResult Cancelled(Guid requestId, ArticleId articleId, int acceptedDataBytes = 0) => + new(VatpFetchKind.Cancelled, default, null, VatpErrorCode.Cancelled, requestId, articleId, acceptedDataBytes); + + /// Creates an invalid ArticleId result. + public static VatpFetchResult InvalidArticleId(string? error) => + new(VatpFetchKind.InvalidArticleId, default, error, null, null, null, 0); + + /// Creates an incomplete or malformed article result. + public static VatpFetchResult IncompleteOrMalformed( + string? error, + VatpErrorCode? errorCode, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new( + VatpFetchKind.IncompleteOrMalformedArticle, + default, + error, + errorCode, + requestId, + articleId, + acceptedDataBytes); + } +} diff --git a/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpTlsClient.cs b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpTlsClient.cs new file mode 100644 index 00000000..e463ff09 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/Transport/Vatp/VatpTlsClient.cs @@ -0,0 +1,56 @@ +using System.Net.Security; +using System.Security.Authentication; + +namespace VectorNNTPD.ReaderService.Transport.Vatp +{ + /// + /// Shared outbound TLS client options and handshake for VATP cache connections. + /// + internal static class VatpTlsClient + { + internal static SslClientAuthenticationOptions CreateClientOptions( + string host, + RemoteCertificateValidationCallback? serverCertificateValidationCallback) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + var options = new SslClientAuthenticationOptions + { + TargetHost = host, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }; + if (serverCertificateValidationCallback is not null) + { + options.RemoteCertificateValidationCallback = serverCertificateValidationCallback; + } + + return options; + } + + internal static async Task AuthenticateAsClientAsync( + Stream inner, + string host, + TimeSpan timeout, + RemoteCertificateValidationCallback? serverCertificateValidationCallback, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(inner); + var ssl = new SslStream(inner, leaveInnerStreamOpen: false); + try + { + using var tlsCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + tlsCts.CancelAfter(timeout); + await ssl.AuthenticateAsClientAsync( + CreateClientOptions(host, serverCertificateValidationCallback), + tlsCts.Token) + .ConfigureAwait(false); + return ssl; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.ReaderService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.ReaderService/VectorNNTPD.RabbitMQ.json new file mode 100644 index 00000000..934ea129 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/VectorNNTPD.RabbitMQ.json @@ -0,0 +1,38 @@ +{ + "RabbitMQ": { + "ChannelLeaseTimeoutSeconds": 60, + "ChannelPoolSize": 512, + "ConnectionBlockedTimeoutSeconds": 30, + "ConnectionScaleDownIdleSeconds": 300, + "DegradedThreshold": 0.75, + "EnableSsl": false, + "Hosts": [ + "198.18.0.3" + ], + "Management": { + "BaseUrl": "http://198.18.0.3:15672", + "RequestTimeoutSeconds": 5 + }, + "MaxConnections": 16, + "MaximumShutdownDrainTimeoutSeconds": 30, + "MaxPendingLeaseWaiters": 1024, + "MinConnections": 4, + "MinimumConnectionLifetimeSeconds": 300, + "NetworkRecoveryIntervalSeconds": 5, + "PoolReconnectBaseDelayMs": 250, + "PoolReconnectMaxDelayMs": 30000, + "Port": 5672, + "PublishConfirmTimeoutSeconds": 10, + "RequestedChannelMax": 2047, + "RequestedHeartbeatSeconds": 60, + "RpcTimeoutSeconds": 30, + "ScaleDownCooldownSeconds": 30, + "SocketTimeoutSeconds": 30, + "UnhealthyThreshold": 5, + "VirtualHost": "/", + "WorkRequestMaxPayloadBytes": 1024, + "ArticleWorkDeliveryLimit": 5, + "ArticleWorkQueueMaxLength": 10000, + "ArticleWorkQueueMessageTtlMilliseconds": 60000 + } +} diff --git a/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.csproj b/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.csproj new file mode 100644 index 00000000..eae11717 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.csproj @@ -0,0 +1,64 @@ + + + + + + net10.0 + Exe + true + VectorNNTPD.ReaderService + VectorNNTPD.ReaderService + NNTP reader-server host and implementation (VectorNNTPD.ReaderService). + false + x64 + + + + + + + + + + + + + PreserveNewest + PreserveNewest + + + PreserveNewest + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json b/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json new file mode 100644 index 00000000..5537b417 --- /dev/null +++ b/src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json @@ -0,0 +1,2987 @@ +{ + "Moderation": { + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", + "RetrievedFrom": "InnUrl", + "Url": "https://github.com/InterNetNews/inn/blob/main/samples/moderators" + } + }, + "Email": { + "Enabled": false, + "DefaultFrom": "noreply@usenet.ninja", + "EnvelopeSender": "noreply@usenet.ninja", + "Smtp": { + "Host": "", + "Port": 587, + "Security": "StartTls", + "Username": "", + "Password": "", + "RequireTlsForAuthentication": true, + "ConnectTimeout": "00:00:15", + "CommandTimeout": "00:00:30", + "MaxAttempts": 3, + "InitialRetryDelay": "00:00:02", + "MaximumRetryDelay": "00:01:00" + }, + "Spool": { + "Directory": "spool/smtp", + "ShutdownTimeout": "00:00:15", + "ScanInterval": "00:00:01" + } + }, + "Control": { + "PgpAuthorities": { + "Authorities": [ + { + "AdminGroup": "aioe.system", + "Authorizations": [ + { + "From": "usenet@aioe.org", + "Message": "checkgroups", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "newgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "rmgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + } + ], + "Contact": "usenet@aioe.org", + "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8", + "KeyUrl": "http://news.aioe.org/hierarchy/aioe.txt", + "Name": "AIOE", + "SyncableServer": "nntp.aioe.org", + "Url": "http://news.aioe.org/hierarchy/" + }, + { + "AdminGroup": "alabama.config", + "Authorizations": [ + { + "From": "jpc@suespammers.org", + "Message": "checkgroups", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "newgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "rmgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + } + ], + "Contact": "jpc@suespammers.org", + "Name": "ALABAMA & HSV" + }, + { + "AdminGroup": "at.usenet.gruppen", + "Authorizations": [ + { + "From": "control@usenet.backbone.at", + "Message": "checkgroups", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "newgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "rmgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + } + ], + "KeyUrl": "http://www.usenet.at/pgpkey.asc", + "Name": "AT", + "Url": "http://www.usenet.at/" + }, + { + "AdminGroup": "aus.net.news", + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "AUS", + "Url": "http://aus.news-admin.org/" + }, + { + "AdminGroup": "be.announce", + "Authorizations": [ + { + "From": "group-admin@usenet.be", + "Message": "checkgroups", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "newgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "rmgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + } + ], + "Contact": "be-hierarchy-admin@usenet.be", + "KeyFingerprint": "302A459470DE1FD5818C5864D2F70871", + "KeyUrl": "http://usenet.be/be.announce.newgroups.asc", + "Name": "BE", + "Url": "http://usenet.be/" + }, + { + "AdminGroup": "belwue.infos", + "Authorizations": [ + { + "From": "news@news.belwue.de", + "Message": "checkgroups", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "newgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "rmgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + } + ], + "Contact": "news@belwue.de", + "KeyFingerprint": "2EFF2B8A891FE5AA6F89672450B542D9", + "KeyUrl": "http://www.belwue.de/fileadmin/belwue/Dokumente/Newsserver/belwue-hir-control.asc", + "Name": "BELWUE", + "Url": "http://www.belwue.de/produkte/dienste/dienste-usenet1.html" + }, + { + "AdminGroup": "bionet.general", + "Authorizations": [ + { + "From": "Biosci-control-key@net.bio.net", + "Message": "checkgroups", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "newgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "rmgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + } + ], + "KeyFingerprint": "EBC0F1BA260BC6D6FB8DEDC4AE5D1054", + "Name": "BIONET", + "Url": "http://www.bio.net/" + }, + { + "AdminGroup": "bit.admin", + "Authorizations": [ + { + "From": "bit@newsadmin.com", + "Message": "checkgroups", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "newgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "rmgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + } + ], + "Name": "BIT", + "Url": "http://www.newsadmin.com/bit/bit.htm" + }, + { + "AdminGroup": "bln.misc", + "Authorizations": [ + { + "From": "news@fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + } + ], + "Contact": "news@fu-berlin.de", + "KeyFingerprint": "2B24B5C78A4861407BCD5A88281D90763F368EA0", + "KeyUrl": "https://ftp.fu-berlin.de/doc/news/bln/bln-key.txt", + "Name": "BLN", + "Url": "https://ftp.fu-berlin.de/doc/news/bln/bln-info.txt" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "BNE", + "Url": "http://bne.news-admin.org/" + }, + { + "Contact": "bofh-control@lists.killfile.org", + "KeyFingerprint": "40B58A561E006152083E38B3CEF569807DC1A266", + "Name": "BOFH" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "CANB", + "Url": "http://canb.news-admin.org/" + }, + { + "Authorizations": [ + { + "From": "felix.rauch@nice.ch", + "Message": "checkgroups", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "newgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "rmgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + } + ], + "Contact": "ch-news-admin@use-net.ch", + "KeyFingerprint": "7180D68CA7DE2C70624A486ED99602DF", + "KeyUrl": "http://www.use-net.ch/Usenet/adminkey.html", + "Name": "CH", + "Url": "http://www.use-net.ch/Usenet/" + }, + { + "Contact": "koordination@cl-netz.de", + "KeyUrl": "http://www.cl-netz.de/control.txt", + "Name": "CL", + "Url": "http://www.cl-netz.de/" + }, + { + "AdminGroup": "clari.net.admin", + "Authorizations": [ + { + "From": "cl*@clarinet.com", + "Message": "rmgroup", + "Newsgroups": "clari.*", + "VerificationIdentity": "ClariNet.Group" + } + ], + "Contact": "support@clari.net", + "KeyUrl": "http://www.clari.net/tech/clarikey.txt", + "Name": "CLARI" + }, + { + "AdminGroup": "cn.announce", + "Authorizations": [ + { + "From": "control@bentium.com", + "Message": "checkgroups", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "newgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "rmgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + } + ], + "KeyFingerprint": "6297EE33F71625C1A49E47BAC53E5E9E", + "Name": "CN", + "Url": "http://news.yaako.com/" + }, + { + "AdminGroup": "cn.bbs.admin.announce", + "Authorizations": [ + { + "From": "control@cn-bbs.org", + "Message": "checkgroups", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "newgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "rmgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + } + ], + "Name": "CN.BBS", + "Url": "http://bbs.cn.news-admin.org/" + }, + { + "AdminGroup": "news.announce.newgroups", + "Authorizations": [ + { + "From": "group-admin@isc.org", + "Message": "checkgroups", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "newgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "rmgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + } + ], + "Contact": "board@big-8.org", + "KeyFingerprint": "8E181BEF0BE638962C1A5C98FAFE7B550C18C8B7", + "Name": "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK", + "Url": "https://www.big-8.org/" + }, + { + "AdminGroup": "de.admin.news.announce", + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "checkgroups", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "newgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Contact": "moderator@dana.de", + "KeyFingerprint": "5BB05288BF55194F667DC2AE16262825", + "KeyUrl": "http://www.dana.de/pgp/dana.txt", + "Name": "DE", + "Url": "http://www.dana.de/" + }, + { + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.alt.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Name": "DE.ALT" + }, + { + "AdminGroup": "demon.news", + "Authorizations": [ + { + "From": "newsmaster@demon.net", + "Message": "checkgroups", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "newgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "rmgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + } + ], + "Contact": "newsmaster@demon.net", + "KeyUrl": "ftp://ftp.demon.co.uk/pub/news/doc/demon.news.txt", + "Name": "DEMON" + }, + { + "Authorizations": [ + { + "From": "news@news.dknet.dk", + "Message": "checkgroups", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "newgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "rmgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + } + ], + "KeyFingerprint": "7CB2C750F37D5D738CEE2E3F558072FF", + "KeyUrl": "http://www.usenet.dk/dk-admin/pubkey.html", + "Name": "DK", + "Url": "http://www.usenet.dk/dk-admin/" + }, + { + "AdminGroup": "easynet.support", + "Authorizations": [ + { + "From": "newsmaster@easynet.net", + "Message": "checkgroups", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "newgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "rmgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + } + ], + "Contact": "Christiaan Keet ", + "Name": "EASYNET" + }, + { + "Authorizations": [ + { + "From": "news@news.ut.ee", + "Message": "checkgroups", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "newgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "rmgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + } + ], + "Contact": "usenet@news.ut.ee", + "KeyUrl": "http://news.ut.ee/pubkey.asc", + "Name": "EE", + "Url": "http://news.ut.ee/" + }, + { + "AdminGroup": "eug.config", + "Authorizations": [ + { + "From": "newsadmin@efn.org", + "Message": "checkgroups", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "newgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "rmgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + } + ], + "Name": "EFN & EUG" + }, + { + "AdminGroup": "england.news.policy", + "Authorizations": [ + { + "From": "admin@england.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "newgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + } + ], + "Contact": "admin@england.news-admin.org", + "KeyFingerprint": "DA3EC20146E561CBA24309CA136D311F", + "Name": "ENGLAND" + }, + { + "AdminGroup": "es.news.anuncios", + "Authorizations": [ + { + "From": "moderador@corus-es.org", + "Message": "checkgroups", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "newgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "rmgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + } + ], + "Contact": "moderador@corus-es.org", + "KeyUrl": "http://www.corus-es.org/docs/esnews.asc", + "Name": "ES", + "Url": "http://www.corus-es.org/docs/es_newsadmins_faq.txt" + }, + { + "Authorizations": [ + { + "From": "mod-ena@ennui.org", + "Message": "checkgroups", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "newgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "rmgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + } + ], + "Contact": "", + "KeyUrl": "http://ennui.org/esp/mod-ena.asc", + "Name": "ESP", + "Url": "http://ennui.org/esp/" + }, + { + "Authorizations": [ + { + "From": "news@eternal-september.org", + "Message": "checkgroups", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "newgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "rmgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + } + ], + "Contact": "news@eternal-september.org", + "KeyFingerprint": "B79A2BC67D5AFF7918E2AC914BC125F1", + "KeyUrl": "http://www.eternal-september.org/control/pgpkey.txt", + "Name": "ETERNAL-SEPTEMBER", + "Url": "http://www.eternal-september.org/" + }, + { + "AdminGroup": "europa.usenet.admin", + "Authorizations": [ + { + "From": "group-admin@usenet.eu.org", + "Message": "checkgroups", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "newgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "rmgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + } + ], + "KeyFingerprint": "3A05A849FB16292575E3DEBB69E01DB4", + "KeyUrl": "http://www.europa.usenet.eu.org/pgp/index.html", + "Name": "EUROPA", + "Url": "http://www.europa.usenet.eu.org/" + }, + { + "AdminGroup": "ffm.admin", + "Authorizations": [ + { + "From": "ffm.admin@arcor.de", + "Message": "checkgroups", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "newgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "rmgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + } + ], + "KeyFingerprint": "53A082626FC781C9CF53AB00A3F8C211", + "KeyUrl": "ftp://ftp.arcor-online.net/pub/news/PGPKEY.FFM", + "Name": "FFM", + "Url": "http://ffm.arcornews.de/" + }, + { + "AdminGroup": "fido.belg.news", + "Authorizations": [ + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "checkgroups", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "newgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "rmgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + } + ], + "Name": "FIDO.BELG" + }, + { + "Authorizations": [ + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + } + ], + "KeyUrl": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/PGP-Key", + "Name": "FIDO.GER", + "Url": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/fido.ger-info.english" + }, + { + "AdminGroup": "fido7.postmasters", + "Authorizations": [ + { + "From": "newgroups-request@fido7.org", + "Message": "checkgroups", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "newgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "rmgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + } + ], + "KeyUrl": "http://www.fido7.org/pgpcontrol.html", + "Name": "FIDO7", + "Url": "http://www.fido7.org/" + }, + { + "AdminGroup": "fj.news.announce", + "Authorizations": [ + { + "From": "committee@fj-news.org", + "Message": "checkgroups", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "newgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "rmgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + } + ], + "Contact": "committee@fj-news.org", + "KeyUrl": "http://www.is.tsukuba.ac.jp/~yas/fj/fj.asc", + "Name": "FJ", + "Url": "http://www.fj-news.org/index.html.en" + }, + { + "AdminGroup": "flora.general", + "Authorizations": [ + { + "From": "news@flora.ottawa.on.ca", + "Message": "checkgroups", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "newgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "rmgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + } + ], + "Contact": "russell@flora.org", + "Name": "FLORA" + }, + { + "AdminGroup": "fr.usenet.forums.annonces", + "Authorizations": [ + { + "From": "control@usenet-fr.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "newgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + } + ], + "KeyFingerprint": "D5F369B297573622015354E7FA423E89970EB10B", + "KeyUrl": "https://www.usenet-fr.net/pgp-fr-2020.txt", + "Name": "FR", + "Url": "https://www.usenet-fr.net/" + }, + { + "AdminGroup": "france.admin.evolutions", + "Authorizations": [ + { + "From": "control@usenet-france.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "newgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + } + ], + "Contact": "control@usenet-france.news.eu.org", + "Name": "FRANCE" + }, + { + "AdminGroup": "gnu.gnusenet.config", + "Authorizations": [ + { + "From": "usenet@gnu.org", + "Message": "checkgroups", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "newgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "rmgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + } + ], + "KeyUrl": "http://www.gnu.org/usenet/usenet-pgp-key.txt", + "Name": "GNU", + "Url": "http://www.gnu.org/usenet/usenet.html" + }, + { + "Authorizations": [ + { + "From": "news@gnuu.de", + "Message": "rmgroup", + "Newsgroups": "gnuu.*", + "VerificationIdentity": "news@gnuu.de" + } + ], + "KeyUrl": "http://www.gnuu.de/config/PGPKEY.GNUU", + "Name": "GNUU", + "Url": "http://www.gnuu.de/" + }, + { + "AdminGroup": "grisbi.admin", + "Authorizations": [ + { + "From": "grisbi-control@grisbi.org", + "Message": "checkgroups", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "newgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "rmgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + } + ], + "Contact": "newsmaster@grisbi.org", + "KeyFingerprint": "EB350C030080BD2A7E0CA4C9F2C62A3DC86CC6E1", + "KeyUrl": "http://news.grisbi.org/public-key.asc", + "Name": "GRISBI", + "Url": "http://news.grisbi.org/" + }, + { + "Authorizations": [ + { + "From": "news@zhaum.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "newgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + } + ], + "Name": "HACKTIC" + }, + { + "AdminGroup": "hamburg.koordination", + "Authorizations": [ + { + "From": "hamburg@steering-group.net", + "Message": "checkgroups", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "newgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "rmgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + } + ], + "Contact": "hamburg@steering-group.net", + "KeyFingerprint": "3EE70CBB6E0194EE456FC557F4B9548E", + "KeyUrl": "http://www.steering-group.net/hamburg/hamburg.koordination.txt", + "Name": "HAMBURG", + "Url": "http://www.steering-group.net/hamburg/" + }, + { + "AdminGroup": "hamster.de.config", + "Authorizations": [ + { + "From": "hamster-control@snafu.de", + "Message": "checkgroups", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "newgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "rmgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + } + ], + "Contact": "hamster-contact@snafu.de", + "KeyFingerprint": "1275A9428AD61F776ACFB40C79155F93", + "Name": "HAMSTER" + }, + { + "AdminGroup": "han.news.admin", + "Authorizations": [ + { + "From": "newgroups-request@usenet.or.kr", + "Message": "checkgroups", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "newgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "rmgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + } + ], + "Contact": "newgroups-request@usenet.or.kr", + "KeyUrl": "ftp://ftp.usenet.or.kr/pub/korea/usenet/pgp/PGPKEY.han", + "Name": "HAN" + }, + { + "AdminGroup": "hiv.config", + "Authorizations": [ + { + "From": "news@hivnet.org", + "Message": "checkgroups", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "newgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "rmgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + } + ], + "Contact": "news@hivnet.org", + "KeyFingerprint": "5DD60EDC1E2DEA0BB0564DD65253D7A4", + "Name": "HIV" + }, + { + "AdminGroup": "houston.usenet.config", + "Authorizations": [ + { + "From": "news@academ.com", + "Message": "checkgroups", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "newgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "rmgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + } + ], + "Name": "HOUSTON" + }, + { + "AdminGroup": "hr.news.admin", + "Authorizations": [ + { + "From": "newsmaster@carnet.hr", + "Message": "checkgroups", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "newgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "rmgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + } + ], + "Contact": "newsmaster@carnet.hr", + "KeyFingerprint": "0EE574FB1C407ADB0AACA52F71921BA3ED63AD9A", + "KeyUrl": "http://newsfeed.carnet.hr/control/key.txt", + "Name": "HR", + "SyncableServer": "news.carnet.hr", + "Url": "http://newsfeed.carnet.hr/control/" + }, + { + "Authorizations": [ + { + "From": "news-admin@humanityquest.com", + "Message": "checkgroups", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "newgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "rmgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + } + ], + "Contact": "news-admin@humanityquest.com", + "KeyFingerprint": "BA3DB306B6F552AABA8F32F08C4F504016F9C046", + "KeyUrl": "http://www.humanityquest.com/projects/newsgroups/PGP.htm", + "Name": "HUMANITYQUEST", + "Url": "http://www.humanityquest.com/projects/newsgroups/" + }, + { + "AdminGroup": "hun.admin.news", + "Authorizations": [ + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "checkgroups", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "newgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "rmgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + } + ], + "KeyUrl": "http://gatling.ikk.sztaki.hu/~kissg/news/hun.admin.news.asc", + "Name": "HUN", + "Url": "http://gatling.ikk.sztaki.hu/~kissg/news/" + }, + { + "AdminGroup": "ie.news.group", + "Authorizations": [ + { + "From": "control@usenet.ie", + "Message": "checkgroups", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "newgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "rmgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + } + ], + "Contact": "control@usenet.ie", + "Name": "IE" + }, + { + "AdminGroup": "is.isnet", + "Authorizations": [ + { + "From": "group-admin@usenet.is", + "Message": "checkgroups", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "newgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "rmgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + } + ], + "Contact": "IS Group Admins ", + "KeyFingerprint": "33328D461E5E1C7F48608E72E53ECAEA", + "KeyUrl": "http://www.usenet.is/group-admin.asc", + "Name": "IS", + "Url": "http://www.usenet.is/" + }, + { + "AdminGroup": "isu.newgroups", + "Authorizations": [ + { + "From": "news@news.isu.edu.tw", + "Message": "checkgroups", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "newgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "rmgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + } + ], + "Contact": "news@news.isu.edu.tw", + "KeyUrl": "http://news.isu.edu.tw/isu.asc", + "Name": "ISU", + "Url": "http://news.isu.edu.tw/" + }, + { + "AdminGroup": "it.news.annunci", + "Authorizations": [ + { + "From": "gcn@news.nic.it", + "Message": "checkgroups", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "newgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "rmgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + } + ], + "Contact": "gcn@news.nic.it", + "KeyFingerprint": "94A4F7B54696D6C7A673F298C48CD0E0", + "KeyUrl": "http://www.news.nic.it/pgp.txt", + "Name": "IT", + "Url": "http://www.news.nic.it/" + }, + { + "AdminGroup": "italia.announce.newgroups", + "Authorizations": [ + { + "From": "news@news.cineca.it", + "Message": "checkgroups", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "newgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "rmgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + } + ], + "Contact": "news@news.cineca.it", + "KeyFingerprint": "0FBB7162DA5D5DB8D586FC2802671A6B", + "KeyUrl": "http://news.cineca.it/italia/italia-pgp.txt", + "Name": "ITALIA", + "Url": "http://news.cineca.it/italia/" + }, + { + "AdminGroup": "japan.admin.announce", + "Authorizations": [ + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "checkgroups", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "newgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "rmgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + } + ], + "Contact": "Tsuneo Tanaka ", + "KeyFingerprint": "6AFA1947691B107438534B1BD8BA3E85", + "KeyUrl": "http://grex.cyberspace.org/~tt/japan.admin.announce.asc", + "Name": "JAPAN", + "Url": "http://www.asahi-net.or.jp/~AE5T-KSN/japan-e.html" + }, + { + "AdminGroup": "jlug.config", + "Authorizations": [ + { + "From": "news@linux.or.jp", + "Message": "checkgroups", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "newgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "rmgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + } + ], + "Contact": "news@linux.or.jp", + "KeyUrl": "http://www.linux.or.jp/pgpkey/news", + "Name": "JLUG", + "Url": "http://www.linux.or.jp/community/news/index.html" + }, + { + "AdminGroup": "ka.admin", + "Contact": "usenet@karlsruhe.org", + "KeyFingerprint": "DE19BB2576198117F067D223E8C87C90", + "KeyUrl": "http://www.karlsruhe.org/pubkey-news.karlsruhe.org.asc", + "Name": "KA", + "Url": "http://www.karlsruhe.org/" + }, + { + "Authorizations": [ + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "checkgroups", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + }, + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "rmgroup", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + } + ], + "Name": "KANTO" + }, + { + "AdminGroup": "kassel.admin", + "Authorizations": [ + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + } + ], + "Name": "KASSEL" + }, + { + "AdminGroup": "linux.admin.news", + "Authorizations": [ + { + "From": "linux-admin@bofh.it", + "Message": "checkgroups", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "newgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "rmgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + } + ], + "Contact": "Marco d'Itri ", + "KeyFingerprint": "81B327994FCE32D11BC9010DBBB32E41", + "Name": "LINUX" + }, + { + "AdminGroup": "malta.config", + "Authorizations": [ + { + "From": "cmeli@cis.um.edu.mt", + "Message": "checkgroups", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "newgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "rmgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + } + ], + "Contact": "cmeli@cis.um.edu.mt", + "KeyFingerprint": "2017015CF0D01A42E41330580B1448A6", + "KeyUrl": "http://www.cis.um.edu.mt/news-malta/PGP.PUBLICKEY", + "Name": "MALTA", + "Url": "http://www.malta.news-admin.org/" + }, + { + "AdminGroup": "maus.info", + "Authorizations": [ + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + } + ], + "KeyFingerprint": "8252C77026B972A1379855983F26623E", + "Name": "MAUS" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "MELB", + "Url": "http://melb.news-admin.org/" + }, + { + "AdminGroup": "mensa.config", + "Authorizations": [ + { + "From": "usenet@newsgate.mensa.org", + "Message": "checkgroups", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "newgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "rmgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + } + ], + "Contact": "usenet@newsgate.mensa.org", + "KeyFingerprint": "52B9396385D908068E197344973C5005DC7DB7A7", + "Name": "MENSA" + }, + { + "AdminGroup": "microsoft.public.news.server", + "Authorizations": [ + { + "From": "control-microsoft@trigofacile.com", + "Message": "checkgroups", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "newgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "rmgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + } + ], + "Contact": "control-microsoft@trigofacile.com", + "KeyFingerprint": "DF705FC9F615D52E02DBA3CB63A98D13E60E2FAA", + "KeyUrl": "http://www.trigofacile.com/divers/usenet/clefs/pgpkey-microsoft.asc", + "Name": "MICROSOFT", + "SyncableServer": "msnews.microsoft.com", + "Url": "http://www.trigofacile.com/divers/usenet/clefs/index.htm" + }, + { + "AdminGroup": "milw.config", + "Authorizations": [ + { + "From": "milw@usenet.mil.wi.us", + "Message": "checkgroups", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "newgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "rmgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + } + ], + "Contact": "milw@usenet.mil.wi.us", + "KeyFingerprint": "6E9B9F7098AB9CE5C3C00582215BF49E", + "KeyUrl": "http://usenet.mil.wi.us/pgpkey", + "Name": "MILW", + "Url": "http://usenet.mil.wi.us/" + }, + { + "AdminGroup": "muc.admin", + "Authorizations": [ + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "checkgroups", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "newgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "rmgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + } + ], + "KeyFingerprint": "43C70E7C45C706E0BD6F76CE07395E66", + "KeyUrl": "http://home.arcor.de/andreas-barth/muc-admin.html", + "Name": "MUC" + }, + { + "AdminGroup": "netscape.public.admin", + "Authorizations": [ + { + "From": "news@netscape.com", + "Message": "checkgroups", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "newgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "rmgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + } + ], + "Contact": "news@netscape.com", + "KeyFingerprint": "B78055121F9C170B8666AD3BDB6835EC", + "Name": "NETSCAPE", + "Url": "http://www.mozilla.org/community.html" + }, + { + "AdminGroup": "nippon.news.group", + "Authorizations": [ + { + "From": "news@gcd.org", + "Message": "rmgroup", + "Newsgroups": "nippon.*", + "VerificationIdentity": "nippon.news.group" + } + ], + "KeyFingerprint": "BCCF15CDB13CDFB3C3DE356F2FF746DB", + "KeyUrl": "http://www.gcd.org/news/nippon/", + "Name": "NIPPON", + "Url": "http://www.gcd.org/news/nippon/" + }, + { + "AdminGroup": "nl.newsgroups", + "Authorizations": [ + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "checkgroups", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "newgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "rmgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + } + ], + "Contact": "nl-admin@nic.surfnet.nl", + "KeyFingerprint": "45200BD5A121EA7CEFB2956C25754D27", + "Name": "NL", + "Url": "http://nl.news-admin.org/info/nladmin.html" + }, + { + "Authorizations": [ + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "news@kink.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + } + ], + "KeyFingerprint": "6B62EB534D5D2F9635D9C89CB0650E4C", + "Name": "NL-ALT" + }, + { + "Authorizations": [ + { + "From": "news@nl.linux.org", + "Message": "checkgroups", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "newgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "rmgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + } + ], + "KeyFingerprint": "63DCB2510AF3DD72C2BDC6FDC1C544CF", + "Name": "NLO", + "SyncableServer": "news.nl.linux.org", + "Url": "http://mail.nl.linux.org/" + }, + { + "AdminGroup": "nntpworld.config", + "Authorizations": [ + { + "From": "usenet@nntpworld.net", + "Message": "checkgroups", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "newgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "rmgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + } + ], + "Contact": "usenet@nntpworld.net", + "KeyFingerprint": "C7BBE0567A8A5A8609B5E329E1BB8CE1", + "KeyUrl": "http://www.nntpworld.net/nntpworld.asc", + "Name": "NNTPWORLD", + "Url": "http://www.nntpworld.net/" + }, + { + "AdminGroup": "no.usenet.admin", + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "checkgroups", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "newgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + } + ], + "KeyUrl": "http://www.usenet.no/pgp-key.txt", + "Name": "NO", + "Url": "http://www.usenet.no/" + }, + { + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.alt.*", + "VerificationIdentity": "no-hir-control" + } + ], + "Name": "NO.ALT" + }, + { + "AdminGroup": "nrw.admin.announce", + "Authorizations": [ + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "checkgroups", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "newgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "rmgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + } + ], + "Contact": "moderator@nrw.usenetverwaltung.de", + "KeyFingerprint": "134A80FED634B464AF32083F620EB1E2", + "KeyUrl": "http://nrw.usenetverwaltung.de/pgp/nrw.asc", + "Name": "NRW", + "Url": "http://nrw.usenetverwaltung.de/" + }, + { + "AdminGroup": "nz.net.announce", + "Authorizations": [ + { + "From": "root@usenet.net.nz", + "Message": "checkgroups", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "newgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "rmgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + } + ], + "Contact": "root@usenet.net.nz", + "KeyFingerprint": "07DF48AAD0EDAA881670C591653D1A28", + "Name": "NZ", + "Url": "http://www.faqs.org/faqs/usenet/nz-news-hierarchy" + }, + { + "Authorizations": [ + { + "From": "news@news2.open-news-network.org", + "Message": "checkgroups", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "newgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "rmgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + } + ], + "Name": "OPENNEWS", + "Url": "http://www.open-news-network.org/" + }, + { + "AdminGroup": "openwatcom.contributors", + "Authorizations": [ + { + "From": "admin@openwatcom.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "newgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + } + ], + "Contact": "admin@openwatcom.news-admin.org", + "KeyUrl": "http://cmeerw.org/files/openwatcom/pgp-openwatcom.asc", + "Name": "OPENWATCOM", + "SyncableServer": "news.openwatcom.org", + "Url": "http://www.openwatcom.org/" + }, + { + "Authorizations": [ + { + "From": "*@opera.com", + "Message": "checkgroups", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "newgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "rmgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + } + ], + "Contact": "usenet@opera.com", + "Name": "OPERA", + "SyncableServer": "news.opera.com" + }, + { + "Authorizations": [ + { + "From": "postmaster@upb.de", + "Message": "checkgroups", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "newgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "rmgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + } + ], + "Contact": "news@uni-paderborn.de", + "Name": "PBINFO" + }, + { + "Authorizations": [ + { + "From": "newsadmin@perl.org", + "Message": "checkgroups", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "newgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "rmgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + } + ], + "Contact": "newsadmin@perl.org", + "KeyFingerprint": "438FD1BA4DCC3B1ABED82BCC32988A7D8B2ACFBB", + "KeyUrl": "http://www.nntp.perl.org/about/newsadmin@perl.org.pgp", + "Name": "PERL", + "Url": "http://www.nntp.perl.org/about/" + }, + { + "AdminGroup": "pgh.config", + "Authorizations": [ + { + "From": "pgh-config@psc.edu", + "Message": "checkgroups", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "newgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "rmgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + } + ], + "Name": "PGH" + }, + { + "Authorizations": [ + { + "From": "news@postgresql.org", + "Message": "checkgroups", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "newgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "rmgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + } + ], + "Contact": "news@postgresql.org", + "KeyUrl": "http://news.hub.org/gpg_public_keys.html", + "Name": "PGSQL", + "Url": "http://news.hub.org/gpg_public_keys.html" + }, + { + "AdminGroup": "pl.news.admin", + "Authorizations": [ + { + "From": "michalj@*fuw.edu.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + } + ], + "KeyUrl": "http://www.usenet.pl/doc/news-pl-new-site-faq.html#pgp", + "Name": "PL", + "Url": "http://www.usenet.pl/doc/news-pl-new-site-faq.html" + }, + { + "AdminGroup": "pt.internet.usenet", + "Authorizations": [ + { + "From": "pmelo@*.inescc.pt", + "Message": "checkgroups", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "newgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "rmgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + } + ], + "KeyUrl": "http://www.usenet-pt.org/control@usenet-pt.org.asc", + "Name": "PT", + "Url": "http://www.usenet-pt.org/" + }, + { + "AdminGroup": "relcom.netnews", + "Authorizations": [ + { + "From": "coord@*.relcom.ru", + "Message": "checkgroups", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "newgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "rmgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + } + ], + "KeyUrl": "ftp://ftp.relcom.ru/pub/relcom/netinfo/coordpubkey.txt", + "Name": "RELCOM", + "Url": "ftp://ftp.relcom.ru/pub/relcom/netinfo/" + }, + { + "AdminGroup": "rocksolid.shared.rocksolid", + "Authorizations": [ + { + "From": "usenet@novabbs.com", + "Message": "checkgroups", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "newgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "rmgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + } + ], + "Contact": "usenet@novabbs.com", + "KeyFingerprint": "FF846D730B72CC0CB69B64802A1C655BF6D10B13", + "KeyUrl": "http://news.novabbs.com/hierarchy/rocksolid.txt", + "Name": "ROCKSOLID", + "Url": "http://news.novabbs.com/hierarchy/" + }, + { + "AdminGroup": "saar.admin.news", + "Authorizations": [ + { + "From": "control@saar-admin-news.de", + "Message": "checkgroups", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "newgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "rmgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + } + ], + "KeyUrl": "http://www.saar-admin-news.de/saar-control.asc", + "Name": "SAAR", + "Url": "http://www.saar-admin-news.de/" + }, + { + "AdminGroup": "sat.usenet.config", + "Authorizations": [ + { + "From": "satgroup@endicor.com", + "Message": "checkgroups", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "newgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "rmgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + } + ], + "Contact": "satgroup@endicor.com", + "Name": "SAT" + }, + { + "AdminGroup": "schule.admin", + "Authorizations": [ + { + "From": "newsctrl@schule.de", + "Message": "checkgroups", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "newgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "rmgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + } + ], + "Contact": "schule-admin@roxel.ms.sub.org", + "KeyFingerprint": "6406F0AEE146850CBDCA0E538B1E73D2", + "KeyUrl": "http://www.afaik.de/usenet/admin/schule/control/schule.asc", + "Name": "SCHULE", + "Url": "http://home.pages.de/~schule-admin/" + }, + { + "AdminGroup": "scot.newsgroups.discuss", + "Authorizations": [ + { + "From": "control@scot.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "newgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + } + ], + "KeyUrl": "http://scot.news-admin.org/signature.html", + "Name": "SCOT", + "Url": "http://scot.news-admin.org/" + }, + { + "AdminGroup": "scout.admin", + "Authorizations": [ + { + "From": "control@news.scoutnet.org", + "Message": "checkgroups", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "newgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "rmgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + } + ], + "KeyUrl": "http://news.scoutnet.org/scout-pgpkey.asc", + "Name": "SCOUT", + "Url": "http://news.scoutnet.org/" + }, + { + "AdminGroup": "se.internet.news.meddelanden", + "Authorizations": [ + { + "From": "usenet@usenet-se.net", + "Message": "checkgroups", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "newgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "rmgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + } + ], + "Contact": "usenet@usenet-se.net", + "KeyFingerprint": "6803F0FD0CC34E696F0D0C603C586396", + "Name": "SE" + }, + { + "AdminGroup": "sfnet.ryhmat+listat", + "Authorizations": [ + { + "From": "sfnet@*cs.tut.fi", + "Message": "checkgroups", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "newgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "rmgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + } + ], + "Contact": "sfnet@cs.tut.fi", + "KeyFingerprint": "DE7933C2D359D12844E56A0CB6E30E536933A636", + "Name": "SFNET", + "Url": "http://www.cs.tut.fi/sfnet/" + }, + { + "AdminGroup": "si.news.announce.newsgroups", + "Authorizations": [ + { + "From": "news-admin@arnes.si", + "Message": "checkgroups", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "newgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "rmgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + } + ], + "KeyUrl": "http://www.arnes.si/news/config/", + "Name": "SI", + "Url": "http://www.arnes.si/news/config/" + }, + { + "AdminGroup": "staroffice.admin", + "Authorizations": [ + { + "From": "news@stardivision.de", + "Message": "checkgroups", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "newgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "rmgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + } + ], + "Contact": "news@starnews.sun.com", + "KeyFingerprint": "C63E816F2A19D3847251F91BE3B9B2C9", + "Name": "STAROFFICE", + "SyncableServer": "starnews.sun.com" + }, + { + "AdminGroup": "stgt.net", + "Authorizations": [ + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "checkgroups", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "newgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "rmgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + } + ], + "Contact": "news@news.uni-stuttgart.de", + "KeyFingerprint": "BAA40A548FF5F51E724851AE0951DE44", + "KeyUrl": "http://news.uni-stuttgart.de/hierarchie/stgt/stgt-control.txt", + "Name": "STGT", + "Url": "http://news.uni-stuttgart.de/hierarchie/stgt/" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "SYD", + "Url": "http://syd.news-admin.org/" + }, + { + "AdminGroup": "szaf.admin", + "Authorizations": [ + { + "From": "hirtenrat@szaf.org", + "Message": "rmgroup", + "Newsgroups": "szaf.*", + "VerificationIdentity": "Hirtenrat" + } + ], + "Contact": "hirtenrat@szaf.org", + "KeyUrl": "http://news.szaf.org/szaf/szaf-key.txt", + "Name": "SZAF" + }, + { + "AdminGroup": "thur.net.news.groups", + "Authorizations": [ + { + "From": "usenet@thur.de", + "Message": "checkgroups", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "newgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "rmgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + } + ], + "Contact": "usenet@thur.de", + "KeyFingerprint": "7E3D731393D4CA7839DE3CE737EE22F1", + "Name": "THUR", + "SyncableServer": "news.thur.de", + "Url": "http://www.thur.de/thurnet/old/thurnews.html" + }, + { + "Authorizations": [ + { + "From": "usenet@agate.berkeley.edu", + "Message": "checkgroups", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "newgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "rmgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + } + ], + "Contact": "Chris van den Berg ", + "KeyFingerprint": "96B88E9A9809377D0EEC8188DB9029BF", + "KeyUrl": "http://www.net.berkeley.edu/usenet/usenet.asc", + "Name": "UCB", + "Url": "http://www.net.berkeley.edu/usenet/" + }, + { + "AdminGroup": "ufra.admin", + "KeyFingerprint": "F7AD96D87A3F7E84020C839ADB8FEBB8", + "Name": "UFRA" + }, + { + "AdminGroup": "uk.net.news.announce", + "Authorizations": [ + { + "From": "control@usenet.org.uk", + "Message": "checkgroups", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "newgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "rmgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + } + ], + "KeyUrl": "http://www.usenet.org.uk/newsadmins.html", + "Name": "UK", + "Url": "http://www.usenet.org.uk/" + }, + { + "AdminGroup": "un.public.usenet.admin", + "Authorizations": [ + { + "From": "news@news.itu.int", + "Message": "checkgroups", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "newgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "rmgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + } + ], + "Name": "UN" + }, + { + "AdminGroup": "us.config", + "Authorizations": [ + { + "From": "us-control@lists.killfile.org", + "Message": "checkgroups", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "newgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "rmgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + } + ], + "Contact": "us-control@lists.killfile.org", + "KeyFingerprint": "BB96EB2CCFD075C8E9DEC2C21DA29D87B73CAF1B", + "Name": "US" + }, + { + "AdminGroup": "uw.newsgroups", + "Authorizations": [ + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "checkgroups", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "newgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "rmgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + } + ], + "Name": "UW", + "SyncableServer": "news.uwaterloo.ca" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "WA", + "Url": "http://wa.news-admin.org/" + }, + { + "AdminGroup": "wales.usenet.config", + "Authorizations": [ + { + "From": "control@wales-usenet.org", + "Message": "checkgroups", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "newgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "rmgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + } + ], + "Contact": "committee@wales-usenet.org", + "KeyFingerprint": "2D9EDEDF12DA345C49E1EE28E3AB0DAD", + "KeyUrl": "http://www.wales-usenet.org/english/newsadmin.txt", + "Name": "WALES", + "Url": "http://www.wales-usenet.org/" + }, + { + "AdminGroup": "x-privat.info", + "Authorizations": [ + { + "From": "dmitry@x-privat.org", + "Message": "checkgroups", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "newgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "rmgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + } + ], + "Contact": "dmitry@x-privat.org", + "KeyFingerprint": "9B0A7E682780C796476B039051056843", + "KeyUrl": "http://www.x-privat.org/dmitry.asc", + "Name": "X-PRIVAT", + "Url": "http://www.x-privat.org/" + }, + { + "AdminGroup": "z-netz.koordination.user+sysops", + "Authorizations": [ + { + "From": "teko@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + } + ], + "Contact": "teko@dinoex.sub.org", + "KeyUrl": "ftp://ftp.dinoex.de/pub/keys/z-netz.koordination.user+sysops.asc", + "Name": "Z-NETZ" + } + ], + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl", + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl" + } + } + }, + "Nntpd": { + "AcmeStateDir": "certs/", + "AllowCleartextAuth": true, + "BindAddress": [ + "198.18.0.66", + "2c0f:f030:1442:501:198:18:0:66" + ], + "BindPort": 1199, + "BindPortTls": 5633, + "GracefulShutdownTimeout": "00:01:00", + "HistoryTime": "02:00:00", + "IdleTime": 300, + "LogDir": "logs/", + "MailComplaintsTo": "abuse@usenet.ninja", + "MaxArticleSize": 5242880, + "ProxyHosts": [], + "ServerId": 1, + "StartupTimeout": null, + "StopHostOnUnexpectedServiceTermination": true, + "Top1000": [ + "top1000@anthologeek.net", + "cknipe@opticnetworks.net" + ], + "Transit": { + "StreamOutstandingArticleDepth": 8 + }, + "Systemd": { + "EnableWatchdog": true, + "NotifyReadyOnApplicationRunning": true, + "NotifyStoppingOnApplicationShutdown": true, + "ReportLifecycleStatus": true, + "WatchdogIntervalFraction": 0.5 + }, + "ArticleIngestion": { + "MaxArticleBytes": 4194304, + "MinWorkers": 2, + "MaxWorkers": 32, + "MaxPublishConcurrency": 32, + "OverviewDbWorkQueueMemoryLimit": 1073741824, + "OverviewDbMinPublisherWorkers": 2, + "OverviewDbMaxPublisherWorkers": 32, + "OverviewDbPublisherBatchSize": 100, + "OverviewDbPublisherShutdownSeconds": 2, + "ScaleIntervalSeconds": 2, + "ScaleUpPressureThreshold": 0.4, + "ScaleDownPressureThreshold": 0.1, + "ScaleUpConsecutiveIntervals": 2, + "ScaleDownConsecutiveIntervals": 3 + }, + "TransitQueueMemoryLimit": 4294967296 + }, + "NntpDb": { + "StartupTimeout": "00:00:15" + }, + "Redis": { + "Host": [ "198.18.0.70" ], + "Port": 6379 + }, + "Serilog": { + "Enrich": [ "FromLogContext" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information", + "System": "Warning", + "VectorNNTPD.ReaderService": "Debug" + } + }, + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File", "Serilog.Sinks.Async" ], + "WriteTo": [ + { + "Args": { + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "restrictedToMinimumLevel": "Information" + }, + "Name": "Console" + }, + { + "Args": { + "blockWhenFull": true, + "bufferSize": 50000, + "configure": [ + { + "Args": { + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "path": "logs/pending-.log", + "restrictedToMinimumLevel": "Debug", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Name": "File" + } + ] + }, + "Name": "Async" + } + ], + "News": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/news-.log", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Inpaths": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/inpaths-.log", + "retainedFileCountLimit": 1, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + } + } +} diff --git a/src/VectorNNTPD.TransitService/.editorconfig b/src/VectorNNTPD.TransitService/.editorconfig new file mode 100644 index 00000000..c774aab7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/.editorconfig @@ -0,0 +1,3 @@ +# All C# namespaces must be block-scoped; file-scoped namespaces are not permitted. +[*.cs] +csharp_style_namespace_declarations = block_scoped:warning diff --git a/src/VectorNNTPD.TransitService/Acme/AcmeCertificateApplicationService.cs b/src/VectorNNTPD.TransitService/Acme/AcmeCertificateApplicationService.cs new file mode 100644 index 00000000..1fac8671 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Acme/AcmeCertificateApplicationService.cs @@ -0,0 +1,35 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Acme +{ + /// + /// NNTPD lifecycle wrapper around the shared ACME certificate implementation. + /// + public sealed class AcmeCertificateApplicationService : IApplicationService, IAsyncDisposable + { + private readonly AcmeCertificateService _inner; + + /// Initializes a new wrapper. + public AcmeCertificateApplicationService(AcmeCertificateService inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + /// + public string Name => AcmeCertificateService.Name; + + /// + public Task? Execution => _inner.Execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) => _inner.StartAsync(cancellationToken); + + /// + public Task StopAsync(CancellationToken cancellationToken) => _inner.StopAsync(cancellationToken); + + /// + public ValueTask DisposeAsync() => _inner.DisposeAsync(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/Article.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/Article.cs new file mode 100644 index 00000000..1e21d6ae --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/Article.cs @@ -0,0 +1,83 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Owned destuffed article produced by downstream IHAVE interpretation. + /// + /// + /// + /// and are owned destuffed copies built by + /// for benches and classifier + /// tests. Production IHAVE queues CanonicalV1 ArticleRecord instead. + /// + /// + /// is the destuffed complete-article length (headers, the header/body + /// blank line, and body). The NNTP terminating . line is not included. This matches + /// ArticleIngestionOptions.MaxArticleBytes (“after dot-unstuffing”) and is the + /// value later Transit routing should use. + /// + /// + /// Headers are raw destuffed header field lines (CRLF-terminated), not a parsed object + /// model. The body is the destuffed received representation (wire yEnc/BASE64/uuencode + /// text when those encodings were used — not decoded). + /// + /// + public readonly struct Article + { + /// Initializes a new owned article from destuffed header and body bytes. + public Article(ReadOnlyMemory headers, ReadOnlyMemory body, ArticleType type) + { + Headers = headers; + Body = body; + Type = type == ArticleType.None ? ArticleType.Default : type; + var separator = headers.Length > 0 || body.Length > 0 ? 2 : 0; + Size = checked(headers.Length + separator + body.Length); + Payload = Combine(headers, body, separator); + } + + /// Gets destuffed header field lines, each CRLF-terminated, without the blank separator. + public ReadOnlyMemory Headers { get; } + + /// Gets destuffed body bytes (received representation; not decoded). + public ReadOnlyMemory Body { get; } + + /// Gets the destuffed complete article (headers + blank line + body). + public ReadOnlyMemory Payload { get; } + + /// + /// Gets destuffed complete-article bytes (headers + blank line + body). + /// Does not include the NNTP multiline terminator. + /// + public int Size { get; } + + /// Gets the deterministic Diablo-mapped classification. + public ArticleType Type { get; } + + private static ReadOnlyMemory Combine( + ReadOnlyMemory headers, + ReadOnlyMemory body, + int separator) + { + if (headers.IsEmpty && body.IsEmpty) + { + return ReadOnlyMemory.Empty; + } + + var buffer = new byte[headers.Length + separator + body.Length]; + headers.Span.CopyTo(buffer); + if (separator > 0) + { + buffer[headers.Length] = (byte)'\r'; + buffer[headers.Length + 1] = (byte)'\n'; + } + + if (!body.IsEmpty) + { + body.Span.CopyTo(buffer.AsSpan(headers.Length + separator)); + } + + return buffer; + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleEnqueueResult.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleEnqueueResult.cs new file mode 100644 index 00000000..3469a108 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleEnqueueResult.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// Outcome of attempting to accept an article into the ingestion queue. + public enum ArticleEnqueueResult + { + /// Article was accepted into the bounded ingestion queue (239 path). + Accepted = 0, + + /// + /// Queue is unavailable (completed/shutting down or otherwise not accepting). + /// Maps to RFC 4644 temporary failure (400 + close) for TAKETHIS. + /// + Unavailable = 1, + + /// + /// Article cannot be admitted because its payload exceeds + /// Nntpd:TransitQueueMemoryLimit. Waiting cannot free enough budget. + /// Maps to IHAVE 437 and TAKETHIS 439. + /// + Rejected = 2, + + /// + /// Article cannot be reserved immediately without waiting for capacity. + /// IHAVE maps this to 436 and must not wait. TAKETHIS still uses + /// (waits) and does not + /// observe this result on the production path. + /// + Full = 3, + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleIngestionQueue.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleIngestionQueue.cs new file mode 100644 index 00000000..bfa34b04 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleIngestionQueue.cs @@ -0,0 +1,505 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Byte-budgeted -backed Transit article ingestion queue. + /// + /// + /// + /// The admission bound is + /// (Nntpd:TransitQueueMemoryLimit, default 1 GiB). Accounting uses each + /// article's owned payload length (.Length): + /// canonical / ArtData length. Every queued + /// item is a CanonicalV1 ArticleRecord. Object overhead is not counted. + /// This is not a process-wide memory cap. + /// + /// + /// Producers reserve payload bytes atomically before the article is visible + /// to consumers. Multiple producers therefore cannot oversubscribe the + /// budget. An article larger than the configured budget is rejected immediately + /// so admission cannot deadlock waiting for capacity. The historical + /// 256-article count bound is not an admission limit; the channel is unbounded + /// and the byte reservation is the resource bound. + /// + /// + /// Every successful reservation has exactly one release: consumer dequeue, + /// failed write after reserve, or (for waiters) cancel/shutdown before admit. + /// + /// + /// The channel is multi-reader. Concurrent callers are + /// supported. Items leave the channel in FIFO order; worker processing/completion + /// order may differ. Byte-budget accounting still serializes on the existing + /// reservation gate. A successful dequeue transfers ownership and releases the + /// reservation inside ; consumers must not release again. + /// + /// + public sealed class ArticleIngestionQueue : IArticleIngestionQueue + { + private readonly Channel _channel; + private readonly object _gate = new(); + private readonly Queue _waiters = new(); + private readonly long _memoryLimit; + private readonly int _maxArticleBytes; + private long _queuedBytes; + private long _peakQueuedBytes; + private int _count; + private int _peakCount; + private int _completed; + private long _admissionFailures; + private long _admissionWaitTicks; + + /// Initializes a new instance of the class. + public ArticleIngestionQueue(IOptions options) + : this(GetIngestionOptions(options), GetMemoryLimit(options)) + { + } + + /// + /// Initializes a new instance with explicit ingestion options and the default 1 GiB budget (tests). + /// + public ArticleIngestionQueue(ArticleIngestionOptions options) + : this(options, NntpdOptions.DefaultTransitQueueMemoryLimit) + { + } + + /// Initializes a new instance with an explicit Transit queue memory budget (tests). + public ArticleIngestionQueue(ArticleIngestionOptions options, long transitQueueMemoryLimit) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentOutOfRangeException.ThrowIfLessThan(options.MaxArticleBytes, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(transitQueueMemoryLimit, 1); + + _memoryLimit = transitQueueMemoryLimit; + _maxArticleBytes = options.MaxArticleBytes; + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public long MemoryLimitBytes => _memoryLimit; + + /// + public long QueuedBytes => Volatile.Read(ref _queuedBytes); + + /// + public long PeakQueuedBytes => Volatile.Read(ref _peakQueuedBytes); + + /// + public int MaxArticleBytes => _maxArticleBytes; + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public int PeakCount => Math.Max(0, Volatile.Read(ref _peakCount)); + + /// + public bool IsAccepting => Volatile.Read(ref _completed) == 0; + + /// + public int WaitingProducerCount => DebugWaiterCount; + + /// + public long AdmissionFailureCount => Volatile.Read(ref _admissionFailures); + + /// + public long AdmissionWaitTicks => Volatile.Read(ref _admissionWaitTicks); + + /// Gets the number of producers waiting for byte-budget capacity (tests). + internal int DebugWaiterCount + { + get + { + lock (_gate) + { + var n = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var waiter in _waiters) + { + if (waiter.IsWaiting) + { + n++; + } + } + + return n; + } + } + } + + /// + public async ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + EnsureCanonical(article); + + var bytes = article.Payload.Length; + if (bytes > _memoryLimit) + { + RecordAdmission(ArticleEnqueueResult.Rejected, waitTicks: 0); + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + RecordAdmission(ArticleEnqueueResult.Unavailable, waitTicks: 0); + return ArticleEnqueueResult.Unavailable; + } + + var waitStarted = System.Diagnostics.Stopwatch.GetTimestamp(); + var reserved = await ReserveAsync(bytes, cancellationToken).ConfigureAwait(false); + var waitTicks = System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted; + IngestionPipelineMetrics.Shared.RecordEnqueueWaitTicks(waitTicks); + if (reserved.WaitedForBudget) + { + IngestionPipelineMetrics.Shared.RecordBudgetWaitTicks(waitTicks); + } + + if (reserved.Result != ArticleEnqueueResult.Accepted) + { + RecordAdmission(reserved.Result, waitTicks); + return reserved.Result; + } + + var written = TryWriteReserved(article, bytes) + ? ArticleEnqueueResult.Accepted + : ArticleEnqueueResult.Unavailable; + RecordAdmission(written, waitTicks); + return written; + } + + /// + public bool TryProbeCapacity() + { + lock (_gate) + { + return _completed == 0 && _queuedBytes < _memoryLimit; + } + } + + /// + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + EnsureCanonical(article); + + var bytes = article.Payload.Length; + if (bytes > _memoryLimit) + { + RecordAdmission(ArticleEnqueueResult.Rejected, waitTicks: 0); + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + RecordAdmission(ArticleEnqueueResult.Unavailable, waitTicks: 0); + return ArticleEnqueueResult.Unavailable; + } + + if (!TryReserveImmediate(bytes)) + { + RecordAdmission(ArticleEnqueueResult.Full, waitTicks: 0); + return ArticleEnqueueResult.Full; + } + + var written = TryWriteReserved(article, bytes) + ? ArticleEnqueueResult.Accepted + : ArticleEnqueueResult.Unavailable; + RecordAdmission(written, waitTicks: 0); + return written; + } + + /// + public bool TryEnqueue(InboundArticle article) => TryAdmit(article) == ArticleEnqueueResult.Accepted; + + /// + public void Complete() + { + List? pending = null; + lock (_gate) + { + Interlocked.Exchange(ref _completed, 1); + _channel.Writer.TryComplete(); + if (_waiters.Count > 0) + { + pending = new List(_waiters.Count); + while (_waiters.TryDequeue(out var waiter)) + { + if (waiter.TryCancel()) + { + pending.Add(waiter); + } + } + } + } + + if (pending is null) + { + return; + } + + foreach (var waiter in pending) + { + waiter.Tcs.TrySetResult(false); + } + } + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + // WaitToReadAsync + TryRead is the Channel multi-reader contract: another + // consumer may take the item between the wait and the read, so retry. + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (!_channel.Reader.TryRead(out var article) || article is null) + { + continue; + } + + Interlocked.Decrement(ref _count); + Release(article.Payload.Length); + return article; + } + + return null; + } + + /// + /// Completes the channel writer without marking the queue unavailable (tests). + /// The next reserve-then-write fails and must release the reservation. + /// + internal void CompleteChannelWithoutMarkingUnavailableForTests() => _channel.Writer.TryComplete(); + + private async ValueTask<(ArticleEnqueueResult Result, bool WaitedForBudget)> ReserveAsync( + int bytes, + CancellationToken cancellationToken) + { + ByteWaiter? waiter; + lock (_gate) + { + if (_completed != 0) + { + return (ArticleEnqueueResult.Unavailable, false); + } + + if (bytes <= _memoryLimit - _queuedBytes) + { + AddReservation(bytes); + return (ArticleEnqueueResult.Accepted, false); + } + + waiter = new ByteWaiter(bytes); + _waiters.Enqueue(waiter); + } + + using var registration = cancellationToken.Register( + static state => ((ByteWaiter)state!).CancelFromToken(), + waiter); + + try + { + var admitted = await waiter.Tcs.Task.ConfigureAwait(false); + return ( + admitted ? ArticleEnqueueResult.Accepted : ArticleEnqueueResult.Unavailable, + true); + } + catch (OperationCanceledException) + { + waiter.CancelFromToken(); + return (ArticleEnqueueResult.Unavailable, true); + } + } + + private void RecordAdmission(ArticleEnqueueResult result, long waitTicks) + { + if (result is ArticleEnqueueResult.Rejected + or ArticleEnqueueResult.Unavailable + or ArticleEnqueueResult.Full) + { + Interlocked.Increment(ref _admissionFailures); + } + + if (waitTicks > 0) + { + Interlocked.Add(ref _admissionWaitTicks, waitTicks); + } + } + + private bool TryReserveImmediate(int bytes) + { + lock (_gate) + { + if (_completed != 0 || bytes > _memoryLimit - _queuedBytes) + { + return false; + } + + AddReservation(bytes); + return true; + } + } + + private void AddReservation(int bytes) + { + _queuedBytes += bytes; + if (_queuedBytes > _peakQueuedBytes) + { + _peakQueuedBytes = _queuedBytes; + } + } + + private static void EnsureCanonical(InboundArticle article) + { + if (article.Record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException( + "The ingress queue accepts CanonicalV1 ArticleRecord items only.", + nameof(article)); + } + } + + private bool TryWriteReserved(InboundArticle article, int bytes) + { + if (_channel.Writer.TryWrite(article)) + { + var now = Interlocked.Increment(ref _count); + UpdatePeakCount(now); + return true; + } + + Release(bytes); + return false; + } + + private void Release(int bytes) + { + List? admitted; + lock (_gate) + { + if (bytes > 0) + { + var next = _queuedBytes - bytes; + _queuedBytes = next < 0 ? 0 : next; + } + + admitted = AdmitWaiters(); + } + + if (admitted is null) + { + return; + } + + foreach (var waiter in admitted) + { + waiter.Tcs.TrySetResult(true); + } + } + + private List? AdmitWaiters() + { + List? admitted = null; + while (_waiters.TryPeek(out var waiter)) + { + if (!waiter.IsWaiting) + { + _waiters.Dequeue(); + continue; + } + + if (_completed != 0) + { + break; + } + + if (waiter.Bytes > _memoryLimit - _queuedBytes) + { + break; + } + + if (!waiter.TryAdmit()) + { + _waiters.Dequeue(); + continue; + } + + _waiters.Dequeue(); + AddReservation(waiter.Bytes); + admitted ??= []; + admitted.Add(waiter); + } + + return admitted; + } + + private void UpdatePeakCount(int now) + { + var peak = Volatile.Read(ref _peakCount); + while (now > peak) + { + var previous = Interlocked.CompareExchange(ref _peakCount, now, peak); + if (previous == peak) + { + break; + } + + peak = previous; + } + } + + private static ArticleIngestionOptions GetIngestionOptions(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Value.ArticleIngestion ?? new ArticleIngestionOptions(); + } + + private static long GetMemoryLimit(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Value.TransitQueueMemoryLimit; + } + + private sealed class ByteWaiter + { + private const int Waiting = 0; + private const int Admitted = 1; + private const int Cancelled = 2; + + private int _state; + + public ByteWaiter(int bytes) + { + Bytes = bytes; + } + + public int Bytes { get; } + + public TaskCompletionSource Tcs { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool IsWaiting => Volatile.Read(ref _state) == Waiting; + + public bool TryAdmit() => Interlocked.CompareExchange(ref _state, Admitted, Waiting) == Waiting; + + public bool TryCancel() => Interlocked.CompareExchange(ref _state, Cancelled, Waiting) == Waiting; + + public void CancelFromToken() + { + if (TryCancel()) + { + Tcs.TrySetResult(false); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/ArticlePlacementLogMessages.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticlePlacementLogMessages.cs new file mode 100644 index 00000000..affcdffd --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticlePlacementLogMessages.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Source-generated placement logs. NNTPD places one copy. A + /// result + /// is logged as attempt 1 and may be followed by one same-target retry logged as attempt 2. + /// + internal static partial class ArticlePlacementLogMessages + { + [LoggerMessage( + EventId = 2620, + Level = LogLevel.Information, + Message = "Article placement accepted artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Accepted elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Accepted(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2621, + Level = LogLevel.Information, + Message = "Article placement duplicate artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Duplicate elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Duplicate(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2622, + Level = LogLevel.Warning, + Message = "Article placement conflict artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=Conflict elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Conflict(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2623, + Level = LogLevel.Warning, + Message = "Article placement rejected capacity artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedCapacity elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedCapacity(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2624, + Level = LogLevel.Warning, + Message = "Article placement rejected pressure artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedPressure elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedPressure(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2625, + Level = LogLevel.Warning, + Message = "Article placement rejected invalid artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} outcome=RejectedInvalid elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void RejectedInvalid(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2626, + Level = LogLevel.Warning, + Message = "Article placement found no active server artId={ArtId} outcome=NoActiveServer elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void NoActiveServer(ILogger logger, string ArtId, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2627, + Level = LogLevel.Warning, + Message = "Article placement transport failure artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} failure={Failure} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void TransportFailed( + ILogger logger, + string ArtId, + int ServerId, + int VatpPort, + string Fqdn, + string Failure, + long ElapsedMs, + int Attempt); + + [LoggerMessage( + EventId = 2628, + Level = LogLevel.Information, + Message = "Article placement cancelled artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void Cancelled(ILogger logger, string ArtId, int ServerId, int VatpPort, string Fqdn, long ElapsedMs, int Attempt); + + [LoggerMessage( + EventId = 2629, + Level = LogLevel.Warning, + Message = "Article placement acknowledgement not observed artId={ArtId} serverId={ServerId} fqdn={Fqdn} vatpPort={VatpPort} elapsedMs={ElapsedMs} attempt={Attempt}")] + public static partial void AcknowledgementNotObserved( + ILogger logger, + string ArtId, + int ServerId, + int VatpPort, + string Fqdn, + long ElapsedMs, + int Attempt); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleRecordIngress.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleRecordIngress.cs new file mode 100644 index 00000000..87c5c442 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/ArticleRecordIngress.cs @@ -0,0 +1,155 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Builds a Common at the NNTP ingress boundary. + /// + /// + /// Destuff is this type's only extra step and only for stuffed wire. Parsing, + /// classification, Date/Path materialization, ArtId, and ArtHash stay in + /// . The destuffed source is discarded after + /// materialize; the queued record references the factory ArtData buffer. + /// + internal static class ArticleRecordIngress + { + /// + /// Parses already-destuffed article bytes into a CanonicalV1 record. + /// + /// Session-scoped Common parser (local identity for Path hops). + /// Unstuffed article bytes. Not retained after success. + /// Factory result. Accepted is the canonical buffer. + public static ArticleRecordCreateResult TryCreateFromDestuffed( + NntpArticleParser parser, + ReadOnlyMemory destuffedArticle) + { + ArgumentNullException.ThrowIfNull(parser); + return ArticleRecordFactory.TryCreate(parser, destuffedArticle, ArticlePathMode.Traverse); + } + + /// + /// Parses already-destuffed article bytes with one captured size limit and Path tracker. + /// + /// Session parser. Its local identity remains the application hop. + /// Unstuffed article bytes. + /// Captured maxartsize. + /// Captured sitename bytes. + /// Factory result. + public static ArticleRecordCreateResult TryCreateFromDestuffed( + NntpArticleParser parser, + ReadOnlyMemory destuffedArticle, + int maxArticleBytes, + ReadOnlySpan siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(parser); + return ArticleRecordFactory.TryCreate( + parser, + destuffedArticle, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + } + + /// + /// Destuffs terminator-omitted wire once, then builds a CanonicalV1 record. + /// + /// Session-scoped Common parser (local identity for Path hops). + /// Stuffed article bytes without the NNTP terminator. + /// Destuff and materialize ceiling. + /// + /// Factory result, or parse-rejected + /// when destuff exceeds . + /// + public static ArticleRecordCreateResult TryCreateFromStuffedWire( + NntpArticleParser parser, + ReadOnlyMemory stuffedWire, + int maxArticleBytes) + { + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (!NntpArticleDestuffer.TryDestuffStuffedWire(stuffedWire.Span, maxArticleBytes, out var destuffed)) + { + return ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.ArticleTooLarge); + } + + return ArticleRecordFactory.TryCreate( + parser, + destuffed, + ArticlePathMode.Traverse, + maxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + } + + /// + /// Destuffs stuffed wire and materializes it with one captured size limit and Path tracker. + /// + /// Session parser. + /// Stuffed article bytes. + /// Captured maxartsize. Also the destuff ceiling. + /// Captured sitename bytes. + /// Factory result, or when destuff exceeds the limit. + public static ArticleRecordCreateResult TryCreateFromStuffedWire( + NntpArticleParser parser, + ReadOnlyMemory stuffedWire, + int maxArticleBytes, + ReadOnlySpan siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (!NntpArticleDestuffer.TryDestuffStuffedWire(stuffedWire.Span, maxArticleBytes, out var destuffed)) + { + return ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.ArticleTooLarge); + } + + return ArticleRecordFactory.TryCreate( + parser, + destuffed, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + } + + /// + /// Creates a queue item whose aliases + /// .ArtData (no article-sized copy). + /// + /// Command or POST Message-ID text. + /// Accepted CanonicalV1 record. + /// Session client identity at admission. + /// Queue-admission timestamp. + /// TAKETHIS, POST, IHAVE, or BackFiller. + /// Inbound Transit identifier bytes captured from the session. + /// Queue item carrying . + /// + /// Throws only when is empty or + /// is not CanonicalV1. POST calls this only after + /// TryCreateFromDestuffed accepted CanonicalV1 with a non-empty + /// Message-ID; those throws are unreachable on that path. + /// + public static InboundArticle CreateQueued( + string messageId, + in ArticleRecord record, + ConnectionClientIdentity clientIdentity, + DateTimeOffset receivedAtUtc, + InboundArticleProducer producer, + ReadOnlyMemory feed = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + if (record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException("Queued ArticleRecord must be CanonicalV1.", nameof(record)); + } + + return new InboundArticle( + messageId, + clientIdentity, + receivedAtUtc, + producer, + record, + feed); + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/DisabledArticleIngestionQueue.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/DisabledArticleIngestionQueue.cs new file mode 100644 index 00000000..db7d2e7e --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/DisabledArticleIngestionQueue.cs @@ -0,0 +1,72 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// No-op ingestion queue used when no real queue is supplied (tests / early wiring). + /// + /// Always reports unavailable so TAKETHIS follows the RFC 4644 400 temporary path. + public sealed class DisabledArticleIngestionQueue : IArticleIngestionQueue + { + /// Shared disabled instance. + public static DisabledArticleIngestionQueue Instance { get; } = new(); + + private DisabledArticleIngestionQueue() + { + } + + /// + public long MemoryLimitBytes => 0; + + /// + public long QueuedBytes => 0; + + /// + public long PeakQueuedBytes => 0; + + /// + public int MaxArticleBytes => Configuration.ArticleIngestionOptions.DefaultMaxArticleBytes; + + /// + public int Count => 0; + + /// + public int PeakCount => 0; + + /// + public bool IsAccepting => false; + + /// + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + return ValueTask.FromResult(ArticleEnqueueResult.Unavailable); + } + + /// + public bool TryProbeCapacity() => false; + + /// + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + return ArticleEnqueueResult.Unavailable; + } + + /// + public bool TryEnqueue(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + return false; + } + + /// + public void Complete() + { + } + + /// + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + ValueTask.FromResult(null); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IArticleIngestionQueue.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IArticleIngestionQueue.cs new file mode 100644 index 00000000..0fe76035 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IArticleIngestionQueue.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Byte-budgeted in-memory ingestion boundary between NNTP receive handlers and the spool writer. + /// + /// + /// Enqueue completes when payload-byte capacity is available (or fails when the queue is + /// completed, or when a single article exceeds the configured budget). Callers must not + /// perform disk I/O; persistence is owned by the background spool writer. + /// + public interface IArticleIngestionQueue + { + /// + /// Gets the configured Transit article-queue memory budget in bytes + /// (Nntpd:TransitQueueMemoryLimit). + /// + long MemoryLimitBytes { get; } + + /// + /// Gets reserved queued article payload bytes + /// (sum of owned lengths). + /// + long QueuedBytes { get; } + + /// Gets the high-water mark of since construction. + long PeakQueuedBytes { get; } + + /// Gets the maximum accepted article payload size in bytes. + int MaxArticleBytes { get; } + + /// Gets the approximate number of articles currently queued. + int Count { get; } + + /// Gets the high-water mark of since construction. + int PeakCount { get; } + + /// + /// Gets the number of producers waiting for byte-budget capacity. + /// + int WaitingProducerCount => 0; + + /// + /// Gets lifetime admission failures (rejected, full, or unavailable). + /// + long AdmissionFailureCount => 0; + + /// + /// Gets accumulated producer wait ticks while blocked on byte-budget capacity. + /// + long AdmissionWaitTicks => 0; + + /// Gets a value indicating whether the queue is still accepting articles. + bool IsAccepting { get; } + + /// + /// Enqueues an accepted article, waiting for byte-budget capacity when the queue is full. + /// + /// + /// when queued; + /// when the article is larger than the budget; + /// when the queue is completed/shutting down + /// or the wait is cancelled. + /// + ValueTask EnqueueAsync(InboundArticle article, CancellationToken cancellationToken); + + /// + /// Returns whether the queue is accepting and has at least one free payload byte. + /// + /// + /// Does not reserve bytes. IHAVE uses this before 335 because the article + /// size is unknown. A later still performs the atomic + /// reservation against the actual payload length. + /// + bool TryProbeCapacity(); + + /// + /// Attempts to reserve the article's payload bytes and enqueue it without waiting. + /// + /// + /// when queued; + /// when the article is larger than the budget; + /// when the remaining budget cannot admit it now; + /// when the queue is completed/shutting down. + /// + ArticleEnqueueResult TryAdmit(InboundArticle article); + + /// + /// Attempts to enqueue without waiting. Returns when the budget + /// cannot admit the article immediately, the article exceeds the budget, or the queue + /// is unavailable. + /// + bool TryEnqueue(InboundArticle article); + + /// + /// Completes the writer side so no further articles are accepted. + /// Already-queued articles remain readable until drained. + /// + void Complete(); + + /// + /// Reads the next queued article, or when the queue is completed and empty. + /// + /// + /// Safe for concurrent callers. A successful read transfers consumer ownership and + /// releases reserved payload bytes exactly once. Callers must not release again. + /// Items leave the queue in FIFO order; concurrent processing order may differ. + /// + ValueTask DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/INewsLogWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/INewsLogWriter.cs new file mode 100644 index 00000000..84066e61 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/INewsLogWriter.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Emits an already-decided INN news event. + /// + /// + /// Production uses a dedicated Serilog pipeline + /// (). File lifecycle is Serilog's + /// responsibility. Implementations must throw on write failure rather than + /// pretending the event was recorded. + /// + public interface INewsLogWriter + { + /// + /// Appends one complete INN news line for . + /// + /// Already-decided news event. + /// The line could not be written. + void Write(in NewsLogEvent evt); + + /// Flushes buffered news lines when the implementation buffers independently of Serilog. + void Flush(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IPathSurveyWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IPathSurveyWriter.cs new file mode 100644 index 00000000..137cda8b --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IPathSurveyWriter.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Persists one canonical Path observation to the durable Path-survey stream. + /// + /// + /// Production uses a dedicated Serilog pipeline + /// (). File lifecycle is Serilog's + /// responsibility. Implementations must throw on write failure rather than + /// pretending the observation was recorded. Callers treat this as an + /// operational survey stream: a failure must not change NNTP responses or + /// ingestion. + /// + public interface IPathSurveyWriter + { + /// + /// Appends one INN-style Path-survey line for . + /// + /// + /// Canonical ArticleRecord.Path value bytes. Empty follows + /// semantics + /// (absent or zero-length field) and still writes a Path line. + /// + /// The line could not be written. + void Write(ReadOnlySpan canonicalPath); + + /// Flushes buffered Path lines when the implementation buffers independently of Serilog. + void Flush(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IhaveArticleInterpreter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IhaveArticleInterpreter.cs new file mode 100644 index 00000000..ee7e0427 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IhaveArticleInterpreter.cs @@ -0,0 +1,77 @@ +using System.Buffers; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Destuffs IHAVE/TAKETHIS wire into for benches and + /// classifier tests. + /// + /// + /// Production IHAVE no longer uses this type. After receive, IHAVE destuffs + /// once in ArticleRecordIngress.TryCreateFromStuffedWire and + /// materializes CanonicalV1 through . + /// Remaining IHAVE-specific behaviour lives in the IHAVE command (History, + /// 335/235/435/436/437, non-blocking probe/admit). Common owns parse, + /// Date/Path materialize, ArtId, ArtHash, and ArtType. + /// + public static class IhaveArticleInterpreter + { + /// + /// Destuffs stuffed IHAVE wire (no terminator) into . + /// + public static Article DestuffToArticle(ReadOnlySpan stuffedWire, int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var headers = new ArrayBufferWriter(1024); + var body = new ArrayBufferWriter(1024); + var type = ArticleType.None; + var exceeded = false; + var inHeaders = true; + var offset = 0; + while (offset < stuffedWire.Length && !exceeded) + { + var remaining = stuffedWire[offset..]; + var crlf = remaining.IndexOf(NntpDelimiterSearch.Crlf); + if (crlf < 0) + { + break; + } + + var line = remaining[..crlf]; + var destuffed = DestuffLine(line); + if (inHeaders) + { + if (destuffed.IsEmpty) + { + inHeaders = false; + } + else + { + NntpArticleDestuffer.AppendUnstuffedLine(headers, line, maxArticleBytes, ref exceeded); + ArticleTypeClassifier.ObserveLine(destuffed, inHeader: true, ref type); + } + } + else + { + NntpArticleDestuffer.AppendUnstuffedLine(body, line, maxArticleBytes, ref exceeded); + ArticleTypeClassifier.ObserveLine(destuffed, inHeader: false, ref type); + } + + offset += crlf + 2; + } + + var headerBytes = headers.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : headers.WrittenMemory.ToArray(); + var bodyBytes = body.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : body.WrittenMemory.ToArray(); + return new Article(headerBytes, bodyBytes, type); + } + + private static ReadOnlySpan DestuffLine(ReadOnlySpan stuffedLine) => + stuffedLine.Length > 0 && stuffedLine[0] == (byte)'.' ? stuffedLine[1..] : stuffedLine; + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticle.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticle.cs new file mode 100644 index 00000000..a556e733 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticle.cs @@ -0,0 +1,65 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// An article accepted into the ingestion queue as a CanonicalV1 . + /// + /// + /// TAKETHIS, POST, and IHAVE all construct the record before admission. + /// aliases . The queue + /// rejects items that are not CanonicalV1. Workers must not destuff or parse. + /// + public sealed class InboundArticle + { + /// Initializes a CanonicalV1 queue item. + public InboundArticle( + string messageId, + ConnectionClientIdentity clientIdentity, + DateTimeOffset receivedAtUtc, + InboundArticleProducer producer, + ArticleRecord record, + ReadOnlyMemory feed = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentNullException.ThrowIfNull(clientIdentity); + if (record.ParseStatus != ArticleParseStatus.CanonicalV1) + { + throw new ArgumentException("Queued ArticleRecord must be CanonicalV1.", nameof(record)); + } + + MessageId = messageId; + ClientIdentity = clientIdentity; + ReceivedAtUtc = receivedAtUtc; + Producer = producer; + Record = record; + Payload = record.ArtData; + Feed = feed; + } + + /// Gets the message-id supplied with the transfer command or POST. + public string MessageId { get; } + + /// Gets canonical ArtData (alias of ). + public ReadOnlyMemory Payload { get; } + + /// Gets the effective client identity at acceptance time. + public ConnectionClientIdentity ClientIdentity { get; } + + /// Gets the UTC timestamp when the article was accepted into the queue. + public DateTimeOffset ReceivedAtUtc { get; } + + /// Gets which command produced this item. + public InboundArticleProducer Producer { get; } + + /// + /// Gets the inbound Transit identifier captured at queue admission, or empty + /// when the session was not a named peer. + /// + public ReadOnlyMemory Feed { get; } + + /// Gets the CanonicalV1 ingress record. + public ArticleRecord Record { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticleProducer.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticleProducer.cs new file mode 100644 index 00000000..65cbc562 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/InboundArticleProducer.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// Identifies which ingest command produced an . + public enum InboundArticleProducer + { + /// + /// TAKETHIS. Ingress builds before admission; + /// aliases canonical ArtData. + /// + TakeThis = 0, + + /// + /// IHAVE. Ingress destuffs stuffed wire and builds + /// before admission. + /// aliases canonical ArtData. + /// + IHave = 1, + + /// + /// POST. Ingress builds after existing POST + /// receive/validate/normalize and before admission. + /// aliases canonical ArtData. + /// + Post = 2, + + /// + /// BackFiller VATP retrieval. The CanonicalV1 + /// is already validated; aliases ArtData with no destuff/parse. + /// + BackFiller = 3, + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IncomingSpoolWriterService.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IncomingSpoolWriterService.cs new file mode 100644 index 00000000..8498ee78 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IncomingSpoolWriterService.cs @@ -0,0 +1,648 @@ +using System.Buffers; +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.OverviewDb; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Storage; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Background application service that drains CanonicalV1 queued articles through + /// a dynamically sized and hands OverviewDB + /// payloads to an independent publisher stage. + /// + /// + /// + /// Article workers encode OverviewArticleV1, enqueue the owned bytes onto a bounded + /// in-process , then write the news log and Path + /// survey. The article body is then discarded. StorageServer/VATP placement remains + /// on this type and is temporarily not invoked. + /// Workers do not await RabbitMQ publication or publisher confirmation. + /// Successful enqueue is not durable RabbitMQ handoff; broker confirmation remains + /// owned by via asynchronous outstanding + /// confirms on . + /// + /// + /// OverviewDB publisher workers scale independently from article workers based on + /// OverviewDB work-queue pressure. Publisher confirms, mandatory publishing, + /// persistent delivery, and the 2-second AMQP expiration are unchanged. + /// + /// + public sealed class IncomingSpoolWriterService : IApplicationService + { + private static readonly TimeSpan RejectedPressureRetryDelay = TimeSpan.FromSeconds(5); + + /// + /// Gets whether StorageServer placement runs after Path survey. + /// + /// + /// Temporarily . Placement methods stay on this type so the + /// bypass is reversed by returning . + /// + private static bool ArticlePlacementEnabled => false; + + private readonly IArticleIngestionQueue _queue; + private readonly INewsLogWriter _newsLog; + private readonly IPathSurveyWriter _pathSurvey; + private readonly INewsgroupCatalogue? _catalogue; + private readonly IOverviewDbHandoffPublisher _overviewHandoff; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly Func? _samplePressure; + private readonly IArticlePlacementClient? _placement; + private readonly IStorageServerRegistry? _placementRegistry; + private readonly CancellationTokenSource _articleRunCts = new(); + private readonly CancellationTokenSource _overviewRunCts = new(); + private Task? _execution; + private int _started; + private IngestionWorkerPool? _pool; + private OverviewDbWorkQueue? _overviewWorkQueue; + private OverviewDbPublisherPool? _overviewPublisherPool; + private Task? _overviewPublisherExecution; + + /// Initializes a new instance of the class. + public IncomingSpoolWriterService( + IArticleIngestionQueue queue, + IOptions options, + ILogger logger, + IFeedDiagnostics? feedDiagnostics = null, + INewsLogWriter? newsLog = null, + INewsgroupCatalogue? catalogue = null, + TimeProvider? timeProvider = null, + IOverviewDbHandoffPublisher? overviewHandoff = null, + IngestionPipelineMetrics? pipelineMetrics = null, + IPathSurveyWriter? pathSurvey = null, + Func? samplePressure = null, + IArticlePlacementClient? placement = null, + IStorageServerRegistry? placementRegistry = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _options = options; + _logger = logger; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _pathSurvey = pathSurvey ?? NullPathSurveyWriter.Instance; + _catalogue = catalogue; + _time = timeProvider ?? TimeProvider.System; + _overviewHandoff = overviewHandoff ?? NullOverviewDbHandoffPublisher.Instance; + _pipeline = pipelineMetrics; + _samplePressure = samplePressure; + _placement = placement; + _placementRegistry = placementRegistry; + PressureRetryDelay = (delay, cancellationToken) => Task.Delay(delay, _time, cancellationToken); + } + + /// + public string Name => "IncomingSpoolWriter"; + + /// + public Task? Execution => _execution; + + /// Gets the active article worker pool (tests). + internal IngestionWorkerPool? Pool => _pool; + + /// + /// Wait between a result and the one retry. + /// Production waits five seconds on . + /// + internal Func PressureRetryDelay { get; set; } + + /// Gets the OverviewDB work queue (tests). + internal IOverviewDbWorkQueue? OverviewWorkQueue => _overviewWorkQueue; + + /// Gets the OverviewDB publisher pool (tests). + internal OverviewDbPublisherPool? OverviewPublisherPool => _overviewPublisherPool; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + var ingestion = _options.Value.ArticleIngestion ?? new ArticleIngestionOptions(); + SpoolLogMessages.WriterStarted( + _logger, + _queue.MemoryLimitBytes, + _queue.MaxArticleBytes); + + _overviewWorkQueue = new OverviewDbWorkQueue(ingestion.OverviewDbWorkQueueMemoryLimit); + _overviewPublisherPool = new OverviewDbPublisherPool( + _overviewWorkQueue, + _overviewHandoff, + ingestion, + _logger, + _pipeline, + _time); + _overviewPublisherExecution = Task.Run( + () => _overviewPublisherPool.RunAsync(_overviewRunCts.Token), + CancellationToken.None); + + _pool = new IngestionWorkerPool( + _queue, + ProcessArticleAsync, + ingestion, + _logger, + _pipeline, + _samplePressure, + _time); + _execution = Task.Run(() => _pool.RunAsync(_articleRunCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + // 1) Stop admitting article ingress and drain article workers (they may still + // enqueue OverviewDB work while draining). Publisher pool keeps running. + _queue.Complete(); + await _articleRunCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + SpoolLogMessages.StopCanceledWithBufferedArticles(_logger, _queue.Count); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.WriterStoppedWithError(_logger, ex); + } + } + + // 2) Complete OverviewDB work queue and drain publisher workers for a bounded period. + // Outstanding OverviewDB confirms may be abandoned after the configured timeout. + _overviewWorkQueue?.Complete(); + await _overviewRunCts.CancelAsync().ConfigureAwait(false); + var overviewExecution = _overviewPublisherExecution; + if (overviewExecution is not null) + { + var shutdownSeconds = Math.Max( + 1, + (_options.Value.ArticleIngestion ?? new ArticleIngestionOptions()) + .OverviewDbPublisherShutdownSeconds); + using var overviewDrainCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + overviewDrainCts.CancelAfter(TimeSpan.FromSeconds(shutdownSeconds)); + try + { + await overviewExecution.WaitAsync(overviewDrainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.WriterStoppedWithError(_logger, ex); + } + } + + _overviewHandoff.AbandonOutstanding(); + + try + { + _newsLog.Flush(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.NewsLogFlushFailed(_logger, ex); + } + + try + { + _pathSurvey.Flush(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.PathSurveyFlushFailed(_logger, ex); + } + + SpoolLogMessages.WriterStopped(_logger); + } + + private async Task ProcessArticleAsync( + InboundArticle article, + long itemStart, + CancellationToken cancellationToken) + { + var overviewAccepted = false; + _feedDiagnostics.BeginSpoolWork(); + try + { + await EnqueueOverviewWorkAsync(article, itemStart, cancellationToken).ConfigureAwait(false); + overviewAccepted = true; + var newsStart = System.Diagnostics.Stopwatch.GetTimestamp(); + WriteNewsLog(article); + _pipeline?.RecordNews(newsStart); + WritePathSurvey(article); + if (ArticlePlacementEnabled) + { + if (article.Producer == InboundArticleProducer.BackFiller) + { + await StoreBackFillerAsync(article, cancellationToken).ConfigureAwait(false); + } + else + { + await PlaceAfterPersistAsync(article, cancellationToken).ConfigureAwait(false); + } + } + else + { + DiscardAcceptedArticle(article); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (!overviewAccepted) + { + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + ex, + article.MessageId, + article.Payload.Length); + await RequeueAsync(article).ConfigureAwait(false); + } + else + { + SpoolLogMessages.FinishFailed( + _logger, + ex, + article.MessageId, + article.Payload.Length); + } + } + finally + { + _feedDiagnostics.EndSpoolWork(article.Payload.Length, persisted: false); + } + } + + /// + /// One VATP STORE of a BackFiller article to one active StorageServer. + /// + /// + /// The target is the active registry entry with a dialable VATP port and the lowest + /// , then ordinal FQDN. Advertised free + /// space is ignored. This method does not call + /// or . + /// Accepted and Duplicate are terminal success. + /// is logged and retried once, + /// after five seconds, against the same server and the same article. Every other + /// is logged once and not retried. NNTPD does not + /// place a second copy. + /// + private async Task StoreBackFillerAsync(InboundArticle article, CancellationToken cancellationToken) + { + if (_placement is null || _placementRegistry is null) + { + return; + } + + var started = Stopwatch.GetTimestamp(); + var artId = article.Record.ArtId.ToLowerHexString(); + try + { + if (cancellationToken.IsCancellationRequested) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + return; + } + + if (!TrySelectBackFillerTarget(_placementRegistry.GetActive(_time.GetUtcNow()), out var target) + || target.VatpPort is not int port) + { + ArticlePlacementLogMessages.NoActiveServer(_logger, artId, Elapsed(started), Attempt: 1); + return; + } + + await PlaceOnSelectedTargetAsync(article.Record, target, port, artId, started, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + 0, + 0, + string.Empty, + ex.GetType().Name, + Elapsed(started), + Attempt: 1); + } + } + + /// + /// Chooses one dialable active StorageServer for . + /// + /// + /// Lowest , then ordinal FQDN. + /// Entries without a VATP port from 1 through 65535 are skipped. Free space + /// and placement persistence are not used. + /// + private static bool TrySelectBackFillerTarget( + IReadOnlyList active, + out StorageServerFleetEntry selected) + { + ArgumentNullException.ThrowIfNull(active); + selected = default; + var found = false; + foreach (var entry in active) + { + if (entry.VatpPort is not (>= 1 and <= 65535)) + { + continue; + } + + if (!found + || entry.ServerId < selected.ServerId + || (entry.ServerId == selected.ServerId + && string.CompareOrdinal(entry.Fqdn, selected.Fqdn) < 0)) + { + selected = entry; + found = true; + } + } + + return found; + } + + /// + /// One STORE after persist. Accepted and Duplicate complete placement. + /// is retried once against the same + /// target and article. Failures stay in this method so they cannot requeue OverviewDB + /// work or undo persistence. NNTPD does not select or store a second copy. + /// + private async Task PlaceAfterPersistAsync(InboundArticle article, CancellationToken cancellationToken) + { + if (article.Producer is not ( + InboundArticleProducer.TakeThis + or InboundArticleProducer.IHave + or InboundArticleProducer.Post)) + { + return; + } + + if (_placement is null || _placementRegistry is null) + { + return; + } + + var started = Stopwatch.GetTimestamp(); + var artId = article.Record.ArtId.ToLowerHexString(); + try + { + if (cancellationToken.IsCancellationRequested) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + return; + } + + var active = _placementRegistry.GetActive(_time.GetUtcNow()); + if (!StorageServerPlacementSelector.TrySelect(active, out var target) || target.VatpPort is not int port) + { + ArticlePlacementLogMessages.NoActiveServer(_logger, artId, Elapsed(started), Attempt: 1); + return; + } + + await PlaceOnSelectedTargetAsync(article.Record, target, port, artId, started, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ArticlePlacementLogMessages.Cancelled(_logger, artId, 0, 0, string.Empty, Elapsed(started), Attempt: 1); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + 0, + 0, + string.Empty, + ex.GetType().Name, + Elapsed(started), + Attempt: 1); + } + } + + /// + /// Stores on . + /// is logged as attempt 1 and + /// retried once, after , with the same record + /// and target. The second result is settled by . + /// + private async Task PlaceOnSelectedTargetAsync( + ArticleRecord record, + StorageServerFleetEntry target, + int port, + string artId, + long started, + CancellationToken cancellationToken) + { + var result = await _placement!.PlaceAsync(record, target, cancellationToken).ConfigureAwait(false); + if (result.Kind != ArticlePlacementKind.RejectedPressure) + { + LogPlacementResult(result, artId, target, port, Elapsed(started), attempt: 1); + return; + } + + LogPlacementResult(result, artId, target, port, Elapsed(started), attempt: 1); + await PressureRetryDelay(RejectedPressureRetryDelay, cancellationToken).ConfigureAwait(false); + var retryStarted = Stopwatch.GetTimestamp(); + result = await _placement.PlaceAsync(record, target, cancellationToken).ConfigureAwait(false); + LogPlacementResult(result, artId, target, port, Elapsed(retryStarted), attempt: 2); + } + + /// Logs one placement result. The attempt number is the STORE call that produced it. + private void LogPlacementResult( + ArticlePlacementResult result, + string artId, + StorageServerFleetEntry target, + int port, + long elapsedMs, + int attempt) + { + switch (result.Kind) + { + case ArticlePlacementKind.Accepted: + ArticlePlacementLogMessages.Accepted(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Duplicate: + ArticlePlacementLogMessages.Duplicate(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Conflict: + ArticlePlacementLogMessages.Conflict(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedCapacity: + ArticlePlacementLogMessages.RejectedCapacity(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedPressure: + ArticlePlacementLogMessages.RejectedPressure(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.RejectedInvalid: + ArticlePlacementLogMessages.RejectedInvalid(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.Cancelled: + ArticlePlacementLogMessages.Cancelled(_logger, artId, target.ServerId, port, target.Fqdn, elapsedMs, attempt); + break; + case ArticlePlacementKind.AcknowledgementNotObserved: + ArticlePlacementLogMessages.AcknowledgementNotObserved( + _logger, + artId, + target.ServerId, + port, + target.Fqdn, + elapsedMs, + attempt); + break; + default: + ArticlePlacementLogMessages.TransportFailed( + _logger, + artId, + target.ServerId, + port, + target.Fqdn, + result.Failure ?? result.Kind.ToString(), + elapsedMs, + attempt); + break; + } + } + + private static long Elapsed(long started) => + (long)Stopwatch.GetElapsedTime(started).TotalMilliseconds; + + /// + /// Writes the post-queue INN news event. Failures are logged and + /// swallowed so the already-accepted article still reaches Path survey. + /// + private void WriteNewsLog(InboundArticle article) + { + try + { + var transit = _options.Value.Transit ?? new TransitOptions(); + if (!IngressNewsDisposition.TryCreateEvent( + article, + transit, + _catalogue, + _time.GetLocalNow(), + out var evt)) + { + return; + } + + _newsLog.Write(in evt); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.NewsLogFailed(_logger, ex, article.MessageId); + } + } + + /// + /// Writes the canonical Path-survey observation. Failures are logged and + /// swallowed so the worker still discards the article and continues. + /// + private void WritePathSurvey(InboundArticle article) + { + try + { + _pathSurvey.Write(article.Record.Path); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SpoolLogMessages.PathSurveyFailed(_logger, ex, article.MessageId); + } + } + + /// + /// Drops after Path survey. The body is not copied, + /// written, or passed to StorageServer selection or VATP placement. + /// + private static void DiscardAcceptedArticle(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + } + + /// + /// Encodes OverviewArticleV1 and enqueues it onto the in-process OverviewDB work + /// queue. Does not call RabbitMQ. + /// + private async Task EnqueueOverviewWorkAsync( + InboundArticle article, + long itemStart, + CancellationToken cancellationToken) + { + var workQueue = _overviewWorkQueue + ?? throw new InvalidOperationException("OverviewDB work queue is not started."); + + var max = OverviewArticleV1Codec.GetMaxEncodedSize(article.Record); + var rented = ArrayPool.Shared.Rent(Math.Max(max, 1)); + try + { + var encodeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var written = OverviewArticleV1Codec.Encode(article.Record, rented); + _pipeline?.RecordEncode(encodeStart); + _pipeline?.RecordToPublishStart(itemStart); + + var owned = new byte[written]; + rented.AsSpan(0, written).CopyTo(owned); + var item = new OverviewDbWorkItem(owned, article.MessageId); + var result = await workQueue.EnqueueAsync(item, cancellationToken).ConfigureAwait(false); + if (result != ArticleEnqueueResult.Accepted) + { + throw new InvalidOperationException( + $"OverviewDB work queue rejected handoff enqueue ({result})."); + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + private async Task RequeueAsync(InboundArticle article) + { + var result = await _queue.EnqueueAsync(article, CancellationToken.None).ConfigureAwait(false); + if (result == ArticleEnqueueResult.Accepted) + { + OverviewDbHandoffLogMessages.Requeued(_logger, article.MessageId); + return; + } + + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, article.MessageId); + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionPressureSnapshot.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionPressureSnapshot.cs new file mode 100644 index 00000000..93d39382 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionPressureSnapshot.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// One sample of Transit ingestion-queue pressure used by the worker-pool scaler. + /// + /// + /// Combined pressure in [0, 1]: the greater of byte utilisation and a + /// waiting-producer signal (1 when any producer waits for byte budget). + /// + /// Current reserved payload bytes in the ingestion queue. + /// Configured Transit queue byte budget. + /// Current queued article count. + /// Producers blocked on the byte budget. + public readonly record struct IngestionPressureSnapshot( + double Pressure, + long QueuedBytes, + long MemoryLimitBytes, + int ArticleCount, + int WaitingProducers) + { + /// Builds a snapshot from the live ingestion queue. + public static IngestionPressureSnapshot FromQueue(IArticleIngestionQueue queue) + { + ArgumentNullException.ThrowIfNull(queue); + var limit = queue.MemoryLimitBytes; + var bytes = queue.QueuedBytes; + var waiting = queue.WaitingProducerCount; + var utilisation = limit <= 0 ? 0d : Math.Clamp(bytes / (double)limit, 0d, 1d); + var waitingSignal = waiting > 0 ? 1d : 0d; + return new IngestionPressureSnapshot( + Math.Max(utilisation, waitingSignal), + bytes, + limit, + queue.Count, + waiting); + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPool.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPool.cs new file mode 100644 index 00000000..2ad567c9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPool.cs @@ -0,0 +1,415 @@ +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Dynamically sized pool of ingestion workers that drain + /// concurrently. + /// + /// + /// Each worker processes one article at a time. OverviewDB RabbitMQ publication + /// is performed by a separate publisher pool; article workers only enqueue onto + /// the in-process OverviewDB work queue. Scaling uses sustained queue pressure + /// with hysteresis. A single worker fault does not stop the pool. + /// + internal sealed class IngestionWorkerPool + { + private readonly IArticleIngestionQueue _queue; + private readonly Func _processArticleAsync; + private readonly Func _samplePressure; + private readonly ArticleIngestionOptions _options; + private readonly ILogger _logger; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly List _workers = []; + private int _highPressureStreak; + private int _lowPressureStreak; + private int _scaleUps; + private int _scaleDowns; + + /// Initializes a new ingestion worker pool. + public IngestionWorkerPool( + IArticleIngestionQueue queue, + Func processArticleAsync, + ArticleIngestionOptions options, + ILogger logger, + IngestionPipelineMetrics? pipelineMetrics = null, + Func? samplePressure = null, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(processArticleAsync); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _processArticleAsync = processArticleAsync; + _options = options; + _logger = logger; + _pipeline = pipelineMetrics; + _samplePressure = samplePressure ?? (() => IngestionPressureSnapshot.FromQueue(queue)); + _time = timeProvider ?? TimeProvider.System; + } + + /// Gets the current worker count. + public int WorkerCount + { + get + { + lock (_gate) + { + return _workers.Count; + } + } + } + + /// Gets how many scale-up decisions have been applied (tests). + public int ScaleUpCount => Volatile.Read(ref _scaleUps); + + /// Gets how many scale-down decisions have been applied (tests). + public int ScaleDownCount => Volatile.Read(ref _scaleDowns); + + /// Runs workers and the scaler until is cancelled. + public async Task RunAsync(CancellationToken cancellationToken) + { + IngestionWorkerPoolLogMessages.PoolStarting( + _logger, + _options.MinWorkers, + _options.MaxWorkers, + _options.MaxPublishConcurrency); + + for (var i = 0; i < _options.MinWorkers; i++) + { + AddWorker(cancellationToken); + } + + PublishPoolObservation(); + + var scaler = Task.Run(() => ScaleLoopAsync(cancellationToken), CancellationToken.None); + try + { + await scaler.ConfigureAwait(false); + } + finally + { + List snapshot; + lock (_gate) + { + snapshot = [.. _workers]; + _workers.Clear(); + } + + foreach (var worker in snapshot) + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + } + + foreach (var worker in snapshot) + { + try + { + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + } + } + + PublishPoolObservation(); + IngestionWorkerPoolLogMessages.PoolStopped(_logger, snapshot.Count); + } + } + + /// Applies one scaling decision from (tests). + internal void ApplyScaleDecisionForTests(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + ApplyScaleDecision(snapshot, workerToken); + PublishPoolObservation(); + } + + private async Task ScaleLoopAsync(CancellationToken cancellationToken) + { + var interval = TimeSpan.FromSeconds(Math.Max(1, _options.ScaleIntervalSeconds)); + using var timer = new PeriodicTimer(interval, _time); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + var snapshot = _samplePressure(); + ApplyScaleDecision(snapshot, cancellationToken); + PublishPoolObservation(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private void ApplyScaleDecision(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + if (snapshot.Pressure >= _options.ScaleUpPressureThreshold) + { + _highPressureStreak++; + _lowPressureStreak = 0; + if (_highPressureStreak >= _options.ScaleUpConsecutiveIntervals) + { + _highPressureStreak = 0; + if (TryAddWorker(workerToken)) + { + Interlocked.Increment(ref _scaleUps); + _pipeline?.RecordScaleUp(); + IngestionWorkerPoolLogMessages.ScaledUp(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + if (snapshot.Pressure <= _options.ScaleDownPressureThreshold) + { + _lowPressureStreak++; + _highPressureStreak = 0; + if (_lowPressureStreak >= _options.ScaleDownConsecutiveIntervals) + { + _lowPressureStreak = 0; + if (TryRemoveWorker()) + { + Interlocked.Increment(ref _scaleDowns); + _pipeline?.RecordScaleDown(); + IngestionWorkerPoolLogMessages.ScaledDown(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + _highPressureStreak = 0; + _lowPressureStreak = 0; + } + + private bool TryAddWorker(CancellationToken workerToken) + { + lock (_gate) + { + if (_workers.Count >= _options.MaxWorkers) + { + return false; + } + + AddWorkerAlreadyLocked(workerToken); + return true; + } + } + + private void AddWorker(CancellationToken workerToken) + { + lock (_gate) + { + AddWorkerAlreadyLocked(workerToken); + } + } + + private void AddWorkerAlreadyLocked(CancellationToken workerToken) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(workerToken); + var worker = new Worker(cts) + { + Execution = Task.CompletedTask, + }; + worker.Execution = Task.Run(() => WorkerLoopAsync(worker, cts.Token), CancellationToken.None); + _workers.Add(worker); + } + + private bool TryRemoveWorker() + { + Worker? victim; + lock (_gate) + { + if (_workers.Count <= _options.MinWorkers) + { + return false; + } + + victim = _workers[^1]; + _workers.RemoveAt(_workers.Count - 1); + } + + _ = RetireWorkerAsync(victim); + return true; + } + + private async Task RetireWorkerAsync(Worker worker) + { + try + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + PublishPoolObservation(); + } + } + + /// + /// Processes queued articles until this worker is cancelled. + /// + /// + /// Graceful shutdown completes the queue before cancelling workers. Cancellation + /// observed while blocked in , or + /// between articles, drains leftovers without the worker token. + /// is approximate and is not proof the + /// channel is empty. Scale-down cancels one worker while the queue is still + /// accepting; that worker exits without draining. + /// + private async Task WorkerLoopAsync(Worker worker, CancellationToken cancellationToken) + { + try + { + while (true) + { + InboundArticle? article; + var idleStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + article = await _queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _pipeline?.AddIdleTicks(System.Diagnostics.Stopwatch.GetTimestamp() - idleStart); + _pipeline?.RecordDequeueWait(idleStart); + await DrainIfShuttingDownAsync().ConfigureAwait(false); + break; + } + + _pipeline?.AddIdleTicks(System.Diagnostics.Stopwatch.GetTimestamp() - idleStart); + _pipeline?.RecordDequeueWait(idleStart); + + if (article is null) + { + break; + } + + await ProcessOneAsync(article).ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + continue; + } + + await DrainIfShuttingDownAsync().ConfigureAwait(false); + break; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + } + + /// + /// Processes articles still queued after graceful shutdown has stopped admission. + /// + /// + /// No-ops while the queue is accepting, so scale-down cancellation does not steal + /// work from the remaining workers. Concurrent callers are safe: each article is + /// dequeued by one caller, and a completed empty queue returns null. + /// + private async Task DrainIfShuttingDownAsync() + { + if (!_queue.IsAccepting) + { + await DrainRemainingAsync().ConfigureAwait(false); + } + } + + /// + /// Dequeues and processes until the queue is completed and empty. + /// + /// + /// Uses so worker cancellation cannot abandon + /// an article that was already accepted. Safe to run from more than one worker. + /// + private async Task DrainRemainingAsync() + { + while (true) + { + InboundArticle? article; + try + { + article = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (article is null) + { + break; + } + + await ProcessOneAsync(article).ConfigureAwait(false); + } + } + + private async Task ProcessOneAsync(InboundArticle article) + { + var itemStart = System.Diagnostics.Stopwatch.GetTimestamp(); + _pipeline?.ObserveQueueDepth(_queue.Count + 1); + try + { + await _processArticleAsync(article, itemStart, CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + IngestionWorkerPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + _pipeline?.AddBusyTicks(System.Diagnostics.Stopwatch.GetTimestamp() - itemStart); + _pipeline?.RecordWorkerItem(itemStart); + } + } + + private void PublishPoolObservation() + { + _pipeline?.ObserveWorkerPool(WorkerCount, _options.MinWorkers, _options.MaxWorkers); + } + + private sealed class Worker(CancellationTokenSource cts) + { + public CancellationTokenSource Cts { get; } = cts; + + public required Task Execution { get; set; } + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs new file mode 100644 index 00000000..59e7a1f3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngestionWorkerPoolLogMessages.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// Source-generated ingestion worker-pool log messages. + internal static partial class IngestionWorkerPoolLogMessages + { + [LoggerMessage( + EventId = 2010, + Level = LogLevel.Information, + Message = "Ingestion worker pool starting (min={MinWorkers}, max={MaxWorkers}, publishConcurrency={PublishConcurrency})")] + public static partial void PoolStarting( + ILogger logger, + int MinWorkers, + int MaxWorkers, + int PublishConcurrency); + + [LoggerMessage( + EventId = 2011, + Level = LogLevel.Information, + Message = "Ingestion worker pool scaled up to {WorkerCount} (pressure={Pressure:F3})")] + public static partial void ScaledUp(ILogger logger, int WorkerCount, double Pressure); + + [LoggerMessage( + EventId = 2012, + Level = LogLevel.Information, + Message = "Ingestion worker pool scaled down to {WorkerCount} (pressure={Pressure:F3})")] + public static partial void ScaledDown(ILogger logger, int WorkerCount, double Pressure); + + [LoggerMessage( + EventId = 2013, + Level = LogLevel.Warning, + Message = "Ingestion worker faulted; remaining workers continue")] + public static partial void WorkerFaulted(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2014, + Level = LogLevel.Information, + Message = "Ingestion worker pool stopped (workers={WorkerCount})")] + public static partial void PoolStopped(ILogger logger, int WorkerCount); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsDisposition.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsDisposition.cs new file mode 100644 index 00000000..b055cac8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsDisposition.cs @@ -0,0 +1,297 @@ +using System.Text; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Post-queue ingress disposition: decides + vs j and whether to log. + /// + /// + /// + /// Uses in-memory . A successful lookup + /// means the group exists in the published catalogue. + /// is the RFC 6048 j status (peer articles permitted, nothing locally filed). + /// Unknown means returned false. A locally + /// carried non-junk group is a catalogue hit whose status is not PeerOnly + /// (y/n/m/x). + /// + /// + /// POST is never WantTrash junk. An unavailable catalogue is treated as + /// accepted rather than invented junk. Newsgroups bytes come from the + /// CanonicalV1 field table; the header is not rewritten. + /// + /// + internal static class IngressNewsDisposition + { + /// Classifies without writing a news line. + public static NewsLogDisposition Classify( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue) => + Classify(article, transit, catalogue, out _); + + /// + /// Classifies and, for junk, the source-backed reason + /// including the Newsgroups tokens that caused that decision. + /// + public static NewsLogDisposition Classify( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + out string? reason) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + reason = null; + if (article.Producer == InboundArticleProducer.Post || catalogue is null) + { + return NewsLogDisposition.Accepted; + } + + var snapshot = catalogue.Current; + var newsgroups = article.Record.Newsgroups; + if (HasLocalNonJunkGroup(newsgroups, snapshot)) + { + return NewsLogDisposition.Accepted; + } + + if (HasPeerOnlyGroup(newsgroups, snapshot)) + { + reason = FormatResponsibleGroups( + IngressNewsReasons.PeerOnly, + newsgroups, + snapshot, + peerOnly: true); + return NewsLogDisposition.Junk; + } + + if (transit.WantTrash) + { + reason = FormatResponsibleGroups( + IngressNewsReasons.NewsgroupNotCarried, + newsgroups, + snapshot, + peerOnly: false); + return NewsLogDisposition.Junk; + } + + return NewsLogDisposition.Accepted; + } + + /// + /// TAKETHIS/IHAVE WantTrash=false rejection: every listed group is unknown. + /// PeerOnly catalogue hits remain accepted junk. POST and a missing catalogue + /// do not reject. + /// + public static bool IsUncarriedWantTrashRejection( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue) => + IsUncarriedWantTrashRejection(article, transit, catalogue, out _); + + /// + /// TAKETHIS/IHAVE WantTrash=false rejection, with the uncarried group names + /// that caused it. Classification is unchanged from the bool overload. + /// + /// Queued inbound article already constructed. + /// Transit options; WantTrash=true never rejects here. + /// Published catalogue; missing catalogue does not reject. + /// + /// When the method returns , newsgroup not carried + /// plus the unknown header tokens in article order. + /// + public static bool IsUncarriedWantTrashRejection( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + out string reason) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + reason = string.Empty; + if (transit.WantTrash + || article.Producer == InboundArticleProducer.Post + || catalogue is null) + { + return false; + } + + var snapshot = catalogue.Current; + var newsgroups = article.Record.Newsgroups; + if (HasLocalNonJunkGroup(newsgroups, snapshot) + || HasPeerOnlyGroup(newsgroups, snapshot)) + { + return false; + } + + reason = FormatResponsibleGroups( + IngressNewsReasons.NewsgroupNotCarried, + newsgroups, + snapshot, + peerOnly: false); + return true; + } + + /// + /// Builds a news event when the disposition should be recorded. + /// + public static bool TryCreateEvent( + InboundArticle article, + TransitOptions transit, + INewsgroupCatalogue? catalogue, + DateTimeOffset timestamp, + out NewsLogEvent evt) + { + ArgumentNullException.ThrowIfNull(article); + transit ??= new TransitOptions(); + var disposition = Classify(article, transit, catalogue, out var reason); + if (disposition == NewsLogDisposition.Junk && !transit.LogTrash) + { + evt = default; + return false; + } + + evt = new NewsLogEvent( + disposition, + MessageIdMemory(article), + feed: article.Feed, + sites: default, + timestamp, + reason: Ascii(reason), + size: article.Payload.Length); + return true; + } + + private static ReadOnlyMemory Ascii(string? value) => + string.IsNullOrEmpty(value) ? default : Encoding.ASCII.GetBytes(value); + + private static ReadOnlyMemory MessageIdMemory(InboundArticle article) + { + var range = article.Record.Fields.MessageId; + if (!range.IsPresent || range.Length == 0) + { + return ReadOnlyMemory.Empty; + } + + return article.Record.ArtData.Slice(range.Offset, range.Length); + } + + private static bool HasLocalNonJunkGroup(ReadOnlySpan newsgroups, NewsgroupSnapshot snapshot) + { + foreach (var token in EnumerateGroups(newsgroups)) + { + if (snapshot.TryGet(token, out var group) + && group.PostingStatus != NewsgroupPostingStatus.PeerOnly) + { + return true; + } + } + + return false; + } + + /// + /// Builds plus the header tokens that match the + /// already-decided junk/rejection kind, in article Newsgroups order. + /// + /// Already-decided reason prefix. + /// Article Newsgroups field bytes. + /// Published catalogue snapshot used by the existing decision. + /// + /// collects PeerOnly catalogue hits; + /// collects unknown tokens. + /// + /// The prefix, or prefix plus responsible header tokens. + private static string FormatResponsibleGroups( + string prefix, + ReadOnlySpan newsgroups, + NewsgroupSnapshot snapshot, + bool peerOnly) + { + List? names = null; + foreach (var token in EnumerateGroups(newsgroups)) + { + var hit = snapshot.TryGet(token, out var group); + var include = peerOnly + ? hit && group.PostingStatus == NewsgroupPostingStatus.PeerOnly + : !hit; + if (!include) + { + continue; + } + + names ??= new List(2); + names.Add(Encoding.ASCII.GetString(token)); + } + + return names is null + ? prefix + : IngressNewsReasons.WithGroups(prefix, names); + } + + private static bool HasPeerOnlyGroup(ReadOnlySpan newsgroups, NewsgroupSnapshot snapshot) + { + foreach (var token in EnumerateGroups(newsgroups)) + { + if (snapshot.TryGet(token, out var group) + && group.PostingStatus == NewsgroupPostingStatus.PeerOnly) + { + return true; + } + } + + return false; + } + + private static GroupTokenEnumerable EnumerateGroups(ReadOnlySpan newsgroups) => new(newsgroups); + + private readonly ref struct GroupTokenEnumerable(ReadOnlySpan newsgroups) + { + private readonly ReadOnlySpan _newsgroups = newsgroups; + + public Enumerator GetEnumerator() => new(_newsgroups); + + public ref struct Enumerator(ReadOnlySpan newsgroups) + { + private ReadOnlySpan _remaining = newsgroups; + private ReadOnlySpan _current; + + public ReadOnlySpan Current => _current; + + public bool MoveNext() + { + while (!_remaining.IsEmpty) + { + var comma = _remaining.IndexOf((byte)','); + var token = comma < 0 ? _remaining : _remaining[..comma]; + _remaining = comma < 0 ? default : _remaining[(comma + 1)..]; + token = Trim(token); + if (!token.IsEmpty) + { + _current = token; + return true; + } + } + + return false; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + while (!value.IsEmpty && (value[0] == (byte)' ' || value[0] == (byte)'\t')) + { + value = value[1..]; + } + + while (!value.IsEmpty && (value[^1] == (byte)' ' || value[^1] == (byte)'\t')) + { + value = value[..^1]; + } + + return value; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsEvents.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsEvents.cs new file mode 100644 index 00000000..d38655b1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsEvents.cs @@ -0,0 +1,73 @@ +using System.Text; +using VectorNNTPD.TransitService.Session; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Builds already-decided news events at ingress decision points. + /// + internal static class IngressNewsEvents + { + /// WantTrash=false reason for unknown/non-carried Newsgroups. + internal const string NewsgroupNotCarried = IngressNewsReasons.NewsgroupNotCarried; + + /// Writes a rejection news event. Failures do not change protocol state. + public static void TryWriteRejected( + NntpSession session, + string? messageId, + int responseCode, + string reason, + int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Rejected, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + responseCode: responseCode, + reason: Ascii(reason), + size: size)); + } + + /// Writes an accepted news event for a path that never enters the queue. + public static void TryWriteAccepted(NntpSession session, string? messageId, int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Accepted, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + size: size)); + } + + /// Writes a moderated news event for a successful POST moderation submission. + public static void TryWriteModerated(NntpSession session, string? messageId, int size = 0) + { + ArgumentNullException.ThrowIfNull(session); + session.TryWriteNews( + new NewsLogEvent( + NewsLogDisposition.Moderated, + Ascii(messageId), + feed: SnapshotInboundFeed(session), + timestamp: session.Time.GetLocalNow(), + size: size)); + } + + /// + /// Copies the session's inbound Transit identifier for a self-contained news event. + /// + internal static ReadOnlyMemory SnapshotInboundFeed(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var name = session.Authorization.TransitPeerName; + return string.IsNullOrEmpty(name) ? default : Encoding.ASCII.GetBytes(name); + } + + private static ReadOnlyMemory Ascii(string? value) => + string.IsNullOrEmpty(value) ? default : Encoding.ASCII.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsReasons.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsReasons.cs new file mode 100644 index 00000000..313b264f --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/IngressNewsReasons.cs @@ -0,0 +1,237 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Session.Commands.Posting; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Maps already-decided ingress failures to concise operator-facing news reasons. + /// + /// + /// The formatter does not inspect parser enums, response codes, or validator + /// state. Callers supply the existing decision value; this type only chooses + /// the news-journal text. It does not change NNTP responses or admission. + /// + internal static class IngressNewsReasons + { + /// Invalid or missing Message-ID. + internal const string MessageIdInvalid = "message-id invalid"; + + /// Invalid or missing Date. + internal const string DateInvalid = "date invalid"; + + /// + /// Unknown/non-carried Newsgroups: WantTrash=false rejection, or WantTrash=true junk. + /// Decision sites append : group[, group...] via . + /// + internal const string NewsgroupNotCarried = "newsgroup not carried"; + + /// Receive newsfeeds expression did not subscribe the article. + internal const string NewsgroupPatternRejected = "newsgroup pattern rejected"; + + /// RFC 6048 j / catalogue hit. + internal const string PeerOnly = "peer-only"; + + /// Receive-size or ArticleRecord size rejection. + internal const string ArticleTooLarge = "article too large"; + + /// Session article-type capability rejection. + internal const string ArticleTypeNotPermitted = "article type not permitted"; + + /// Hard queue memory-budget rejection (). + internal const string QueueCapacityExceeded = "queue capacity exceeded"; + + /// yEnc markers present and validation failed. + internal const string YEncodingInvalid = "yEncoding invalid"; + + /// + /// Formats an already-decided prefix plus the responsible group names. + /// + /// Already-decided reason prefix. + /// Responsible group names in header order. + /// The prefix, or prefix: name[, name...]. + /// + /// Does not inspect catalogue or header state. Callers supply the groups + /// that caused the existing decision, in header order. + /// + internal static string WithGroups(string prefix, IReadOnlyList groups) + { + ArgumentException.ThrowIfNullOrEmpty(prefix); + ArgumentNullException.ThrowIfNull(groups); + if (groups.Count == 0) + { + return prefix; + } + + var builder = new StringBuilder(prefix.Length + 2 + (groups.Count * 16)); + builder.Append(prefix); + builder.Append(": "); + for (var i = 0; i < groups.Count; i++) + { + if (i > 0) + { + builder.Append(", "); + } + + builder.Append(groups[i]); + } + + return builder.ToString(); + } + + /// Maps an ArticleRecord construction failure to a news reason. + public static string ForArticleRecord(in ArticleRecordCreateResult created) + { + if (created.ParseFailure != NntpArticleParseFailureCode.None) + { + return ForParseFailure(created.ParseFailure); + } + + return ForMaterializeFailure(created.MaterializeFailure); + } + + /// Maps a parser failure code already produced by ArticleRecord construction. + public static string ForParseFailure(NntpArticleParseFailureCode code) => + code switch + { + NntpArticleParseFailureCode.YEncDecodingFailed => YEncodingInvalid, + NntpArticleParseFailureCode.InvalidMessageId + or NntpArticleParseFailureCode.MissingMessageId => MessageIdInvalid, + NntpArticleParseFailureCode.MissingOrInvalidDate => DateInvalid, + NntpArticleParseFailureCode.ArticleTooLarge => ArticleTooLarge, + NntpArticleParseFailureCode.EmptyArticle => "empty article", + NntpArticleParseFailureCode.MissingHeaderBodySeparator => "missing header/body separator", + NntpArticleParseFailureCode.HeaderSectionTooLarge => "header too large", + NntpArticleParseFailureCode.TooManyHeaders => "too many headers", + NntpArticleParseFailureCode.HeaderLineTooLong => "header line too long", + NntpArticleParseFailureCode.MalformedHeader + or NntpArticleParseFailureCode.MalformedHeaderContinuation => "malformed header", + NntpArticleParseFailureCode.HeaderNameTooLong => "header name too long", + NntpArticleParseFailureCode.HeaderValueTooLong => "header value too long", + NntpArticleParseFailureCode.ContainsNul => "embedded NUL", + NntpArticleParseFailureCode.ContainsIllegalControlByte => "illegal control byte", + NntpArticleParseFailureCode.MissingNewsgroups => "newsgroups missing", + NntpArticleParseFailureCode.InvalidNewsgroups => "newsgroups invalid", + NntpArticleParseFailureCode.InvalidFrom => "from invalid", + NntpArticleParseFailureCode.InvalidPath => "path invalid", + NntpArticleParseFailureCode.DuplicateMessageId => "duplicate message-id", + NntpArticleParseFailureCode.DuplicateNewsgroups => "duplicate newsgroups", + NntpArticleParseFailureCode.DuplicatePath => "duplicate path", + NntpArticleParseFailureCode.BodyLineTooLong => "body line too long", + _ => "article rejected", + }; + + /// Maps a materializer failure code already produced by ArticleRecord construction. + public static string ForMaterializeFailure(NntpArticleCanonicalFailureCode code) => + code switch + { + NntpArticleCanonicalFailureCode.ArticleTooLarge => ArticleTooLarge, + NntpArticleCanonicalFailureCode.MissingSelectedDateHeader => DateInvalid, + NntpArticleCanonicalFailureCode.EmptyArticle => "empty article", + NntpArticleCanonicalFailureCode.DuplicatePath => "duplicate path", + NntpArticleCanonicalFailureCode.InvalidHeaderSeparator => "malformed header", + NntpArticleCanonicalFailureCode.DateLineTooLong => "date line too long", + NntpArticleCanonicalFailureCode.PathRewriteLineTooLong + or NntpArticleCanonicalFailureCode.PathInsertionLineTooLong => "path invalid", + _ => "article record rejected", + }; + + /// + /// Maps an existing IHAVE/TAKETHIS reject-detail string (kept for command + /// completion / TX logs) to the news-journal reason. + /// + public static string ForExistingRejectDetail(string? detail) + { + if (string.IsNullOrEmpty(detail)) + { + return "article rejected"; + } + + if (detail is "rejected too large") + { + return ArticleTooLarge; + } + + if (detail is "rejected article type") + { + return ArticleTypeNotPermitted; + } + + if (detail is "rejected newsgroup pattern") + { + return NewsgroupPatternRejected; + } + + if (detail is "rejected exceeds queue budget") + { + return QueueCapacityExceeded; + } + + if (detail.StartsWith("rejected article record ", StringComparison.Ordinal)) + { + var token = detail["rejected article record ".Length..]; + if (Enum.TryParse(token, out NntpArticleParseFailureCode parse) + && parse != NntpArticleParseFailureCode.None) + { + return ForParseFailure(parse); + } + + if (Enum.TryParse(token, out NntpArticleCanonicalFailureCode materialize) + && materialize != NntpArticleCanonicalFailureCode.None) + { + return ForMaterializeFailure(materialize); + } + } + + return detail; + } + + /// Maps a POST already produced by the command path. + public static string ForPostingFailure(in PostingFailure failure) + { + if (TryMapArticleRecordEnumName(failure.Detail, out var fromEnum)) + { + return fromEnum; + } + + return failure.Category switch + { + PostingFailureCategory.ArticleTooLarge => ArticleTooLarge, + PostingFailureCategory.InvalidMessageId => MessageIdInvalid, + PostingFailureCategory.InvalidDate => DateInvalid, + PostingFailureCategory.PolicyRejected when failure.Detail == "arttype-capability" + => ArticleTypeNotPermitted, + PostingFailureCategory.PolicyRejected when failure.Detail == "unknown newsgroup" + => NewsgroupNotCarried, + PostingFailureCategory.PersistenceFailure when failure.Detail is "Rejected" or "Full" + => QueueCapacityExceeded, + _ => string.IsNullOrEmpty(failure.Detail) ? "article rejected" : failure.Detail, + }; + } + + private static bool TryMapArticleRecordEnumName(string? detail, out string reason) + { + if (!string.IsNullOrEmpty(detail) + && Enum.TryParse(detail, out NntpArticleParseFailureCode parse) + && parse != NntpArticleParseFailureCode.None) + { + reason = ForParseFailure(parse); + return true; + } + + if (!string.IsNullOrEmpty(detail) + && Enum.TryParse(detail, out NntpArticleCanonicalFailureCode materialize) + && materialize != NntpArticleCanonicalFailureCode.None) + { + reason = ForMaterializeFailure(materialize); + return true; + } + + reason = string.Empty; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogDisposition.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogDisposition.cs new file mode 100644 index 00000000..de539c6a --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogDisposition.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// INN news log disposition codes implemented for v1 ingress. + /// + /// + /// Values match innd(8) LOGGING / newslog(5) where VectorNNTP actually + /// produces the event: + accepted for propagation, j accepted + /// locally as junk (not propagated), - deliberately rejected. m + /// is a VectorNNTP disposition for a POST sealed for moderation (not an INN + /// innd code) and is not currently propagated by egress routing. Cancel + /// processing (c) is not produced until a cancel path exists. INN's + /// informational ? (isolated CR/LF) is not emitted. Only + /// includes the outbound-site field. + /// + public enum NewsLogDisposition : byte + { + /// Accepted for local store and propagation (+); includes the outbound-site field. + Accepted = (byte)'+', + + /// Accepted locally as junk (j); not propagated, so no outbound-site field. + Junk = (byte)'j', + + /// Deliberately rejected article (-); no outbound-site field. + Rejected = (byte)'-', + + /// POST sealed for moderation (m); not propagated, so no outbound-site field. + Moderated = (byte)'m', + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogEvent.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogEvent.cs new file mode 100644 index 00000000..32feb789 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogEvent.cs @@ -0,0 +1,89 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Domain-neutral INN news log event already decided by ingress processing. + /// + /// + /// This type does not classify junk or invent rejection reasons. The writer + /// serializes the fields supplied here. An empty is written + /// as INN's unavailable token ? in the inbound-peer position. Outbound + /// exist only for articles accepted for propagation + /// (); empty Sites then emit ? + /// until egress routing exists. Junk, rejected, moderated, and future cancel + /// lines omit the outbound-site field. is the INN article + /// size field after the Message-ID (canonical payload bytes). Timestamp is the + /// news-log event time, not the article Date header. + /// may be stored for but is not + /// rendered. is the already-decided operator-facing text + /// for and + /// , including any responsible newsgroup + /// names supplied by the decision site. + /// + public readonly struct NewsLogEvent + { + /// Initializes a news-log event. + /// Already-decided +, j, -, or m. + /// Article Message-ID bytes, including angle brackets when present. + /// Authoritative inbound feed identity, or empty to emit ?. + /// + /// Outbound site list for ; empty emits + /// ? on that disposition only and is omitted on j/-/m. + /// + /// + /// Event time. lets the writer stamp + /// processing time. + /// + /// NNTP response code for -; retained on the event, not rendered. + /// Already-decided operator-facing reason bytes for - or j. + /// INN article size in bytes (canonical payload length). + public NewsLogEvent( + NewsLogDisposition disposition, + ReadOnlyMemory messageId, + ReadOnlyMemory feed = default, + ReadOnlyMemory sites = default, + DateTimeOffset timestamp = default, + int responseCode = 0, + ReadOnlyMemory reason = default, + int size = 0) + { + Disposition = disposition; + MessageId = messageId; + Feed = feed; + Sites = sites; + Timestamp = timestamp; + ResponseCode = responseCode; + Reason = reason; + Size = size; + } + + /// Gets the news-log event timestamp (processing time, not article Date). + public DateTimeOffset Timestamp { get; } + + /// Gets the already-decided disposition. + public NewsLogDisposition Disposition { get; } + + /// Gets the inbound peer/feed identity captured at decision time. + public ReadOnlyMemory Feed { get; } + + /// Gets the article Message-ID value bytes. + public ReadOnlyMemory MessageId { get; } + + /// Gets the INN article size in bytes (canonical payload length). + public int Size { get; } + + /// + /// Gets whether this event includes the outbound-site field. + /// Only accepted-for-propagation (+) articles have it. + /// + public bool HasOutboundSiteField => Disposition == NewsLogDisposition.Accepted; + + /// Gets optional future outbound site tokens; empty emits ? when is true. + public ReadOnlyMemory Sites { get; } + + /// Gets the NNTP response code carried for a rejection; 0 means none. Not rendered. + public int ResponseCode { get; } + + /// Gets the already-decided rejection or junk reason; empty omits it. + public ReadOnlyMemory Reason { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogLineFormatter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogLineFormatter.cs new file mode 100644 index 00000000..249b9f93 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/NewsLogLineFormatter.cs @@ -0,0 +1,189 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Serializes one INN news line per innd(8) LOGGING / ARTlog. + /// + /// + /// Verified INN field order (innd.pod LOGGING and innd/art.c ARTlog): + /// mon dd hh:mm:ss.mmm disposition feedsite message-id size [sites|reason]. + /// Timestamp uses English month abbreviations and local wall time at millisecond + /// resolution. Empty inbound Feed is INN's unavailable token ?. + /// Outbound Sites are written only for + /// (+); empty Sites then emit ? until routing exists. Junk, rejected, + /// and moderated lines omit Sites and append the already-decided reason after size + /// when present. The NNTP response code is not a news field. Disposition characters + /// and field order are hard-coded. + /// + internal static class NewsLogLineFormatter + { + private static ReadOnlySpan Months => "JanFebMarAprMayJunJulAugSepOctNovDec"u8; + + private static ReadOnlySpan UnknownFeed => "?"u8; + + /// Formats into as one LF-terminated line. + /// Bytes written. + public static int Write(Span destination, in NewsLogEvent evt, DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(timestamp.Offset); + var dt = local.DateTime; + var written = 0; + written += WriteMonth(destination[written..], dt.Month); + destination[written++] = (byte)' '; + written += WriteSpacePaddedDay(destination[written..], dt.Day); + destination[written++] = (byte)' '; + written += WriteTwoDigits(destination[written..], dt.Hour); + destination[written++] = (byte)':'; + written += WriteTwoDigits(destination[written..], dt.Minute); + destination[written++] = (byte)':'; + written += WriteTwoDigits(destination[written..], dt.Second); + destination[written++] = (byte)'.'; + written += WriteThreeDigits(destination[written..], dt.Millisecond); + destination[written++] = (byte)' '; + destination[written++] = (byte)evt.Disposition; + destination[written++] = (byte)' '; + var feed = evt.Feed.IsEmpty ? UnknownFeed : evt.Feed.Span; + feed.CopyTo(destination[written..]); + written += feed.Length; + destination[written++] = (byte)' '; + evt.MessageId.Span.CopyTo(destination[written..]); + written += evt.MessageId.Length; + destination[written++] = (byte)' '; + written += WriteUnsignedDecimal(destination[written..], evt.Size); + if (evt.HasOutboundSiteField) + { + destination[written++] = (byte)' '; + var sites = evt.Sites.IsEmpty ? UnknownFeed : evt.Sites.Span; + sites.CopyTo(destination[written..]); + written += sites.Length; + } + + if (WritesReason(evt.Disposition) && !evt.Reason.IsEmpty) + { + destination[written++] = (byte)' '; + evt.Reason.Span.CopyTo(destination[written..]); + written += evt.Reason.Length; + } + + destination[written++] = (byte)'\n'; + return written; + } + + /// Byte count required to format including the trailing LF. + public static int RequiredLength(in NewsLogEvent evt) + { + var feedLength = evt.Feed.IsEmpty ? 1 : evt.Feed.Length; + var sites = evt.HasOutboundSiteField + ? 1 + (evt.Sites.IsEmpty ? 1 : evt.Sites.Length) + : 0; + var reason = WritesReason(evt.Disposition) && !evt.Reason.IsEmpty + ? 1 + evt.Reason.Length + : 0; + return 24 + feedLength + evt.MessageId.Length + 1 + DecimalDigitCount(evt.Size) + + sites + reason; + } + + private static bool WritesReason(NewsLogDisposition disposition) => + disposition is NewsLogDisposition.Rejected or NewsLogDisposition.Junk; + + private static int WriteMonth(Span dest, int month) + { + var index = (month - 1) * 3; + Months.Slice(index, 3).CopyTo(dest); + return 3; + } + + private static int WriteSpacePaddedDay(Span dest, int day) + { + if (day < 10) + { + dest[0] = (byte)' '; + dest[1] = (byte)('0' + day); + } + else + { + dest[0] = (byte)('0' + (day / 10)); + dest[1] = (byte)('0' + (day % 10)); + } + + return 2; + } + + private static int WriteTwoDigits(Span dest, int value) + { + dest[0] = (byte)('0' + (value / 10)); + dest[1] = (byte)('0' + (value % 10)); + return 2; + } + + private static int WriteThreeDigits(Span dest, int value) + { + dest[0] = (byte)('0' + (value / 100)); + dest[1] = (byte)('0' + ((value / 10) % 10)); + dest[2] = (byte)('0' + (value % 10)); + return 3; + } + + private static int WriteUnsignedDecimal(Span dest, int value) + { + var digits = DecimalDigitCount(value); + var remaining = value; + for (var i = digits - 1; i >= 0; i--) + { + dest[i] = (byte)('0' + (remaining % 10)); + remaining /= 10; + } + + return digits; + } + + private static int DecimalDigitCount(int value) + { + if (value < 10) + { + return 1; + } + + if (value < 100) + { + return 2; + } + + if (value < 1_000) + { + return 3; + } + + if (value < 10_000) + { + return 4; + } + + if (value < 100_000) + { + return 5; + } + + if (value < 1_000_000) + { + return 6; + } + + if (value < 10_000_000) + { + return 7; + } + + if (value < 100_000_000) + { + return 8; + } + + if (value < 1_000_000_000) + { + return 9; + } + + return 10; + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/NullNewsLogWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/NullNewsLogWriter.cs new file mode 100644 index 00000000..1b12c133 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/NullNewsLogWriter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// No-op for tests that do not assert news output. + public sealed class NullNewsLogWriter : INewsLogWriter + { + /// Gets the shared no-op instance. + public static NullNewsLogWriter Instance { get; } = new(); + + /// + public void Write(in NewsLogEvent evt) + { + } + + /// + public void Flush() + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/NullPathSurveyWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/NullPathSurveyWriter.cs new file mode 100644 index 00000000..844c130e --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/NullPathSurveyWriter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// No-op for tests that do not assert Path-survey output. + public sealed class NullPathSurveyWriter : IPathSurveyWriter + { + /// Gets the shared no-op instance. + public static NullPathSurveyWriter Instance { get; } = new(); + + /// + public void Write(ReadOnlySpan canonicalPath) + { + } + + /// + public void Flush() + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..9e36c841 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbHandoffPublisher.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// One-way OverviewDB RabbitMQ handoff with asynchronous publisher confirms. + /// + /// + /// returns after the message is written into the outstanding + /// confirmation window. Broker confirmation is not awaited on the publish path. + /// Crash loss of outstanding unconfirmed work is acceptable for OverviewDB. + /// + public interface IOverviewDbHandoffPublisher + { + /// + /// Publishes a compact OverviewDB payload to overviewdb.queue without waiting + /// for the broker confirmation. + /// + /// Owned OverviewDB work item (already-encoded payload). + /// Token used to cancel waiting for outstanding capacity or the write. + Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken); + + /// Gets the number of publishes awaiting broker confirmation. + int OutstandingCount { get; } + + /// + /// Tries to dequeue one nack/return failure for in-process requeue handling. + /// + /// Must not be called from inside a RabbitMQ ack/nack/return callback. + bool TryDequeuePublishFailure(out OverviewDbWorkItem item); + + /// + /// Abandons all outstanding unconfirmed work (bounded shutdown / crash-loss path). + /// + void AbandonOutstanding(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs new file mode 100644 index 00000000..d5d677b3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/IOverviewDbWorkQueue.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// Bounded, byte-budgeted in-process queue between article workers and OverviewDB + /// RabbitMQ publisher workers. + /// + /// + /// This queue is not durable. Enqueue success does not imply RabbitMQ confirmation. + /// + internal interface IOverviewDbWorkQueue + { + /// Gets the configured byte budget. + long MemoryLimitBytes { get; } + + /// Gets reserved queued payload bytes. + long QueuedBytes { get; } + + /// Gets the approximate number of queued work items. + int Count { get; } + + /// Gets producers waiting for byte-budget capacity. + int WaitingProducerCount { get; } + + /// Gets whether the queue still accepts new work. + bool IsAccepting { get; } + + /// + /// Enqueues work, waiting for byte-budget capacity when full. + /// + ValueTask EnqueueAsync(OverviewDbWorkItem item, CancellationToken cancellationToken); + + /// + /// Dequeues the next item, or when completed and empty. + /// Releases the byte reservation on successful dequeue. + /// + ValueTask DequeueAsync(CancellationToken cancellationToken); + + /// Stops accepting new work; queued items remain readable until drained. + void Complete(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..d0290e46 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/NullOverviewDbHandoffPublisher.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// Test-only OverviewDB publisher that completes without contacting a broker. + /// Production DI registers instead. + /// + internal sealed class NullOverviewDbHandoffPublisher : IOverviewDbHandoffPublisher + { + /// Shared succeeding instance used when tests construct the worker directly. + public static NullOverviewDbHandoffPublisher Instance { get; } = new(); + + private NullOverviewDbHandoffPublisher() + { + } + + /// + public Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + + /// + public int OutstandingCount => 0; + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + item = null!; + return false; + } + + /// + public void AbandonOutstanding() + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs new file mode 100644 index 00000000..e20e0ff8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffLogMessages.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// Source-generated OverviewDB handoff log messages. + internal static partial class OverviewDbHandoffLogMessages + { + [LoggerMessage( + EventId = 2100, + Level = LogLevel.Error, + Message = "OverviewDB RabbitMQ handoff failed for {MessageId} ({Bytes} payload bytes)")] + public static partial void PublishFailed(ILogger logger, Exception exception, string MessageId, int Bytes); + + [LoggerMessage( + EventId = 2101, + Level = LogLevel.Warning, + Message = "OverviewDB handoff was requeued for {MessageId}")] + public static partial void Requeued(ILogger logger, string MessageId); + + [LoggerMessage( + EventId = 2102, + Level = LogLevel.Error, + Message = "OverviewDB handoff could not be requeued for {MessageId}; queue is unavailable")] + public static partial void RequeueUnavailable(ILogger logger, string MessageId); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..86042ffc --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbHandoffPublisher.cs @@ -0,0 +1,448 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Text; +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// Publishes OverviewDB protobuf payloads to overviewdb.queue with an asynchronous + /// outstanding-confirmation window. + /// + /// + /// + /// Uses one confirm-enabled RabbitMQ channel (library tracking disabled) so + /// BasicPublishAsync returns after the write. Confirms arrive via + /// BasicAcksAsync / BasicNacksAsync / BasicReturnAsync and are + /// correlated by publish sequence number. Channel publishes are serialized; callbacks + /// only update outstanding state and must not publish. + /// + /// + /// Crash or shutdown loss of outstanding unconfirmed OverviewDB work is acceptable. + /// + /// + internal sealed class OverviewDbHandoffPublisher : IOverviewDbHandoffPublisher, IAsyncDisposable + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly ConcurrentDictionary _outstanding = new(); + private readonly ConcurrentDictionary _returned = new(); + private readonly Channel _failures = Channel.CreateUnbounded( + new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + private SemaphoreSlim? _outstandingWindow; + private int _windowSize; + private IRabbitMqAsyncConfirmPublishChannel? _channel; + private int _disposed; + + /// Initializes a new OverviewDB handoff publisher. + public OverviewDbHandoffPublisher( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IOptions rabbitMqOptions, + IngestionPipelineMetrics? pipelineMetrics = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(rabbitMqOptions); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _pipeline = pipelineMetrics; + _ = rabbitMqOptions; + } + + /// + public int OutstandingCount => _outstanding.Count; + + /// Gets the configured outstanding window size (tests). + internal int WindowSize => Volatile.Read(ref _windowSize); + + /// + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + var fqdn = _nntpdOptions.Value.Fqdn; + if (string.IsNullOrWhiteSpace(fqdn)) + { + throw new InvalidOperationException("OverviewDB handoff requires the generated application FQDN."); + } + + var window = EnsureOutstandingWindow(); + var publishStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await window.WaitAsync(cancellationToken).ConfigureAwait(false); + _pipeline?.RecordGateWait(publishStart); + _pipeline?.ObserveInFlightPublishes(OutstandingCount + 1); + + try + { + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var channel = await EnsureChannelAlreadyLockedAsync(cancellationToken).ConfigureAwait(false); + var sequence = await channel + .GetNextPublishSequenceNumberAsync(cancellationToken) + .ConfigureAwait(false); + if (!_outstanding.TryAdd(sequence, item)) + { + throw new InvalidOperationException( + $"Duplicate OverviewDB publish sequence number {sequence}."); + } + + try + { + var writeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await channel.PublishAsync( + OverviewDbTopology.DefaultExchange, + OverviewDbTopology.RoutingKey, + Guid.NewGuid().ToString(), + fqdn, + OverviewDbTopology.ExpirationMilliseconds, + sequence, + item.Payload, + cancellationToken) + .ConfigureAwait(false); + // Write latency only; broker confirms are asynchronous. + _pipeline?.RecordPublishAndConfirm(writeStart); + _pipeline?.RecordHandoff(publishStart); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + _outstanding.TryRemove(sequence, out _); + _returned.TryRemove(sequence, out _); + throw; + } + } + finally + { + _publishGate.Release(); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + ReleaseWindowSlot(); + _pipeline?.RecordConfirmFailure(); + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + throw; + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) => + _failures.Reader.TryRead(out item!); + + /// + public void AbandonOutstanding() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _outstanding) + { + if (_outstanding.TryRemove(pair.Key, out _)) + { + _returned.TryRemove(pair.Key, out _); + ReleaseWindowSlot(); + } + } + + while (_failures.Reader.TryRead(out _)) + { + // Drain residual failure notifications before observe. + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + AbandonOutstanding(); + await DisposeChannelAsync().ConfigureAwait(false); + _publishGate.Dispose(); + Interlocked.Exchange(ref _outstandingWindow, null)?.Dispose(); + } + + private SemaphoreSlim EnsureOutstandingWindow() + { + var existing = Volatile.Read(ref _outstandingWindow); + if (existing is not null) + { + return existing; + } + + var size = _nntpdOptions.Value.ArticleIngestion?.OverviewDbPublisherBatchSize + ?? ArticleIngestionOptions.DefaultOverviewDbPublisherBatchSize; + if (size < 1) + { + size = 1; + } + + var created = new SemaphoreSlim(size, size); + // codeql[cs/missed-using-statement] Ownership transfers into _outstandingWindow on successful CompareExchange; using would dispose the live window. toDispose is cleared only after publish succeeds. + SemaphoreSlim? toDispose = created; + try + { + var prior = Interlocked.CompareExchange(ref _outstandingWindow, created, null); + if (prior is not null) + { + return prior; + } + + toDispose = null; + Volatile.Write(ref _windowSize, size); + return created; + } + finally + { + toDispose?.Dispose(); + } + } + + /// + /// Ensures a usable async-confirm channel. Caller must hold . + /// + private async Task EnsureChannelAlreadyLockedAsync( + CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + throw new InvalidOperationException("RabbitMQ connection is not ready for OverviewDB handoff."); + } + + var existing = _channel; + if (existing is not null + && existing.Generation == handle.Generation + && existing.IsOpen) + { + return existing; + } + + if (existing is not null) + { + DetachHandlers(existing); + try + { + await existing.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort channel dispose must not fail channel recovery. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored during channel recovery dispose + } + + FailOutstandingForChannelLoss(); + _channel = null; + } + + var created = await handle.Connection + .CreateAsyncConfirmPublishChannelAsync(handle.Generation, cancellationToken) + .ConfigureAwait(false); + AttachHandlers(created); + _channel = created; + return created; + } + + private void AttachHandlers(IRabbitMqAsyncConfirmPublishChannel channel) + { + channel.BasicAcksAsync += OnBasicAckAsync; + channel.BasicNacksAsync += OnBasicNackAsync; + channel.BasicReturnAsync += OnBasicReturnAsync; + } + + private void DetachHandlers(IRabbitMqAsyncConfirmPublishChannel channel) + { + channel.BasicAcksAsync -= OnBasicAckAsync; + channel.BasicNacksAsync -= OnBasicNackAsync; + channel.BasicReturnAsync -= OnBasicReturnAsync; + } + + private Task OnBasicAckAsync(object sender, BasicAckEventArgs args) + { + // Lightweight: settle outstanding only — never publish from this callback. + CompleteOutstanding(args.DeliveryTag, args.Multiple, nack: false); + return Task.CompletedTask; + } + + private Task OnBasicNackAsync(object sender, BasicNackEventArgs args) + { + CompleteOutstanding(args.DeliveryTag, args.Multiple, nack: true); + return Task.CompletedTask; + } + + private Task OnBasicReturnAsync(object sender, BasicReturnEventArgs args) + { + // Mark returned; settlement (window release + failure dequeue) waits for ack/nack. + // RabbitMQ typically acks after returning a mandatory unroutable message. + if (TryReadPublishSequence(args.BasicProperties, out var sequence)) + { + _returned[sequence] = 0; + } + + return Task.CompletedTask; + } + + private void CompleteOutstanding(ulong deliveryTag, bool multiple, bool nack) + { + if (multiple) + { + // codeql[cs/linq/missed-where] ConcurrentDictionary keys with SettleOne side effects. + foreach (var key in _outstanding.Keys) + { + if (key <= deliveryTag) + { + SettleOne(key, nack); + } + } + } + else + { + SettleOne(deliveryTag, nack); + } + + _pipeline?.ObserveInFlightPublishes(OutstandingCount); + } + + private void SettleOne(ulong sequence, bool nack) + { + if (!_outstanding.TryRemove(sequence, out var item)) + { + return; + } + + var wasReturned = _returned.TryRemove(sequence, out _); + ReleaseWindowSlot(); + + if (nack || wasReturned) + { + _ = _failures.Writer.TryWrite(item); + _pipeline?.RecordConfirmFailure(); + } + } + + private void FailOutstandingForChannelLoss() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _outstanding) + { + if (_outstanding.TryRemove(pair.Key, out var item)) + { + _returned.TryRemove(pair.Key, out _); + _ = _failures.Writer.TryWrite(item); + ReleaseWindowSlot(); + _pipeline?.RecordConfirmFailure(); + } + } + } + + private void ReleaseWindowSlot() + { + try + { + EnsureOutstandingWindow().Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (SemaphoreFullException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task DisposeChannelAsync() + { + var channel = Interlocked.Exchange(ref _channel, null); + if (channel is null) + { + return; + } + + DetachHandlers(channel); + try + { + await channel.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort channel dispose must not fail publisher teardown. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored during publisher teardown dispose + } + } + + /// + /// Recovers the publish sequence from Basic.Return headers. + /// + /// + /// Matches RabbitMQ.Client 7.2.2 HandleReturn representations: + /// long, decimal string, and longstr byte[] (ASCII digits). + /// + internal static bool TryReadPublishSequence(IReadOnlyBasicProperties? properties, out ulong sequence) + { + sequence = 0; + if (properties?.Headers is null + || !properties.Headers.TryGetValue(Constants.PublishSequenceNumberHeader, out var raw) + || raw is null) + { + return false; + } + + return TryParsePublishSequenceNumberHeader(raw, out sequence); + } + + /// + /// Parses an AMQP table value written for . + /// + internal static bool TryParsePublishSequenceNumberHeader(object raw, out ulong sequence) + { + switch (raw) + { + case long l when l >= 0: + sequence = (ulong)l; + return true; + case int i when i >= 0: + sequence = (ulong)i; + return true; + case string s: + return ulong.TryParse( + s, + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence); + case byte[] bytes: + // Wire longstr ('S') for a decimal string arrives as UTF-8/ASCII bytes. + return ulong.TryParse( + Encoding.ASCII.GetString(bytes), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence); + default: + sequence = 0; + return false; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs new file mode 100644 index 00000000..db26b29e --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPool.cs @@ -0,0 +1,447 @@ +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// Dynamically sized pool that drains and + /// publishes through with asynchronous confirms. + /// + /// + /// Workers do not await broker confirmation per message. They wait only for outstanding + /// confirmation-window capacity and the network write. Nack/return failures are drained + /// from the publisher and requeued in-process (bounded; abandoned on shutdown). + /// + internal sealed class OverviewDbPublisherPool + { + private readonly IOverviewDbWorkQueue _queue; + private readonly IOverviewDbHandoffPublisher _publisher; + private readonly ArticleIngestionOptions _options; + private readonly ILogger _logger; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly List _workers = []; + private int _highPressureStreak; + private int _lowPressureStreak; + private int _scaleUps; + private int _scaleDowns; + + /// Initializes a new OverviewDB publisher pool. + public OverviewDbPublisherPool( + IOverviewDbWorkQueue queue, + IOverviewDbHandoffPublisher publisher, + ArticleIngestionOptions options, + ILogger logger, + IngestionPipelineMetrics? pipelineMetrics = null, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(publisher); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _publisher = publisher; + _options = options; + _logger = logger; + _pipeline = pipelineMetrics; + _time = timeProvider ?? TimeProvider.System; + } + + /// Gets the current publisher worker count. + public int WorkerCount + { + get + { + lock (_gate) + { + return _workers.Count; + } + } + } + + /// Gets scale-up count (tests). + public int ScaleUpCount => Volatile.Read(ref _scaleUps); + + /// Gets scale-down count (tests). + public int ScaleDownCount => Volatile.Read(ref _scaleDowns); + + /// Runs publisher workers and the scaler until cancelled. + public async Task RunAsync(CancellationToken cancellationToken) + { + OverviewDbPublisherPoolLogMessages.PoolStarting( + _logger, + _options.OverviewDbMinPublisherWorkers, + _options.OverviewDbMaxPublisherWorkers, + _options.OverviewDbPublisherBatchSize); + + for (var i = 0; i < _options.OverviewDbMinPublisherWorkers; i++) + { + AddWorker(cancellationToken); + } + + PublishPoolObservation(); + + var scaler = Task.Run(() => ScaleLoopAsync(cancellationToken), CancellationToken.None); + try + { + await scaler.ConfigureAwait(false); + } + finally + { + List snapshot; + lock (_gate) + { + snapshot = [.. _workers]; + _workers.Clear(); + } + + foreach (var worker in snapshot) + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + } + + var drainTimeout = TimeSpan.FromSeconds( + Math.Max(1, _options.OverviewDbPublisherShutdownSeconds)); + using var drainCts = new CancellationTokenSource(drainTimeout); + foreach (var worker in snapshot) + { + try + { + await worker.Execution.WaitAsync(drainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + } + } + + // Bounded shutdown: abandon any remaining outstanding confirms. + _publisher.AbandonOutstanding(); + PublishPoolObservation(); + OverviewDbPublisherPoolLogMessages.PoolStopped(_logger, snapshot.Count); + } + } + + /// Applies one scaling decision (tests). + internal void ApplyScaleDecisionForTests(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + ApplyScaleDecision(snapshot, workerToken); + PublishPoolObservation(); + } + + private async Task ScaleLoopAsync(CancellationToken cancellationToken) + { + var interval = TimeSpan.FromSeconds(Math.Max(1, _options.ScaleIntervalSeconds)); + using var timer = new PeriodicTimer(interval, _time); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + ApplyScaleDecision(SamplePressure(), cancellationToken); + PublishPoolObservation(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private IngestionPressureSnapshot SamplePressure() + { + var limit = _queue.MemoryLimitBytes; + var bytes = _queue.QueuedBytes; + var waiting = _queue.WaitingProducerCount; + var utilisation = limit <= 0 ? 0d : Math.Clamp(bytes / (double)limit, 0d, 1d); + var waitingSignal = waiting > 0 ? 1d : 0d; + return new IngestionPressureSnapshot( + Math.Max(utilisation, waitingSignal), + bytes, + limit, + _queue.Count, + waiting); + } + + private void ApplyScaleDecision(IngestionPressureSnapshot snapshot, CancellationToken workerToken) + { + if (snapshot.Pressure >= _options.ScaleUpPressureThreshold) + { + _highPressureStreak++; + _lowPressureStreak = 0; + if (_highPressureStreak >= _options.ScaleUpConsecutiveIntervals) + { + _highPressureStreak = 0; + if (TryAddWorker(workerToken)) + { + Interlocked.Increment(ref _scaleUps); + _pipeline?.RecordOverviewDbPublisherScaleUp(); + OverviewDbPublisherPoolLogMessages.ScaledUp(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + if (snapshot.Pressure <= _options.ScaleDownPressureThreshold) + { + _lowPressureStreak++; + _highPressureStreak = 0; + if (_lowPressureStreak >= _options.ScaleDownConsecutiveIntervals) + { + _lowPressureStreak = 0; + if (TryRemoveWorker()) + { + Interlocked.Increment(ref _scaleDowns); + _pipeline?.RecordOverviewDbPublisherScaleDown(); + OverviewDbPublisherPoolLogMessages.ScaledDown(_logger, WorkerCount, snapshot.Pressure); + } + } + + return; + } + + _highPressureStreak = 0; + _lowPressureStreak = 0; + } + + private bool TryAddWorker(CancellationToken workerToken) + { + lock (_gate) + { + if (_workers.Count >= _options.OverviewDbMaxPublisherWorkers) + { + return false; + } + + AddWorkerAlreadyLocked(workerToken); + return true; + } + } + + private void AddWorker(CancellationToken workerToken) + { + lock (_gate) + { + AddWorkerAlreadyLocked(workerToken); + } + } + + private void AddWorkerAlreadyLocked(CancellationToken workerToken) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(workerToken); + var worker = new Worker(cts) { Execution = Task.CompletedTask }; + worker.Execution = Task.Run(() => WorkerLoopAsync(worker, cts.Token), CancellationToken.None); + _workers.Add(worker); + } + + private bool TryRemoveWorker() + { + Worker? victim; + lock (_gate) + { + if (_workers.Count <= _options.OverviewDbMinPublisherWorkers) + { + return false; + } + + victim = _workers[^1]; + _workers.RemoveAt(_workers.Count - 1); + } + + _ = RetireWorkerAsync(victim); + return true; + } + + private async Task RetireWorkerAsync(Worker worker) + { + try + { + await worker.Cts.CancelAsync().ConfigureAwait(false); + await worker.Execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + finally + { + worker.Cts.Dispose(); + PublishPoolObservation(); + } + } + + private async Task WorkerLoopAsync(Worker worker, CancellationToken cancellationToken) + { + try + { + while (true) + { + DrainPublishFailures(); + + OverviewDbWorkItem? item; + try + { + item = await _queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (!_queue.IsAccepting) + { + await DrainRemainingAsync().ConfigureAwait(false); + } + + DrainPublishFailures(); + break; + } + + if (item is null) + { + DrainPublishFailures(); + break; + } + + await PublishOneAsync(item, cancellationToken).ConfigureAwait(false); + DrainPublishFailures(); + + if (cancellationToken.IsCancellationRequested && _queue.Count == 0) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + OverviewDbPublisherPoolLogMessages.WorkerFaulted(_logger, ex); + } + } + + private async Task DrainRemainingAsync() + { + while (true) + { + DrainPublishFailures(); + OverviewDbWorkItem? item; + try + { + item = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (item is null) + { + break; + } + + await PublishOneAsync(item, CancellationToken.None).ConfigureAwait(false); + } + + DrainPublishFailures(); + } + + private async Task PublishOneAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + try + { + // Does not await broker confirmation — only outstanding-window capacity + write. + await _publisher.PublishAsync(item, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown: abandon in-flight work. Do not claim the OverviewDB work queue + // is unavailable — cancellation is not a queue-admission failure. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // Local serialization / publish-write failures (e.g. WireFormattingException) + // never reached the broker; treat as publish failure + in-memory requeue. + OverviewDbHandoffLogMessages.PublishFailed(_logger, ex, item.MessageId, item.ByteLength); + await TryRequeueFailureAsync(item).ConfigureAwait(false); + } + } + + private void DrainPublishFailures() + { + while (_publisher.TryDequeuePublishFailure(out var failed)) + { + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + new InvalidOperationException("OverviewDB publish was nacked or returned."), + failed.MessageId, + failed.ByteLength); + // Fire-and-forget requeue onto the in-process work queue while accepting. + _ = TryRequeueFailureAsync(failed); + } + } + + private async Task TryRequeueFailureAsync(OverviewDbWorkItem item) + { + if (!_queue.IsAccepting) + { + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, item.MessageId); + return; + } + + var result = await _queue.EnqueueAsync(item, CancellationToken.None).ConfigureAwait(false); + if (result == ArticleEnqueueResult.Accepted) + { + OverviewDbHandoffLogMessages.Requeued(_logger, item.MessageId); + return; + } + + if (result == ArticleEnqueueResult.Unavailable) + { + OverviewDbHandoffLogMessages.RequeueUnavailable(_logger, item.MessageId); + return; + } + + // Rejected (e.g. oversized) — not an OverviewDB work-queue unavailable condition. + OverviewDbHandoffLogMessages.PublishFailed( + _logger, + new InvalidOperationException($"OverviewDB work-queue rejected requeue ({result})."), + item.MessageId, + item.ByteLength); + } + + private void PublishPoolObservation() + { + _pipeline?.ObserveOverviewDbPublisherPool( + WorkerCount, + _options.OverviewDbMinPublisherWorkers, + _options.OverviewDbMaxPublisherWorkers); + _pipeline?.ObserveOverviewDbWorkQueue(_queue.Count, _queue.QueuedBytes, _queue.WaitingProducerCount); + } + + private sealed class Worker(CancellationTokenSource cts) + { + public CancellationTokenSource Cts { get; } = cts; + + public Task Execution { get; set; } = Task.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs new file mode 100644 index 00000000..2c243fe0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbPublisherPoolLogMessages.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// Source-generated OverviewDB publisher-pool log messages. + internal static partial class OverviewDbPublisherPoolLogMessages + { + [LoggerMessage( + EventId = 2110, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool starting (min={MinWorkers}, max={MaxWorkers}, publishConcurrency={PublishConcurrency})")] + public static partial void PoolStarting( + ILogger logger, + int MinWorkers, + int MaxWorkers, + int PublishConcurrency); + + [LoggerMessage( + EventId = 2111, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool stopped (workers={Workers})")] + public static partial void PoolStopped(ILogger logger, int Workers); + + [LoggerMessage( + EventId = 2112, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool scaled up to {Workers} (pressure={Pressure:F3})")] + public static partial void ScaledUp(ILogger logger, int Workers, double Pressure); + + [LoggerMessage( + EventId = 2113, + Level = LogLevel.Information, + Message = "OverviewDB publisher pool scaled down to {Workers} (pressure={Pressure:F3})")] + public static partial void ScaledDown(ILogger logger, int Workers, double Pressure); + + [LoggerMessage( + EventId = 2114, + Level = LogLevel.Error, + Message = "OverviewDB publisher worker faulted")] + public static partial void WorkerFaulted(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs new file mode 100644 index 00000000..86104352 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkItem.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// One in-process OverviewDB handoff unit: an already-encoded + /// OverviewArticleV1 payload awaiting RabbitMQ publication. + /// + /// + /// Ownership of transfers with the item. The publisher + /// stage publishes these bytes and does not re-encode. Successful enqueue into + /// the OverviewDB work queue is not equivalent to a RabbitMQ publisher + /// confirmation; broker durability still occurs only after the OverviewDB publisher + /// stage receives a broker confirmation for the outstanding publish. + /// + public sealed class OverviewDbWorkItem + { + /// Initializes a work item that owns . + public OverviewDbWorkItem(byte[] payload, string messageId) + { + ArgumentNullException.ThrowIfNull(payload); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + Payload = payload; + MessageId = messageId; + } + + /// Gets the owned encoded OverviewArticleV1 bytes. + public byte[] Payload { get; } + + /// Gets the article Message-ID for diagnostics. + public string MessageId { get; } + + /// Gets the reserved byte-budget length (payload length). + public int ByteLength => Payload.Length; + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs new file mode 100644 index 00000000..94256afa --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/OverviewDb/OverviewDbWorkQueue.cs @@ -0,0 +1,241 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.TransitService.ArticleIngestion.OverviewDb +{ + /// + /// Byte-budgeted -backed OverviewDB handoff work queue. + /// + /// + /// In-process only. The admission bound is . + /// Accounting uses each item's . + /// + internal sealed class OverviewDbWorkQueue : IOverviewDbWorkQueue + { + private readonly Channel _channel; + private readonly object _gate = new(); + private readonly Queue _waiters = new(); + private readonly long _memoryLimit; + private long _queuedBytes; + private int _count; + private int _completed; + + /// Initializes a new OverviewDB work queue with the given byte budget. + public OverviewDbWorkQueue(long memoryLimitBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(memoryLimitBytes, 1); + _memoryLimit = memoryLimitBytes; + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public long MemoryLimitBytes => _memoryLimit; + + /// + public long QueuedBytes => Volatile.Read(ref _queuedBytes); + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public bool IsAccepting => Volatile.Read(ref _completed) == 0; + + /// + public int WaitingProducerCount + { + get + { + lock (_gate) + { + var n = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var waiter in _waiters) + { + if (waiter.IsWaiting) + { + n++; + } + } + + return n; + } + } + } + + /// + public async ValueTask EnqueueAsync( + OverviewDbWorkItem item, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + var bytes = item.ByteLength; + if (bytes > _memoryLimit) + { + return ArticleEnqueueResult.Rejected; + } + + if (!IsAccepting) + { + return ArticleEnqueueResult.Unavailable; + } + + var reserved = await ReserveAsync(bytes, cancellationToken).ConfigureAwait(false); + if (reserved != ArticleEnqueueResult.Accepted) + { + return reserved; + } + + if (!_channel.Writer.TryWrite(item)) + { + Release(bytes); + return ArticleEnqueueResult.Unavailable; + } + + Interlocked.Increment(ref _count); + return ArticleEnqueueResult.Accepted; + } + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var item)) + { + Release(item.ByteLength); + Interlocked.Decrement(ref _count); + return item; + } + } + + return null; + } + + /// + public void Complete() + { + if (Interlocked.Exchange(ref _completed, 1) == 1) + { + return; + } + + _channel.Writer.TryComplete(); + List waiters; + lock (_gate) + { + waiters = [.. _waiters]; + _waiters.Clear(); + } + + foreach (var waiter in waiters) + { + waiter.TrySetUnavailable(); + } + } + + private async ValueTask ReserveAsync(int bytes, CancellationToken cancellationToken) + { + while (true) + { + ByteWaiter waiter; + lock (_gate) + { + if (_completed != 0) + { + return ArticleEnqueueResult.Unavailable; + } + + if (_queuedBytes + bytes <= _memoryLimit) + { + _queuedBytes += bytes; + return ArticleEnqueueResult.Accepted; + } + + waiter = new ByteWaiter(bytes); + _waiters.Enqueue(waiter); + } + + try + { + return await waiter.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + lock (_gate) + { + waiter.Cancel(); + } + + throw; + } + } + } + + private void Release(int bytes) + { + lock (_gate) + { + _queuedBytes = Math.Max(0, _queuedBytes - bytes); + while (_waiters.Count > 0) + { + var next = _waiters.Peek(); + if (!next.IsWaiting) + { + _waiters.Dequeue(); + continue; + } + + if (_queuedBytes + next.Bytes > _memoryLimit) + { + break; + } + + _waiters.Dequeue(); + _queuedBytes += next.Bytes; + next.TrySetAccepted(); + } + } + } + + private sealed class ByteWaiter + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _state; // 0 waiting, 1 completed + + public ByteWaiter(int bytes) => Bytes = bytes; + + public int Bytes { get; } + + public bool IsWaiting => Volatile.Read(ref _state) == 0; + + public Task Task => _tcs.Task; + + public void TrySetAccepted() + { + if (Interlocked.Exchange(ref _state, 1) == 0) + { + _tcs.TrySetResult(ArticleEnqueueResult.Accepted); + } + } + + public void TrySetUnavailable() + { + if (Interlocked.Exchange(ref _state, 1) == 0) + { + _tcs.TrySetResult(ArticleEnqueueResult.Unavailable); + } + } + + public void Cancel() + { + Interlocked.Exchange(ref _state, 1); + _tcs.TrySetCanceled(); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/PathSurveyLineFormatter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/PathSurveyLineFormatter.cs new file mode 100644 index 00000000..0815881e --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/PathSurveyLineFormatter.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Serializes one INN-style Path-survey observation as Path: {canonical-path}\n. + /// + /// + /// The wire format is an application invariant. This type copies + /// ArticleRecord.Path bytes as-is: it does not parse Path, invent hops, + /// or add Message-ID, Newsgroups, size, timestamp, peer, or disposition. + /// + internal static class PathSurveyLineFormatter + { + /// ASCII Path: prefix including the header-style space. + internal static ReadOnlySpan Prefix => "Path: "u8; + + /// Formats into as one LF-terminated line. + /// Bytes written. + public static int Write(Span destination, ReadOnlySpan canonicalPath) + { + var prefix = Prefix; + prefix.CopyTo(destination); + var written = prefix.Length; + canonicalPath.CopyTo(destination[written..]); + written += canonicalPath.Length; + destination[written++] = (byte)'\n'; + return written; + } + + /// Byte count required to format including the trailing LF. + public static int RequiredLength(ReadOnlySpan canonicalPath) => + Prefix.Length + canonicalPath.Length + 1; + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogNewsLogWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogNewsLogWriter.cs new file mode 100644 index 00000000..d2ffa299 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogNewsLogWriter.cs @@ -0,0 +1,58 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Emits already-decided news events through the dedicated Serilog news pipeline. + /// + /// + /// File path, rolling, retention, compression, and buffering come from + /// Serilog:News. The INN line is produced only by + /// . This writer does not open, rotate, or + /// compress files itself. Serilog host disposal flushes the pipeline. + /// + public sealed class SerilogNewsLogWriter : INewsLogWriter, IDisposable + { + private readonly Serilog.Core.Logger _logger; + + /// Initializes a writer that owns a dedicated Serilog news logger. + public SerilogNewsLogWriter(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _logger = NntpdNewsLogging.CreateLogger(configuration); + } + + /// Initializes a writer that uses an already-constructed news logger (tests). + public SerilogNewsLogWriter(Serilog.Core.Logger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// + public void Write(in NewsLogEvent evt) + { + var messageId = Encoding.ASCII.GetString(evt.MessageId.Span); + var feed = evt.Feed.IsEmpty ? "?" : Encoding.ASCII.GetString(evt.Feed.Span); + var sites = evt.Sites.IsEmpty ? string.Empty : Encoding.ASCII.GetString(evt.Sites.Span); + var reason = evt.Reason.IsEmpty ? string.Empty : Encoding.ASCII.GetString(evt.Reason.Span); + _logger + .ForContext(NntpdNewsLogging.DispositionProperty, (char)evt.Disposition) + .ForContext(NntpdNewsLogging.FeedProperty, feed) + .ForContext(NntpdNewsLogging.MessageIdProperty, messageId) + .ForContext(NntpdNewsLogging.SitesProperty, sites) + .ForContext(NntpdNewsLogging.SizeProperty, evt.Size) + .ForContext(NntpdNewsLogging.ResponseCodeProperty, evt.ResponseCode) + .ForContext(NntpdNewsLogging.ReasonProperty, reason) + .Information("news"); + } + + /// + public void Flush() => _logger.Dispose(); + + /// + public void Dispose() => _logger.Dispose(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogPathSurveyWriter.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogPathSurveyWriter.cs new file mode 100644 index 00000000..2ebc9e6e --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/SerilogPathSurveyWriter.cs @@ -0,0 +1,61 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// + /// Emits canonical Path observations through the dedicated Serilog Path-survey pipeline. + /// + /// + /// File path, rolling, retention, compression, and buffering come from + /// Serilog:Inpaths. The Path line is produced only by + /// . This writer does not open, rotate, or + /// compress files itself. Serilog host disposal flushes the pipeline. Path + /// bytes are copied at so later ArtData release cannot + /// mutate a buffered event. Observations are not retained in an in-memory + /// collection; the Path-survey file is the persistence mechanism. + /// + public sealed class SerilogPathSurveyWriter : IPathSurveyWriter, IDisposable + { + private readonly Serilog.Core.Logger _logger; + + /// Initializes a writer that owns a dedicated Serilog Path-survey logger. + public SerilogPathSurveyWriter( + IConfiguration configuration, + ICompletedPathSurveyFileHandler? completedFileHandler = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(configuration); + _logger = NntpdPathSurveyLogging.CreateLogger( + configuration, + completedFileHandler ?? NullCompletedPathSurveyFileHandler.Instance, + logger ?? NullLogger.Instance); + } + + /// Initializes a writer that uses an already-constructed Path-survey logger (tests). + public SerilogPathSurveyWriter(Serilog.Core.Logger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// + public void Write(ReadOnlySpan canonicalPath) + { + var path = canonicalPath.IsEmpty + ? string.Empty + : Encoding.Latin1.GetString(canonicalPath); + _logger + .ForContext(NntpdPathSurveyLogging.PathProperty, path) + .Information("inpaths"); + } + + /// + public void Flush() => _logger.Dispose(); + + /// + public void Dispose() => _logger.Dispose(); + } +} diff --git a/src/VectorNNTPD.TransitService/ArticleIngestion/SpoolLogMessages.cs b/src/VectorNNTPD.TransitService/ArticleIngestion/SpoolLogMessages.cs new file mode 100644 index 00000000..093cb954 --- /dev/null +++ b/src/VectorNNTPD.TransitService/ArticleIngestion/SpoolLogMessages.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.TransitService.ArticleIngestion +{ + /// Source-generated incoming spool writer log messages. + internal static partial class SpoolLogMessages + { + [LoggerMessage( + EventId = 2000, + Level = LogLevel.Warning, + Message = "Incoming spool writer stop canceled with {Queued} article(s) still buffered")] + public static partial void StopCanceledWithBufferedArticles(ILogger logger, int Queued); + + [LoggerMessage( + EventId = 2001, + Level = LogLevel.Error, + Message = "Incoming spool writer stopped with an error")] + public static partial void WriterStoppedWithError(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2002, + Level = LogLevel.Information, + Message = "Incoming spool writer started (transit queue memory limit {MemoryLimit} bytes, max article {MaxBytes} bytes)")] + public static partial void WriterStarted(ILogger logger, long MemoryLimit, int MaxBytes); + + [LoggerMessage( + EventId = 2003, + Level = LogLevel.Error, + Message = "Incoming article {MessageId} ({Bytes} bytes) failed after OverviewDB handoff and was not requeued")] + public static partial void FinishFailed(ILogger logger, Exception exception, string MessageId, int Bytes); + + [LoggerMessage( + EventId = 2004, + Level = LogLevel.Information, + Message = "Incoming spool writer stopped")] + public static partial void WriterStopped(ILogger logger); + + [LoggerMessage( + EventId = 2005, + Level = LogLevel.Error, + Message = "Failed to write news log for incoming article {MessageId}; article remains accepted and is not reprocessed")] + public static partial void NewsLogFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2006, + Level = LogLevel.Error, + Message = "Failed to flush the news log during incoming spool writer stop")] + public static partial void NewsLogFlushFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2007, + Level = LogLevel.Error, + Message = "Failed to write Path-survey log for incoming article {MessageId}; article remains accepted and is not reprocessed")] + public static partial void PathSurveyFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2008, + Level = LogLevel.Error, + Message = "Failed to flush the Path-survey log during incoming spool writer stop")] + public static partial void PathSurveyFlushFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/AccountCacheKeys.cs b/src/VectorNNTPD.TransitService/Authentication/AccountCacheKeys.cs new file mode 100644 index 00000000..e70cfc38 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/AccountCacheKeys.cs @@ -0,0 +1,55 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Redis key material for the short-lived AUTHINFO account-record cache. + /// + /// + /// Key: nntpd:account:{sha256hex(accountName)}. The plaintext AUTHINFO + /// username is hashed with the same UTF-8 SHA-256 convention as other NNTPD + /// Redis account keys so listings do not expose account names. The value is a + /// complete MySQL nntpusers snapshot with a Redis-native TTL. This is + /// not a session, lease, or byte-ledger key. + /// + internal static class AccountCacheKeys + { + /// ASCII namespace for account-record cache keys. + public static ReadOnlySpan NamespacePrefix => "nntpd:account:"u8; + + /// Builds the account-record cache key for . + public static byte[] Create(string accountName) + { + var hex = HashHex(accountName); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + + /// + /// Returns lowercase hex SHA-256 of the UTF-8 wire account name. + /// The caller supplies the same string used for the MySQL lookup. + /// + public static string HashHex(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + return Convert.ToHexStringLower(hash); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/AccountCacheLogMessages.cs b/src/VectorNNTPD.TransitService/Authentication/AccountCacheLogMessages.cs new file mode 100644 index 00000000..9d273f32 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/AccountCacheLogMessages.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Authentication +{ + /// Source-generated account-cache diagnostics. Never includes secrets or payloads. + internal static partial class AccountCacheLogMessages + { + [LoggerMessage(EventId = 2340, Level = LogLevel.Debug, Message = "Account cache HIT accountHash={AccountHash}")] + public static partial void Hit(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2341, Level = LogLevel.Debug, Message = "Account cache MISS accountHash={AccountHash}")] + public static partial void Miss(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2342, Level = LogLevel.Debug, Message = "Account cache unavailable; falling back to MySQL accountHash={AccountHash}")] + public static partial void Unavailable(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2343, Level = LogLevel.Debug, Message = "Account cache populate skipped accountHash={AccountHash}")] + public static partial void PopulateSkipped(ILogger logger, string accountHash); + + [LoggerMessage(EventId = 2344, Level = LogLevel.Debug, Message = "Account cache payload rejected accountHash={AccountHash}")] + public static partial void PayloadRejected(ILogger logger, string accountHash); + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/ArticleTypeAccessPolicy.cs b/src/VectorNNTPD.TransitService/Authentication/ArticleTypeAccessPolicy.cs new file mode 100644 index 00000000..9e7640dd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/ArticleTypeAccessPolicy.cs @@ -0,0 +1,45 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Session; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Shared account ArtType capability checks. Distinct from PostFilter policy + /// art_type ENUM rows and from flags. + /// + public static class ArticleTypeAccessPolicy + { + /// + /// Returns whether the session may inject . + /// Unauthenticated sessions and accounts without a capability snapshot are + /// unrestricted. Authenticated accounts use the flags loaded at AUTHINFO. + /// + public static bool CanPostArticleType(NntpSession session, ArticleType articleType) + { + ArgumentNullException.ThrowIfNull(session); + return Allows(session.AccountPolicy, articleType); + } + + /// + /// Returns whether the session may read . + /// Uses the same capability mask as posting. Retrieval cannot call this + /// until stored articles expose . + /// + public static bool CanReadArticleType(NntpSession session, ArticleType articleType) + { + ArgumentNullException.ThrowIfNull(session); + return Allows(session.AccountPolicy, articleType); + } + + /// Evaluates a captured account policy, or unrestricted when none applies. + public static bool Allows(NntpAccountPolicy? policy, ArticleType articleType) + { + if (policy is null) + { + return true; + } + + return ArticleTypeCapabilities.Allows(policy.AllowedArtTypes, articleType); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/AuthenticationLogMessages.cs b/src/VectorNNTPD.TransitService/Authentication/AuthenticationLogMessages.cs new file mode 100644 index 00000000..9e7ee6e6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/AuthenticationLogMessages.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.TransitService.Authentication +{ + /// Source-generated authentication diagnostics. Never includes secrets. + internal static partial class AuthenticationLogMessages + { + [LoggerMessage(EventId = 2302, Level = LogLevel.Debug, Message = "Authentication rejected: invalid credentials mechanism={Mechanism} user={Username} ip={ClientIp}")] + public static partial void InvalidCredentials(ILogger logger, string mechanism, string username, string clientIp); + + [LoggerMessage(EventId = 2303, Level = LogLevel.Information, Message = "Authentication succeeded mechanism={Mechanism} user={Username} ip={ClientIp} byteLimit={ByteLimit} rateLimitBps={RateLimitBps} customer={CustomerId}")] + public static partial void Succeeded( + ILogger logger, + string mechanism, + string username, + string clientIp, + long byteLimit, + int rateLimitBps, + string customerId); + + [LoggerMessage(EventId = 2304, Level = LogLevel.Error, Message = "Authentication backend failure mechanism={Mechanism} user={Username}")] + public static partial void BackendFailed(ILogger logger, Exception exception, string mechanism, string username); + + [LoggerMessage(EventId = 2305, Level = LogLevel.Error, Message = "nntpusers lookup failed user={Username}")] + public static partial void UserLookupFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2306, Level = LogLevel.Information, Message = "Authentication admission rejected user={Username} reason={Reason} ip={ClientIp}")] + public static partial void AdmissionRejected(ILogger logger, string username, string reason, string clientIp); + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/CachedNntpUserRecordStore.cs b/src/VectorNNTPD.TransitService/Authentication/CachedNntpUserRecordStore.cs new file mode 100644 index 00000000..7aa813f4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/CachedNntpUserRecordStore.cs @@ -0,0 +1,227 @@ +using System.Collections.Concurrent; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Short-lived Redis cache around MySQL SELECT. + /// + /// + /// Caches a complete account-record snapshot only. Does not cache password + /// verification or session admission. Redis GET/SET failure falls back to the + /// inner store and does not trip the shared Redis availability circuit. + /// Concurrent cold lookups for the same wire account name share one inner + /// SELECT. Missing accounts are not cached. Idle single-flight state is + /// removed when the shared lookup completes. + /// + internal sealed class CachedNntpUserRecordStore : INntpUserRecordStore + { + internal static readonly TimeSpan CacheTtl = TimeSpan.FromSeconds(10); + + private readonly INntpUserRecordStore _inner; + private readonly IRedisService _redis; + private readonly ILogger _logger; + private readonly ConcurrentDictionary> _inflight = + new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + public CachedNntpUserRecordStore( + INntpUserRecordStore inner, + IRedisService redis, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + _inner = inner; + _redis = redis; + _logger = logger; + } + + /// Gets the number of in-flight account-scoped lookups (tests). + internal int InFlightCount => _inflight.Count; + + /// + public async ValueTask TryGetUserAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + + var cached = await TryReadCacheAsync(accountName, cancellationToken).ConfigureAwait(false); + if (cached.Status == CacheReadStatus.Hit) + { + AccountCacheLogMessages.Hit(_logger, AccountCacheKeys.HashHex(accountName)); + return cached.Record; + } + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var shared = _inflight.GetOrAdd(accountName, tcs.Task); + if (!ReferenceEquals(shared, tcs.Task)) + { + return await shared.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + try + { + var record = await LoadAndPopulateAsync(accountName, cached.Status).ConfigureAwait(false); + tcs.SetResult(record); + return record; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + tcs.SetException(ex); + throw; + } + finally + { + _ = _inflight.TryRemove(new KeyValuePair>(accountName, tcs.Task)); + } + } + + private async Task LoadAndPopulateAsync(string accountName, CacheReadStatus priorStatus) + { + var again = await TryReadCacheAsync(accountName, CancellationToken.None).ConfigureAwait(false); + if (again.Status == CacheReadStatus.Hit) + { + AccountCacheLogMessages.Hit(_logger, AccountCacheKeys.HashHex(accountName)); + return again.Record; + } + + if (priorStatus == CacheReadStatus.Miss || again.Status == CacheReadStatus.Miss) + { + AccountCacheLogMessages.Miss(_logger, AccountCacheKeys.HashHex(accountName)); + } + + var record = await _inner.TryGetUserAsync(accountName, CancellationToken.None).ConfigureAwait(false); + if (record is not null) + { + await TryWriteCacheAsync(accountName, record).ConfigureAwait(false); + } + + return record; + } + + private async ValueTask TryReadCacheAsync(string accountName, CancellationToken cancellationToken) + { + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + byte[]? payload; + try + { + payload = await database.GetAsync(AccountCacheKeys.Create(accountName), cancellationToken) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + // Cache is an optimization. Do not trip the shared Redis circuit. + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.Unavailable(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Unavailable; + } + + if (payload is null) + { + return CacheRead.Missed; + } + + if (!NntpAccountCacheCodec.TryDecode(payload, accountName, out var record) || record is null) + { + AccountCacheLogMessages.PayloadRejected(_logger, AccountCacheKeys.HashHex(accountName)); + return CacheRead.Missed; + } + + return CacheRead.Found(record); + } + + private async ValueTask TryWriteCacheAsync(string accountName, NntpUserRecord record) + { + byte[] payload; + try + { + payload = NntpAccountCacheCodec.Encode(record); + } + catch (ArgumentOutOfRangeException) + { + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + return; + } + + try + { + await database.SetAsync( + AccountCacheKeys.Create(accountName), + payload, + CacheTtl, + CancellationToken.None) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _redis.AbandonOperation(isRecoveryProbe); + AccountCacheLogMessages.PopulateSkipped(_logger, AccountCacheKeys.HashHex(accountName)); + } + } + + private enum CacheReadStatus + { + Hit, + Miss, + Unavailable, + } + + private readonly record struct CacheRead(CacheReadStatus Status, NntpUserRecord? Record) + { + public static CacheRead Found(NntpUserRecord record) => new(CacheReadStatus.Hit, record); + + public static CacheRead Missed { get; } = new(CacheReadStatus.Miss, null); + + public static CacheRead Unavailable { get; } = new(CacheReadStatus.Unavailable, null); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/CompositeNntpAuthenticationProvider.cs b/src/VectorNNTPD.TransitService/Authentication/CompositeNntpAuthenticationProvider.cs new file mode 100644 index 00000000..8a56c4fd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/CompositeNntpAuthenticationProvider.cs @@ -0,0 +1,71 @@ +using System.Net; +using VectorNNTPD.TransitService.Session.Authentication; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// AUTHINFO USER/PASS provider: newsmaster first (when configured), then MySQL reader accounts. + /// + /// + /// A newsmaster username match never falls through to MySQL. Transit peer AUTHINFO is + /// selected only when the session AUTHINFO authority is Transit (MODE STREAM) and is + /// handled by , + /// not this type. This provider is Reader-authority only. + /// + public sealed class CompositeNntpAuthenticationProvider : INntpAuthenticationProvider + { + private readonly INntpAuthenticationProvider _newsmaster; + private readonly string? _newsmasterUsername; + private readonly MySqlNntpCredentialValidator _mysql; + + /// Initializes a composite provider. + public CompositeNntpAuthenticationProvider( + INntpAuthenticationProvider newsmaster, + string? newsmasterUsername, + MySqlNntpCredentialValidator mysql) + { + ArgumentNullException.ThrowIfNull(newsmaster); + ArgumentNullException.ThrowIfNull(mysql); + _newsmaster = newsmaster; + _newsmasterUsername = string.IsNullOrWhiteSpace(newsmasterUsername) ? null : newsmasterUsername.Trim(); + _mysql = mysql; + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + AuthenticateAsync(username, password, IPAddress.None, cancellationToken); + + /// + public async ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(clientIp); + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + if (_newsmasterUsername is not null + && string.Equals(username, _newsmasterUsername, StringComparison.Ordinal)) + { + return await _newsmaster.AuthenticateAsync(username, password, cancellationToken).ConfigureAwait(false); + } + + return await _mysql + .ValidatePasswordAsync( + NntpAuthMechanisms.AuthInfoUserPass, + username, + password, + clientIp, + cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/INntpUserRecordStore.cs b/src/VectorNNTPD.TransitService/Authentication/INntpUserRecordStore.cs new file mode 100644 index 00000000..05827eb8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/INntpUserRecordStore.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.TransitService.Authentication +{ + /// Looks up an NNTP account row by plaintext wire username. + public interface INntpUserRecordStore + { + /// + /// Returns the account snapshot, or when no row exists. + /// Disabled accounts are still returned; enablement is enforced by the validator. + /// + /// Backend failure. Must not be mapped to invalid credentials. + ValueTask TryGetUserAsync(string accountName, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/MySqlNntpCredentialValidator.cs b/src/VectorNNTPD.TransitService/Authentication/MySqlNntpCredentialValidator.cs new file mode 100644 index 00000000..998596f2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/MySqlNntpCredentialValidator.cs @@ -0,0 +1,217 @@ +using System.Net; +using VectorNNTPD.TransitService.Authentication.Sasl; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.SessionState; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Validates AUTHINFO PASS / SASL password mechanisms and finalizes SCRAM/CRAM against MySQL. + /// + /// + /// Account rows come from (Redis 10s cache, then + /// MySQL). This type does not keep a process-local full-record cache. Password + /// and SASL proofs are verified on every attempt. + /// + public sealed class MySqlNntpCredentialValidator + { + /// Reader + posting privileges granted to every successful MySQL account. + public static NntpAuthorization ReaderPrivileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private readonly INntpUserRecordStore _store; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlNntpCredentialValidator(INntpUserRecordStore store, ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _logger = logger; + } + + /// Validates a cleartext password for AUTHINFO PASS, PLAIN, or LOGIN. + public async ValueTask ValidatePasswordAsync( + string mechanism, + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(mechanism); + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(clientIp); + var ip = FormatClientIp(clientIp); + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !record.AllowAuthPlain + || !NntpPasswordComparer.EqualsAscii(record.AccountPassword, password)) + { + AuthenticationLogMessages.InvalidCredentials(_logger, mechanism, username, ip); + return NntpAuthenticationResult.Failed; + } + + return Succeed(mechanism, record, ip); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.BackendFailed(_logger, ex, mechanism, username); + return NntpAuthenticationResult.TransientFailure; + } + } + + /// + /// Loads the nntpusers row for a SCRAM username without applying + /// enabled/SCRAM policy. Callers decide real vs dummy verifier. + /// + public async ValueTask TryLoadUserAsync( + string username, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + try + { + return await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + } + + /// Loads SCRAM material when the account may use SCRAM-SHA-256. + public async ValueTask TryGetScramRecordAsync( + string username, + CancellationToken cancellationToken = default) + { + var record = await TryLoadUserAsync(username, cancellationToken).ConfigureAwait(false); + if (!ScramStoredCredential.TryGetAuthorized(record, out _)) + { + return null; + } + + return record; + } + + /// Loads a CRAM-MD5 HMAC key from the decrypted password. + public async ValueTask TryGetCramRecordAsync( + string username, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !record.AllowAuthPlain + || !NntpPasswordComparer.IsAscii(record.AccountPassword) + || record.AccountPassword.Length == 0) + { + return null; + } + + return record; + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + } + + /// Finalizes SCRAM or CRAM after wire-level proof verification. + public async ValueTask CompleteSaslAccountAsync( + string mechanism, + string username, + IPAddress clientIp, + bool requireScram, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(username)) + { + return NntpAuthenticationResult.Failed; + } + + ArgumentNullException.ThrowIfNull(clientIp); + var ip = FormatClientIp(clientIp); + try + { + var record = await _store.TryGetUserAsync(username, cancellationToken).ConfigureAwait(false); + if (record is null + || !record.IsEnabled + || !(requireScram ? record.AllowAuthScram256 : record.AllowAuthPlain)) + { + AuthenticationLogMessages.InvalidCredentials(_logger, mechanism, username, ip); + return NntpAuthenticationResult.Failed; + } + + return Succeed(mechanism, record, ip); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.BackendFailed(_logger, ex, mechanism, username); + return NntpAuthenticationResult.TransientFailure; + } + } + + private NntpAuthenticationResult Succeed(string mechanism, NntpUserRecord record, string clientIp) + { + AuthenticationLogMessages.Succeeded( + _logger, + mechanism, + record.AccountName, + clientIp, + record.ByteLimit, + record.RateLimitBps, + record.CustomerId); + return NntpAuthenticationResult.Success( + record.AccountName, + ReaderPrivileges, + NntpAccountPolicy.FromRecord(record)); + } + + internal static string FormatClientIp(IPAddress clientIp) => + SourceAddressIdentity.Format(clientIp); + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/MySqlUserRecordStore.cs b/src/VectorNNTPD.TransitService/Authentication/MySqlUserRecordStore.cs new file mode 100644 index 00000000..5c92e2ba --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/MySqlUserRecordStore.cs @@ -0,0 +1,50 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Loads nntpusers through the existing NntpDB connection pool. + /// + public sealed class MySqlUserRecordStore : INntpUserRecordStore + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlUserRecordStore(NntpDbService nntpDb, ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask TryGetUserAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryUserAccountAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AuthenticationLogMessages.UserLookupFailed(_logger, ex, accountName); + throw new NntpDbUnavailableException("MySQL nntpusers lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpAccountCacheCodec.cs b/src/VectorNNTPD.TransitService/Authentication/NntpAccountCacheCodec.cs new file mode 100644 index 00000000..57455b50 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpAccountCacheCodec.cs @@ -0,0 +1,237 @@ +using System.Buffers.Binary; +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Explicit binary snapshot of one MySQL nntpusers row for Redis. + /// + /// + /// Versioned little-endian layout. Preserves nullability as empty strings or + /// empty byte spans, booleans as 0/1, and integer widths without truncation. + /// Does not include account_type. The payload contains credential + /// material and must not be logged. + /// + internal static class NntpAccountCacheCodec + { + internal const byte Version = 2; + + internal const byte Version1 = 1; + + private const int MaxAccountNameBytes = 512; + private const int MaxPasswordBytes = 4096; + private const int MaxScramBytes = 1024; + private const int MaxCustomerIdBytes = 128; + + /// Serializes to the version-1 cache payload. + public static byte[] Encode(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + var accountName = Encoding.UTF8.GetBytes(record.AccountName); + var password = Encoding.UTF8.GetBytes(record.AccountPassword); + var customerId = Encoding.UTF8.GetBytes(record.CustomerId); + var salt = record.ScramSalt; + var storedKey = record.ScramStoredKey; + var serverKey = record.ScramServerKey; + if (accountName.Length > MaxAccountNameBytes + || password.Length > MaxPasswordBytes + || salt.Length > MaxScramBytes + || storedKey.Length > MaxScramBytes + || serverKey.Length > MaxScramBytes + || customerId.Length > MaxCustomerIdBytes) + { + throw new ArgumentOutOfRangeException(nameof(record), "Account cache field exceeds encoding limits."); + } + + var length = 1 + + 4 + accountName.Length + + 4 + password.Length + + 1 + + 1 + + 4 + salt.Length + + 4 + + 4 + storedKey.Length + + 4 + serverKey.Length + + 4 + + 8 + + 4 + + 4 + + 1 + + 4 + customerId.Length + + 4; + var buffer = new byte[length]; + var span = buffer.AsSpan(); + span[0] = Version; + var offset = 1; + WriteBytes(span, ref offset, accountName); + WriteBytes(span, ref offset, password); + span[offset++] = record.AllowAuthPlain ? (byte)1 : (byte)0; + span[offset++] = record.AllowAuthScram256 ? (byte)1 : (byte)0; + WriteBytes(span, ref offset, salt.Span); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.ScramIterations); + offset += 4; + WriteBytes(span, ref offset, storedKey.Span); + WriteBytes(span, ref offset, serverKey.Span); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.RateLimitBps); + offset += 4; + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(offset), record.ByteLimit); + offset += 8; + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.SessionLimit); + offset += 4; + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), record.SrcIpLimit); + offset += 4; + span[offset++] = record.IsEnabled ? (byte)1 : (byte)0; + WriteBytes(span, ref offset, customerId); + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(offset), (int)record.AllowedArtTypes); + return buffer; + } + + /// + /// Decodes when it is a well-formed snapshot for + /// . Returns for any + /// corrupt, truncated, or account-mismatched payload. + /// + public static bool TryDecode(ReadOnlySpan payload, string accountName, out NntpUserRecord? record) + { + record = null; + if (payload.Length < 2 || (payload[0] != Version && payload[0] != Version1)) + { + return false; + } + + var version = payload[0]; + var offset = 1; + if (!TryReadBytes(payload, ref offset, MaxAccountNameBytes, out var cachedName) + || !TryReadBytes(payload, ref offset, MaxPasswordBytes, out var password) + || !TryReadByte(payload, ref offset, out var allowPlain) + || allowPlain > 1 + || !TryReadByte(payload, ref offset, out var allowScram) + || allowScram > 1 + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var salt) + || !TryReadInt32(payload, ref offset, out var iterations) + || iterations < 0 + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var storedKey) + || !TryReadBytes(payload, ref offset, MaxScramBytes, out var serverKey) + || !TryReadInt32(payload, ref offset, out var rateLimitBps) + || !TryReadInt64(payload, ref offset, out var byteLimit) + || !TryReadInt32(payload, ref offset, out var sessionLimit) + || !TryReadInt32(payload, ref offset, out var srcIpLimit) + || !TryReadByte(payload, ref offset, out var enabled) + || enabled > 1 + || !TryReadBytes(payload, ref offset, MaxCustomerIdBytes, out var customerId)) + { + return false; + } + + var allowedArtTypes = ArticleTypeCapabilities.All; + if (version == Version) + { + if (!TryReadInt32(payload, ref offset, out var artTypeBits) || offset != payload.Length) + { + return false; + } + + allowedArtTypes = (ArticleType)artTypeBits; + } + else if (offset != payload.Length) + { + return false; + } + + var decodedName = Encoding.UTF8.GetString(cachedName); + if (!string.Equals(decodedName, accountName, StringComparison.Ordinal)) + { + return false; + } + + record = new NntpUserRecord( + decodedName, + Encoding.UTF8.GetString(password), + allowPlain == 1, + allowScram == 1, + salt.ToArray(), + iterations, + storedKey.ToArray(), + serverKey.ToArray(), + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + enabled == 1, + Encoding.UTF8.GetString(customerId), + allowedArtTypes); + return true; + } + + private static void WriteBytes(Span destination, ref int offset, ReadOnlySpan value) + { + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset), value.Length); + offset += 4; + value.CopyTo(destination.Slice(offset)); + offset += value.Length; + } + + private static bool TryReadBytes( + ReadOnlySpan source, + ref int offset, + int maxLength, + out ReadOnlySpan value) + { + value = default; + if (offset > source.Length - 4) + { + return false; + } + + var length = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(offset)); + offset += 4; + if (length < 0 || length > maxLength || offset > source.Length - length) + { + return false; + } + + value = source.Slice(offset, length); + offset += length; + return true; + } + + private static bool TryReadByte(ReadOnlySpan source, ref int offset, out byte value) + { + if (offset >= source.Length) + { + value = 0; + return false; + } + + value = source[offset++]; + return true; + } + + private static bool TryReadInt32(ReadOnlySpan source, ref int offset, out int value) + { + if (offset > source.Length - 4) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(offset)); + offset += 4; + return true; + } + + private static bool TryReadInt64(ReadOnlySpan source, ref int offset, out long value) + { + if (offset > source.Length - 8) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt64LittleEndian(source.Slice(offset)); + offset += 8; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpAccountPolicy.cs b/src/VectorNNTPD.TransitService/Authentication/NntpAccountPolicy.cs new file mode 100644 index 00000000..7fc1e985 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpAccountPolicy.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// Authenticated account policy copied from nntpusers after a successful credential check. + /// + /// + /// Every authenticated account has both remaining-byte and rate policies. + /// Session and source-IP limits are enforced by + /// . + /// is enforced by + /// + /// after SessionState admission. is the AUTHINFO-time + /// remaining-byte snapshot and is not enforced here; live remaining is observed by + /// . + /// + public sealed class NntpAccountPolicy + { + /// Initializes a new account policy snapshot. + public NntpAccountPolicy( + string username, + int rateLimitBps, + long byteLimit, + int sessionLimit, + int srcIpLimit, + string customerId, + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + Username = username; + RateLimitBps = rateLimitBps; + ByteLimit = byteLimit; + SessionLimit = sessionLimit; + SrcIpLimit = srcIpLimit; + CustomerId = customerId ?? string.Empty; + AllowedArtTypes = allowedArtTypes; + } + + /// Gets the plaintext wire username. + public string Username { get; } + + /// + /// Gets account_rate_limit in bits per second. + /// Examples: 240 = 240 bps, 2_400 = 2,400 bps = 300 B/s, + /// 1_000_000 = 1 Mbps, 10_000_000 = 10 Mbps. + /// 0 is unlimited. Negative values are treated as unlimited. + /// + public int RateLimitBps { get; } + + /// + /// Gets the AUTHINFO-time snapshot of account_byte_limit. + /// This is remaining bytes at last user-record load: 0 is exhausted, + /// not unlimited. Live remaining is observed by + /// and must not + /// be taken from this cached snapshot. + /// + public long ByteLimit { get; } + + /// Gets the cluster-wide authenticated session cap. 0 is unlimited. + public int SessionLimit { get; } + + /// Gets the cluster-wide distinct source-IP cap. 0 is unlimited. + public int SrcIpLimit { get; } + + /// Gets the customer/tenant identifier. + public string CustomerId { get; } + + /// + /// Gets the account ArtType capability flags loaded at AUTHINFO. + /// is unrestricted and is the + /// default for existing users. + /// + public ArticleType AllowedArtTypes { get; } + + /// + /// Gets whether cluster admission must run: session limit, source-IP limit, + /// or a positive rate that needs a cluster session count. + /// + public bool RequiresAdmission => SessionLimit > 0 || SrcIpLimit > 0 || RequiresRateTracking; + + /// + /// Gets whether this account has a positive account_rate_limit. + /// 0 remains unlimited and does not participate in rate allocation. + /// + public bool RequiresRateTracking => RateLimitBps > 0; + + /// Builds policy from a validated user record. + public static NntpAccountPolicy FromRecord(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return new( + record.AccountName, + record.RateLimitBps, + record.ByteLimit, + record.SessionLimit, + record.SrcIpLimit, + record.CustomerId, + record.AllowedArtTypes); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpAuthMechanisms.cs b/src/VectorNNTPD.TransitService/Authentication/NntpAuthMechanisms.cs new file mode 100644 index 00000000..17d27d7d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpAuthMechanisms.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.TransitService.Authentication +{ + /// Canonical authentication mechanism labels used in logs and SASL dispatch. + internal static class NntpAuthMechanisms + { + public const string AuthInfoUserPass = "AUTHINFO USER/PASS"; + public const string SaslPlain = "SASL PLAIN"; + public const string SaslLogin = "SASL LOGIN"; + public const string SaslCramMd5 = "SASL CRAM-MD5"; + public const string SaslScramSha256 = "SASL SCRAM-SHA-256"; + + public const string WirePlain = "PLAIN"; + public const string WireLogin = "LOGIN"; + public const string WireCramMd5 = "CRAM-MD5"; + public const string WireScramSha256 = "SCRAM-SHA-256"; + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpPasswordComparer.cs b/src/VectorNNTPD.TransitService/Authentication/NntpPasswordComparer.cs new file mode 100644 index 00000000..8a529187 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpPasswordComparer.cs @@ -0,0 +1,76 @@ +using System.Buffers; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// Constant-time ASCII password comparison used by AUTHINFO PASS, PLAIN, and LOGIN. + internal static class NntpPasswordComparer + { + private const int StackallocThresholdBytes = 4096; + + /// + /// Returns only when both strings are non-null ASCII of equal length + /// and identical content. + /// + public static bool EqualsAscii(string? storedPassword, string? suppliedPassword) + { + if (storedPassword is null || suppliedPassword is null) + { + return false; + } + + if (!IsAscii(storedPassword) || !IsAscii(suppliedPassword)) + { + return false; + } + + var storedLength = storedPassword.Length; + var suppliedLength = suppliedPassword.Length; + var maxLength = Math.Max(storedLength, suppliedLength); + if (maxLength <= StackallocThresholdBytes) + { + Span left = stackalloc byte[maxLength]; + Span right = stackalloc byte[maxLength]; + left.Clear(); + right.Clear(); + Encoding.ASCII.GetBytes(storedPassword, left); + Encoding.ASCII.GetBytes(suppliedPassword, right); + return CryptographicOperations.FixedTimeEquals(left, right) && storedLength == suppliedLength; + } + + var leftArray = ArrayPool.Shared.Rent(maxLength); + var rightArray = ArrayPool.Shared.Rent(maxLength); + try + { + var left = leftArray.AsSpan(0, maxLength); + var right = rightArray.AsSpan(0, maxLength); + left.Clear(); + right.Clear(); + Encoding.ASCII.GetBytes(storedPassword, left); + Encoding.ASCII.GetBytes(suppliedPassword, right); + return CryptographicOperations.FixedTimeEquals(left, right) && storedLength == suppliedLength; + } + finally + { + ArrayPool.Shared.Return(leftArray, clearArray: true); + ArrayPool.Shared.Return(rightArray, clearArray: true); + } + } + + /// Returns whether contains only ASCII code points. + public static bool IsAscii(string value) + { + ArgumentNullException.ThrowIfNull(value); + foreach (var c in value) + { + if (c > 0x7F) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpSaslExchange.cs b/src/VectorNNTPD.TransitService/Authentication/NntpSaslExchange.cs new file mode 100644 index 00000000..4fbaf560 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpSaslExchange.cs @@ -0,0 +1,54 @@ +using VectorNNTPD.TransitService.Authentication.Sasl; +using VectorNNTPD.TransitService.Session.Authentication; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// In-progress AUTHINFO SASL exchange isolated to one NNTP session. + internal sealed class NntpSaslExchange + { + public NntpSaslExchange(string mechanism) + { + ArgumentException.ThrowIfNullOrWhiteSpace(mechanism); + Mechanism = mechanism; + } + + public string Mechanism { get; } + public string? LoginUsername { get; set; } + /// Gets or sets the native RFC 2195 CRAM-MD5 challenge (not Base64). + public string? CramChallenge { get; set; } + public ScramMechanism? Scram { get; set; } + public string? ScramUsername { get; set; } + + /// + /// Gets or sets whether this exchange used the server-local dummy verifier. + /// Dummy success must never authenticate. + /// + public bool ScramUsedDummy { get; set; } + + public bool WaitingPlainCredentials { get; set; } + } + + /// SASL start/continuation outcome for the AUTHINFO handler. + internal sealed class NntpSaslReply + { + public NntpSaslReply( + ReadOnlyMemory wire, + string statusLine, + bool completed, + NntpSaslExchange? exchange = null, + NntpAuthenticationResult? authentication = null) + { + Wire = wire; + StatusLine = statusLine; + Completed = completed; + Exchange = exchange; + Authentication = authentication; + } + + public ReadOnlyMemory Wire { get; } + public string StatusLine { get; } + public bool Completed { get; } + public NntpSaslExchange? Exchange { get; } + public NntpAuthenticationResult? Authentication { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpSaslService.cs b/src/VectorNNTPD.TransitService/Authentication/NntpSaslService.cs new file mode 100644 index 00000000..fdc7a73f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpSaslService.cs @@ -0,0 +1,427 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Authentication.Sasl; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// AUTHINFO SASL exchange: PLAIN, LOGIN, CRAM-MD5, and SCRAM-SHA-256. + public sealed class NntpSaslService + { + private readonly INntpAuthenticationProvider _passwordProvider; + private readonly MySqlNntpCredentialValidator? _mysql; + private readonly string _fqdn; + private readonly ScramDummyVerifier _dummyScram; + + /// Initializes a SASL service. SCRAM/CRAM require . + public NntpSaslService( + INntpAuthenticationProvider passwordProvider, + MySqlNntpCredentialValidator? mysql = null, + IOptions? nntpd = null) + : this(passwordProvider, mysql, nntpd, dummyScram: null) + { + } + + /// Initializes a SASL service with an injected dummy SCRAM verifier (tests). + internal NntpSaslService( + INntpAuthenticationProvider passwordProvider, + MySqlNntpCredentialValidator? mysql, + IOptions? nntpd, + ScramDummyVerifier? dummyScram) + { + ArgumentNullException.ThrowIfNull(passwordProvider); + _passwordProvider = passwordProvider; + _mysql = mysql; + _fqdn = ResolveFqdn(nntpd?.Value); + _dummyScram = dummyScram ?? ScramDummyVerifier.Create(); + } + + /// Starts AUTHINFO SASL. is the optional raw token. + internal async ValueTask StartAsync( + string mechanism, + string initialResponse, + IPAddress clientIp, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(mechanism); + ArgumentNullException.ThrowIfNull(initialResponse); + ArgumentNullException.ThrowIfNull(clientIp); + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WirePlain)) + { + if (IsEmptyInitialResponse(initialResponse)) + { + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WirePlain) { WaitingPlainCredentials = true }; + return ContinueReply(NntpResponses.SaslEmptyChallenge, NntpResponseStatus.SaslEmptyChallenge, completed: false, exchange); + } + + if (!TryDecodeBase64(initialResponse, out var decoded)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + return await CompletePlainAsync(decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireLogin)) + { + if (!IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslProtocolError, NntpResponseStatus.SaslProtocolError); + } + + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireLogin); + return ContinueReply( + NntpResponses.SaslLoginUsernameChallenge, + NntpResponseStatus.SaslLoginUsernameChallenge, + completed: false, + exchange); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireCramMd5)) + { + if (_mysql is null) + { + return FailReply(NntpResponses.SaslCramUnavailable, NntpResponseStatus.SaslCramUnavailable); + } + + if (!IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslProtocolError, NntpResponseStatus.SaslProtocolError); + } + + var nativeChallenge = CramMd5Mechanism.CreateChallenge(_fqdn); + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireCramMd5) { CramChallenge = nativeChallenge }; + var wire = EncodeLine(383, Convert.ToBase64String(Encoding.ASCII.GetBytes(nativeChallenge))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslChallenge, completed: false, exchange); + } + + if (EqualsMechanism(mechanism, NntpAuthMechanisms.WireScramSha256)) + { + if (_mysql is null) + { + return FailReply(NntpResponses.SaslScramUnavailable, NntpResponseStatus.SaslScramUnavailable); + } + + if (IsEmptyInitialResponse(initialResponse)) + { + return FailReply(NntpResponses.SaslScramUnavailable, NntpResponseStatus.SaslScramUnavailable); + } + + if (!TryDecodeBase64(initialResponse, out var clientFirst)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + return await BeginScramAsync(clientFirst, clientIp, cancellationToken).ConfigureAwait(false); + } + + return FailReply(NntpResponses.SaslMechanismNotSupported, NntpResponseStatus.SaslMechanismNotSupported); + } + + /// Continues an in-progress SASL exchange. Payload * cancels. + internal async ValueTask ContinueAsync( + NntpSaslExchange exchange, + string line, + IPAddress clientIp, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(exchange); + ArgumentNullException.ThrowIfNull(line); + ArgumentNullException.ThrowIfNull(clientIp); + var payload = line.Trim(); + if (payload == "*") + { + return FailReply(NntpResponses.AuthenticationCancelled, NntpResponseStatus.AuthenticationCancelled); + } + + if (!TryDecodeBase64(payload, out var decoded)) + { + return FailReply(NntpResponses.SaslInvalidBase64, NntpResponseStatus.SaslInvalidBase64); + } + + if (exchange.WaitingPlainCredentials) + { + return await CompletePlainAsync(decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireLogin)) + { + return await ContinueLoginAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireCramMd5)) + { + return await ContinueCramAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + if (EqualsMechanism(exchange.Mechanism, NntpAuthMechanisms.WireScramSha256)) + { + return await FinishScramAsync(exchange, decoded, clientIp, cancellationToken).ConfigureAwait(false); + } + + return FailReply(NntpResponses.SaslContinuationNotSupported, NntpResponseStatus.SaslContinuationNotSupported); + } + + private async ValueTask ContinueLoginAsync( + NntpSaslExchange exchange, + string decoded, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (exchange.LoginUsername is null) + { + exchange.LoginUsername = decoded; + return ContinueReply( + NntpResponses.SaslLoginPasswordChallenge, + NntpResponseStatus.SaslLoginPasswordChallenge, + completed: false, + exchange); + } + + var result = await _passwordProvider + .AuthenticateAsync(exchange.LoginUsername, decoded, clientIp, cancellationToken) + .ConfigureAwait(false); + return FromPasswordResult(result); + } + + private async ValueTask CompletePlainAsync( + string decoded, + IPAddress clientIp, + CancellationToken cancellationToken) + { + var parts = decoded.Split('\0'); + if (parts.Length < 3 || string.IsNullOrEmpty(parts[1])) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _passwordProvider + .AuthenticateAsync(parts[1], parts[2], clientIp, cancellationToken) + .ConfigureAwait(false); + return FromPasswordResult(result); + } + + private async ValueTask ContinueCramAsync( + NntpSaslExchange exchange, + string response, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (_mysql is null || exchange.CramChallenge is null) + { + return FailReply(NntpResponses.SaslCramUnavailable, NntpResponseStatus.SaslCramUnavailable); + } + + var space = response.IndexOf(' '); + if (space <= 0) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var username = response[..space]; + NntpUserRecord? record; + try + { + record = await _mysql.TryGetCramRecordAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (NntpDbUnavailableException) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + if (record is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var secret = CramMd5Mechanism.SecretFromPassword(record.AccountPassword); + if (!CramMd5Mechanism.Verify(username, response, exchange.CramChallenge, secret)) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _mysql + .CompleteSaslAccountAsync(NntpAuthMechanisms.SaslCramMd5, username, clientIp, requireScram: false, cancellationToken) + .ConfigureAwait(false); + return FromSaslResult(result, NntpResponses.AuthenticationAccepted, NntpResponseStatus.AuthenticationAccepted); + } + + private async ValueTask BeginScramAsync( + string clientFirst, + IPAddress clientIp, + CancellationToken cancellationToken) + { + _ = clientIp; + if (!ScramMechanism.TryGetUsername(clientFirst, out var username) || _mysql is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + NntpUserRecord? record; + try + { + record = await _mysql.TryLoadUserAsync(username, cancellationToken).ConfigureAwait(false); + } + catch (NntpDbUnavailableException) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + var useReal = ScramStoredCredential.TryGetAuthorized(record, out var credential); + if (!useReal) + { + credential = _dummyScram.Credential; + } + + try + { + var (state, serverFirst) = ScramMechanism.Begin(clientFirst, credential); + var exchange = new NntpSaslExchange(NntpAuthMechanisms.WireScramSha256) + { + Scram = state, + ScramUsername = username, + ScramUsedDummy = !useReal, + }; + var wire = EncodeLine(383, Convert.ToBase64String(Encoding.UTF8.GetBytes(serverFirst))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslChallenge, completed: false, exchange); + } + catch (ArgumentException) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + } + + private async ValueTask FinishScramAsync( + NntpSaslExchange exchange, + string clientFinal, + IPAddress clientIp, + CancellationToken cancellationToken) + { + if (exchange.Scram is null || exchange.ScramUsername is null || _mysql is null) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var serverFinal = exchange.Scram.TryFinish(clientFinal); + if (serverFinal is null || exchange.ScramUsedDummy) + { + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + var result = await _mysql + .CompleteSaslAccountAsync( + NntpAuthMechanisms.SaslScramSha256, + exchange.ScramUsername, + clientIp, + requireScram: true, + cancellationToken) + .ConfigureAwait(false); + if (!result.Succeeded) + { + return FromPasswordResult(result); + } + + var wire = EncodeLine(283, Convert.ToBase64String(Encoding.UTF8.GetBytes(serverFinal))); + return new NntpSaslReply(wire, NntpResponseStatus.SaslSuccessWithData, completed: true, authentication: result); + } + + private static NntpSaslReply FromPasswordResult(NntpAuthenticationResult result) + { + if (result.Succeeded) + { + return new NntpSaslReply( + NntpResponses.AuthenticationAccepted, + NntpResponseStatus.AuthenticationAccepted, + completed: true, + authentication: result); + } + + if (result.Failure == NntpAuthenticationFailureKind.TransientFailure) + { + return FailReply(NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure); + } + + return FailReply(NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed); + } + + private static NntpSaslReply FromSaslResult( + NntpAuthenticationResult result, + ReadOnlyMemory successWire, + string successStatus) + { + if (result.Succeeded) + { + return new NntpSaslReply(successWire, successStatus, completed: true, authentication: result); + } + + return FromPasswordResult(result); + } + + private static NntpSaslReply ContinueReply( + ReadOnlyMemory wire, + string status, + bool completed, + NntpSaslExchange exchange) => + new(wire, status, completed, exchange); + + private static NntpSaslReply FailReply(ReadOnlyMemory wire, string status) => + new(wire, status, completed: true); + + internal static bool EqualsMechanism(string actual, string expected) => + actual.Equals(expected, StringComparison.OrdinalIgnoreCase); + + internal static bool IsEmptyInitialResponse(string token) + { + ArgumentNullException.ThrowIfNull(token); + var text = token.Trim(); + return text.Length == 0 || text == "="; + } + + /// + /// Decodes a SASL token. RFC 4643 = is an empty payload. Invalid BASE64 is rejected + /// (504) rather than treated as plaintext. + /// + internal static bool TryDecodeBase64(string token, out string decoded) + { + ArgumentNullException.ThrowIfNull(token); + var text = token.Trim(); + if (text.Length == 0 || text == "=") + { + decoded = string.Empty; + return true; + } + + try + { + decoded = Encoding.UTF8.GetString(Convert.FromBase64String(text)); + return true; + } + catch (FormatException) + { + decoded = string.Empty; + return false; + } + } + + private static string ResolveFqdn(NntpdOptions? options) + { + if (options is not null && !string.IsNullOrWhiteSpace(options.Fqdn)) + { + return options.Fqdn; + } + + return NntpdOptions.FormatFqdn(1, "usenet.ninja"); + } + + private static ReadOnlyMemory EncodeLine(int code, string text) + { + var line = string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{code} {text}\r\n"); + return Encoding.ASCII.GetBytes(line); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/NntpUserRecord.cs b/src/VectorNNTPD.TransitService/Authentication/NntpUserRecord.cs new file mode 100644 index 00000000..e443fb57 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/NntpUserRecord.cs @@ -0,0 +1,117 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Authentication +{ + /// + /// In-process snapshot of one nntpusers row after SQL decrypt. + /// + /// + /// , , , + /// and are sensitive. They must not be logged or copied onto the + /// session identity snapshot. account_type is not selected for NNTP policy. + /// Every account has both remaining-byte and rate fields. + /// + public sealed class NntpUserRecord + { + /// Creates a validated user-record snapshot. + public NntpUserRecord( + string accountName, + string accountPassword, + bool allowAuthPlain, + bool allowAuthScram256, + ReadOnlyMemory scramSalt, + int scramIterations, + ReadOnlyMemory scramStoredKey, + ReadOnlyMemory scramServerKey, + int rateLimitBps, + long byteLimit, + int sessionLimit, + int srcIpLimit, + bool isEnabled, + string customerId, + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) + { + ArgumentException.ThrowIfNullOrEmpty(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(scramIterations); + AccountName = accountName; + AccountPassword = accountPassword ?? string.Empty; + AllowAuthPlain = allowAuthPlain; + AllowAuthScram256 = allowAuthScram256; + ScramSalt = scramSalt; + ScramIterations = scramIterations; + ScramStoredKey = scramStoredKey; + ScramServerKey = scramServerKey; + RateLimitBps = rateLimitBps; + ByteLimit = byteLimit; + SessionLimit = sessionLimit; + SrcIpLimit = srcIpLimit; + IsEnabled = isEnabled; + CustomerId = customerId ?? string.Empty; + AllowedArtTypes = allowedArtTypes; + } + + /// Gets the plaintext wire username (not the MD5 column value). + public string AccountName { get; } + + /// Gets the decrypted password. Sensitive. Empty when SQL decrypt is NULL. + public string AccountPassword { get; } + + /// Gets whether AUTHINFO PASS, PLAIN, LOGIN, and CRAM-MD5 are permitted. + public bool AllowAuthPlain { get; } + + /// Gets whether SCRAM-SHA-256 is permitted. + public bool AllowAuthScram256 { get; } + + /// Gets SCRAM salt bytes, or empty when not provisioned. + public ReadOnlyMemory ScramSalt { get; } + + /// Gets SCRAM iteration count. 0 means SCRAM is not provisioned. + public int ScramIterations { get; } + + /// Gets SCRAM StoredKey. Sensitive. + public ReadOnlyMemory ScramStoredKey { get; } + + /// Gets SCRAM ServerKey. Sensitive. + public ReadOnlyMemory ScramServerKey { get; } + + /// + /// Gets account_rate_limit in bits per second. + /// Examples: 240 = 240 bps, 1_000_000 = 1 Mbps, 10_000_000 = 10 Mbps. + /// 0 is unlimited. NULL at map time becomes 0. This is not megabits. + /// + public int RateLimitBps { get; } + + /// + /// Gets account_byte_limit remaining bytes. + /// 0 is exhausted. NULL at map time becomes 0. This snapshot is not + /// live cluster remaining. + /// + public long ByteLimit { get; } + + /// Gets account_session_limit. 0 is unlimited. + public int SessionLimit { get; } + + /// Gets account_srcip_limit. 0 is unlimited. + public int SrcIpLimit { get; } + + /// Gets whether is_enabled is Y. + public bool IsEnabled { get; } + + /// Gets customer_id. + public string CustomerId { get; } + + /// + /// Gets account_art_type flags loaded at AUTHINFO. + /// is unrestricted. + /// + public ArticleType AllowedArtTypes { get; } + + /// Gets whether stored SCRAM material is complete enough to begin an exchange. + public bool HasScramMaterial => + AllowAuthScram256 + && !ScramSalt.IsEmpty + && ScramIterations > 0 + && !ScramStoredKey.IsEmpty + && !ScramServerKey.IsEmpty; + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/Sasl/CramMd5Mechanism.cs b/src/VectorNNTPD.TransitService/Authentication/Sasl/CramMd5Mechanism.cs new file mode 100644 index 00000000..cf61c1d1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/Sasl/CramMd5Mechanism.cs @@ -0,0 +1,96 @@ +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace VectorNNTPD.TransitService.Authentication.Sasl +{ + /// + /// RFC 2195 CRAM-MD5 native challenge and verification. + /// + /// + /// The mechanism challenge is an RFC 822 msg-id + /// <random-digits.timestamp@fqdn>. HMAC-MD5 is computed over that + /// native string. NNTP AUTHINFO SASL Base64-encodes the same octets in a + /// 383 reply (RFC 4643); that encoding is not part of CRAM-MD5. + /// + internal static partial class CramMd5Mechanism + { + [GeneratedRegex(@"^<[0-9]+\.[0-9]+@[^@<>\s]+>$", RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture)] + private static partial Regex ChallengeForm(); + + /// + /// Creates a native RFC 2195 challenge using random digits, a UTC Unix + /// timestamp, and the application's Nntpd FQDN. + /// + public static string CreateChallenge(string fqdn, TimeProvider? timeProvider = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + var host = NormalizeFqdn(fqdn); + var unique = BinaryPrimitives.ReadUInt64LittleEndian(RandomNumberGenerator.GetBytes(8)); + var timestamp = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeSeconds(); + return string.Create( + CultureInfo.InvariantCulture, + $"<{unique}.{timestamp}@{host}>"); + } + + /// Returns when matches RFC 2195 msg-id form. + public static bool IsRfc2195Challenge(ReadOnlySpan challenge) => + ChallengeForm().IsMatch(challenge); + + /// Verifies username hexhmac against HMAC-MD5 of the native challenge octets. + public static bool Verify(string username, string response, string challenge, ReadOnlySpan secret) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(response); + ArgumentException.ThrowIfNullOrEmpty(challenge); + var space = response.IndexOf(' '); + if (space <= 0) + { + return false; + } + + var respUser = response[..space]; + if (!string.Equals(respUser, username, StringComparison.Ordinal)) + { + return false; + } + + var expectedHex = ComputeHexDigest(secret, challenge); + return string.Equals(response[(space + 1)..], expectedHex, StringComparison.OrdinalIgnoreCase); + } + + /// Computes the lowercase hex HMAC-MD5 of the native CRAM challenge. + public static string ComputeHexDigest(ReadOnlySpan secret, string nativeChallenge) + { + ArgumentException.ThrowIfNullOrEmpty(nativeChallenge); +#pragma warning disable CA5351 + using var hmac = new HMACMD5(secret.ToArray()); +#pragma warning restore CA5351 + return Convert.ToHexString(hmac.ComputeHash(Encoding.ASCII.GetBytes(nativeChallenge))).ToLowerInvariant(); + } + + /// Encodes an ASCII password as HMAC key bytes. + public static byte[] SecretFromPassword(string password) + { + ArgumentNullException.ThrowIfNull(password); + return Encoding.ASCII.GetBytes(password); + } + + private static string NormalizeFqdn(string fqdn) + { + var host = fqdn.Trim().TrimEnd('.'); + if (host.Length == 0 + || host.Contains('@', StringComparison.Ordinal) + || host.Contains('<', StringComparison.Ordinal) + || host.Contains('>', StringComparison.Ordinal) + || host.Contains(' ', StringComparison.Ordinal)) + { + throw new ArgumentException("CRAM-MD5 host identity must be the Nntpd FQDN.", nameof(fqdn)); + } + + return host; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramDummyVerifier.cs b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramDummyVerifier.cs new file mode 100644 index 00000000..4322d782 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramDummyVerifier.cs @@ -0,0 +1,56 @@ +using System.Security.Cryptography; + +namespace VectorNNTPD.TransitService.Authentication.Sasl +{ + /// + /// Server-local dummy SCRAM-SHA-256 verifier used so the 383 server-first + /// exchange does not reveal whether a username exists or has SCRAM material. + /// + /// + /// Material is generated once and is not derived from any client-controlled + /// input. A client that somehow obtained these keys still cannot authenticate: + /// the SASL exchange records that the dummy verifier was used and forces 481. + /// Never log this material. + /// + internal sealed class ScramDummyVerifier + { + /// RFC 7677 minimum iteration count. Same shape as a real server-first. + public const int IterationCount = 4096; + + /// Salt length used for dummy server-first s=. + public const int SaltLength = 16; + + /// SHA-256 key length for dummy StoredKey and ServerKey. + public const int KeyLength = 32; + + /// Creates a dummy verifier with cryptographically random keys. + public static ScramDummyVerifier Create() => + new(new ScramStoredCredential( + RandomNumberGenerator.GetBytes(SaltLength), + IterationCount, + RandomNumberGenerator.GetBytes(KeyLength), + RandomNumberGenerator.GetBytes(KeyLength))); + + /// Creates a dummy verifier from injected material (tests). + public static ScramDummyVerifier Create(ScramStoredCredential credential) + { + if (credential.Salt.Length != SaltLength + || credential.IterationCount != IterationCount + || credential.StoredKey.Length != KeyLength + || credential.ServerKey.Length != KeyLength) + { + throw new ArgumentException("Dummy SCRAM material must match the production dummy shape.", nameof(credential)); + } + + return new ScramDummyVerifier(credential); + } + + private ScramDummyVerifier(ScramStoredCredential credential) + { + Credential = credential; + } + + /// Gets the immutable dummy stored credential. Do not log. + public ScramStoredCredential Credential { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramMechanism.cs b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramMechanism.cs new file mode 100644 index 00000000..e584b4d0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramMechanism.cs @@ -0,0 +1,157 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.Authentication.Sasl +{ + /// RFC 5802 SCRAM-SHA-256 server-side exchange using stored keys (no PBKDF2 at auth time). + internal sealed class ScramMechanism + { + private readonly ScramStoredCredential _credential; + private readonly string _gs2Header; + private readonly string _clientFirstBare; + private readonly string _serverFirst; + private readonly string _combinedNonce; + + private ScramMechanism( + ScramStoredCredential credential, + string gs2Header, + string clientFirstBare, + string serverFirst, + string combinedNonce) + { + _credential = credential; + _gs2Header = gs2Header; + _clientFirstBare = clientFirstBare; + _serverFirst = serverFirst; + _combinedNonce = combinedNonce; + } + + /// Begins SCRAM with a client-first message and returns server-first. + public static (ScramMechanism State, string ServerFirst) Begin(string clientFirst, ScramStoredCredential credential) + { + if (!TrySplitClientFirst(clientFirst, out var gs2Header, out var clientFirstBare) + || !TryParseAttribute(clientFirstBare, 'r', out var clientNonce) + || string.IsNullOrEmpty(clientNonce)) + { + throw new ArgumentException("Invalid SCRAM client-first message.", nameof(clientFirst)); + } + + var serverNonce = Convert.ToBase64String(RandomNumberGenerator.GetBytes(18)); + var combinedNonce = clientNonce + serverNonce; + var saltB64 = Convert.ToBase64String(credential.Salt.Span); + var serverFirst = $"r={combinedNonce},s={saltB64},i={credential.IterationCount}"; + return (new ScramMechanism(credential, gs2Header, clientFirstBare, serverFirst, combinedNonce), serverFirst); + } + + /// Verifies client-final and returns v= server-final, or on failure. + public string? TryFinish(string clientFinal) + { + if (!TryParseAttribute(clientFinal, 'p', out var proofB64) + || !TryParseAttribute(clientFinal, 'r', out var combinedNonce) + || !TryParseAttribute(clientFinal, 'c', out var channelBinding)) + { + return null; + } + + if (!string.Equals(combinedNonce, _combinedNonce, StringComparison.Ordinal)) + { + return null; + } + + if (!string.Equals(channelBinding, Convert.ToBase64String(Encoding.ASCII.GetBytes(_gs2Header)), StringComparison.Ordinal)) + { + return null; + } + + byte[] clientProof; + try + { + clientProof = Convert.FromBase64String(proofB64); + } + catch (FormatException) + { + return null; + } + + var clientFinalWithoutProof = clientFinal.Replace($",p={proofB64}", string.Empty, StringComparison.Ordinal); + var authMessage = $"{_clientFirstBare},{_serverFirst},{clientFinalWithoutProof}"; + var storedKey = _credential.StoredKey.ToArray(); + var clientSignature = HmacSha256(storedKey, Encoding.UTF8.GetBytes(authMessage)); + var clientKey = Xor(clientProof, clientSignature); + var computedStoredKey = SHA256.HashData(clientKey); + if (computedStoredKey.Length != storedKey.Length + || !CryptographicOperations.FixedTimeEquals(computedStoredKey, storedKey)) + { + return null; + } + + var serverSignature = HmacSha256(_credential.ServerKey.ToArray(), Encoding.UTF8.GetBytes(authMessage)); + return $"v={Convert.ToBase64String(serverSignature)}"; + } + + /// Extracts the n= username from a client-first message. + public static bool TryGetUsername(string clientFirst, [NotNullWhen(true)] out string? username) + { + username = null; + if (!TrySplitClientFirst(clientFirst, out _, out var bare)) + { + return false; + } + + return TryParseAttribute(bare, 'n', out username) && !string.IsNullOrEmpty(username); + } + + private static byte[] HmacSha256(byte[] key, byte[] data) + { + using var hmac = IncrementalHash.CreateHMAC(HashAlgorithmName.SHA256, key); + hmac.AppendData(data); + return hmac.GetHashAndReset(); + } + + private static byte[] Xor(byte[] a, byte[] b) + { + var len = Math.Min(a.Length, b.Length); + var result = new byte[len]; + for (var i = 0; i < len; i++) + { + result[i] = (byte)(a[i] ^ b[i]); + } + + return result; + } + + private static bool TrySplitClientFirst( + string clientFirst, + [NotNullWhen(true)] out string? gs2Header, + [NotNullWhen(true)] out string? clientFirstBare) + { + var idx = clientFirst.IndexOf(",,", StringComparison.Ordinal); + if (idx < 0) + { + gs2Header = null; + clientFirstBare = null; + return false; + } + + gs2Header = clientFirst[..(idx + 2)]; + clientFirstBare = clientFirst[(idx + 2)..]; + return !string.IsNullOrEmpty(clientFirstBare); + } + + private static bool TryParseAttribute(string message, char key, [NotNullWhen(true)] out string? value) + { + foreach (var part in message.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (part.Length >= 2 && part[0] == key && part[1] == '=') + { + value = part[2..]; + return true; + } + } + + value = null; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramStoredCredential.cs b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramStoredCredential.cs new file mode 100644 index 00000000..4cd77289 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Authentication/Sasl/ScramStoredCredential.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.TransitService.Authentication.Sasl +{ + /// Stored SCRAM-SHA-256 material from nntpusers. + internal readonly struct ScramStoredCredential + { + /// Initializes stored SCRAM fields. + public ScramStoredCredential( + ReadOnlyMemory salt, + int iterationCount, + ReadOnlyMemory storedKey, + ReadOnlyMemory serverKey) + { + Salt = salt; + IterationCount = iterationCount; + StoredKey = storedKey; + ServerKey = serverKey; + } + + public ReadOnlyMemory Salt { get; } + public int IterationCount { get; } + public ReadOnlyMemory StoredKey { get; } + public ReadOnlyMemory ServerKey { get; } + + public static ScramStoredCredential FromRecord(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return new(record.ScramSalt, record.ScramIterations, record.ScramStoredKey, record.ScramServerKey); + } + + /// + /// Returns the real stored verifier only for an enabled account with usable + /// SCRAM-SHA-256 material. Disabled, denied, missing, or malformed rows + /// must use the dummy verifier so server-first does not enumerate accounts. + /// + public static bool TryGetAuthorized(NntpUserRecord? record, out ScramStoredCredential credential) + { + credential = default; + if (record is null + || !record.IsEnabled + || !record.AllowAuthScram256 + || record.ScramSalt.IsEmpty + || record.ScramIterations <= 0 + || record.ScramStoredKey.Length != 32 + || record.ScramServerKey.Length != 32) + { + return false; + } + + credential = FromRecord(record); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ArticleIngestionOptions.cs b/src/VectorNNTPD.TransitService/Configuration/ArticleIngestionOptions.cs new file mode 100644 index 00000000..6c25cd74 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ArticleIngestionOptions.cs @@ -0,0 +1,222 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Article ingestion worker and per-article size settings for the Transit ingestion path. + /// + /// + /// Used by TAKETHIS and IHAVE as the + /// network → queue → background ingestion worker boundary. + /// The worker does not write the article body to local durable storage. + /// Queue admission is bounded by , + /// not by . + /// + public sealed class ArticleIngestionOptions + { + /// + /// Legacy default article-count setting. Not an admission bound. + /// + public const int DefaultQueueCapacity = 256; + + /// Default maximum article payload size in bytes (4 MiB). + public const int DefaultMaxArticleBytes = 4 * 1024 * 1024; + + /// + /// Gets or sets a leftover article-count setting retained for binding compatibility. + /// + /// + /// Not an admission bound. The historical 256-entry cap was a memory-safety + /// choke and has been replaced by . + /// Valid range remains 1–100000 so existing configuration files still validate. + /// + [Range(1, 100_000)] + public int QueueCapacity { get; set; } = DefaultQueueCapacity; + + /// + /// Gets or sets the maximum accepted IHAVE/TAKETHIS article size in bytes + /// (after dot-unstuffing). + /// + /// + /// Valid range is 1–104857600 (100 MiB). Default is 4 MiB. + /// This is not the POST destuffed limit; POST uses + /// . + /// + [Range(1, 100 * 1024 * 1024)] + public int MaxArticleBytes { get; set; } = DefaultMaxArticleBytes; + + /// Default outstanding STORE streams on one placement connection. + public const int DefaultMaxOutstandingStoresPerConnection = 2; + + /// Default minimum concurrent ingestion workers. + public const int DefaultMinWorkers = 2; + + /// Default maximum concurrent ingestion workers. + public const int DefaultMaxWorkers = 32; + + /// Default maximum outstanding OverviewDB publisher confirms. + public const int DefaultMaxPublishConcurrency = 32; + + /// Default worker-pool scaling sample interval in seconds. + public const int DefaultScaleIntervalSeconds = 2; + + /// Default sustained high-pressure threshold (byte utilisation / wait signal). + public const double DefaultScaleUpPressureThreshold = 0.40; + + /// Default sustained low-pressure threshold. + public const double DefaultScaleDownPressureThreshold = 0.10; + + /// Default consecutive high-pressure intervals required before +1 worker. + public const int DefaultScaleUpConsecutiveIntervals = 2; + + /// Default consecutive low-pressure intervals required before −1 worker. + public const int DefaultScaleDownConsecutiveIntervals = 3; + + /// Default OverviewDB work-queue byte budget (1 GiB). + public const long DefaultOverviewDbWorkQueueMemoryLimit = 1L * 1024 * 1024 * 1024; + + /// Default minimum OverviewDB RabbitMQ publisher workers. + public const int DefaultOverviewDbMinPublisherWorkers = 2; + + /// Default maximum OverviewDB RabbitMQ publisher workers. + public const int DefaultOverviewDbMaxPublisherWorkers = 32; + + /// Default outstanding OverviewDB publisher confirmation window. + public const int DefaultOverviewDbPublisherBatchSize = 100; + + /// Default bounded OverviewDB publisher shutdown drain seconds. + public const int DefaultOverviewDbPublisherShutdownSeconds = 2; + + /// + /// Gets or sets how many STORE streams may be outstanding on one placement connection. + /// + /// + /// Valid values are 1 and 2. One keeps a single STORE in flight. Two is the measured safe overlap. + /// This does not enable placement. The spool writer still skips placement while that path is disabled. + /// + [Range(1, 2)] + public int MaxOutstandingStoresPerConnection { get; set; } = DefaultMaxOutstandingStoresPerConnection; + + /// + /// Gets or sets the minimum number of concurrent ingestion workers. + /// + /// Valid range is 1–512. Default is . + [Range(1, 512)] + public int MinWorkers { get; set; } = DefaultMinWorkers; + + /// + /// Gets or sets the maximum number of concurrent ingestion workers. + /// + /// + /// Must be greater than or equal to . Valid range is 1–512. + /// Default is . + /// + [Range(1, 512)] + public int MaxWorkers { get; set; } = DefaultMaxWorkers; + + /// + /// Gets or sets the maximum number of outstanding OverviewDB publisher confirms + /// (legacy channel-pool bound retained for binding compatibility). + /// + /// + /// Prefer for the asynchronous + /// confirmation window. Valid range is 1–512. + /// Default is . + /// + [Range(1, 512)] + public int MaxPublishConcurrency { get; set; } = DefaultMaxPublishConcurrency; + + /// + /// Gets or sets the outstanding OverviewDB RabbitMQ confirmation window. + /// + /// + /// Maximum unconfirmed publishes in flight. Publishers stop dequeuing/publishing + /// when the window is full and resume as confirms free capacity. + /// Valid range is 1–10000. Default is . + /// + [Range(1, 10_000)] + public int OverviewDbPublisherBatchSize { get; set; } = DefaultOverviewDbPublisherBatchSize; + + /// + /// Gets or sets the bounded OverviewDB publisher shutdown drain period in seconds. + /// + /// + /// After this period, outstanding OverviewDB confirms are abandoned. + /// Valid range is 1–60. Default is . + /// + [Range(1, 60)] + public int OverviewDbPublisherShutdownSeconds { get; set; } = DefaultOverviewDbPublisherShutdownSeconds; + + /// + /// Gets or sets the in-process OverviewDB work-queue byte budget. + /// + /// + /// Bounded handoff between article workers and OverviewDB publisher workers. + /// Not durable; not a RabbitMQ queue. Valid range is 1–9223372036854775807. + /// Default is . + /// + [Range(1, long.MaxValue)] + public long OverviewDbWorkQueueMemoryLimit { get; set; } = DefaultOverviewDbWorkQueueMemoryLimit; + + /// + /// Gets or sets the minimum OverviewDB RabbitMQ publisher workers. + /// + /// Valid range is 1–512. Default is . + [Range(1, 512)] + public int OverviewDbMinPublisherWorkers { get; set; } = DefaultOverviewDbMinPublisherWorkers; + + /// + /// Gets or sets the maximum OverviewDB RabbitMQ publisher workers. + /// + /// + /// Must be greater than or equal to . + /// Scales independently of /. + /// Valid range is 1–512. Default is . + /// + [Range(1, 512)] + public int OverviewDbMaxPublisherWorkers { get; set; } = DefaultOverviewDbMaxPublisherWorkers; + + /// + /// Gets or sets the worker-pool scaling sample interval in seconds. + /// + /// + /// Shared by article and OverviewDB publisher pool scalers. + /// Valid range is 1–3600. Default is . + /// + [Range(1, 3600)] + public int ScaleIntervalSeconds { get; set; } = DefaultScaleIntervalSeconds; + + /// + /// Gets or sets the sustained pressure threshold that triggers scale-up. + /// + /// + /// Pressure is the greater of queue byte utilisation and a waiting-producer + /// signal. Must be strictly greater than . + /// Valid range is 0–1. + /// + [Range(0, 1)] + public double ScaleUpPressureThreshold { get; set; } = DefaultScaleUpPressureThreshold; + + /// + /// Gets or sets the sustained pressure threshold that triggers scale-down. + /// + /// + /// Must be strictly less than . + /// Valid range is 0–1. + /// + [Range(0, 1)] + public double ScaleDownPressureThreshold { get; set; } = DefaultScaleDownPressureThreshold; + + /// + /// Gets or sets how many consecutive high-pressure samples are required before adding one worker. + /// + [Range(1, 100)] + public int ScaleUpConsecutiveIntervals { get; set; } = DefaultScaleUpConsecutiveIntervals; + + /// + /// Gets or sets how many consecutive low-pressure samples are required before removing one worker. + /// + [Range(1, 100)] + public int ScaleDownConsecutiveIntervals { get; set; } = DefaultScaleDownConsecutiveIntervals; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ControlOptions.cs b/src/VectorNNTPD.TransitService/Configuration/ControlOptions.cs new file mode 100644 index 00000000..0b4b12fd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ControlOptions.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Top-level control-message infrastructure options. + /// + /// + /// + /// The configuration section name is (Control; + /// case-insensitive). This section is a passive catalogue. It does not enable + /// control-message processing, dispatch, authorization, or PGP verification. + /// + /// + /// An omitted or empty section is valid. NNTPD startup does not require a + /// populated authority catalogue. + /// + /// + public sealed class ControlOptions + { + /// Top-level configuration section name. + public const string SectionName = "Control"; + + /// + /// Gets or sets the authoritative Usenet PGP control-authority catalogue. + /// + public PgpAuthoritiesOptions PgpAuthorities { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ControlOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/ControlOptionsValidator.cs new file mode 100644 index 00000000..9e4aaccd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ControlOptionsValidator.cs @@ -0,0 +1,108 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Validates when the section is bound. + /// + /// + /// An omitted or empty catalogue is valid. This validator does not authorize + /// control messages and does not load PGP keys. + /// + public sealed class ControlOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, ControlOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + var authorities = options.PgpAuthorities?.Authorities; + if (authorities is null || authorities.Length == 0) + { + return ValidateOptionsResult.Success; + } + + for (var i = 0; i < authorities.Length; i++) + { + var authority = authorities[i]; + if (authority is null) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}] must not be null."); + continue; + } + + if (string.IsNullOrWhiteSpace(authority.Name)) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.Name)} must not be empty."); + } + + if (authority.KeyFingerprint is not null + && !string.IsNullOrWhiteSpace(authority.KeyFingerprint) + && !PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out _)) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.KeyFingerprint)} must be hexadecimal."); + } + + var authorizations = authority.Authorizations; + if (authorizations is null) + { + continue; + } + + for (var j = 0; j < authorizations.Length; j++) + { + var rule = authorizations[j]; + if (rule is null) + { + failures.Add($"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{i}].{nameof(PgpAuthorityOptions.Authorizations)}[{j}] must not be null."); + continue; + } + + RequireNonEmpty( + failures, + rule.Message, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.Message)); + RequireNonEmpty( + failures, + rule.From, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.From)); + RequireNonEmpty( + failures, + rule.Newsgroups, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.Newsgroups)); + RequireNonEmpty( + failures, + rule.VerificationIdentity, + i, + j, + nameof(PgpAuthorityAuthorizationOptions.VerificationIdentity)); + } + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void RequireNonEmpty( + List failures, + string? value, + int authorityIndex, + int ruleIndex, + string propertyName) + { + if (string.IsNullOrWhiteSpace(value)) + { + failures.Add( + $"{nameof(ControlOptions.PgpAuthorities)}.{nameof(PgpAuthoritiesOptions.Authorities)}[{authorityIndex}].{nameof(PgpAuthorityOptions.Authorizations)}[{ruleIndex}].{propertyName} must not be empty."); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/EmailOptions.cs b/src/VectorNNTPD.TransitService/Configuration/EmailOptions.cs new file mode 100644 index 00000000..08f9cd77 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/EmailOptions.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Top-level outbound email options for the generic Email subsystem. + /// + /// + /// + /// The configuration section name is (Email; + /// case-insensitive). This is application email infrastructure, not an NNTP or + /// moderation setting. Moderation is one producer. + /// + /// + /// When is (the production default), + /// IEmailService.SendAsync rejects immediately. No spool file is written + /// and no SMTP connection is opened. SMTP host and credentials are not required. + /// + /// + /// Credentials must be supplied via environment variables + /// (Email__Smtp__Username, Email__Smtp__Password) or a secret store. + /// Do not place real credentials in VectorNNTPD.TransitService.json. + /// + /// + public sealed class EmailOptions + { + /// Top-level configuration section name. + public const string SectionName = "Email"; + + /// + /// Gets or sets whether the outbound email subsystem accepts work. + /// Default . + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the default RFC 5322 From mailbox used when a producer + /// does not supply one. Required when is . + /// + public string DefaultFrom { get; set; } = string.Empty; + + /// + /// Gets or sets the SMTP envelope sender (MAIL FROM). + /// When empty, is used. Never derived from recipients. + /// + public string EnvelopeSender { get; set; } = string.Empty; + + /// Gets or sets SMTP transport options. + public SmtpOptions Smtp { get; set; } = new(); + + /// Gets or sets the durable filesystem spool options. + public EmailSpoolOptions Spool { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/EmailOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/EmailOptionsValidator.cs new file mode 100644 index 00000000..b48f657b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/EmailOptionsValidator.cs @@ -0,0 +1,186 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// Validates at bind / startup time. + /// Never includes credentials in failure messages. + public sealed class EmailOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, EmailOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + + var failures = new List(); + ValidateSpool(options.Spool, failures); + + if (!options.Enabled) + { + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + ValidateMailbox( + options.DefaultFrom, + $"{EmailOptions.SectionName}:{nameof(EmailOptions.DefaultFrom)}", + failures); + + if (!string.IsNullOrWhiteSpace(options.EnvelopeSender)) + { + ValidateMailbox( + options.EnvelopeSender, + $"{EmailOptions.SectionName}:{nameof(EmailOptions.EnvelopeSender)}", + failures); + } + + ValidateSmtp(options.Smtp, failures); + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateSpool(EmailSpoolOptions spool, List failures) + { + if (string.IsNullOrWhiteSpace(spool.Directory) || spool.Directory.Trim().Length > 512) + { + failures.Add($"{EmailOptions.SectionName}:Spool:Directory must be a non-empty filesystem path of 512 characters or fewer."); + } + + if (spool.ShutdownTimeout < TimeSpan.FromSeconds(1) + || spool.ShutdownTimeout > TimeSpan.FromMinutes(5)) + { + failures.Add( + $"{EmailOptions.SectionName}:Spool:ShutdownTimeout must be between 1 second and 5 minutes."); + } + + if (spool.ScanInterval < TimeSpan.FromMilliseconds(20) + || spool.ScanInterval > TimeSpan.FromMinutes(5)) + { + failures.Add( + $"{EmailOptions.SectionName}:Spool:ScanInterval must be between 20ms and 5 minutes."); + } + } + + private static void ValidateSmtp(SmtpOptions smtp, List failures) + { + if (string.IsNullOrWhiteSpace(smtp.Host)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Host is required when Email is enabled."); + } + else if (ContainsControl(smtp.Host) || smtp.Host.Trim().Length > 253) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Host must be a hostname or IP without control characters."); + } + + if (smtp.Port is < 1 or > 65535) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Port must be between 1 and 65535."); + } + + if (!Enum.IsDefined(smtp.Security)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Security must be None, StartTls, or ImplicitTls."); + } + + if (smtp.ConnectTimeout < TimeSpan.FromMilliseconds(100) + || smtp.ConnectTimeout > TimeSpan.FromMinutes(5)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:ConnectTimeout must be between 100ms and 5 minutes."); + } + + if (smtp.CommandTimeout < TimeSpan.FromMilliseconds(100) + || smtp.CommandTimeout > TimeSpan.FromMinutes(10)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:CommandTimeout must be between 100ms and 10 minutes."); + } + + if (smtp.MaxAttempts is < 1 or > 20) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:MaxAttempts must be between 1 and 20."); + } + + if (smtp.InitialRetryDelay < TimeSpan.Zero || smtp.InitialRetryDelay > TimeSpan.FromHours(1)) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:InitialRetryDelay must be between 0 and 1 hour."); + } + + if (smtp.MaximumRetryDelay < smtp.InitialRetryDelay || smtp.MaximumRetryDelay > TimeSpan.FromHours(1)) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:MaximumRetryDelay must be at least InitialRetryDelay and at most 1 hour."); + } + + var userSet = !string.IsNullOrWhiteSpace(smtp.Username); + var passwordSet = !string.IsNullOrEmpty(smtp.Password); + if (userSet != passwordSet) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:Username and Password must both be set or both omitted " + + "(use Email__Smtp__Password or secrets; never commit the value)."); + } + else if (userSet) + { + if (ContainsControl(smtp.Username) || smtp.Username.Trim().Length > 256) + { + failures.Add($"{EmailOptions.SectionName}:Smtp:Username must be 1–256 characters without control characters."); + } + + if (smtp.RequireTlsForAuthentication && smtp.Security == SmtpSecurityMode.None) + { + failures.Add( + $"{EmailOptions.SectionName}:Smtp:RequireTlsForAuthentication is true, so Security cannot be None. " + + "Set RequireTlsForAuthentication to false only when a controlled plaintext relay is intended."); + } + } + } + + internal static bool TryValidateMailbox(string? value, out string mailbox) + { + mailbox = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (trimmed.Length is 0 or > 254 || ContainsControl(trimmed)) + { + return false; + } + + var at = trimmed.IndexOf('@'); + if (at <= 0 || at != trimmed.LastIndexOf('@') || at == trimmed.Length - 1) + { + return false; + } + + mailbox = trimmed; + return true; + } + + private static void ValidateMailbox(string value, string key, List failures) + { + if (!TryValidateMailbox(value, out _)) + { + failures.Add($"{key} must be a mailbox address."); + } + } + + private static bool ContainsControl(string value) + { + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/EmailSpoolOptions.cs b/src/VectorNNTPD.TransitService/Configuration/EmailSpoolOptions.cs new file mode 100644 index 00000000..604dd8a2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/EmailSpoolOptions.cs @@ -0,0 +1,31 @@ +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// Filesystem outbound email spool settings nested under . + public sealed class EmailSpoolOptions + { + /// Default application-relative spool directory. + public const string DefaultDirectory = "spool/smtp"; + + /// + /// Gets or sets the spool directory. Relative paths resolve through Common + /// against + /// . Default . + /// Created automatically when email is enabled. + /// + public string Directory { get; set; } = DefaultDirectory; + + /// + /// Gets or sets how long shutdown waits for an in-flight SMTP operation. + /// Default 15 seconds. Pending .eml files are left on disk. + /// + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(15); + + /// + /// Gets or sets how often the worker rescans the spool when idle. + /// Default 1 second. A write also wakes the worker. + /// + public TimeSpan ScanInterval { get; set; } = TimeSpan.FromSeconds(1); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/FeedDiagnosticsOptions.cs b/src/VectorNNTPD.TransitService/Configuration/FeedDiagnosticsOptions.cs new file mode 100644 index 00000000..91489568 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/FeedDiagnosticsOptions.cs @@ -0,0 +1,69 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Temporary opt-in real-feed observability under Nntpd:FeedDiagnostics. + /// + /// + /// Off by default. When disabled, the reporter does not run and session probes are not + /// attached. Enable with Nntpd:FeedDiagnostics:Enabled or environment variable + /// . This is diagnostic instrumentation, not a + /// permanent per-article log path. + /// + public sealed class FeedDiagnosticsOptions + { + /// Environment variable that enables feed diagnostics (VECTORNNTP_FEED_DIAGNOSTICS). + public const string EnvironmentVariableName = "VECTORNNTP_FEED_DIAGNOSTICS"; + + /// Default snapshot interval in seconds. + public const int DefaultIntervalSeconds = 5; + + /// Minimum accepted interval in seconds. + public const int MinIntervalSeconds = 1; + + /// Maximum accepted interval in seconds. + public const int MaxIntervalSeconds = 60; + + /// Gets or sets whether the periodic feed snapshot reporter is enabled. + public bool Enabled { get; set; } + + /// + /// Gets or sets the snapshot interval in seconds. + /// + /// Default is 10. Valid range is 1–60. + [Range(MinIntervalSeconds, MaxIntervalSeconds)] + public int IntervalSeconds { get; set; } = DefaultIntervalSeconds; + + /// Gets or sets whether each snapshot includes compact per-session lines. + public bool IncludeSessions { get; set; } = true; + + /// + /// Returns whether feed diagnostics should run for . + /// The environment variable wins when set to a truthy value. + /// + public static bool ResolveEnabled(NntpdOptions? options) + { + if (IsEnvironmentEnabled()) + { + return true; + } + + return options?.FeedDiagnostics?.Enabled == true; + } + + /// Returns whether is a truthy value. + public static bool IsEnvironmentEnabled() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + return value.Equals("1", StringComparison.OrdinalIgnoreCase) + || value.Equals("true", StringComparison.OrdinalIgnoreCase) + || value.Equals("yes", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/IpPrefix.cs b/src/VectorNNTPD.TransitService/Configuration/IpPrefix.cs new file mode 100644 index 00000000..1bac1f4f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/IpPrefix.cs @@ -0,0 +1,203 @@ +using System.Net; +using System.Net.Sockets; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// An IPv4 or IPv6 network prefix used by Transit AllowFrom matching. + /// + /// + /// Does not apply routability tests such as IPAddress.IsGlobal. Private, ULA, + /// documentation, and lab addresses are valid. + /// + public readonly struct IpPrefix : IEquatable + { + /// Initializes a prefix from a network address and length. + public IpPrefix(IPAddress network, int prefixLength) + { + ArgumentNullException.ThrowIfNull(network); + network = TrustedProxyHosts.Canonicalize(network); + var max = network.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + if (prefixLength < 0 || prefixLength > max) + { + throw new ArgumentOutOfRangeException(nameof(prefixLength)); + } + + Network = Mask(network, prefixLength); + PrefixLength = prefixLength; + AddressFamily = Network.AddressFamily; + } + + /// Gets the masked network address. + public IPAddress Network { get; } + + /// Gets the prefix length in bits. + public int PrefixLength { get; } + + /// Gets the address family of this prefix. + public AddressFamily AddressFamily { get; } + + /// Creates a host prefix (/32 or /128) for . + public static IpPrefix Host(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + address = TrustedProxyHosts.Canonicalize(address); + var length = address.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + return new IpPrefix(address, length); + } + + /// + /// Parses a literal IPv4/IPv6 address or CIDR prefix. + /// + public static bool TryParse(string text, out IpPrefix prefix) + { + prefix = default; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var trimmed = text.Trim(); + var slash = trimmed.LastIndexOf('/'); + if (slash < 0) + { + if (!IPAddress.TryParse(trimmed, out var address)) + { + return false; + } + + prefix = Host(address); + return true; + } + + if (slash == 0 || slash == trimmed.Length - 1) + { + return false; + } + + if (!IPAddress.TryParse(trimmed[..slash], out var network)) + { + return false; + } + + if (!int.TryParse(trimmed[(slash + 1)..], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out var length)) + { + return false; + } + + var max = network.AddressFamily == AddressFamily.InterNetwork ? 32 : 128; + if (network.AddressFamily == AddressFamily.InterNetworkV6 && network.IsIPv4MappedToIPv6) + { + max = 32; + } + + if (length < 0 || length > max) + { + return false; + } + + prefix = new IpPrefix(network, length); + return true; + } + + /// Returns whether is contained by this prefix. + public bool Contains(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + address = TrustedProxyHosts.Canonicalize(address); + if (address.AddressFamily != AddressFamily) + { + return false; + } + + return PrefixBitsEqual(Network, address, PrefixLength); + } + + /// + /// Returns whether this prefix and share any address. + /// + public bool Overlaps(IpPrefix other) + { + if (AddressFamily != other.AddressFamily) + { + return false; + } + + var length = Math.Min(PrefixLength, other.PrefixLength); + return PrefixBitsEqual(Network, other.Network, length); + } + + /// + public bool Equals(IpPrefix other) => + AddressFamily == other.AddressFamily + && PrefixLength == other.PrefixLength + && Network.Equals(other.Network); + + /// + public override bool Equals(object? obj) => obj is IpPrefix other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(AddressFamily, PrefixLength, Network); + + /// + public override string ToString() => + $"{Network}/{PrefixLength}"; + + private static IPAddress Mask(IPAddress address, int prefixLength) + { + var bytes = address.GetAddressBytes(); + ApplyPrefixMask(bytes, prefixLength); + return new IPAddress(bytes); + } + + private static bool PrefixBitsEqual(IPAddress left, IPAddress right, int prefixLength) + { + if (prefixLength == 0) + { + return true; + } + + var leftBytes = left.GetAddressBytes(); + var rightBytes = right.GetAddressBytes(); + if (leftBytes.Length != rightBytes.Length) + { + return false; + } + + var fullBytes = prefixLength / 8; + for (var i = 0; i < fullBytes; i++) + { + if (leftBytes[i] != rightBytes[i]) + { + return false; + } + } + + var remaining = prefixLength % 8; + if (remaining == 0) + { + return true; + } + + var mask = (byte)(0xFF << (8 - remaining)); + return (leftBytes[fullBytes] & mask) == (rightBytes[fullBytes] & mask); + } + + private static void ApplyPrefixMask(byte[] bytes, int prefixLength) + { + var fullBytes = prefixLength / 8; + var remaining = prefixLength % 8; + if (remaining != 0) + { + bytes[fullBytes] &= (byte)(0xFF << (8 - remaining)); + fullBytes++; + } + + for (var i = fullBytes; i < bytes.Length; i++) + { + bytes[i] = 0; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ModerationOptions.cs b/src/VectorNNTPD.TransitService/Configuration/ModerationOptions.cs new file mode 100644 index 00000000..a3021c3c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ModerationOptions.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Top-level ordinary-newsgroup moderation options. + /// + /// + /// + /// The configuration section name is (Moderation; + /// case-insensitive). This section is not Control:PgpAuthorities. PGP control + /// authorities do not authorize moderated-newsgroup approval. + /// + /// + /// An omitted or empty section is valid. Runtime authorization is loaded from + /// nntpmoderators. A leftover list fails startup. + /// Missing database routes reject moderated POST; they do not bypass moderation. + /// + /// + public sealed class ModerationOptions + { + /// Top-level configuration section name. + public const string SectionName = "Moderation"; + + /// + /// Gets or sets provenance for the imported INN samples/moderators snapshot. + /// + public ModerationSourceOptions Source { get; set; } = new(); + + /// + /// Gets or sets a leftover static mapping list. Must be empty. + /// Runtime authorization is loaded from nntpmoderators. + /// + public ModeratorMappingOptions[] Moderators { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ModerationOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/ModerationOptionsValidator.cs new file mode 100644 index 00000000..16e4d8be --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ModerationOptionsValidator.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Validates . Runtime authorization is loaded from + /// nntpmoderators; a leftover list is rejected. + /// + public sealed class ModerationOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, ModerationOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Moderators is { Length: > 0 }) + { + return ValidateOptionsResult.Fail( + "Moderation:Moderators is no longer a runtime authorization source. " + + "Load moderator routes from nntpmoderators."); + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ModerationSourceOptions.cs b/src/VectorNNTPD.TransitService/Configuration/ModerationSourceOptions.cs new file mode 100644 index 00000000..f558e141 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ModerationSourceOptions.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Provenance for the historical INN samples/moderators import. + /// Runtime authorization is nntpmoderators, not this metadata. + /// + public sealed class ModerationSourceOptions + { + /// + /// Gets or sets the human-facing INN samples/moderators URL. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the INN GitHub raw samples/moderators URL. + /// + public string? InnUrl { get; set; } + + /// + /// Gets or sets which configured URL was actually retrieved. + /// + public string? RetrievedFrom { get; set; } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/ModeratorMappingOptions.cs b/src/VectorNNTPD.TransitService/Configuration/ModeratorMappingOptions.cs new file mode 100644 index 00000000..f68fff3f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/ModeratorMappingOptions.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Leftover static moderator-route shape retained only so a non-empty + /// Moderation:Moderators list can fail startup validation. + /// + /// + /// + /// is the unapproved-article submission destination. + /// A static mailbox is used as-is. An INN template containing %s is + /// expanded to the matched newsgroup name with dots changed to dashes. + /// The expanded mailbox does not authenticate an NNTP client. + /// + /// + /// is optional. It is required only for local + /// authenticated reinjection. Routing-only INN destinations omit it. + /// Credentials are not stored here. + /// + /// + public sealed class ModeratorMappingOptions + { + /// + /// Gets or sets the RFC 3977 wildmat matched against the newsgroup name. + /// + public string Pattern { get; set; } = string.Empty; + + /// + /// Gets or sets the routing mailbox or INN %s template for groups + /// matching . + /// + public string Address { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO username authorized to approve groups matching + /// , or empty when this entry is routing-only. + /// + public string Username { get; set; } = string.Empty; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/NntpServerJsonFiles.cs b/src/VectorNNTPD.TransitService/Configuration/NntpServerJsonFiles.cs new file mode 100644 index 00000000..32c94655 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/NntpServerJsonFiles.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// Shared JSON filenames used by Reader and Transit hosts. + public static class NntpServerJsonFiles + { + /// Shared RabbitMQ connectivity JSON registered by each NNTP server host. + public const string SharedRabbitMqFileName = "VectorNNTPD.RabbitMQ.json"; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs b/src/VectorNNTPD.TransitService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs new file mode 100644 index 00000000..8d3ad3f1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/NntpdAcmeCloudflareOptionsOverlay.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Applies root-level ACME directory and renewal values without replacing + /// NNTPD-owned bind addresses, ports, or ACME state directory. Account email, + /// certificate password, and the Cloudflare API key are published from + /// nntpsharedconfig and are not read from VECTOR__ variables. + /// + internal static class NntpdAcmeCloudflareOptionsOverlay + { + /// + /// Overlays shared root directory and renewal settings, then restores + /// bound from Nntpd. + /// Bind addresses and ports remain the values bound from Nntpd. + /// + /// The NNTPD options already bound from the Nntpd section. + /// The full configuration root. + public static void OverlaySharedFromRootPreservingApplicationAcme( + NntpdOptions options, + IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(configuration); + + var stateDir = options.AcmeStateDir; + + AcmeCloudflareOptions.OverlaySharedFromRoot(options, configuration, includeSharedSecrets: false); + + options.AcmeStateDir = stateDir; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/NntpdOptions.cs b/src/VectorNNTPD.TransitService/Configuration/NntpdOptions.cs new file mode 100644 index 00000000..7c23cbb4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/NntpdOptions.cs @@ -0,0 +1,436 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Strongly typed lifecycle, hosting, and NNTPD listener options for VectorNNTPD.TransitService. + /// + /// + /// + /// NNTPD-specific settings bind from section and + /// NNTPD__* environment variables. Bind addresses and ports and the + /// ACME state directory bind from Nntpd only. ACME directory URL, + /// renewal threshold, account email, certificate password, Cloudflare API key, + /// zone id, and DNS suffix are published from nntpsharedconfig. + /// RabbitMQ hosts and Management settings bind from the top-level + /// RabbitMQ section. RabbitMQ username and password are published from + /// nntpsharedconfig and are not read from VECTOR__RABBITMQ__USERNAME + /// or VECTOR__RABBITMQ__PASSWORD. + /// + /// + /// is generated from and and cannot + /// be bound or overridden from configuration or environment variables. + /// + /// + /// Never log complete instances: + /// , + /// , , + /// / , and + /// are secrets. + /// + /// + public sealed class NntpdOptions : AcmeCloudflareOptions, IApplicationLifecycleOptions + { + /// NNTPD-specific configuration section name. + public new const string SectionName = "Nntpd"; + + /// + /// Legacy combined-host FQDN prefix retained for documentation and older fixtures. + /// Live Reader/Transit hosts use . + /// + public const string ApplicationPrefix = "nntpd"; + + /// FQDN host-label prefix for the Reader service. + public const string ReaderApplicationPrefix = "reader"; + + /// FQDN host-label prefix for the Transit service. + public const string TransitApplicationPrefix = "transit"; + + /// + /// Returns the FQDN host-label prefix for . + /// + /// Reader or Transit host role. + /// The role-specific prefix. + public static string ApplicationPrefixFor(Hosting.NntpServerRole role) => + role switch + { + Hosting.NntpServerRole.Transit => TransitApplicationPrefix, + _ => ReaderApplicationPrefix, + }; + + /// + /// Gets or sets the FQDN host-label prefix for this process. + /// Set by hosting registration; not bound from configuration. + /// + public string HostApplicationPrefix { get; set; } = ApplicationPrefix; + + /// + /// Environment variable that supplies + /// (NNTPD__SERVERID). + /// + public const string ServerIdEnvironmentVariable = "NNTPD__SERVERID"; + + /// Configuration key for the POST X-Trace AES-256 key secret. + public const string XTraceKeyConfigurationKey = "XTraceKey"; + + /// Configuration key for the optional previous POST X-Trace AES-256 key. + public const string XTracePreviousKeyConfigurationKey = "XTracePreviousKey"; + + /// + /// Environment variable that supplies + /// (NNTPD__XTRACEKEY). + /// + public const string XTraceKeyEnvironmentVariable = "NNTPD__XTRACEKEY"; + + /// + /// Environment variable that supplies + /// (NNTPD__XTRACEPREVIOUSKEY). + /// + public const string XTracePreviousKeyEnvironmentVariable = "NNTPD__XTRACEPREVIOUSKEY"; + + /// Configuration key for the optional newsmaster AUTHINFO username. + public const string NewsmasterUserConfigurationKey = "NewsmasterUser"; + + /// Configuration key for the optional newsmaster AUTHINFO password secret. + public const string NewsmasterPasswordConfigurationKey = "NewsmasterPassword"; + + /// + /// Environment variable that supplies + /// (NNTPD__NEWSMASTERUSER). + /// + public const string NewsmasterUserEnvironmentVariable = "NNTPD__NEWSMASTERUSER"; + + /// + /// Environment variable that supplies + /// (NNTPD__NEWSMASTERPASSWORD). + /// + public const string NewsmasterPasswordEnvironmentVariable = "NNTPD__NEWSMASTERPASSWORD"; + + /// + /// Gets or sets the application display name used by lifecycle messages. + /// + /// + /// Defaults to . + /// Log file names, the Serilog application property, the Windows service name, + /// and the RabbitMQ connection prefix use the entry assembly name directly. + /// + [Required(AllowEmptyStrings = false)] + [MaxLength(128)] + public string ApplicationName { get; set; } = ApplicationJsonConfiguration.EntryAssemblyName; + + /// + /// Gets or sets the maximum time allowed for the overall graceful shutdown of application services. + /// + /// + /// This is a single wall-clock budget for the entire reverse-order stop sequence, not a fresh + /// timeout granted independently to each service. + /// awaits each service stop (no abandon). Services that ignore cancellation can block that await + /// until they return; the host ShutdownTimeout (configured from this value) and external + /// supervisors remain the process-level backstop. + /// + public TimeSpan GracefulShutdownTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the maximum time allowed for application startup before cancellation is considered a failure. + /// + /// + /// When , startup is bounded only by the host cancellation token. + /// + public TimeSpan? StartupTimeout { get; set; } + + /// + /// Gets or sets a value indicating whether the process should exit if an application service + /// terminates unexpectedly while running. + /// + public bool StopHostOnUnexpectedServiceTermination { get; set; } = true; + + /// + /// Gets or sets systemd-specific notification and watchdog options. + /// + [Required] + public SystemdOptions Systemd { get; set; } = new(); + + /// + /// Gets or sets trusted HAProxy PROXY-protocol peer addresses. + /// + /// + /// + /// Optional. Each entry must be a literal IPv4 or IPv6 address (not a DNS name, not a CIDR). + /// When empty or omitted, PROXY protocol processing is disabled and every connection's + /// effective client endpoint is the TCP peer. + /// + /// + /// When non-empty, a TCP peer whose address matches an entry is treated as a trusted proxy + /// and must present a valid PROXY v1/v2 header before TLS/NNTP. Untrusted peers are not + /// permitted to supply PROXY metadata that replaces their TCP identity. + /// + /// + public string[] ProxyHosts { get; set; } = []; + + /// + /// Gets or sets whether AUTHINFO USER/PASS is permitted when the connection is not TLS-protected. + /// + /// + /// + /// Default is so cleartext NNTP can authenticate. AUTHINFO PASS + /// transmits the password in clear text at the NNTP protocol layer; prefer TLS in production. + /// + /// + /// When and the connection is not TLS-protected, AUTHINFO USER/PASS + /// return 483 and AUTHINFO USER is not advertised in CAPABILITIES. + /// TLS connections always permit AUTHINFO USER/PASS regardless of this setting. + /// + /// + public bool AllowCleartextAuth { get; set; } = true; + + /// Default relative Serilog file-log and INN news directory. + public const string DefaultLogDir = "logs/"; + + /// + /// Gets or sets the filesystem directory for Serilog daily rolling application logs + /// and the INN-compatible news file. + /// + public string LogDir { get; set; } = DefaultLogDir; + + /// + /// Gets or sets the numeric server identity used when generating . + /// + /// + /// + /// Required. No default. Must be explicitly configured as an integer in 1–255. + /// + /// + /// Typed as so a missing configuration value remains distinguishable + /// from an explicitly configured 0 (both fail validation). + /// + /// + [Required] + [Range(ServerIdRules.MinimumInclusive, ServerIdRules.MaximumInclusive)] + public int? ServerId { get; set; } + + /// + /// Gets the generated server FQDN nntpd{ServerId:00}.{DnsSuffix}. + /// + /// + /// Not independently configurable. There is no dot between nntpd and the two-digit id. + /// Example: ServerId 1 → nntpd01.usenet.ninja. + /// Returns an empty string when is unset so data-annotation validation + /// can inspect the object; dependent services must use this only after validation succeeds. + /// + public override string Fqdn => + ServerId is { } serverId + && ServerIdRules.IsInRange(serverId) + && !string.IsNullOrWhiteSpace(DnsSuffix) + ? FormatFqdn( + string.IsNullOrWhiteSpace(HostApplicationPrefix) ? ApplicationPrefix : HostApplicationPrefix, + serverId, + DnsSuffix) + : string.Empty; + + /// + /// Formats the generated FQDN for a validated server id and DNS suffix. + /// + /// Server id in 1–255. + /// DNS suffix without a trailing dot. + /// The FQDN string. + public static string FormatFqdn(int serverId, string dnsSuffix) => + ApplicationFqdn.Build(ApplicationPrefix, serverId, dnsSuffix); + + /// + /// Formats the generated FQDN for a role-specific host-label prefix. + /// + /// Host-label prefix (reader or transit). + /// Server id in 1–255. + /// DNS suffix without a trailing dot. + /// The FQDN string. + public static string FormatFqdn(string applicationPrefix, int serverId, string dnsSuffix) => + ApplicationFqdn.Build(applicationPrefix, serverId, dnsSuffix); + + /// + /// Gets or sets NNTPD-local transit runtime options (STREAM TX depth and + /// WantTrash/LogTrash). + /// + /// + /// Named Transit peers bind from the top-level Transit section + /// (), not from this property. + /// + [Required] + public TransitOptions Transit { get; set; } = new(); + + /// + /// Gets or sets SPEEDTEST diagnostic limits. + /// + /// + /// Optional. Defaults cap duration, bytes, and concurrency for the VectorNNTP + /// SPEEDTEST extension. Not part of the article data plane. + /// + [Required] + public SpeedTestOptions SpeedTest { get; set; } = new(); + + /// + /// Gets or sets temporary real-feed observability. + /// + /// + /// Off by default. Periodic snapshots only; not per-article logging. + /// Also enabled by environment variable VECTORNNTP_FEED_DIAGNOSTICS. + /// + [Required] + public FeedDiagnosticsOptions FeedDiagnostics { get; set; } = new(); + + /// + /// Gets or sets how long a HistoryDB marker is retained in local memory and Redis. + /// + /// + /// Default is two hours. Redis uses native key TTL. The in-memory store expires the same + /// digest after this period. HistoryDB policy lives here, not on . + /// + public TimeSpan HistoryTime { get; set; } = TimeSpan.FromHours(2); + + /// Default NNTP command idle timeout in seconds. + public const int DefaultIdleTime = 300; + + /// Minimum accepted in seconds. + public const int MinIdleTime = 1; + + /// Maximum accepted in seconds (24 hours). + public const int MaxIdleTime = 86_400; + + /// + /// Gets or sets how many seconds an established NNTP session may remain without + /// executing an NNTP command before the session is disconnected. + /// + /// + /// Unit is seconds. Default is . Valid range is + /// –. 0 is invalid + /// (not “disabled”); it follows the same minimum-1-second rule as other + /// required timeouts such as . + /// The timer starts after session setup (greeting) and resets when a command + /// is accepted for processing. In-flight commands (including pipelined CHECK + /// and TAKETHIS) keep the session non-idle. + /// + public int IdleTime { get; set; } = DefaultIdleTime; + + /// Default POST article size limit: 5 MiB. + public const int DefaultMaxArticleSize = 5 * 1024 * 1024; + + /// Minimum accepted in bytes. + public const int MinMaxArticleSize = 1; + + /// Maximum accepted in bytes (100 MiB). + public const int MaxMaxArticleSize = 100 * 1024 * 1024; + + /// + /// Gets or sets the maximum destuffed POST article size in bytes. + /// + /// + /// Default is (5,242,880). Valid range is + /// –. The limit is + /// enforced while the multiline article is streamed (destuffed headers + blank + /// separator + body; NNTP terminator excluded; stuffing dots are not counted). + /// Distinct from ArticleIngestion:MaxArticleBytes, which bounds + /// IHAVE/TAKETHIS ingest and IHAVE worker destuff only. + /// + public int MaxArticleSize { get; set; } = DefaultMaxArticleSize; + + /// Default POST Injection-Info complaint mailbox. + public const string DefaultMailComplaintsTo = "abuse@usenet.ninja"; + + /// + /// Gets or sets the mailbox emitted as mail-complaints-to on server-owned + /// POST Injection-Info. + /// + /// + /// Default is . Required; must be a plausible + /// mailbox. Client-supplied Injection-Info is never used. + /// + public string MailComplaintsTo { get; set; } = DefaultMailComplaintsTo; + + /// + /// Gets or sets Path-survey (ninpaths) report recipient mailboxes. + /// + /// + /// Missing, null, or empty disables internal ninpaths reporting. Whitespace-only + /// entries are ignored. There is no separate enable flag and no environment-variable + /// overlay. Reports are emailed through the existing Email subsystem. + /// + public string[] Top1000 { get; set; } = []; + + /// + /// Gets or sets the persisted AES-256 key used to protect POST X-Trace values. + /// + /// + /// Required secret. Supply 32 bytes as 64 hex characters or Base64 via + /// or deployment secrets — never commit + /// this value. The process does not generate a key at startup; restarting with the + /// same key keeps previously issued tokens decryptable. + /// + public string XTraceKey { get; set; } = string.Empty; + + /// + /// Gets or sets the previous AES-256 key retained for one-generation X-Trace rotation. + /// + /// + /// Optional. When set, trusted decrypt accepts tokens produced with this key. + /// New tokens always use . After this value is removed, + /// tokens produced only with the retired key cannot be decrypted. + /// + public string XTracePreviousKey { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO username that may POST a well-formed cancel control article. + /// + /// + /// Optional. When omitted with , AUTHINFO remains deny-all + /// (unless another provider is registered). When set, is + /// required. Ordinary POST clients never receive this identity. + /// + public string NewsmasterUser { get; set; } = string.Empty; + + /// + /// Gets or sets the AUTHINFO password for . + /// + /// + /// Optional secret. Required when is set. Supply via + /// or deployment secrets — never commit + /// this value. Do not log it or include it in exception messages. + /// + public string NewsmasterPassword { get; set; } = string.Empty; + + /// + /// Default Transit article-queue memory budget: 1 GiB (1,073,741,824 bytes). + /// + public const long DefaultTransitQueueMemoryLimit = 1_073_741_824L; + + /// + /// Gets or sets the Transit article-queue memory budget in bytes. + /// + /// + /// + /// Bounds the sum of owned queued article payload lengths + /// (InboundArticle.Payload.Length): complete NNTP article bytes as + /// queued (IHAVE: stuffed wire, terminating CRLF . CRLF excluded). + /// It does not include object overhead and is not a process-wide memory cap. + /// Memory is released when queued articles are consumed. + /// + /// + /// Default is 1 GiB. Valid range is 1 through + /// (signed 64-bit accounting). An individual article larger than this budget + /// is rejected rather than waited for, so admission cannot deadlock. + /// + /// + public long TransitQueueMemoryLimit { get; set; } = DefaultTransitQueueMemoryLimit; + + /// + /// Gets or sets article ingestion worker options. + /// + /// + /// Defaults: max article 4 MiB. The ingestion worker does not write article bodies to disk. + /// Queue admission is bounded by , not + /// an article-count cap. Used by TAKETHIS and IHAVE. + /// + [Required] + public ArticleIngestionOptions ArticleIngestion { get; set; } = new(); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/NntpdOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/NntpdOptionsValidator.cs new file mode 100644 index 00000000..85feba74 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/NntpdOptionsValidator.cs @@ -0,0 +1,609 @@ +using System.Globalization; +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Validates at options bind / startup time. + /// + /// + /// Does not bind sockets or call Cloudflare APIs. Never includes secret values in failure messages. + /// + public sealed class NntpdOptionsValidator : IValidateOptions + { + private readonly ILocalIpAddressAssignee _localIpAddressAssignee; + + /// + /// Initializes a new instance of the class. + /// + /// Probe used to verify explicit bind addresses. + public NntpdOptionsValidator(ILocalIpAddressAssignee localIpAddressAssignee) + { + ArgumentNullException.ThrowIfNull(localIpAddressAssignee); + _localIpAddressAssignee = localIpAddressAssignee; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + + ValidateApplicationName(options, failures); + ValidateTimeouts(options, failures); + ValidateTop1000(options, failures); + ValidateSystemd(options, failures); + AcmeCloudflareOptionsValidator.CollectBindAddressFailures(options, _localIpAddressAssignee, failures); + ValidateProxyHosts(options, failures); + ValidatePorts(options, failures); + ValidateDnsSuffixAndServerId(options, failures); + ValidateAcme(options, failures); + ValidateLogDir(options, failures); + ValidateArticleIngestion(options, failures); + ValidateTransitQueueMemoryLimit(options, failures); + ValidateTransit(options, failures); + ValidateSpeedTest(options, failures); + ValidateFeedDiagnostics(options, failures); + ValidateXTraceKeys(options, failures); + ValidateNewsmaster(options, failures); + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateSpeedTest(NntpdOptions options, List failures) + { + var speed = options.SpeedTest ?? new SpeedTestOptions(); + if (speed.MaxDurationSeconds is < SpeedTestOptions.MinDurationSeconds + or > SpeedTestOptions.MaxDurationSecondsLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxDurationSeconds)} must be between {SpeedTestOptions.MinDurationSeconds} and {SpeedTestOptions.MaxDurationSecondsLimit}."); + } + + if (speed.MaxBytes is < SpeedTestOptions.MinBytes or > SpeedTestOptions.MaxBytesLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxBytes)} must be between {SpeedTestOptions.MinBytes} and {SpeedTestOptions.MaxBytesLimit}."); + } + + if (speed.MaxConcurrent is < SpeedTestOptions.MinConcurrent + or > SpeedTestOptions.MaxConcurrentLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxConcurrent)} must be between {SpeedTestOptions.MinConcurrent} and {SpeedTestOptions.MaxConcurrentLimit}."); + } + + if (speed.MaxConcurrentPerPeer is < SpeedTestOptions.MinConcurrentPerPeer + or > SpeedTestOptions.MaxConcurrentPerPeerLimit) + { + failures.Add( + $"{nameof(NntpdOptions.SpeedTest)}.{nameof(SpeedTestOptions.MaxConcurrentPerPeer)} must be between {SpeedTestOptions.MinConcurrentPerPeer} and {SpeedTestOptions.MaxConcurrentPerPeerLimit}."); + } + } + + private static void ValidateFeedDiagnostics(NntpdOptions options, List failures) + { + var feed = options.FeedDiagnostics ?? new FeedDiagnosticsOptions(); + if (feed.IntervalSeconds is < FeedDiagnosticsOptions.MinIntervalSeconds + or > FeedDiagnosticsOptions.MaxIntervalSeconds) + { + failures.Add( + $"{nameof(NntpdOptions.FeedDiagnostics)}.{nameof(FeedDiagnosticsOptions.IntervalSeconds)} must be between {FeedDiagnosticsOptions.MinIntervalSeconds} and {FeedDiagnosticsOptions.MaxIntervalSeconds}."); + } + } + + private static void ValidateTransit(NntpdOptions options, List failures) + { + var transit = options.Transit ?? new TransitOptions(); + var depth = transit.StreamOutstandingArticleDepth; + if (!NntpStreamArticleTxScheduler.IsValidDepth(depth)) + { + failures.Add( + $"{nameof(NntpdOptions.Transit)}.{nameof(TransitOptions.StreamOutstandingArticleDepth)} must be between {NntpStreamArticleTxScheduler.MinDepth} and {NntpStreamArticleTxScheduler.MaxDepth}."); + } + } + + private static void ValidateArticleIngestion(NntpdOptions options, List failures) + { + var ingestion = options.ArticleIngestion ?? new ArticleIngestionOptions(); + if (ingestion.QueueCapacity is < 1 or > 100_000) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.QueueCapacity)} must be between 1 and 100000."); + } + + if (ingestion.MaxArticleBytes is < 1 or > 100 * 1024 * 1024) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxArticleBytes)} must be between 1 and 104857600."); + } + + if (ingestion.MaxOutstandingStoresPerConnection is not (1 or 2)) + { + failures.Add( + $"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxOutstandingStoresPerConnection)} must be 1 or 2."); + } + + if (ingestion.MinWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MinWorkers)} must be between 1 and 512."); + } + + if (ingestion.MaxWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxWorkers)} must be between 1 and 512."); + } + + if (ingestion.MaxWorkers < ingestion.MinWorkers) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxWorkers)} must be greater than or equal to {nameof(ArticleIngestionOptions.MinWorkers)}."); + } + + if (ingestion.MaxPublishConcurrency is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.MaxPublishConcurrency)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbPublisherBatchSize is < 1 or > 10_000) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbPublisherBatchSize)} must be between 1 and 10000."); + } + + if (ingestion.OverviewDbPublisherShutdownSeconds is < 1 or > 60) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbPublisherShutdownSeconds)} must be between 1 and 60."); + } + + if (ingestion.OverviewDbWorkQueueMemoryLimit < 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbWorkQueueMemoryLimit)} must be at least 1."); + } + + if (ingestion.OverviewDbMinPublisherWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMinPublisherWorkers)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbMaxPublisherWorkers is < 1 or > 512) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMaxPublisherWorkers)} must be between 1 and 512."); + } + + if (ingestion.OverviewDbMaxPublisherWorkers < ingestion.OverviewDbMinPublisherWorkers) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.OverviewDbMaxPublisherWorkers)} must be greater than or equal to {nameof(ArticleIngestionOptions.OverviewDbMinPublisherWorkers)}."); + } + + if (ingestion.ScaleIntervalSeconds is < 1 or > 3600) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleIntervalSeconds)} must be between 1 and 3600."); + } + + if (ingestion.ScaleUpPressureThreshold is < 0 or > 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpPressureThreshold)} must be between 0 and 1."); + } + + if (ingestion.ScaleDownPressureThreshold is < 0 or > 1) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleDownPressureThreshold)} must be between 0 and 1."); + } + + if (ingestion.ScaleUpPressureThreshold <= ingestion.ScaleDownPressureThreshold) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpPressureThreshold)} must be greater than {nameof(ArticleIngestionOptions.ScaleDownPressureThreshold)}."); + } + + if (ingestion.ScaleUpConsecutiveIntervals is < 1 or > 100) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleUpConsecutiveIntervals)} must be between 1 and 100."); + } + + if (ingestion.ScaleDownConsecutiveIntervals is < 1 or > 100) + { + failures.Add($"{nameof(NntpdOptions.ArticleIngestion)}.{nameof(ArticleIngestionOptions.ScaleDownConsecutiveIntervals)} must be between 1 and 100."); + } + } + + private static void ValidateTransitQueueMemoryLimit(NntpdOptions options, List failures) + { + if (options.TransitQueueMemoryLimit < 1) + { + failures.Add( + $"{nameof(NntpdOptions.TransitQueueMemoryLimit)} must be a positive byte count (1 through {long.MaxValue})."); + } + } + + private static void ValidateApplicationName(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.ApplicationName)) + { + failures.Add($"{nameof(NntpdOptions.ApplicationName)} must be a non-empty string."); + } + else if (options.ApplicationName.Length > 128) + { + failures.Add($"{nameof(NntpdOptions.ApplicationName)} must be 128 characters or fewer."); + } + } + + private static void ValidateTimeouts(NntpdOptions options, List failures) + { + if (options.GracefulShutdownTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.GracefulShutdownTimeout)} must be at least 1 second."); + } + + if (options.GracefulShutdownTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.GracefulShutdownTimeout)} must not exceed 1 hour."); + } + + if (options.StartupTimeout is { } startupTimeout) + { + if (startupTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.StartupTimeout)} must be at least 1 second when specified."); + } + + if (startupTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.StartupTimeout)} must not exceed 1 hour."); + } + } + + if (options.CloudFlareOperationTimeout < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.CloudFlareOperationTimeout)} must be at least 1 second."); + } + + if (options.CloudFlareOperationTimeout > TimeSpan.FromHours(1)) + { + failures.Add($"{nameof(NntpdOptions.CloudFlareOperationTimeout)} must not exceed 1 hour."); + } + + if (options.HistoryTime < TimeSpan.FromSeconds(1)) + { + failures.Add($"{nameof(NntpdOptions.HistoryTime)} must be at least 1 second."); + } + + if (options.HistoryTime > TimeSpan.FromDays(7)) + { + failures.Add($"{nameof(NntpdOptions.HistoryTime)} must not exceed 7 days."); + } + + if (options.IdleTime < NntpdOptions.MinIdleTime) + { + failures.Add( + $"{nameof(NntpdOptions.IdleTime)} must be at least {NntpdOptions.MinIdleTime} second."); + } + + if (options.IdleTime > NntpdOptions.MaxIdleTime) + { + failures.Add( + $"{nameof(NntpdOptions.IdleTime)} must not exceed {NntpdOptions.MaxIdleTime} seconds."); + } + + if (options.MaxArticleSize < NntpdOptions.MinMaxArticleSize + || options.MaxArticleSize > NntpdOptions.MaxMaxArticleSize) + { + failures.Add( + $"{nameof(NntpdOptions.MaxArticleSize)} must be between {NntpdOptions.MinMaxArticleSize} and {NntpdOptions.MaxMaxArticleSize}."); + } + + if (string.IsNullOrWhiteSpace(options.MailComplaintsTo) + || !IsPlausibleEmail(options.MailComplaintsTo)) + { + failures.Add( + $"{nameof(NntpdOptions.MailComplaintsTo)} must be a valid mailbox address."); + } + } + + private static void ValidateTop1000(NntpdOptions options, List failures) + { + if (options.Top1000 is null || options.Top1000.Length == 0) + { + return; + } + + for (var i = 0; i < options.Top1000.Length; i++) + { + var entry = options.Top1000[i]; + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + if (!EmailOptionsValidator.TryValidateMailbox(entry, out _)) + { + failures.Add( + $"{nameof(NntpdOptions.Top1000)}[{i}] must be a mailbox address or whitespace."); + } + } + } + + private static void ValidateXTraceKeys(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.XTraceKey) + || !XTraceKeyParser.TryDecode(options.XTraceKey, out _)) + { + failures.Add( + $"{NntpdOptions.XTraceKeyConfigurationKey} must be a 32-byte AES-256 key encoded as 64 hex characters or Base64 " + + $"(use environment variable {NntpdOptions.XTraceKeyEnvironmentVariable} or secrets; never commit the value)."); + } + + if (!string.IsNullOrWhiteSpace(options.XTracePreviousKey) + && !XTraceKeyParser.TryDecode(options.XTracePreviousKey, out _)) + { + failures.Add( + $"{NntpdOptions.XTracePreviousKeyConfigurationKey} must be a 32-byte AES-256 key encoded as 64 hex characters or Base64 " + + $"when set (use environment variable {NntpdOptions.XTracePreviousKeyEnvironmentVariable} or secrets; never commit the value)."); + } + } + + private static void ValidateNewsmaster(NntpdOptions options, List failures) + { + var userSet = !string.IsNullOrWhiteSpace(options.NewsmasterUser); + var passwordSet = !string.IsNullOrEmpty(options.NewsmasterPassword); + if (userSet != passwordSet) + { + failures.Add( + $"{NntpdOptions.NewsmasterUserConfigurationKey} and {NntpdOptions.NewsmasterPasswordConfigurationKey} must both be set or both omitted " + + $"(use {NntpdOptions.NewsmasterPasswordEnvironmentVariable} or secrets for the password; never commit the value)."); + return; + } + + if (!userSet) + { + return; + } + + var user = options.NewsmasterUser.Trim(); + if (user.Length is < 1 or > 64 || ContainsControl(user)) + { + failures.Add($"{NntpdOptions.NewsmasterUserConfigurationKey} must be 1–64 characters without control characters."); + } + + if (options.NewsmasterPassword.Length > 256) + { + failures.Add($"{NntpdOptions.NewsmasterPasswordConfigurationKey} must not exceed 256 characters."); + } + } + + private static bool ContainsControl(string value) + { + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + return true; + } + } + + return false; + } + + private static void ValidateSystemd(NntpdOptions options, List failures) + { + if (options.Systemd is null) + { + failures.Add($"{nameof(NntpdOptions.Systemd)} must be provided."); + return; + } + + var fraction = options.Systemd.WatchdogIntervalFraction; + if (double.IsNaN(fraction) || double.IsInfinity(fraction) || fraction is <= 0 or >= 1) + { + failures.Add( + $"{nameof(NntpdOptions.Systemd)}.{nameof(SystemdOptions.WatchdogIntervalFraction)} must be in the open interval (0, 1)."); + } + else if (fraction is < 0.05 or > 0.9) + { + failures.Add( + $"{nameof(NntpdOptions.Systemd)}.{nameof(SystemdOptions.WatchdogIntervalFraction)} must be between 0.05 and 0.9 inclusive."); + } + } + + private static void ValidateProxyHosts(NntpdOptions options, List failures) + { + if (options.ProxyHosts is null) + { + return; + } + + for (var i = 0; i < options.ProxyHosts.Length; i++) + { + var entry = options.ProxyHosts[i]; + if (string.IsNullOrWhiteSpace(entry)) + { + failures.Add($"{nameof(NntpdOptions.ProxyHosts)}[{i}] must not be empty."); + continue; + } + + var trimmed = entry.Trim(); + if (trimmed.Contains('/') || trimmed.Contains('*')) + { + failures.Add( + $"{nameof(NntpdOptions.ProxyHosts)}[{i}] must be a literal IPv4 or IPv6 address (CIDR and wildcards are not supported)."); + continue; + } + + if (!IPAddress.TryParse(trimmed, out var address)) + { + failures.Add($"{nameof(NntpdOptions.ProxyHosts)}[{i}] is not a valid IPv4 or IPv6 address."); + continue; + } + + if (address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any)) + { + failures.Add( + $"{nameof(NntpdOptions.ProxyHosts)}[{i}] must not be an any-address wildcard; use an explicit proxy peer address."); + } + } + } + + private static void ValidatePorts(NntpdOptions options, List failures) + { + if (options.BindPort is < 1 or > 65535) + { + failures.Add($"{nameof(NntpdOptions.BindPort)} must be an integer in the range 1–65535."); + } + + if (options.BindPortTls is < 0 or > 65535) + { + failures.Add( + $"{nameof(NntpdOptions.BindPortTls)} must be 0 (TLS disabled) or an integer in the range 1–65535."); + } + } + + private static void ValidateAcme(NntpdOptions options, List failures) + { + // Directory URL, renewal threshold, zone id, DNS suffix, account email, + // certificate password, and Cloudflare API key come from nntpsharedconfig. + // State directory stays on this application. + if (string.IsNullOrWhiteSpace(options.AcmeStateDir)) + { + failures.Add($"{nameof(NntpdOptions.AcmeStateDir)} must be a non-empty filesystem path."); + } + } + + private static void ValidateLogDir(NntpdOptions options, List failures) + { + if (string.IsNullOrWhiteSpace(options.LogDir)) + { + failures.Add($"{nameof(NntpdOptions.LogDir)} must be a non-empty filesystem path."); + } + } + + private static bool IsPlausibleEmail(string email) + { + var trimmed = email.Trim(); + if (trimmed.Length is 0 or > 254) + { + return false; + } + + var at = trimmed.IndexOf('@'); + if (at <= 0 || at != trimmed.LastIndexOf('@') || at == trimmed.Length - 1) + { + return false; + } + + var domain = trimmed[(at + 1)..]; + return domain.Contains('.', StringComparison.Ordinal) && IsValidDnsSuffix(domain); + } + + private static void ValidateDnsSuffixAndServerId(NntpdOptions options, List failures) + { + switch (ServerIdRules.Classify(options.ServerId)) + { + case ServerIdValidationStatus.Missing: + failures.Add( + $"{nameof(NntpdOptions.ServerId)} is required and must be an integer in the range {ServerIdRules.MinimumInclusive}–{ServerIdRules.MaximumInclusive} (no default; set {NntpdOptions.SectionName}:{nameof(NntpdOptions.ServerId)} or {NntpdOptions.ServerIdEnvironmentVariable})."); + break; + case ServerIdValidationStatus.OutOfRange: + failures.Add( + $"{nameof(NntpdOptions.ServerId)} must be an integer in the range {ServerIdRules.MinimumInclusive}–{ServerIdRules.MaximumInclusive}."); + break; + } + } + + /// + /// Validates DNS suffix / name syntax (labels, length, allowed characters). + /// + internal static bool IsValidDnsSuffix(string suffix) + { + if (string.IsNullOrWhiteSpace(suffix)) + { + return false; + } + + var value = suffix.Trim().TrimEnd('.'); + if (value.Length is 0 or > 253) + { + return false; + } + + if (value.Contains("..", StringComparison.Ordinal)) + { + return false; + } + + var labels = value.Split('.'); + if (labels.Length == 0) + { + return false; + } + + foreach (var label in labels) + { + if (label.Length is 0 or > 63) + { + return false; + } + + if (label[0] == '-' || label[^1] == '-') + { + return false; + } + + foreach (var ch in label) + { + if (!IsDnsLabelChar(ch)) + { + return false; + } + } + } + + return true; + } + + private static bool IsDnsLabelChar(char ch) => + char.IsAsciiLetterOrDigit(ch) || ch == '-'; + + /// + /// Returns whether contains a configured secret (for tests / diagnostics hygiene). + /// + public static bool ContainsSecret(string text, string? secret) + { + if (string.IsNullOrEmpty(secret) || string.IsNullOrEmpty(text)) + { + return false; + } + + return text.Contains(secret, StringComparison.Ordinal); + } + + /// + /// Joins validation failures for assertions without exposing caller-supplied secrets. + /// + public static string JoinFailures(ValidateOptionsResult result) + { + if (result.Failures is null) + { + return result.FailureMessage ?? string.Empty; + } + + var sb = new StringBuilder(); + foreach (var failure in result.Failures) + { + if (sb.Length > 0) + { + sb.Append(CultureInfo.InvariantCulture, $"; {failure}"); + } + else + { + sb.Append(failure); + } + } + + return sb.ToString(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/NntpdRabbitMqOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/NntpdRabbitMqOptionsValidator.cs new file mode 100644 index 00000000..57e86a67 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/NntpdRabbitMqOptionsValidator.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// NNTPD RabbitMQ options validator: Common connection rules plus required Management HTTP settings. + /// + /// + /// NNTPD ArticleWork availability discovery requires Management BaseUrl. + /// Broker username and password are published from nntpsharedconfig after this validator runs. + /// Connectivity-only hosts should register alone. + /// + public sealed class NntpdRabbitMqOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, RabbitMqOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + RabbitMqOptionsValidator.ValidateConnection(options, failures); + ValidateRequiredManagement(options, failures); + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + + private static void ValidateRequiredManagement(RabbitMqOptions rabbitMq, List failures) + { + rabbitMq.Management ??= new RabbitMqManagementOptions(); + var management = rabbitMq.Management; + + if (string.IsNullOrWhiteSpace(management.BaseUrl)) + { + failures.Add("RabbitMQ:Management:BaseUrl is required."); + return; + } + + RabbitMqOptionsValidator.ValidateManagementShape(management, failures); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritiesOptions.cs b/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritiesOptions.cs new file mode 100644 index 00000000..5a94b3ee --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritiesOptions.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Authoritative Usenet PGP control-authority catalogue derived from ISC/INN + /// control.ctl. + /// + /// + /// This is trusted reference data for a later control-message implementation. + /// It is not an authorization engine and does not verify signatures. + /// + public sealed class PgpAuthoritiesOptions + { + /// + /// Gets or sets provenance for the control.ctl snapshot used to populate + /// . + /// + public PgpAuthoritySourceOptions Source { get; set; } = new(); + + /// + /// Gets or sets the catalogue entries. An empty array is valid. + /// + public PgpAuthorityOptions[] Authorities { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityAuthorizationOptions.cs b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityAuthorizationOptions.cs new file mode 100644 index 00000000..5c8e12f2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityAuthorizationOptions.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// One explicit PGP control-rule association from ISC/INN control.ctl. + /// + /// + /// Corresponds to a <message>:<from>:<newsgroups>:verify-<identity> + /// line. Stored for a later authorization implementation; not evaluated now. + /// + public sealed class PgpAuthorityAuthorizationOptions + { + /// + /// Gets or sets the control-message type (newgroup, rmgroup, + /// checkgroups, or all). + /// + public string Message { get; set; } = string.Empty; + + /// + /// Gets or sets the source From-address pattern. + /// + public string From { get; set; } = string.Empty; + + /// + /// Gets or sets the source newsgroup/hierarchy pattern. + /// + public string Newsgroups { get; set; } = string.Empty; + + /// + /// Gets or sets the PGP user-id from the source verify- action. + /// + public string VerificationIdentity { get; set; } = string.Empty; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityFingerprint.cs b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityFingerprint.cs new file mode 100644 index 00000000..f9d4bd1c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityFingerprint.cs @@ -0,0 +1,91 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Normalizes PGP key fingerprints from ISC/INN control.ctl grouping + /// into a machine-readable uppercase hexadecimal form. + /// + public static class PgpAuthorityFingerprint + { + /// + /// Attempts to normalize a fingerprint to uppercase hexadecimal without + /// whitespace. + /// + /// Source fingerprint text, possibly grouped with spaces. + /// Normalized fingerprint when the method returns . + /// + /// when contains only hexadecimal + /// digits and whitespace, is non-empty after stripping whitespace, and has an + /// even number of hex digits. + /// + /// + /// Normalization removes whitespace and uppercases hex digits. It does not + /// reinterpret or invent bytes. + /// + public static bool TryNormalize(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = null; + if (value is null) + { + return false; + } + + var hexLength = 0; + foreach (var c in value) + { + if (char.IsWhiteSpace(c)) + { + continue; + } + + if (!IsHex(c)) + { + return false; + } + + hexLength++; + } + + if (hexLength == 0 || (hexLength & 1) != 0) + { + return false; + } + + var chars = new char[hexLength]; + var i = 0; + foreach (var c in value) + { + if (char.IsWhiteSpace(c)) + { + continue; + } + + chars[i++] = char.ToUpperInvariant(c); + } + + normalized = new string(chars); + return true; + } + + /// + /// Normalizes a fingerprint or throws when the value is not hexadecimal. + /// + /// Source fingerprint text. + /// Uppercase hexadecimal without whitespace. + /// The value is not a fingerprint. + public static string Normalize(string value) + { + ArgumentNullException.ThrowIfNull(value); + if (!TryNormalize(value, out var normalized)) + { + throw new ArgumentException("Value is not a hexadecimal PGP key fingerprint.", nameof(value)); + } + + return normalized; + } + + private static bool IsHex(char c) => + c is (>= '0' and <= '9') or (>= 'A' and <= 'F') or (>= 'a' and <= 'f'); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityOptions.cs b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityOptions.cs new file mode 100644 index 00000000..6c700efa --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PgpAuthorityOptions.cs @@ -0,0 +1,67 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// One Usenet PGP control authority from ISC/INN control.ctl. + /// + /// + /// Optional metadata properties are omitted from JSON when the source does not + /// provide them. This type stores public catalogue metadata only: no private + /// keys, passphrases, or armored key blocks. + /// + public sealed class PgpAuthorityOptions + { + /// + /// Gets or sets the hierarchy heading name from control.ctl. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the source Contact value, when present. + /// + public string? Contact { get; set; } + + /// + /// Gets or sets the source Admin group value, when present. + /// + public string? AdminGroup { get; set; } + + /// + /// Gets or sets the source URL value, when present. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the source Key URL value, when present. + /// + public string? KeyUrl { get; set; } + + /// + /// Gets or sets the source PGP key fingerprint, when present. + /// + /// + /// Stored as uppercase hexadecimal without spaces. Source grouping whitespace + /// is discarded only; the hex digits are unchanged. + /// + public string? KeyFingerprint { get; set; } + + /// + /// Gets or sets the source Key mail value, when present. + /// + public string? KeyMail { get; set; } + + /// + /// Gets or sets the source Syncable server value, when present. + /// + public string? SyncableServer { get; set; } + + /// + /// Gets or sets explicit PGP verify-* control rules from the source. + /// + /// + /// Populated only when control.ctl states a control-message type, + /// From pattern, newsgroup pattern, and verification identity. Rules are + /// not collapsed. This array is not evaluated at runtime. + /// + public PgpAuthorityAuthorizationOptions[] Authorizations { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritySourceOptions.cs b/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritySourceOptions.cs new file mode 100644 index 00000000..27218ebc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PgpAuthoritySourceOptions.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Provenance for the ISC/INN control.ctl snapshot that populated + /// . + /// + public sealed class PgpAuthoritySourceOptions + { + /// + /// Gets or sets the ISC canonical control.ctl URL. + /// + public string? Url { get; set; } + + /// + /// Gets or sets the INN GitHub samples/control.ctl URL. + /// + public string? InnUrl { get; set; } + + /// + /// Gets or sets the source Last modified date from control.ctl + /// (yyyy-MM-dd). + /// + public string? LastModified { get; set; } + + /// + /// Gets or sets which configured URL was actually retrieved. + /// + /// + /// InnUrl when the ISC URL was unavailable and the INN copy was used. + /// + public string? RetrievedFrom { get; set; } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PostFilterLeftoverConfigurationValidator.cs b/src/VectorNNTPD.TransitService/Configuration/PostFilterLeftoverConfigurationValidator.cs new file mode 100644 index 00000000..30dd7a08 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PostFilterLeftoverConfigurationValidator.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Rejects leftover Nntpd:PostFilter so cluster policy cannot split from NntpDB. + /// + public sealed class PostFilterLeftoverConfigurationValidator : IValidateOptions + { + private readonly IConfiguration _configuration; + + /// Initializes the leftover-section check. + public PostFilterLeftoverConfigurationValidator(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _configuration = configuration; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var section = _configuration.GetSection("Nntpd:PostFilter"); + if (!section.Exists() || !section.GetChildren().Any()) + { + return ValidateOptionsResult.Success; + } + + return ValidateOptionsResult.Fail( + "Nntpd:PostFilter is not node-local configuration. Cluster PostFilter policy is loaded from NntpDB table nntppostfiltercurrent."); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/PostFilterOptions.cs b/src/VectorNNTPD.TransitService/Configuration/PostFilterOptions.cs new file mode 100644 index 00000000..af7c1559 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/PostFilterOptions.cs @@ -0,0 +1,201 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// Operational gate for NNTPD PostFilter. + public enum PostFilterGateState + { + /// Skip remaining PostFilter stages, including quota and SpamAssassin. + Disabled = 0, + + /// Evaluate deny, ArtType, quotas, and SpamAssassin. + Active = 1, + + /// Reject every POST that reaches PostFilter. + Closed = 2, + } + + /// + /// SpamAssassin scanner-fault action. Required when + /// is . + /// + public enum PostFilterSpamOnFailure + { + /// Scanner fault rejects the article (441). + Reject = 0, + + /// Scanner fault continues with the existing reservation. + Accept = 1, + } + + /// Deterministic SPAMD host selection compiled into the PostFilter snapshot. + public enum PostFilterSpamAssassinHostSelection + { + /// Rotate across ; failover on connect failure only. + RoundRobin = 0, + + /// Always try hosts in list order; failover on connect failure only. + Failover = 1, + } + + /// In-memory PostFilter compile model produced from NntpDB. + /// + /// Not bound from Nntpd:PostFilter. Cluster policy is nntppostfiltercurrent. + /// POST does not parse this object. The compiler produces an immutable snapshot. + /// + public sealed class PostFilterOptions + { + /// Gets or sets the operational gate. + public PostFilterGateState Gate { get; set; } = PostFilterGateState.Disabled; + + /// + /// Gets or sets deny-list entries. Plaintext usernames are hashed to + /// lowercase MD5 hex at compile time. Stored NntpDB values are already hashed. + /// + public string[] DeniedAccounts { get; set; } = []; + + /// Gets or sets client CIDRs that are denied. + public string[] DeniedCidrs { get; set; } = []; + + /// + /// Gets or sets allow-list entries (same MD5 identity rules as + /// ) that skip SpamAssassin only. + /// Does not skip the gate, deny, ArtType, or quotas. + /// + public string[] AllowlistedAccounts { get; set; } = []; + + /// Gets or sets client CIDRs that skip SpamAssassin only. + public string[] AllowlistedCidrs { get; set; } = []; + + /// + /// Gets or sets names rejected by Stage 3. + /// Empty means no type policy. + /// + public string[] RejectArtTypes { get; set; } = []; + + /// Gets or sets accept-quota windows and ceilings. + [Required] + public PostFilterQuotaOptions Quota { get; set; } = new(); + + /// Gets or sets SpamAssassin eligibility and failure policy. + [Required] + public PostFilterSpamAssassinOptions SpamAssassin { get; set; } = new(); + } + + /// Quota windows and ceilings compiled into the PostFilter snapshot. + public sealed class PostFilterQuotaOptions + { + /// Gets or sets the sustained window. Must be positive. + public TimeSpan LongWindow { get; set; } = TimeSpan.FromDays(1); + + /// Gets or sets the burst window. Must be positive. + public TimeSpan ShortWindow { get; set; } = TimeSpan.FromMinutes(10); + + /// Gets or sets the L message ceiling. 0 disables the dimension. + public long MaxMessagesLong { get; set; } + + /// Gets or sets the L byte ceiling. 0 disables the dimension. + public long MaxBytesLong { get; set; } + + /// Gets or sets the L identical-body ceiling. 0 disables the dimension. + public long MaxIdenticalLong { get; set; } + + /// Gets or sets the S message ceiling. 0 disables the dimension. + public long MaxMessagesShort { get; set; } + + /// Gets or sets the S byte ceiling. 0 disables the dimension. + public long MaxBytesShort { get; set; } + + /// Gets or sets the S identical-body ceiling. 0 disables the dimension. + public long MaxIdenticalShort { get; set; } + } + + /// SpamAssassin stage options. Disabled until explicitly enabled. + /// + /// SpamAssassin is a targeted small-article scanner. The default eligibility + /// matches the historical production boundary: + /// (131072) exclusive maximum and excluded. + /// MaxArticleSize = 0 disables the size gate. An empty + /// array disables type exclusion. + /// Ineligible articles skip CHECK; that is not an event. + /// Spam findings always reject in v1 (OnSpam is not independently configurable). + /// is required when is . + /// + public sealed class PostFilterSpamAssassinOptions + { + /// Historical exclusive ArtSize maximum used as the safe default (131072). + public const int DefaultMaxArticleSize = 131_072; + + /// Default pooled concurrent SPAMD connections. + public const int DefaultMaxConnections = 4; + + /// Minimum when SpamAssassin is enabled. + public const int MinMaxConnections = 1; + + /// Maximum when SpamAssassin is enabled. + public const int MaxMaxConnections = 32; + + /// Default SPAMC protocol version sent on CHECK. + public const string DefaultProtocolVersion = "1.5"; + + /// Gets or sets whether CHECK is invoked for eligible articles. + public bool Enabled { get; set; } + + /// + /// Gets or sets scanner-fault behaviour. Required when is true. + /// + public PostFilterSpamOnFailure? OnFailure { get; set; } + + /// + /// Gets or sets the exclusive ArtSize eligibility maximum. + /// Articles with ArtSize >= MaxArticleSize skip CHECK. + /// 0 disables the size gate. Default is . + /// + public int MaxArticleSize { get; set; } = DefaultMaxArticleSize; + + /// + /// Gets or sets names excluded from CHECK. + /// Default is . Empty means no type exclusion. + /// + public string[] ExcludeArtTypes { get; set; } = ["YEncoded"]; + + /// Gets or sets SPAMD hosts. Required when is true. + public string[] Hosts { get; set; } = []; + + /// Gets or sets the shared SPAMD port for every host. + public int Port { get; set; } = 783; + + /// Gets or sets the SPAMC protocol version used in the CHECK request line. + public string ProtocolVersion { get; set; } = DefaultProtocolVersion; + + /// + /// Gets or sets the maximum number of concurrent persistent SPAMD connections. + /// Each connection serializes CHECK requests; concurrency is the pool size. + /// + public int MaxConnections { get; set; } = DefaultMaxConnections; + + /// Gets or sets deterministic host selection. + public PostFilterSpamAssassinHostSelection HostSelection { get; set; } = + PostFilterSpamAssassinHostSelection.RoundRobin; + + /// Gets or sets the connect timeout. + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Gets or sets the CHECK operation timeout. + /// + /// This is the wall-clock budget for one CHECK, including connect. + /// When SpamAssassin is enabled, the PostFilter reservation hold is + /// max(10s, OperationTimeout + 1s) so the reservation cannot expire + /// during a legitimate CHECK. + /// + public TimeSpan OperationTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Minimum when SpamAssassin is enabled. + public static readonly TimeSpan MinOperationTimeout = TimeSpan.FromSeconds(1); + + /// Maximum when SpamAssassin is enabled. + /// Caps derived reservation hold so crash-recovery TTL cannot become unbounded. + public static readonly TimeSpan MaxOperationTimeout = TimeSpan.FromMinutes(2); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/RedisOptions.cs b/src/VectorNNTPD.TransitService/Configuration/RedisOptions.cs new file mode 100644 index 00000000..b00cab96 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/RedisOptions.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Top-level Redis connection options. Hosts are StackExchange.Redis endpoints/seeds + /// for one shared topology, not independently round-robined servers. + /// + public sealed class RedisOptions + { + /// Configuration section name. + public const string SectionName = "Redis"; + + /// Default Redis TCP port. + public const int DefaultPort = 6379; + + /// Gets or sets Redis hostnames or IP addresses used as multiplexer endpoints. + public string[] Host { get; set; } = []; + + /// Gets or sets the Redis TCP port applied to every configured host. + public int Port { get; set; } = DefaultPort; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/RedisOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/RedisOptionsValidator.cs new file mode 100644 index 00000000..d9ff977e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/RedisOptionsValidator.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// Validates at bind / startup time. + public sealed class RedisOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, RedisOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + if (options.Host is null || options.Host.Length == 0) + { + failures.Add($"{nameof(RedisOptions.Host)} must contain at least one Redis hostname or IP address."); + } + else + { + for (var i = 0; i < options.Host.Length; i++) + { + if (string.IsNullOrWhiteSpace(options.Host[i])) + { + failures.Add($"{nameof(RedisOptions.Host)}[{i}] must not be empty."); + } + } + } + + if (options.Port is < 1 or > 65535) + { + failures.Add($"{nameof(RedisOptions.Port)} must be an integer in the range 1–65535."); + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/SmtpOptions.cs b/src/VectorNNTPD.TransitService/Configuration/SmtpOptions.cs new file mode 100644 index 00000000..60dfa9e8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/SmtpOptions.cs @@ -0,0 +1,57 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// SMTP transport settings nested under . + /// + /// Distinguishes SMTP relay (often port 25, possibly plaintext on a private network) + /// from message submission (RFC 6409, typically 587 STARTTLS or 465 implicit TLS). + /// This client speaks SMTP; it does not assume MSA-only rules when the destination + /// is a local relay. + /// + public sealed class SmtpOptions + { + /// Default submission port used when configuration omits . + public const int DefaultPort = 587; + + /// Gets or sets the SMTP server hostname or IP. Required when email is enabled. + public string Host { get; set; } = string.Empty; + + /// Gets or sets the SMTP TCP port (1–65535). Default . + public int Port { get; set; } = DefaultPort; + + /// Gets or sets the explicit TLS mode. Never inferred from . + public SmtpSecurityMode Security { get; set; } = SmtpSecurityMode.StartTls; + + /// + /// Gets or sets the SMTP AUTH username. Empty means no authentication. + /// Override with Email__Smtp__Username; never commit real values. + /// + public string Username { get; set; } = string.Empty; + + /// + /// Gets or sets the SMTP AUTH password. Override with Email__Smtp__Password + /// or a secret store. Never logged. + /// + public string Password { get; set; } = string.Empty; + + /// + /// Gets or sets whether AUTH is refused unless the session is already TLS-protected. + /// Default . Plaintext AUTH requires an explicit . + /// + public bool RequireTlsForAuthentication { get; set; } = true; + + /// Gets or sets the TCP connect timeout. Default 15 seconds. + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(15); + + /// Gets or sets the SMTP command/read/write timeout. Default 30 seconds. + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Gets or sets the maximum delivery attempts including the first. Default 3. + public int MaxAttempts { get; set; } = 3; + + /// Gets or sets the first retry delay. Default 2 seconds. + public TimeSpan InitialRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// Gets or sets the retry delay ceiling. Default 60 seconds. + public TimeSpan MaximumRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/SmtpSecurityMode.cs b/src/VectorNNTPD.TransitService/Configuration/SmtpSecurityMode.cs new file mode 100644 index 00000000..e1a9bf8e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/SmtpSecurityMode.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// SMTP transport security mode. Explicit configuration wins over port defaults. + /// + /// + /// + /// These modes are mutually exclusive. The client never infers a mode from port + /// number alone and never silently downgrades from a required TLS mode. + /// + /// + /// — plaintext TCP. Legitimate for a controlled local relay. + /// — RFC 3207 STARTTLS after the SMTP greeting and first EHLO. + /// — TLS handshake immediately after TCP connect (typically port 465). + /// + /// Port defaults (25 / 587 / 465) are documentation only; is authoritative. + /// + public enum SmtpSecurityMode + { + /// Plaintext SMTP (no TLS). + None = 0, + + /// STARTTLS after EHLO (RFC 3207). Typical message-submission port 587. + StartTls = 1, + + /// Implicit TLS immediately after TCP connect (RFC 8314). Typical port 465. + ImplicitTls = 2, + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/SpeedTestOptions.cs b/src/VectorNNTPD.TransitService/Configuration/SpeedTestOptions.cs new file mode 100644 index 00000000..b25f05ef --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/SpeedTestOptions.cs @@ -0,0 +1,125 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Diagnostic limits for the VectorNNTP SPEEDTEST extension under Nntpd:SpeedTest. + /// + /// + /// + /// These bounds cap synthetic diagnostic traffic. They are not article-ingestion, + /// HistoryDB, Redis, or Transit feed settings. SPEEDTEST does not open outbound + /// peer sockets from ConnectTo. + /// + /// + /// Defaults are sized for a short operator diagnostic, not a general-purpose traffic generator. + /// + /// + public sealed class SpeedTestOptions + { + /// Default maximum test duration in seconds. + public const int DefaultMaxDurationSeconds = 10; + + /// Minimum accepted duration in seconds. + public const int MinDurationSeconds = 1; + + /// Maximum accepted duration in seconds. + public const int MaxDurationSecondsLimit = 60; + + /// Default maximum payload bytes transferred in one test (64 MiB). + public const long DefaultMaxBytes = 64L * 1024 * 1024; + + /// Minimum accepted byte limit (one payload line). + public const long MinBytes = 1024; + + /// Maximum accepted byte limit (1 GiB). + public const long MaxBytesLimit = 1024L * 1024 * 1024; + + /// Default maximum concurrent SPEEDTEST operations on this host. + public const int DefaultMaxConcurrent = 2; + + /// Minimum accepted global concurrency. + public const int MinConcurrent = 1; + + /// Maximum accepted global concurrency. + public const int MaxConcurrentLimit = 8; + + /// Default maximum concurrent SPEEDTEST operations per Transit identifier. + public const int DefaultMaxConcurrentPerPeer = 1; + + /// Minimum accepted per-peer concurrency. + public const int MinConcurrentPerPeer = 1; + + /// Maximum accepted per-peer concurrency. + public const int MaxConcurrentPerPeerLimit = 4; + + /// + /// Gets or sets the maximum wall-clock duration of one SPEEDTEST payload transfer. + /// + /// Default is 10. Valid range is 1–60. + [Range(MinDurationSeconds, MaxDurationSecondsLimit)] + public int MaxDurationSeconds { get; set; } = DefaultMaxDurationSeconds; + + /// + /// Gets or sets the maximum synthetic payload bytes written for one SPEEDTEST. + /// + /// + /// Default is 64 MiB. Valid range is 1024–1073741824. + /// Counts payload body octets (pattern lines including CRLF), not status lines. + /// + [Range(MinBytes, MaxBytesLimit)] + public long MaxBytes { get; set; } = DefaultMaxBytes; + + /// + /// Gets or sets the maximum number of SPEEDTEST operations that may run at once on this host. + /// + /// Default is 2. Valid range is 1–8. + [Range(MinConcurrent, MaxConcurrentLimit)] + public int MaxConcurrent { get; set; } = DefaultMaxConcurrent; + + /// + /// Gets or sets the maximum number of simultaneous SPEEDTEST operations for one Transit identifier. + /// + /// Default is 1. Valid range is 1–4. + [Range(MinConcurrentPerPeer, MaxConcurrentPerPeerLimit)] + public int MaxConcurrentPerPeer { get; set; } = DefaultMaxConcurrentPerPeer; + + /// Returns an immutable copy of the current limits. + public SpeedTestLimits Snapshot() => new(this); + } + + /// Immutable SPEEDTEST limits captured at acquire time. + public readonly struct SpeedTestLimits + { + /// Initializes limits from . + public SpeedTestLimits(SpeedTestOptions options) + { + ArgumentNullException.ThrowIfNull(options); + MaxDuration = TimeSpan.FromSeconds(options.MaxDurationSeconds); + MaxBytes = options.MaxBytes; + MaxConcurrent = options.MaxConcurrent; + MaxConcurrentPerPeer = options.MaxConcurrentPerPeer; + } + + /// Creates limits directly (tests). + public SpeedTestLimits(TimeSpan maxDuration, long maxBytes, int maxConcurrent, int maxConcurrentPerPeer) + { + MaxDuration = maxDuration; + MaxBytes = maxBytes; + MaxConcurrent = maxConcurrent; + MaxConcurrentPerPeer = maxConcurrentPerPeer; + } + + /// Gets the maximum payload duration. + public TimeSpan MaxDuration { get; } + + /// Gets the maximum payload bytes. + public long MaxBytes { get; } + + /// Gets the host-wide concurrency cap. + public int MaxConcurrent { get; } + + /// Gets the per-peer concurrency cap. + public int MaxConcurrentPerPeer { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/SystemdOptions.cs b/src/VectorNNTPD.TransitService/Configuration/SystemdOptions.cs new file mode 100644 index 00000000..06e20bbe --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/SystemdOptions.cs @@ -0,0 +1,56 @@ +using System.ComponentModel.DataAnnotations; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Optional systemd-specific hosting behaviour for Linux deployments. + /// + /// + /// These settings never override systemd-supplied watchdog deadlines + /// (WATCHDOG_USEC / unit WatchdogSec=). Watchdog heartbeats activate only when + /// systemd has configured a watchdog, the process is the intended watchdog target, and + /// is . + /// + public sealed class SystemdOptions + { + /// + /// Gets or sets a value indicating whether watchdog keep-alives may be sent when systemd + /// has configured WatchdogSec= for this process. + /// + public bool EnableWatchdog { get; set; } = true; + + /// + /// Gets or sets a value indicating whether lifecycle STATUS= notifications are sent + /// when systemd notify is enabled. + /// + public bool ReportLifecycleStatus { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the application sends an explicit READY=1 + /// when the application lifecycle reaches . + /// + /// + /// The built-in SystemdLifetime also notifies ready after all hosted services start. + /// Sending ready at the application lifecycle boundary ensures readiness is tied to + /// successful application initialization rather than host construction alone. + /// Duplicate READY=1 notifications are harmless. + /// + public bool NotifyReadyOnApplicationRunning { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the application sends an explicit STOPPING=1 + /// when graceful shutdown begins at the application lifecycle boundary. + /// + public bool NotifyStoppingOnApplicationShutdown { get; set; } = true; + + /// + /// Gets or sets the fraction of the systemd watchdog deadline used as the heartbeat interval. + /// + /// + /// systemd recommends notifying at about half the watchdog timeout. Valid range is (0, 1). + /// + [Range(0.05, 0.9)] + public double WatchdogIntervalFraction { get; set; } = 0.5; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitAllowFromEntry.cs b/src/VectorNNTPD.TransitService/Configuration/TransitAllowFromEntry.cs new file mode 100644 index 00000000..f1da24ff --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitAllowFromEntry.cs @@ -0,0 +1,93 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// A parsed Transit AllowFrom entry: literal prefix or DNS hostname. + /// + public abstract class TransitAllowFromEntry + { + private TransitAllowFromEntry(string original) + { + Original = original; + } + + /// Gets the original configuration text. + public string Original { get; } + + /// Literal IPv4/IPv6 address or CIDR prefix. + public sealed class Literal : TransitAllowFromEntry + { + /// Initializes a literal prefix entry. + public Literal(string original, IpPrefix prefix) + : base(original) + { + Prefix = prefix; + } + + /// Gets the parsed prefix. + public IpPrefix Prefix { get; } + } + + /// DNS hostname that must be resolved to A/AAAA addresses. + public sealed class Hostname : TransitAllowFromEntry + { + /// Initializes a hostname entry. + public Hostname(string original, string hostname) + : base(original) + { + DnsName = hostname; + } + + /// Gets the normalized hostname (no trailing dot). + public string DnsName { get; } + } + + /// Parses one AllowFrom entry. + public static bool TryParse(string text, out TransitAllowFromEntry? entry, out string? error) + { + entry = null; + error = null; + if (string.IsNullOrWhiteSpace(text)) + { + error = "AllowFrom entry must not be empty."; + return false; + } + + var trimmed = text.Trim(); + if (trimmed.Contains('*', StringComparison.Ordinal) || trimmed.Contains('+', StringComparison.Ordinal)) + { + error = "AllowFrom does not support wildcards; use a hostname, IP address, or CIDR prefix."; + return false; + } + + if (IpPrefix.TryParse(trimmed, out var prefix)) + { + entry = new Literal(trimmed, prefix); + return true; + } + + if (trimmed.Contains('/', StringComparison.Ordinal)) + { + error = "AllowFrom CIDR prefix is not a valid IPv4 or IPv6 network."; + return false; + } + + if (IPAddress.TryParse(trimmed, out _)) + { + error = "AllowFrom IP address is not valid."; + return false; + } + + var hostname = trimmed.TrimEnd('.'); + if (!NntpdOptionsValidator.IsValidDnsSuffix(hostname)) + { + error = "AllowFrom entry is not a valid DNS hostname, IP address, or CIDR prefix."; + return false; + } + + entry = new Hostname(trimmed, hostname); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitConnectEndpoint.cs b/src/VectorNNTPD.TransitService/Configuration/TransitConnectEndpoint.cs new file mode 100644 index 00000000..ee8edf08 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitConnectEndpoint.cs @@ -0,0 +1,124 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// A parsed Transit ConnectTo endpoint (host plus required explicit port). + /// + /// + /// Supports DNS names, IPv4, and bracketed IPv6 with a port. This type does not + /// establish outbound connections. + /// + public sealed class TransitConnectEndpoint + { + private TransitConnectEndpoint(string host, int port, bool isIpAddress) + { + Host = host; + Port = port; + IsIpAddress = isIpAddress; + } + + /// Gets the host name or IP address (unbracketed). + public string Host { get; } + + /// Gets the explicit TCP port. + public int Port { get; } + + /// Gets a value indicating whether is a literal IP address. + public bool IsIpAddress { get; } + + /// + /// Parses host:port or [IPv6]:port. An explicit port is required. + /// + public static bool TryParse(string text, out TransitConnectEndpoint? endpoint, out string? error) + { + endpoint = null; + error = null; + if (string.IsNullOrWhiteSpace(text)) + { + error = "ConnectTo entry must not be empty."; + return false; + } + + var trimmed = text.Trim(); + string host; + string portText; + if (trimmed.StartsWith('[')) + { + var close = trimmed.IndexOf(']'); + if (close <= 1) + { + error = "ConnectTo IPv6 endpoint must be written as [address]:port."; + return false; + } + + if (close == trimmed.Length - 1 || trimmed[close + 1] != ':') + { + error = "ConnectTo IPv6 endpoint must include an explicit port after the closing bracket."; + return false; + } + + host = trimmed[1..close]; + portText = trimmed[(close + 2)..]; + } + else + { + var colon = trimmed.LastIndexOf(':'); + if (colon <= 0 || colon == trimmed.Length - 1) + { + error = "ConnectTo endpoint must include an explicit port (host:port or [IPv6]:port)."; + return false; + } + + host = trimmed[..colon]; + portText = trimmed[(colon + 1)..]; + + // Unbracketed IPv6 with a port is ambiguous (multiple colons). Require brackets. + if (host.Contains(':', StringComparison.Ordinal)) + { + error = "ConnectTo IPv6 endpoint must be written as [address]:port."; + return false; + } + } + + if (!int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out var port) + || port is < 1 or > 65535) + { + error = "ConnectTo port must be an integer in the range 1–65535."; + return false; + } + + if (IPAddress.TryParse(host, out var address)) + { + if (address.Equals(IPAddress.Any) || address.Equals(IPAddress.IPv6Any)) + { + error = "ConnectTo must not use an any-address wildcard."; + return false; + } + + var canonical = address.AddressFamily == AddressFamily.InterNetworkV6 + ? address.ToString() + : address.ToString(); + endpoint = new TransitConnectEndpoint(canonical, port, isIpAddress: true); + return true; + } + + if (!NntpdOptionsValidator.IsValidDnsSuffix(host)) + { + error = "ConnectTo host is not a valid DNS hostname or IP address."; + return false; + } + + endpoint = new TransitConnectEndpoint(host.Trim().TrimEnd('.'), port, isIpAddress: false); + return true; + } + + /// + public override string ToString() => + Host.Contains(':', StringComparison.Ordinal) + ? string.Create(CultureInfo.InvariantCulture, $"[{Host}]:{Port}") + : string.Create(CultureInfo.InvariantCulture, $"{Host}:{Port}"); + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitLeftoverConfigurationValidator.cs b/src/VectorNNTPD.TransitService/Configuration/TransitLeftoverConfigurationValidator.cs new file mode 100644 index 00000000..851a49bc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitLeftoverConfigurationValidator.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Rejects JSON that still owns the Transit catalogue or the site-wide junk flags. + /// + public sealed class TransitLeftoverConfigurationValidator : IValidateOptions + { + private readonly IConfiguration _configuration; + + /// Initializes the leftover-section check. + public TransitLeftoverConfigurationValidator(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + _configuration = configuration; + } + + /// + public ValidateOptionsResult Validate(string? name, NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (_configuration.GetSection(TransitPeersOptions.SectionName).GetChildren().Any()) + { + return ValidateOptionsResult.Fail( + "Transit peers are not JSON configuration. The catalogue is loaded from NntpDB table nntptransitcurrent."); + } + + if (_configuration.GetSection("Nntpd:Transit:WantTrash").Exists() + || _configuration.GetSection("Nntpd:Transit:LogTrash").Exists()) + { + return ValidateOptionsResult.Fail( + "Nntpd:Transit:WantTrash and Nntpd:Transit:LogTrash are not node-local settings. They are loaded from nntptransitglobalrevision."); + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypes.cs b/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypes.cs new file mode 100644 index 00000000..66ee140c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypes.cs @@ -0,0 +1,70 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Diablo-style article message-type flags (configuration/policy only). + /// + /// + /// Names follow Diablo ArtTypeConv(). is the value for both + /// binary and binaries. is the union of every concrete flag. + /// Bit values match Common ArticleType. Receive matching uses + /// ArticleTypeCapabilities against ArticleRecord.ArtType. Send matching is not wired. + /// + [Flags] + public enum TransitMessageTypes + { + /// Diablo none — no selected types. + None = 0, + + /// Diablo default. Distinct from . + Default = 1 << 0, + + /// Diablo control. + Control = 1 << 1, + + /// Diablo cancel. + Cancel = 1 << 2, + + /// Diablo mime. + Mime = 1 << 3, + + /// Diablo binary / binaries. + Binary = 1 << 4, + + /// Diablo uuencode. + UuEncode = 1 << 5, + + /// Diablo base64. + Base64 = 1 << 6, + + /// Diablo yenc. + Yenc = 1 << 7, + + /// Diablo bommanews. + BommaNews = 1 << 8, + + /// Diablo unidata. + UniData = 1 << 9, + + /// Diablo multipart. + Multipart = 1 << 10, + + /// Diablo html. + Html = 1 << 11, + + /// Diablo ps. + PostScript = 1 << 12, + + /// Diablo binhex. + BinHex = 1 << 13, + + /// Diablo partial. + Partial = 1 << 14, + + /// Diablo pgp. + PgpMessage = 1 << 15, + + /// Union of every concrete message-type flag (Diablo all). + All = Default | Control | Cancel | Mime | Binary | UuEncode | Base64 | Yenc | BommaNews + | UniData | Multipart | Html | PostScript | BinHex | Partial | PgpMessage, + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypesParser.cs b/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypesParser.cs new file mode 100644 index 00000000..58eb02d3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitMessageTypesParser.cs @@ -0,0 +1,134 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Parses and formats from Diablo ArtTypeConv names. + /// + public static class TransitMessageTypesParser + { + /// Canonical default when MessageTypes is omitted. + public static readonly string[] DefaultConfiguredValues = ["default"]; + + private static readonly (string Name, TransitMessageTypes Value)[] Tokens = + [ + ("none", TransitMessageTypes.None), + ("default", TransitMessageTypes.Default), + ("control", TransitMessageTypes.Control), + ("cancel", TransitMessageTypes.Cancel), + ("mime", TransitMessageTypes.Mime), + ("binary", TransitMessageTypes.Binary), + ("binaries", TransitMessageTypes.Binary), + ("uuencode", TransitMessageTypes.UuEncode), + ("base64", TransitMessageTypes.Base64), + ("yenc", TransitMessageTypes.Yenc), + ("bommanews", TransitMessageTypes.BommaNews), + ("unidata", TransitMessageTypes.UniData), + ("multipart", TransitMessageTypes.Multipart), + ("html", TransitMessageTypes.Html), + ("ps", TransitMessageTypes.PostScript), + ("binhex", TransitMessageTypes.BinHex), + ("partial", TransitMessageTypes.Partial), + ("pgp", TransitMessageTypes.PgpMessage), + ("all", TransitMessageTypes.All), + ]; + + /// + /// Parses a JSON string-array of Diablo names into flags. + /// + /// + /// Comparison is case-insensitive. Surrounding whitespace on each value is ignored. + /// Duplicate names are OR-ed (harmless). Unknown values fail. + /// + public static bool TryParse( + IReadOnlyList? values, + out TransitMessageTypes types, + out string? error) + { + types = TransitMessageTypes.None; + error = null; + if (values is null || values.Count == 0) + { + types = TransitMessageTypes.Default; + return true; + } + + for (var i = 0; i < values.Count; i++) + { + var raw = values[i]; + if (raw is null || !TryParseToken(raw, out var flag)) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"[{i}] is not a recognised MessageTypes value."); + types = TransitMessageTypes.None; + return false; + } + + types |= flag; + } + + return true; + } + + /// Parses one Diablo token (case-insensitive, trimmed). + public static bool TryParseToken(string value, out TransitMessageTypes type) + { + ArgumentNullException.ThrowIfNull(value); + type = TransitMessageTypes.None; + var token = value.AsSpan().Trim(); + if (token.IsEmpty) + { + return false; + } + + foreach (var (name, flag) in Tokens) + { + if (token.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + type = flag; + return true; + } + } + + return false; + } + + /// + /// Returns canonical lowercase Diablo names for diagnostics (never includes passwords). + /// + public static IReadOnlyList ToCanonicalNames(TransitMessageTypes types) + { + if (types == TransitMessageTypes.None) + { + return ["none"]; + } + + if (types == TransitMessageTypes.All) + { + return ["all"]; + } + + var names = new List(); + foreach (var (name, flag) in Tokens) + { + if (flag is TransitMessageTypes.None or TransitMessageTypes.All) + { + continue; + } + + if (name.Equals("binaries", StringComparison.Ordinal)) + { + continue; + } + + if (types.HasFlag(flag)) + { + names.Add(name); + } + } + + return names; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitOptions.cs b/src/VectorNNTPD.TransitService/Configuration/TransitOptions.cs new file mode 100644 index 00000000..88e3a77d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitOptions.cs @@ -0,0 +1,53 @@ +using System.ComponentModel.DataAnnotations; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// NNTPD-local transit/streaming runtime options under Nntpd:Transit. + /// + /// + /// Peer authorization is the published MySQL Transit catalogue. This type holds + /// the process-local STREAM TX depth. and + /// are copied from nntptransitglobalrevision when + /// that catalogue is published. JSON must not set those two flags. + /// + public sealed class TransitOptions + { + /// + /// Gets or sets the maximum number of concurrent outstanding STREAM article TX operations + /// admitted by NntpStreamArticleTxScheduler. + /// + /// + /// Default is 8. Valid range is 4–16 (rejected outside that range; not clamped). + /// Independent of TX Channel capacity, Pipe thresholds, and TAKETHIS ingestion queue capacity. + /// + [Range(4, 16)] + public int StreamOutstandingArticleDepth { get; set; } = 8; + + /// + /// Gets or sets whether TAKETHIS/IHAVE articles posted only to unknown or + /// RFC 6048 j groups are accepted and treated as junk internally. + /// + /// + /// INN wanttrash. Default is true. When , + /// TAKETHIS/IHAVE articles posted only to unknown groups are accepted and + /// classified as junk after dequeue. When , those + /// articles are rejected before enqueue with a - news event. RFC 6048 + /// j (PeerOnly) groups remain accepted junk. The original + /// Newsgroups: header is not rewritten. POST is not subject to this + /// policy. The in-memory catalogue snapshot is consulted only after + /// ArticleRecord exists. + /// + public bool WantTrash { get; set; } = true; + + /// + /// Gets or sets whether accepted-junk events are written to the INN news log. + /// + /// + /// INN logtrash. Default is true. When , + /// junk articles are still accepted; only the j news line is omitted. + /// Accepted (+) and rejected (-) events are still written. + /// + public bool LogTrash { get; set; } = true; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitPeerOptions.cs b/src/VectorNNTPD.TransitService/Configuration/TransitPeerOptions.cs new file mode 100644 index 00000000..956901b9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitPeerOptions.cs @@ -0,0 +1,138 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Bindable configuration for one named Transit peer (dictionary value under top-level Transit). + /// + /// + /// The dictionary key is the protocol-safe Identifier, not the display name. + /// is the required human-readable administrative name. + /// Do not log . + /// + public sealed class TransitPeerOptions + { + /// + /// Gets or sets the human-readable administrative display name for this peer. + /// + /// + /// Required. Preserved exactly as configured (no trimming, case-folding, or hyphenation). + /// Not a protocol argument and not derived from the dictionary-key identifier. + /// May contain spaces and punctuation. Must be non-empty, not whitespace-only, + /// at most characters, + /// and must not contain control characters. + /// + public string PeerName { get; set; } = string.Empty; + + /// + /// Gets or sets the maximum simultaneous inbound connections associated with this peer. + /// + /// + /// Required. Valid range is 0–4096. This is the cluster-wide inbound + /// cap for the peer Identifier. 0 means closed: admit no new + /// inbound connections. It is not unlimited. Existing connections are not + /// dropped when the limit is lowered. + /// + public int? MaxIncomingConnections { get; set; } + + /// + /// Gets or sets the maximum simultaneous outbound connections that may later be + /// established to this peer. + /// + /// + /// Required. Valid range is 0–4096. Stored and validated only; this host does + /// not open outbound peer sockets from . + /// + public int? MaxOutgoingConnections { get; set; } + + /// + /// Gets or sets inbound source authorization entries (DNS hostnames, IPv4/IPv6 addresses, CIDR prefixes). + /// + /// + /// Empty means this peer cannot match inbound sources (outbound-only policy). + /// + public string[] AllowFrom { get; set; } = []; + + /// + /// Gets or sets outbound endpoint strings (host:port or [IPv6]:port). + /// + /// + /// Optional. Parsed and validated only; no outbound connection is established. + /// An explicit port is required. + /// + public string[] ConnectTo { get; set; } = []; + + /// + /// Gets or sets the optional peer-specific AUTHINFO username. + /// + /// + /// Configuration requires and + /// both set or both blank. Peer AUTHINFO succeeds only when both are + /// non-empty and both supplied values match. + /// + public string Username { get; set; } = string.Empty; + + /// + /// Gets or sets the optional peer-specific AUTHINFO password. + /// + /// + /// Never log this value. Authentication requires a matching non-empty + /// as well. + /// + public string Password { get; set; } = string.Empty; + + /// + /// Gets or sets the TLS mode string: blank (none), TLS, or STARTTLS. + /// + public string Ssl { get; set; } = string.Empty; + + /// + /// Gets or sets the newsfeeds(5)-style subscription expression. + /// + /// Default is exactly *. + public string Patterns { get; set; } = "*"; + + /// + /// Gets or sets whether duplicate/in-flight article offers should later be deferred + /// (431/436) rather than rejected (438/435). + /// + /// + /// Default is . CHECK and IHAVE apply it when History + /// already contains the Message-ID. TAKETHIS does not. + /// + public bool DeferOnDuplicate { get; set; } = true; + + /// + /// Gets or sets the Path-header token reserved for outbound feed loop prevention. + /// + /// + /// Preserved exactly (no case-folding, not a DNS name or IP). Empty is allowed + /// because article Path-header matching is not implemented yet. Not consumed by + /// routing code in this change. + /// + public string PathToken { get; set; } = string.Empty; + + /// + /// Gets or sets the maximum allowed incoming article size for this peer, in bytes. + /// + /// + /// When this options object is compiled into a snapshot, this is the Receive ceiling. + /// Inbound acceptance uses the smaller of this value and the published global maxartsize. + /// Default is . Valid range is 1–2147483647. + /// + public long MaxSize { get; set; } = DefaultMaxSize; + + /// Default (10 MiB). + public const long DefaultMaxSize = 10_485_760; + + /// Maximum accepted (2 GiB − 1 byte). + public const long MaxMaxSize = int.MaxValue; + + /// + /// Gets or sets Diablo message-type names accepted for this peer. + /// + /// + /// Default is ["default"]. Not regex, MIME types, or newsgroup Patterns. + /// Article classification is not implemented. + /// + public string[] MessageTypes { get; set; } = []; + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptions.cs b/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptions.cs new file mode 100644 index 00000000..1a2a2be1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptions.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Top-level Transit configuration: peer identifiers as dictionary keys. + /// + /// + /// + /// Binds the exact JSON shape Transit:{identifier}:{fields}. There is no nested + /// Transit:Peers layer. Distinct from under + /// Nntpd:Transit (STREAM TX depth only). + /// + /// + /// The dictionary key is the stable protocol/machine identifier (single NNTP token, + /// exact ordinal match, not normalized). is + /// the human-readable display name. An empty dictionary is valid (no configured peers). + /// + /// + public sealed class TransitPeersOptions : Dictionary + { + /// Top-level configuration section name. + public const string SectionName = "Transit"; + + /// Initializes an empty peer dictionary. + public TransitPeersOptions() + : base(StringComparer.Ordinal) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptionsValidator.cs b/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptionsValidator.cs new file mode 100644 index 00000000..97863168 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitPeersOptionsValidator.cs @@ -0,0 +1,359 @@ +using System.Globalization; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Validates top-level at bind / startup / reload. + /// + /// + /// Failures name the peer and property. A malformed peer is rejected entirely; it never + /// enters the active snapshot. Passwords are never included in failure messages. + /// + public sealed class TransitPeersOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, TransitPeersOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!TryValidate(options, out var failures)) + { + return ValidateOptionsResult.Fail(failures); + } + + return ValidateOptionsResult.Success; + } + + /// Validates options and returns property-qualified failure messages. + public static bool TryValidate(TransitPeersOptions options, out IReadOnlyList failures) + { + ArgumentNullException.ThrowIfNull(options); + var list = new List(); + var parsed = new List<(string Name, TransitPeerOptions Options, List AllowFrom)>(); + + foreach (var (identifier, peer) in options) + { + if (!TryValidateIdentifier(identifier, out var identifierError)) + { + list.Add(identifierError); + continue; + } + + if (peer is null) + { + list.Add(Format(identifier, null, "peer configuration must not be null.")); + continue; + } + + ValidatePeer(identifier, peer, list, parsed); + } + + if (list.Count == 0) + { + ValidateOverlaps(parsed, list); + } + + failures = list; + return list.Count == 0; + } + + private static void ValidatePeer( + string peerName, + TransitPeerOptions peer, + List failures, + List<(string Name, TransitPeerOptions Options, List AllowFrom)> parsed) + { + if (!TryValidatePeerName(peer.PeerName, out var peerNameError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.PeerName), peerNameError)); + } + + ValidateConnectionLimit(peerName, nameof(TransitPeerOptions.MaxIncomingConnections), peer.MaxIncomingConnections, failures); + ValidateConnectionLimit(peerName, nameof(TransitPeerOptions.MaxOutgoingConnections), peer.MaxOutgoingConnections, failures); + ValidateMaxSize(peerName, peer.MaxSize, failures); + ValidatePathToken(peerName, peer.PathToken, failures); + + ValidatePairedCredentials(peerName, peer, failures); + + if (!TransitMessageTypesParser.TryParse(peer.MessageTypes, out _, out var messageTypesError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.MessageTypes), messageTypesError ?? "invalid MessageTypes.")); + } + + if (!TransitSslParser.TryParse(peer.Ssl, out _, out var sslError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.Ssl), sslError!)); + } + + if (!NewsfeedsPattern.TryParse(peer.Patterns, out _, out var patternError)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.Patterns), patternError!)); + } + + var allowFrom = new List(); + var entries = peer.AllowFrom ?? []; + for (var i = 0; i < entries.Length; i++) + { + if (!TransitAllowFromEntry.TryParse(entries[i], out var entry, out var error) || entry is null) + { + failures.Add(Format(peerName, $"{nameof(TransitPeerOptions.AllowFrom)}[{i}]", error ?? "invalid AllowFrom entry.")); + continue; + } + + allowFrom.Add(entry); + } + + var connectTo = peer.ConnectTo ?? []; + for (var i = 0; i < connectTo.Length; i++) + { + if (!TransitConnectEndpoint.TryParse(connectTo[i], out _, out var error)) + { + failures.Add(Format(peerName, $"{nameof(TransitPeerOptions.ConnectTo)}[{i}]", error ?? "invalid ConnectTo endpoint.")); + } + } + + parsed.Add((peerName, peer, allowFrom)); + } + + /// + /// Maximum Transit identifier length in characters (and SPEEDTEST token octets). + /// + public const int MaxPeerIdentifierLength = 256; + + /// Maximum PeerName length in characters (human-readable labels). + public const int MaxPeerNameLength = 256; + + /// Maximum PathToken length in characters. + public const int MaxPathTokenLength = 255; + + /// + /// Transit identifier grammar: 1–256 visible ASCII characters (0x21–0x7E). + /// + /// + /// No space, TAB, or other whitespace; no control characters; no Unicode. + /// Not normalized. The exact configured string is the identity (ordinal). + /// Safe as a single NNTP command token: SPEEDTEST identifier. + /// + public static bool IsPeerIdentifier(ReadOnlySpan identifier) + { + if (identifier.Length is < 1 or > MaxPeerIdentifierLength) + { + return false; + } + + for (var i = 0; i < identifier.Length; i++) + { + if (identifier[i] is < 0x21 or > 0x7E) + { + return false; + } + } + + return true; + } + + /// + public static bool IsPeerIdentifier(ReadOnlySpan identifier) + { + if (identifier.Length is < 1 or > MaxPeerIdentifierLength) + { + return false; + } + + for (var i = 0; i < identifier.Length; i++) + { + if (identifier[i] is < (char)0x21 or > (char)0x7E) + { + return false; + } + } + + return true; + } + + private static bool TryValidateIdentifier(string? identifier, out string error) + { + error = string.Empty; + if (string.IsNullOrEmpty(identifier)) + { + error = "Transit peer identifier must be non-empty."; + return false; + } + + if (!IsPeerIdentifier(identifier.AsSpan())) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"Transit peer identifier '{identifier}' must be 1-{MaxPeerIdentifierLength} visible ASCII characters (0x21-0x7E) with no whitespace or control characters."); + return false; + } + + return true; + } + + private static bool TryValidatePeerName(string? peerName, out string error) + { + error = string.Empty; + if (string.IsNullOrWhiteSpace(peerName)) + { + error = "must be non-empty and not whitespace-only."; + return false; + } + + if (peerName.Length > MaxPeerNameLength) + { + error = string.Create( + CultureInfo.InvariantCulture, + $"exceeds the maximum length of {MaxPeerNameLength} characters."); + return false; + } + + foreach (var ch in peerName) + { + if (char.IsControl(ch)) + { + error = "must not contain a control character."; + return false; + } + } + + return true; + } + + private static void ValidatePathToken(string peerName, string? pathToken, List failures) + { + var value = pathToken ?? string.Empty; + if (value.Length > MaxPathTokenLength) + { + failures.Add( + Format( + peerName, + nameof(TransitPeerOptions.PathToken), + string.Create( + CultureInfo.InvariantCulture, + $"must be at most {MaxPathTokenLength} characters."))); + return; + } + + foreach (var ch in value) + { + if (char.IsControl(ch)) + { + failures.Add(Format(peerName, nameof(TransitPeerOptions.PathToken), "must not contain control characters.")); + return; + } + } + } + + private static void ValidatePairedCredentials(string peerName, TransitPeerOptions peer, List failures) + { + var username = (peer.Username ?? string.Empty).Trim(); + var password = peer.Password ?? string.Empty; + var hasUser = username.Length > 0; + var hasPassword = password.Length > 0; + if (hasUser == hasPassword) + { + return; + } + + failures.Add( + Format( + peerName, + hasUser ? nameof(TransitPeerOptions.Password) : nameof(TransitPeerOptions.Username), + "Username and Password must both be set or both be blank.")); + } + + private static void ValidateMaxSize(string peerName, long maxSize, List failures) + { + if (maxSize is < 1 or > TransitPeerOptions.MaxMaxSize) + { + failures.Add( + Format( + peerName, + nameof(TransitPeerOptions.MaxSize), + string.Create( + CultureInfo.InvariantCulture, + $"must be an integer in the range 1–{TransitPeerOptions.MaxMaxSize}."))); + } + } + + private static void ValidateConnectionLimit(string peerName, string property, int? value, List failures) + { + if (value is null) + { + failures.Add(Format(peerName, property, "is required.")); + return; + } + + if (value < 0 || value > TransitPeerPolicy.MaxConnectionLimit) + { + failures.Add( + Format( + peerName, + property, + string.Create( + CultureInfo.InvariantCulture, + $"must be an integer in the range 0–{TransitPeerPolicy.MaxConnectionLimit}."))); + } + } + + private static void ValidateOverlaps( + List<(string Name, TransitPeerOptions Options, List AllowFrom)> parsed, + List failures) + { + for (var i = 0; i < parsed.Count; i++) + { + for (var j = i + 1; j < parsed.Count; j++) + { + if (TryDescribeOverlap(parsed[i], parsed[j], out var message)) + { + failures.Add(message); + } + } + } + } + + private static bool TryDescribeOverlap( + (string Name, TransitPeerOptions Options, List AllowFrom) left, + (string Name, TransitPeerOptions Options, List AllowFrom) right, + out string message) + { + message = string.Empty; + foreach (var leftEntry in left.AllowFrom) + { + foreach (var rightEntry in right.AllowFrom) + { + if (leftEntry is TransitAllowFromEntry.Hostname leftHost + && rightEntry is TransitAllowFromEntry.Hostname rightHost + && leftHost.DnsName.Equals(rightHost.DnsName, StringComparison.OrdinalIgnoreCase)) + { + message = string.Create( + CultureInfo.InvariantCulture, + $"Transit peers '{left.Name}' and '{right.Name}' have overlapping AllowFrom hostname '{leftHost.DnsName}'."); + return true; + } + + if (leftEntry is TransitAllowFromEntry.Literal leftLiteral + && rightEntry is TransitAllowFromEntry.Literal rightLiteral + && leftLiteral.Prefix.Overlaps(rightLiteral.Prefix)) + { + message = string.Create( + CultureInfo.InvariantCulture, + $"Transit peers '{left.Name}' and '{right.Name}' have overlapping AllowFrom prefixes '{leftLiteral.Original}' and '{rightLiteral.Original}'."); + return true; + } + } + } + + return false; + } + + private static string Format(string peerName, string? property, string detail) + { + return property is null + ? string.Create(CultureInfo.InvariantCulture, $"Transit:{peerName}: {detail}") + : string.Create(CultureInfo.InvariantCulture, $"Transit:{peerName}:{property} {detail}"); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitSslMode.cs b/src/VectorNNTPD.TransitService/Configuration/TransitSslMode.cs new file mode 100644 index 00000000..76942cfa --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitSslMode.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Canonical TLS mode for a configured Transit peer. + /// + /// + /// Configuration accepts these values case-insensitively. Invalid values fail validation; + /// they are never silently treated as plaintext. + /// + public enum TransitSslMode + { + /// No TLS (blank configuration value). + None = 0, + + /// Native TLS on connect (TLS). + Tls = 1, + + /// Cleartext TCP upgraded with STARTTLS (STARTTLS). + StartTls = 2, + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/TransitSslParser.cs b/src/VectorNNTPD.TransitService/Configuration/TransitSslParser.cs new file mode 100644 index 00000000..2da8b500 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/TransitSslParser.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.TransitService.Configuration +{ + /// Parses Transit Ssl configuration values into . + public static class TransitSslParser + { + /// + /// Parses a configuration value case-insensitively and returns the canonical mode. + /// + public static bool TryParse(string? value, out TransitSslMode mode, out string? error) + { + mode = TransitSslMode.None; + error = null; + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + var trimmed = value.Trim(); + if (trimmed.Equals("TLS", StringComparison.OrdinalIgnoreCase)) + { + mode = TransitSslMode.Tls; + return true; + } + + if (trimmed.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + mode = TransitSslMode.StartTls; + return true; + } + + error = "Ssl must be blank, 'TLS', or 'STARTTLS'."; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Configuration/XTraceKeyParser.cs b/src/VectorNNTPD.TransitService/Configuration/XTraceKeyParser.cs new file mode 100644 index 00000000..eb1cfb31 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Configuration/XTraceKeyParser.cs @@ -0,0 +1,56 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.TransitService.Configuration +{ + /// + /// Decodes the persisted POST X-Trace AES-256 key from configuration text. + /// + /// + /// Accepts 64 hex characters or Base64 of exactly 32 bytes. Does not generate keys. + /// Failure paths must not include the supplied secret in messages. + /// + public static class XTraceKeyParser + { + /// Required AES-256 key length in bytes. + public const int KeyLength = 32; + + /// + /// Attempts to decode into a 32-byte AES-256 key. + /// + public static bool TryDecode(string? text, [NotNullWhen(true)] out byte[]? key) + { + key = null; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var trimmed = text.Trim(); + if (trimmed.Length == KeyLength * 2) + { + try + { + var hex = Convert.FromHexString(trimmed); + if (hex.Length == KeyLength) + { + key = hex; + return true; + } + } + catch (FormatException) + { + return false; + } + } + + Span buffer = stackalloc byte[KeyLength]; + if (Convert.TryFromBase64String(trimmed, buffer, out var written) && written == KeyLength) + { + key = buffer.ToArray(); + return true; + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsFormatter.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsFormatter.cs new file mode 100644 index 00000000..8b3a83e9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsFormatter.cs @@ -0,0 +1,155 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// Formats a feed-diagnostics snapshot for a single Information log. + /// + /// Output contains peer names, remote IP/port, counters, and wait totals. + /// It does not include Message-IDs, credentials, or article payloads. + /// + public static class FeedDiagnosticsFormatter + { + /// Builds the periodic snapshot text. + public static string Format(FeedDiagnosticsSnapshot snapshot, bool includeSessions) + { + ArgumentNullException.ThrowIfNull(snapshot); + var sb = new StringBuilder(1024); + sb.Append(CultureInfo.InvariantCulture, $"FEED interval={FormatSeconds(snapshot.Interval)}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={snapshot.IntervalArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" rate={snapshot.ArticleMegabitsPerSecond:F1} Mbps"); + sb.Append(CultureInfo.InvariantCulture, $" articles/sec={snapshot.ArticlesPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage rx_app bytes={snapshot.IntervalTcpBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.TcpMegabitsPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" articles_rx={snapshot.IntervalArticlesReceived}"); + sb.Append(CultureInfo.InvariantCulture, $" articles/sec={snapshot.ArticlesReceivedPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" commands={snapshot.IntervalCommands}"); + sb.Append(CultureInfo.InvariantCulture, $" commands/sec={snapshot.CommandsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage ingest admitted={snapshot.IntervalQueuedArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" bytes={snapshot.IntervalQueuedBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.IngestMegabitsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage process completed={snapshot.IntervalArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.ProcessMegabitsPerSecond:F1}"); + sb.AppendLine(); + sb.Append(" stage routing status=not_implemented"); + sb.AppendLine(); + sb.Append(CultureInfo.InvariantCulture, $" stage spool written={snapshot.IntervalSpoolArticles}"); + sb.Append(CultureInfo.InvariantCulture, $" bytes={snapshot.IntervalSpoolBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" mbps={snapshot.SpoolMegabitsPerSecond:F1}"); + sb.Append(CultureInfo.InvariantCulture, $" workers={snapshot.SpoolActiveWorkers}"); + sb.AppendLine(); + + var queueWaitMs = 0d; + var historyWaitMs = 0d; + foreach (var session in snapshot.Sessions) + { + queueWaitMs += session.QueueWaitTime.TotalMilliseconds; + historyWaitMs += session.HistoryWaitTime.TotalMilliseconds; + } + + sb.Append(CultureInfo.InvariantCulture, $" queue articles={snapshot.QueueCount} bytes={snapshot.QueueBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" peak_articles={snapshot.QueuePeakCount} peak_bytes={snapshot.QueuePeakBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting={snapshot.QueueWaitingProducers}"); + sb.Append(CultureInfo.InvariantCulture, $" limit={snapshot.QueueMemoryLimit}"); + sb.Append(CultureInfo.InvariantCulture, $" admit_fail={snapshot.QueueAdmissionFailures}"); + sb.Append(CultureInfo.InvariantCulture, $" admission_wait_ms={queueWaitMs:F0}"); + sb.AppendLine(); + + var lookups = snapshot.HistoryHits + snapshot.HistoryMisses + snapshot.HistoryErrors; + sb.Append(CultureInfo.InvariantCulture, $" history lookups={lookups} hits={snapshot.HistoryHits} misses={snapshot.HistoryMisses} errors={snapshot.HistoryErrors}"); + sb.Append(CultureInfo.InvariantCulture, $" wait_ms={historyWaitMs:F0}"); + sb.AppendLine(); + + var idle = 0; + var receiving = 0; + var waitingHistory = 0; + var waitingQueue = 0; + var waitingWindow = 0; + var completing = 0; + foreach (var session in snapshot.Sessions) + { + switch (session.State) + { + case FeedSessionState.Receiving: + receiving++; + break; + case FeedSessionState.WaitingHistory: + waitingHistory++; + break; + case FeedSessionState.WaitingQueue: + waitingQueue++; + break; + case FeedSessionState.WaitingWindow: + waitingWindow++; + break; + case FeedSessionState.Completing: + completing++; + break; + default: + idle++; + break; + } + } + + sb.Append(CultureInfo.InvariantCulture, $" sessions active={snapshot.ActiveSessions}"); + sb.Append(CultureInfo.InvariantCulture, $" established={snapshot.ActiveSessions}"); + sb.Append(CultureInfo.InvariantCulture, $" idle={idle} receiving={receiving}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting_history={waitingHistory} waiting_queue={waitingQueue}"); + sb.Append(CultureInfo.InvariantCulture, $" waiting_window={waitingWindow} completing={completing}"); + sb.AppendLine(); + + foreach (var peer in snapshot.Peers) + { + var avg = peer.ArticlesReceived > 0 ? peer.ArticleBytes / (double)peer.ArticlesReceived : 0; + sb.Append(CultureInfo.InvariantCulture, $" {peer.PeerName}"); + sb.Append(CultureInfo.InvariantCulture, $" active={peer.ActiveConnections}/{peer.ConfiguredMaxConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" peak={peer.PeakConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" accepted={peer.AcceptedConnections} rejected={peer.RejectedConnections}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={peer.ArticlesCompleted} bytes={peer.ArticleBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" avg_article={avg:F0} checks={peer.Checks}"); + sb.AppendLine(); + } + + if (!includeSessions || snapshot.Sessions.Count == 0) + { + return sb.ToString(); + } + + var shown = 0; + foreach (var session in snapshot.Sessions) + { + if (shown >= 50) + { + sb.AppendLine(" ..."); + break; + } + + sb.Append(CultureInfo.InvariantCulture, $" {session.Remote} peer={session.PeerName}"); + sb.Append(CultureInfo.InvariantCulture, $" age={FormatSeconds(session.Age)} state={session.State}"); + sb.Append(CultureInfo.InvariantCulture, $" articles={session.ArticlesCompleted} bytes={session.ArticleBytes}"); + sb.Append(CultureInfo.InvariantCulture, $" iv_bytes={session.IntervalBytes}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_mbps={FeedRateCalculator.MegabitsPerSecond(session.IntervalBytes, snapshot.Interval):F1}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_art={session.IntervalArticlesReceived}"); + sb.Append( + CultureInfo.InvariantCulture, + $" iv_art/s={FeedRateCalculator.PerSecond(session.IntervalArticlesReceived, snapshot.Interval):F1}"); + sb.Append(CultureInfo.InvariantCulture, $" last={FormatSeconds(session.SinceLastArticle)} idle={FormatSeconds(session.Idle)}"); + sb.Append(CultureInfo.InvariantCulture, $" rx={FormatSeconds(session.ReceiveTime)} hist={FormatSeconds(session.HistoryWaitTime)} q={FormatSeconds(session.QueueWaitTime)}"); + sb.AppendLine(); + shown++; + } + + return sb.ToString(); + } + + private static string FormatSeconds(TimeSpan value) => + value.TotalSeconds.ToString("F1", CultureInfo.InvariantCulture) + "s"; + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsHub.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsHub.cs new file mode 100644 index 00000000..15acf432 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsHub.cs @@ -0,0 +1,400 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Process-wide feed-diagnostics counters. Safe for concurrent session attach/release. + /// + public sealed class FeedDiagnosticsHub : IFeedDiagnostics + { + private readonly ConcurrentDictionary _sessions = new(); + private readonly ConcurrentDictionary _sessionPriors = new(); + private readonly ConcurrentDictionary _peers = new(StringComparer.Ordinal); + private long _priorArticles; + private long _priorArticlesReceived; + private long _priorBytes; + private long _priorCommands; + private long _priorQueuedArticles; + private long _priorQueuedBytes; + private long _priorTcpBytes; + private long _priorSpoolArticles; + private long _priorSpoolBytes; + private long _priorTimestamp; + private long _historyHits; + private long _historyMisses; + private long _historyErrors; + private long _queueFailures; + private long _tcpBytes; + private long _spoolArticles; + private long _spoolBytes; + private int _spoolBusy; + + /// + public bool IsEnabled => true; + + /// + public void OnRejected(string peerName, string remote) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + _ = remote; + var peer = GetPeer(peerName); + Interlocked.Increment(ref peer.Rejected); + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var peerName = session.Authorization.TransitPeerName ?? "none"; + var remote = string.Create( + CultureInfo.InvariantCulture, + $"{session.ClientAddress}:{session.ClientPort}"); + var probe = new FeedSessionProbe(remote, peerName); + var peer = GetPeer(peerName); + Interlocked.Increment(ref peer.Accepted); + while (true) + { + var current = Volatile.Read(ref peer.Active); + var next = current + 1; + if (Interlocked.CompareExchange(ref peer.Active, next, current) == current) + { + UpdatePeak(ref peer.Peak, next); + break; + } + } + + _sessions[probe] = 0; + return probe; + } + + /// + public void OnReleased(FeedSessionProbe? probe) + { + if (probe is null) + { + return; + } + + if (!_sessions.TryRemove(probe, out _)) + { + return; + } + + _sessionPriors.TryRemove(probe, out _); + AddLifetime(probe); + var peer = GetPeer(probe.PeerName); + while (true) + { + var current = Volatile.Read(ref peer.Active); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref peer.Active, current - 1, current) == current) + { + return; + } + } + } + + /// + public void RecordTcpBytes(int bytes) + { + if (bytes > 0) + { + Interlocked.Add(ref _tcpBytes, bytes); + } + } + + /// + public void BeginSpoolWork() => Interlocked.Increment(ref _spoolBusy); + + /// + public void EndSpoolWork(int bytes, bool persisted) + { + if (persisted) + { + Interlocked.Increment(ref _spoolArticles); + if (bytes > 0) + { + Interlocked.Add(ref _spoolBytes, bytes); + } + } + + Interlocked.Decrement(ref _spoolBusy); + } + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(store); + + var now = Stopwatch.GetTimestamp(); + var priorTs = Interlocked.Exchange(ref _priorTimestamp, now); + var sessions = _sessions.Keys.ToArray(); + long articles = 0; + long articlesReceived = 0; + long bytes = 0; + long commands = 0; + long queuedArticles = 0; + long queuedBytes = 0; + long liveHits = 0; + long liveMisses = 0; + long liveErrors = 0; + long liveQueueFail = 0; + var receiving = 0; + var idle = 0; + foreach (var session in sessions) + { + articles += session.ArticlesCompleted; + articlesReceived += session.ArticlesReceived; + bytes += session.ArticleBytes; + commands += session.Commands; + queuedArticles += session.QueueAccepted; + queuedBytes += session.QueueAdmittedBytes; + liveHits += session.HistoryHits; + liveMisses += session.HistoryMisses; + liveErrors += session.HistoryErrors; + liveQueueFail += session.QueueRejected + session.QueueUnavailable; + if (session.State == FeedSessionState.Receiving) + { + receiving++; + } + else if (session.State == FeedSessionState.Idle) + { + idle++; + } + } + + articles += SumPeer(static p => Volatile.Read(ref p.ReleasedArticles)); + articlesReceived += SumPeer(static p => Volatile.Read(ref p.ReleasedArticlesReceived)); + bytes += SumPeer(static p => Volatile.Read(ref p.ReleasedBytes)); + commands += SumPeer(static p => Volatile.Read(ref p.ReleasedCommands)); + queuedArticles += SumPeer(static p => Volatile.Read(ref p.ReleasedQueuedArticles)); + queuedBytes += SumPeer(static p => Volatile.Read(ref p.ReleasedQueuedBytes)); + + var intervalArticles = FeedRateCalculator.NonNegativeDelta( + articles, + Interlocked.Exchange(ref _priorArticles, articles)); + var intervalArticlesReceived = FeedRateCalculator.NonNegativeDelta( + articlesReceived, + Interlocked.Exchange(ref _priorArticlesReceived, articlesReceived)); + var intervalBytes = FeedRateCalculator.NonNegativeDelta( + bytes, + Interlocked.Exchange(ref _priorBytes, bytes)); + var intervalCommands = FeedRateCalculator.NonNegativeDelta( + commands, + Interlocked.Exchange(ref _priorCommands, commands)); + var intervalQueuedArticles = FeedRateCalculator.NonNegativeDelta( + queuedArticles, + Interlocked.Exchange(ref _priorQueuedArticles, queuedArticles)); + var intervalQueuedBytes = FeedRateCalculator.NonNegativeDelta( + queuedBytes, + Interlocked.Exchange(ref _priorQueuedBytes, queuedBytes)); + var tcpBytes = Volatile.Read(ref _tcpBytes); + var intervalTcpBytes = FeedRateCalculator.NonNegativeDelta( + tcpBytes, + Interlocked.Exchange(ref _priorTcpBytes, tcpBytes)); + var spoolArticles = Volatile.Read(ref _spoolArticles); + var intervalSpoolArticles = FeedRateCalculator.NonNegativeDelta( + spoolArticles, + Interlocked.Exchange(ref _priorSpoolArticles, spoolArticles)); + var spoolBytes = Volatile.Read(ref _spoolBytes); + var intervalSpoolBytes = FeedRateCalculator.NonNegativeDelta( + spoolBytes, + Interlocked.Exchange(ref _priorSpoolBytes, spoolBytes)); + + var interval = priorTs == 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(priorTs, now); + var peers = new List(); + foreach (var (name, counters) in _peers) + { + var max = 0; + if (store.Current.Peers.TryGetValue(name, out var policy)) + { + max = policy.MaxIncomingConnections; + } + + var liveArticles = 0L; + var liveCompleted = 0L; + var liveBytes = 0L; + var liveChecks = 0L; + foreach (var session in sessions) + { + if (!string.Equals(session.PeerName, name, StringComparison.Ordinal)) + { + continue; + } + + liveArticles += session.ArticlesReceived; + liveCompleted += session.ArticlesCompleted; + liveBytes += session.ArticleBytes; + liveChecks += session.Checks; + } + + peers.Add(new FeedPeerSnapshot + { + PeerName = name, + ConfiguredMaxConnections = max, + ActiveConnections = Math.Max(0, Volatile.Read(ref counters.Active)), + PeakConnections = Volatile.Read(ref counters.Peak), + AcceptedConnections = Volatile.Read(ref counters.Accepted), + RejectedConnections = Volatile.Read(ref counters.Rejected), + ArticlesReceived = liveArticles + Volatile.Read(ref counters.ReleasedArticlesReceived), + ArticlesCompleted = liveCompleted + Volatile.Read(ref counters.ReleasedArticles), + ArticleBytes = liveBytes + Volatile.Read(ref counters.ReleasedBytes), + Checks = liveChecks + Volatile.Read(ref counters.ReleasedChecks), + }); + } + + peers.Sort(static (a, b) => string.CompareOrdinal(a.PeerName, b.PeerName)); + + var sessionRows = new List(sessions.Length); + foreach (var session in sessions) + { + _sessionPriors.TryGetValue(session, out var prior); + var sessionCommands = session.Commands; + var sessionReceived = session.ArticlesReceived; + var sessionBytes = session.ArticleBytes; + sessionRows.Add(new FeedSessionSnapshot + { + Remote = session.Remote, + PeerName = session.PeerName, + Age = session.Age, + State = session.State, + Commands = sessionCommands, + Checks = session.Checks, + ArticlesReceived = sessionReceived, + ArticlesCompleted = session.ArticlesCompleted, + ArticleBytes = sessionBytes, + IntervalCommands = FeedRateCalculator.NonNegativeDelta(sessionCommands, prior.Commands), + IntervalArticlesReceived = FeedRateCalculator.NonNegativeDelta(sessionReceived, prior.ArticlesReceived), + IntervalBytes = FeedRateCalculator.NonNegativeDelta(sessionBytes, prior.Bytes), + Idle = session.Idle, + SinceLastArticle = session.SinceLastArticle, + ReceiveTime = session.ReceiveTime, + HistoryWaitTime = session.HistoryWaitTime, + QueueWaitTime = session.QueueWaitTime, + MaxReceiveTime = session.MaxReceiveTime, + MaxHistoryWaitTime = session.MaxHistoryWaitTime, + MaxQueueWaitTime = session.MaxQueueWaitTime, + }); + _sessionPriors[session] = new SessionPrior(sessionCommands, sessionReceived, sessionBytes); + } + + sessionRows.Sort(static (a, b) => string.CompareOrdinal(a.Remote, b.Remote)); + + return new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = interval, + Peers = peers, + Sessions = sessionRows, + IntervalArticles = intervalArticles, + IntervalArticlesReceived = intervalArticlesReceived, + IntervalBytes = intervalBytes, + IntervalCommands = intervalCommands, + IntervalTcpBytes = intervalTcpBytes, + IntervalQueuedBytes = intervalQueuedBytes, + IntervalQueuedArticles = intervalQueuedArticles, + IntervalSpoolArticles = intervalSpoolArticles, + IntervalSpoolBytes = intervalSpoolBytes, + TcpBytes = tcpBytes, + SpoolArticles = spoolArticles, + SpoolBytes = spoolBytes, + SpoolActiveWorkers = Math.Max(0, Volatile.Read(ref _spoolBusy)), + ActiveSessions = sessions.Length, + ReceivingSessions = receiving, + IdleSessions = idle, + QueueCount = queue.Count, + QueueBytes = queue.QueuedBytes, + QueuePeakCount = queue.PeakCount, + QueuePeakBytes = queue.PeakQueuedBytes, + QueueWaitingProducers = queue.WaitingProducerCount, + QueueMemoryLimit = queue.MemoryLimitBytes, + HistoryHits = Volatile.Read(ref _historyHits) + liveHits, + HistoryMisses = Volatile.Read(ref _historyMisses) + liveMisses, + HistoryErrors = Volatile.Read(ref _historyErrors) + liveErrors, + QueueAdmissionFailures = Volatile.Read(ref _queueFailures) + liveQueueFail, + }; + } + + private PeerCounters GetPeer(string peerName) => _peers.GetOrAdd(peerName, static _ => new PeerCounters()); + + private long SumPeer(Func selector) + { + var total = 0L; + foreach (var pair in _peers) + { + total += selector(pair.Value); + } + + return total; + } + + private void AddLifetime(FeedSessionProbe probe) + { + var peer = GetPeer(probe.PeerName); + Interlocked.Add(ref peer.ReleasedArticles, probe.ArticlesCompleted); + Interlocked.Add(ref peer.ReleasedArticlesReceived, probe.ArticlesReceived); + Interlocked.Add(ref peer.ReleasedBytes, probe.ArticleBytes); + Interlocked.Add(ref peer.ReleasedChecks, probe.Checks); + Interlocked.Add(ref peer.ReleasedCommands, probe.Commands); + Interlocked.Add(ref peer.ReleasedQueuedArticles, probe.QueueAccepted); + Interlocked.Add(ref peer.ReleasedQueuedBytes, probe.QueueAdmittedBytes); + Interlocked.Add(ref _historyHits, probe.HistoryHits); + Interlocked.Add(ref _historyMisses, probe.HistoryMisses); + Interlocked.Add(ref _historyErrors, probe.HistoryErrors); + Interlocked.Add(ref _queueFailures, probe.QueueRejected + probe.QueueUnavailable); + } + + private static void UpdatePeak(ref int location, int candidate) + { + while (true) + { + var current = Volatile.Read(ref location); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref location, candidate, current) == current) + { + return; + } + } + } + + private sealed class PeerCounters + { + public int Active; + public int Peak; + public long Accepted; + public long Rejected; + public long ReleasedArticles; + public long ReleasedArticlesReceived; + public long ReleasedBytes; + public long ReleasedChecks; + public long ReleasedCommands; + public long ReleasedQueuedArticles; + public long ReleasedQueuedBytes; + } + + private readonly struct SessionPrior(long commands, long articlesReceived, long bytes) + { + public long Commands { get; } = commands; + + public long ArticlesReceived { get; } = articlesReceived; + + public long Bytes { get; } = bytes; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsLogMessages.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsLogMessages.cs new file mode 100644 index 00000000..be80e544 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsLogMessages.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// Source-generated feed-diagnostics reporter messages. + internal static partial class FeedDiagnosticsLogMessages + { + [LoggerMessage( + EventId = 2300, + Level = LogLevel.Information, + Message = "Feed diagnostics enabled (interval {IntervalSeconds}s, sessions={IncludeSessions})")] + public static partial void Enabled(ILogger logger, int IntervalSeconds, bool IncludeSessions); + + [LoggerMessage( + EventId = 2301, + Level = LogLevel.Information, + Message = "{Snapshot}")] + public static partial void Snapshot(ILogger logger, string Snapshot); + + [LoggerMessage( + EventId = 2302, + Level = LogLevel.Information, + Message = "Feed diagnostics stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsService.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsService.cs new file mode 100644 index 00000000..34f11e5b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsService.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Periodic feed-diagnostics reporter. Does not run when is false. + /// + public sealed class FeedDiagnosticsService : IApplicationService, IAsyncDisposable + { + private readonly IFeedDiagnostics _feed; + private readonly IArticleIngestionQueue _queue; + private readonly TransitConfigurationStore _store; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _disposed; + + /// Initializes a new instance of the class. + public FeedDiagnosticsService( + IFeedDiagnostics feed, + IArticleIngestionQueue queue, + TransitConfigurationStore store, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(feed); + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _feed = feed; + _queue = queue; + _store = store; + _options = options; + _logger = logger; + } + + /// + public string Name => "FeedDiagnostics"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!_feed.IsEnabled) + { + return Task.CompletedTask; + } + + var interval = _options.Value.FeedDiagnostics?.IntervalSeconds + ?? FeedDiagnosticsOptions.DefaultIntervalSeconds; + var includeSessions = _options.Value.FeedDiagnostics?.IncludeSessions ?? true; + FeedDiagnosticsLogMessages.Enabled(_logger, interval, includeSessions); + _execution = RunAsync(TimeSpan.FromSeconds(interval), includeSessions, _runCts.Token); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is null) + { + return; + } + + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + if (_feed.IsEnabled) + { + FeedDiagnosticsLogMessages.Stopped(_logger); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + _runCts.Dispose(); + } + + private async Task RunAsync(TimeSpan interval, bool includeSessions, CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(interval, cancellationToken).ConfigureAwait(false); + var snapshot = _feed.CaptureSnapshot(_queue, _store); + FeedDiagnosticsLogMessages.Snapshot(_logger, FeedDiagnosticsFormatter.Format(snapshot, includeSessions)); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsSnapshot.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsSnapshot.cs new file mode 100644 index 00000000..93ca5ef9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedDiagnosticsSnapshot.cs @@ -0,0 +1,236 @@ +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// Point-in-time feed-diagnostics view (off the hot path). + public sealed class FeedDiagnosticsSnapshot + { + /// Empty snapshot used when diagnostics are disabled. + public static FeedDiagnosticsSnapshot Empty { get; } = new() + { + CapturedAt = DateTimeOffset.UnixEpoch, + Interval = TimeSpan.Zero, + Peers = [], + Sessions = [], + }; + + /// Gets the capture time. + public required DateTimeOffset CapturedAt { get; init; } + + /// Gets the interval since the previous snapshot (zero on the first). + public required TimeSpan Interval { get; init; } + + /// Gets per-peer rows. + public required IReadOnlyList Peers { get; init; } + + /// Gets per-session rows. + public required IReadOnlyList Sessions { get; init; } + + /// Gets articles completed in this interval. + public long IntervalArticles { get; init; } + + /// Gets framed articles received in this interval. + public long IntervalArticlesReceived { get; init; } + + /// Gets article bytes framed in this interval. + public long IntervalBytes { get; init; } + + /// Gets commands observed in this interval. + public long IntervalCommands { get; init; } + + /// Gets application-layer upstream bytes read in this interval. + public long IntervalTcpBytes { get; init; } + + /// Gets ingestion-queue admitted payload bytes in this interval. + public long IntervalQueuedBytes { get; init; } + + /// Gets articles successfully admitted to the ingestion queue in this interval. + public long IntervalQueuedArticles { get; init; } + + /// Gets spool articles persisted in this interval. + public long IntervalSpoolArticles { get; init; } + + /// Gets spool payload bytes persisted in this interval. + public long IntervalSpoolBytes { get; init; } + + /// Gets lifetime application-layer upstream bytes read. + public long TcpBytes { get; init; } + + /// Gets lifetime spool articles persisted. + public long SpoolArticles { get; init; } + + /// Gets lifetime spool payload bytes persisted. + public long SpoolBytes { get; init; } + + /// Gets whether the spool drain loop is currently persisting an article. + public int SpoolActiveWorkers { get; init; } + + /// Gets current admitted sessions (established and running). + public int ActiveSessions { get; init; } + + /// Gets sessions in . + public int ReceivingSessions { get; init; } + + /// Gets sessions in . + public int IdleSessions { get; init; } + + /// Gets current queue depth. + public int QueueCount { get; init; } + + /// Gets current queued payload bytes. + public long QueueBytes { get; init; } + + /// Gets queue peak article count. + public int QueuePeakCount { get; init; } + + /// Gets queue peak payload bytes. + public long QueuePeakBytes { get; init; } + + /// Gets producers waiting for queue byte budget. + public int QueueWaitingProducers { get; init; } + + /// Gets queue memory limit. + public long QueueMemoryLimit { get; init; } + + /// Gets lifetime HistoryDB Seen count (all sessions, including released). + public long HistoryHits { get; init; } + + /// Gets lifetime HistoryDB Unseen count. + public long HistoryMisses { get; init; } + + /// Gets lifetime HistoryDB error count. + public long HistoryErrors { get; init; } + + /// Gets lifetime queue admission failures (rejected + unavailable + full). + public long QueueAdmissionFailures { get; init; } + + /// Gets completed articles/sec over . + public double ArticlesPerSecond => FeedRateCalculator.PerSecond(IntervalArticles, Interval); + + /// Gets framed articles received/sec over . + public double ArticlesReceivedPerSecond => + FeedRateCalculator.PerSecond(IntervalArticlesReceived, Interval); + + /// Gets commands/sec over . + public double CommandsPerSecond => FeedRateCalculator.PerSecond(IntervalCommands, Interval); + + /// Gets bits/sec of framed article payload over . + public double BitsPerSecond => FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval) * 1_000_000d; + + /// Gets framed-article Mbps over . + public double ArticleMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval); + + /// Gets application-layer receive Mbps over . + public double TcpMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalTcpBytes, Interval); + + /// Gets ingestion-queue admit Mbps over . + public double IngestMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalQueuedBytes, Interval); + + /// Gets completed-article Mbps over . + public double ProcessMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalBytes, Interval); + + /// Gets spool persist Mbps over . + public double SpoolMegabitsPerSecond => + FeedRateCalculator.MegabitsPerSecond(IntervalSpoolBytes, Interval); + } + + /// Per-peer connection and article totals. + public sealed class FeedPeerSnapshot + { + /// Gets the Transit identifier. + public required string PeerName { get; init; } + + /// Gets configured MaxIncomingConnections, or 0 when unknown. + public int ConfiguredMaxConnections { get; init; } + + /// Gets current admitted sessions. + public int ActiveConnections { get; init; } + + /// Gets peak admitted sessions. + public int PeakConnections { get; init; } + + /// Gets lifetime admits. + public long AcceptedConnections { get; init; } + + /// Gets lifetime admission rejects. + public long RejectedConnections { get; init; } + + /// Gets lifetime framed articles. + public long ArticlesReceived { get; init; } + + /// Gets lifetime completed articles. + public long ArticlesCompleted { get; init; } + + /// Gets lifetime framed bytes. + public long ArticleBytes { get; init; } + + /// Gets lifetime CHECK commands. + public long Checks { get; init; } + } + + /// Compact per-session row. + public sealed class FeedSessionSnapshot + { + /// Gets ip:port. + public required string Remote { get; init; } + + /// Gets the Transit identifier. + public required string PeerName { get; init; } + + /// Gets session age. + public TimeSpan Age { get; init; } + + /// Gets coarse activity. + public FeedSessionState State { get; init; } + + /// Gets commands observed. + public long Commands { get; init; } + + /// Gets CHECK commands. + public long Checks { get; init; } + + /// Gets framed articles. + public long ArticlesReceived { get; init; } + + /// Gets completed articles. + public long ArticlesCompleted { get; init; } + + /// Gets framed bytes. + public long ArticleBytes { get; init; } + + /// Gets commands observed in this interval. + public long IntervalCommands { get; init; } + + /// Gets framed articles received in this interval. + public long IntervalArticlesReceived { get; init; } + + /// Gets framed article bytes in this interval. + public long IntervalBytes { get; init; } + + /// Gets idle time. + public TimeSpan Idle { get; init; } + + /// Gets time since last completed article. + public TimeSpan SinceLastArticle { get; init; } + + /// Gets total receive time. + public TimeSpan ReceiveTime { get; init; } + + /// Gets total HistoryDB wait. + public TimeSpan HistoryWaitTime { get; init; } + + /// Gets total queue wait. + public TimeSpan QueueWaitTime { get; init; } + + /// Gets max receive time. + public TimeSpan MaxReceiveTime { get; init; } + + /// Gets max HistoryDB wait. + public TimeSpan MaxHistoryWaitTime { get; init; } + + /// Gets max queue wait. + public TimeSpan MaxQueueWaitTime { get; init; } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedRateCalculator.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedRateCalculator.cs new file mode 100644 index 00000000..e7894f34 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedRateCalculator.cs @@ -0,0 +1,45 @@ +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Interval rate helpers for feed-diagnostics snapshots. + /// + /// + /// All methods are allocation-free and safe for zero elapsed time and + /// non-monotonic counters (reset or observed reordering). + /// + public static class FeedRateCalculator + { + /// + /// Returns current - previous when the counter advanced; otherwise 0. + /// + public static long NonNegativeDelta(long current, long previous) + { + if (current < previous) + { + return 0; + } + + return current - previous; + } + + /// + /// Returns per second, or 0 when elapsed time is not positive. + /// + public static double PerSecond(long delta, TimeSpan elapsed) + { + var seconds = elapsed.TotalSeconds; + if (seconds <= 0 || delta <= 0) + { + return 0; + } + + return delta / seconds; + } + + /// + /// Returns megabits/sec from a byte delta (decimal Mbps: bits / 1e6). + /// + public static double MegabitsPerSecond(long byteDelta, TimeSpan elapsed) => + PerSecond(byteDelta, elapsed) * 8d / 1_000_000d; + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionProbe.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionProbe.cs new file mode 100644 index 00000000..2dd56a3b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionProbe.cs @@ -0,0 +1,291 @@ +using System.Diagnostics; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.History; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Per-session counters and wait totals for opt-in feed diagnostics. + /// + /// + /// Hot-path updates are Interlocked / Volatile only. No logging. No Message-IDs or payloads. + /// + public sealed class FeedSessionProbe + { + private readonly long _startedTimestamp; + private int _state; + private long _commands; + private long _checks; + private long _articlesReceived; + private long _articlesCompleted; + private long _articleBytes; + private long _receiveTicks; + private long _historyTicks; + private long _queueWaitTicks; + private long _responseTicks; + private long _maxReceiveTicks; + private long _maxHistoryTicks; + private long _maxQueueWaitTicks; + private long _maxArticleIntervalTicks; + private long _lastActivityTimestamp; + private long _lastArticleCompleteTimestamp; + private long _historyHits; + private long _historyMisses; + private long _historyErrors; + private long _queueAccepted; + private long _queueRejected; + private long _queueUnavailable; + private long _queueAdmittedBytes; + private long _duplicates; + + /// Initializes a probe for one accepted session. + public FeedSessionProbe(string remote, string peerName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(remote); + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + Remote = remote; + PeerName = peerName; + _startedTimestamp = Stopwatch.GetTimestamp(); + _lastActivityTimestamp = _startedTimestamp; + } + + /// Gets the effective client endpoint (ip:port). + public string Remote { get; } + + /// Gets the Transit identifier, or none. + public string PeerName { get; } + + /// Gets the session age. + public TimeSpan Age => Stopwatch.GetElapsedTime(_startedTimestamp); + + /// Gets the coarse activity state. + public FeedSessionState State => (FeedSessionState)Volatile.Read(ref _state); + + /// Gets commands observed on this session. + public long Commands => Volatile.Read(ref _commands); + + /// Gets CHECK commands observed on this session. + public long Checks => Volatile.Read(ref _checks); + + /// Gets framed articles (receive completed). + public long ArticlesReceived => Volatile.Read(ref _articlesReceived); + + /// Gets articles that reached a 239/439/400 completion. + public long ArticlesCompleted => Volatile.Read(ref _articlesCompleted); + + /// Gets framed article payload bytes (terminator omitted). + public long ArticleBytes => Volatile.Read(ref _articleBytes); + + /// Gets HistoryDB Seen outcomes. + public long HistoryHits => Volatile.Read(ref _historyHits); + + /// Gets HistoryDB Unseen outcomes. + public long HistoryMisses => Volatile.Read(ref _historyMisses); + + /// Gets HistoryDB Unavailable / exception outcomes. + public long HistoryErrors => Volatile.Read(ref _historyErrors); + + /// Gets successful ingestion-queue admits. + public long QueueAccepted => Volatile.Read(ref _queueAccepted); + + /// Gets payload bytes successfully admitted to the ingestion queue. + public long QueueAdmittedBytes => Volatile.Read(ref _queueAdmittedBytes); + + /// Gets queue rejects (article larger than budget). + public long QueueRejected => Volatile.Read(ref _queueRejected); + + /// Gets queue unavailable / shutdown outcomes. + public long QueueUnavailable => Volatile.Read(ref _queueUnavailable); + + /// Gets TAKETHIS completions treated as already-seen. + public long Duplicates => Volatile.Read(ref _duplicates); + + /// Gets total time spent in article receive. + public TimeSpan ReceiveTime => TicksToTime(Volatile.Read(ref _receiveTicks)); + + /// Gets total time spent awaiting HistoryDB. + public TimeSpan HistoryWaitTime => TicksToTime(Volatile.Read(ref _historyTicks)); + + /// Gets total time spent awaiting queue admission. + public TimeSpan QueueWaitTime => TicksToTime(Volatile.Read(ref _queueWaitTicks)); + + /// Gets total time spent enqueueing the protocol response. + public TimeSpan ResponseTime => TicksToTime(Volatile.Read(ref _responseTicks)); + + /// Gets the longest single article receive. + public TimeSpan MaxReceiveTime => TicksToTime(Volatile.Read(ref _maxReceiveTicks)); + + /// Gets the longest single HistoryDB wait. + public TimeSpan MaxHistoryWaitTime => TicksToTime(Volatile.Read(ref _maxHistoryTicks)); + + /// Gets the longest single queue-admission wait. + public TimeSpan MaxQueueWaitTime => TicksToTime(Volatile.Read(ref _maxQueueWaitTicks)); + + /// Gets the longest interval between completed articles. + public TimeSpan MaxArticleInterval => TicksToTime(Volatile.Read(ref _maxArticleIntervalTicks)); + + /// Gets time since the last recorded activity. + public TimeSpan Idle => Stopwatch.GetElapsedTime(Volatile.Read(ref _lastActivityTimestamp)); + + /// Gets time since the previous completed article, or . + public TimeSpan SinceLastArticle + { + get + { + var last = Volatile.Read(ref _lastArticleCompleteTimestamp); + return last == 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(last); + } + } + + /// Sets the coarse activity state. + public void SetState(FeedSessionState state) + { + Volatile.Write(ref _state, (int)state); + Touch(); + } + + /// Increments the command counter. + public void RecordCommand() + { + Interlocked.Increment(ref _commands); + Touch(); + } + + /// Increments the CHECK counter. + public void RecordCheck() + { + Interlocked.Increment(ref _checks); + Interlocked.Increment(ref _commands); + Touch(); + } + + /// Records a completed article frame. + public void RecordArticleReceived(int bytes, long receiveTicks) + { + Interlocked.Increment(ref _articlesReceived); + Interlocked.Add(ref _articleBytes, bytes); + AddTicks(ref _receiveTicks, ref _maxReceiveTicks, receiveTicks); + Touch(); + } + + /// Records a HistoryDB Peek/Lookup outcome and wait. + public void RecordHistory(HistoryLookupResult result, long waitTicks) + { + switch (result) + { + case HistoryLookupResult.Seen: + Interlocked.Increment(ref _historyHits); + break; + case HistoryLookupResult.Unseen: + Interlocked.Increment(ref _historyMisses); + break; + default: + Interlocked.Increment(ref _historyErrors); + break; + } + + AddTicks(ref _historyTicks, ref _maxHistoryTicks, waitTicks); + Touch(); + } + + /// Records ingestion-queue admission outcome and wait. + public void RecordQueue(ArticleEnqueueResult result, long waitTicks, int articleBytes = 0) + { + switch (result) + { + case ArticleEnqueueResult.Accepted: + Interlocked.Increment(ref _queueAccepted); + if (articleBytes > 0) + { + Interlocked.Add(ref _queueAdmittedBytes, articleBytes); + } + + break; + case ArticleEnqueueResult.Rejected: + Interlocked.Increment(ref _queueRejected); + break; + default: + Interlocked.Increment(ref _queueUnavailable); + break; + } + + AddTicks(ref _queueWaitTicks, ref _maxQueueWaitTicks, waitTicks); + Touch(); + } + + /// Records a completed TAKETHIS (or serial fallback) outcome. + public void RecordArticleCompleted(bool duplicate, long responseTicks) + { + Interlocked.Increment(ref _articlesCompleted); + if (duplicate) + { + Interlocked.Increment(ref _duplicates); + } + + if (responseTicks > 0) + { + Interlocked.Add(ref _responseTicks, responseTicks); + } + + var now = Stopwatch.GetTimestamp(); + var previous = Interlocked.Exchange(ref _lastArticleCompleteTimestamp, now); + if (previous != 0) + { + var interval = now - previous; + if (interval > 0) + { + UpdateMax(ref _maxArticleIntervalTicks, interval); + } + } + + Volatile.Write(ref _state, (int)FeedSessionState.Idle); + Touch(now); + } + + /// Adds response-enqueue wait without completing an article (CHECK). + public void RecordResponseWait(long responseTicks) + { + if (responseTicks > 0) + { + Interlocked.Add(ref _responseTicks, responseTicks); + } + + Touch(); + } + + private void Touch() => Touch(Stopwatch.GetTimestamp()); + + private void Touch(long timestamp) => Volatile.Write(ref _lastActivityTimestamp, timestamp); + + private static TimeSpan TicksToTime(long ticks) => + ticks <= 0 ? TimeSpan.Zero : Stopwatch.GetElapsedTime(0, ticks); + + private static void AddTicks(ref long total, ref long max, long ticks) + { + if (ticks <= 0) + { + return; + } + + Interlocked.Add(ref total, ticks); + UpdateMax(ref max, ticks); + } + + private static void UpdateMax(ref long location, long candidate) + { + while (true) + { + var current = Volatile.Read(ref location); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref location, candidate, current) == current) + { + return; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionState.cs b/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionState.cs new file mode 100644 index 00000000..cfd0cf68 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/FeedSessionState.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// Coarse activity for one NNTP session in a feed-diagnostics snapshot. + public enum FeedSessionState : byte + { + /// Between commands, or no in-flight article work. + Idle = 0, + + /// Framing or reading an article from the input pipe. + Receiving = 1, + + /// Waiting for HistoryDB Peek/Lookup to complete. + WaitingHistory = 2, + + /// Waiting for ingestion-queue byte-budget admission. + WaitingQueue = 3, + + /// Blocked on CHECK/TAKETHIS pipeline depth. + WaitingWindow = 4, + + /// Emitting the transfer/CHECK response. + Completing = 5, + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/IFeedDiagnostics.cs b/src/VectorNNTPD.TransitService/Diagnostics/IFeedDiagnostics.cs new file mode 100644 index 00000000..b374f16a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/IFeedDiagnostics.cs @@ -0,0 +1,41 @@ +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Opt-in real-feed observability. Disabled implementations are no-ops. + /// + public interface IFeedDiagnostics + { + /// Gets whether probes should be attached and snapshots emitted. + bool IsEnabled { get; } + + /// Records a Transit admission rejection (no session runs). + void OnRejected(string peerName, string remote); + + /// Attaches and registers a probe after a session is admitted. + FeedSessionProbe? OnAccepted(NntpSession session); + + /// Unregisters a probe when the session ends. + void OnReleased(FeedSessionProbe? probe); + + /// + /// Adds application-layer bytes read from an upstream connection (plain TCP payload, + /// or TLS-decrypted octets). No-op when disabled. + /// + void RecordTcpBytes(int bytes); + + /// Marks the single spool drain loop as busy around one persist. + void BeginSpoolWork(); + + /// Records one successfully persisted spool article and clears the busy mark. + void EndSpoolWork(int bytes, bool persisted); + + /// Builds a point-in-time snapshot for the reporter. + FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store); + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/IngestionPipelineMetrics.cs b/src/VectorNNTPD.TransitService/Diagnostics/IngestionPipelineMetrics.cs new file mode 100644 index 00000000..635099bb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/IngestionPipelineMetrics.cs @@ -0,0 +1,692 @@ +using System.Diagnostics; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// + /// Always-on ingestion-pipeline histograms. Samples are lock-free counters; + /// ApplicationTelemetry emits one Information snapshot per minute (no per-article logs). + /// + /// + /// OverviewDB publishes use an asynchronous outstanding-confirmation window. + /// records the publish write duration + /// (not fused with broker ack). tracks the + /// outstanding confirmation count. Confirms remain enabled; crash loss of + /// outstanding OverviewDB work is acceptable. + /// + public sealed class IngestionPipelineMetrics + { + /// Process-wide instance used by TAKETHIS and the ingestion queue. + public static IngestionPipelineMetrics Shared { get; } = new(); + + private readonly DurationHistogram _dequeueWait = new(); + private readonly DurationHistogram _toPublishStart = new(); + private readonly DurationHistogram _encode = new(); + private readonly DurationHistogram _gateWait = new(); + private readonly DurationHistogram _publish = new(); + private readonly DurationHistogram _confirm = new(); + private readonly DurationHistogram _handoff = new(); + private readonly DurationHistogram _news = new(); + private readonly DurationHistogram _workerItem = new(); + private readonly DurationHistogram _emitGateWait = new(); + private readonly DurationHistogram _takeThisEnqueue = new(); + private readonly DurationHistogram _enqueueWait = new(); + private readonly DurationHistogram _budgetWait = new(); + private readonly DurationHistogram _occupiedFullWait = new(); + + private long _workerItems; + private long _confirmFailures; + private long _confirmTimeouts; + private long _busyTicks; + private long _idleTicks; + private long _occupiedFullWaits; + private long _budgetWaits; + private int _maxQueueDepth; + private int _maxTakeThisOccupied; + private int _maxTakeThisProcessing; + private int _currentWorkers; + private int _minWorkers; + private int _maxWorkers; + private int _inFlightPublishes; + private int _maxInFlightPublishes; + private long _scaleUps; + private long _scaleDowns; + private int _overviewPublisherWorkers; + private int _overviewPublisherMinWorkers; + private int _overviewPublisherMaxWorkers; + private int _overviewWorkCount; + private long _overviewWorkBytes; + private int _overviewWorkWaiting; + private long _overviewPublisherScaleUps; + private long _overviewPublisherScaleDowns; + + /// Records time spent blocked in DequeueAsync. + public void RecordDequeueWait(long startTimestamp) => _dequeueWait.Record(startTimestamp); + + /// Records dequeue completion until OverviewDB work-queue enqueue begins. + public void RecordToPublishStart(long startTimestamp) => _toPublishStart.Record(startTimestamp); + + /// Records OverviewArticleV1 encode duration. + public void RecordEncode(long startTimestamp) => _encode.Record(startTimestamp); + + /// Records OverviewDB publisher SemaphoreSlim wait. + public void RecordGateWait(long startTimestamp) => _gateWait.Record(startTimestamp); + + /// + /// Records OverviewDB publish write duration (async confirms; not fused with broker ack). + /// + public void RecordPublishAndConfirm(long startTimestamp) + { + _publish.Record(startTimestamp); + _confirm.Record(startTimestamp); + } + + /// Records the full OverviewDB publisher call including the publish gate. + public void RecordHandoff(long startTimestamp) => _handoff.Record(startTimestamp); + + /// Records news-log Write duration. + public void RecordNews(long startTimestamp) => _news.Record(startTimestamp); + + /// Records an exact news-log duration. Tests use this so the sample is not the wall clock. + internal void RecordNews(TimeSpan elapsed) => _news.RecordExact(elapsed); + + /// Records one worker item from dequeue return through finish or requeue. + public void RecordWorkerItem(long startTimestamp) + { + _workerItem.Record(startTimestamp); + Interlocked.Increment(ref _workerItems); + } + + /// Adds Stopwatch ticks spent processing a worker item. + public void AddBusyTicks(long stopwatchTicks) + { + if (stopwatchTicks > 0) + { + Interlocked.Add(ref _busyTicks, stopwatchTicks); + } + } + + /// Adds Stopwatch ticks spent waiting for the next dequeue. + public void AddIdleTicks(long stopwatchTicks) + { + if (stopwatchTicks > 0) + { + Interlocked.Add(ref _idleTicks, stopwatchTicks); + } + } + + /// Records a broker nack, return, or other publish failure (not a timeout). + public void RecordConfirmFailure() => Interlocked.Increment(ref _confirmFailures); + + /// Records a publisher-confirm timeout. + public void RecordConfirmTimeout() => Interlocked.Increment(ref _confirmTimeouts); + + /// Records TAKETHIS _emitGate.WaitAsync duration. + public void RecordEmitGateWait(long startTimestamp) => _emitGateWait.Record(startTimestamp); + + /// Records TAKETHIS time spent inside EnqueueAsync while holding _emitGate. + public void RecordTakeThisEnqueue(long startTimestamp) => _takeThisEnqueue.Record(startTimestamp); + + /// Records every ingestion-queue EnqueueAsync wait, including immediate admits. + public void RecordEnqueueWaitTicks(long stopwatchTicks) => _enqueueWait.RecordStopwatchTicks(stopwatchTicks); + + /// Records a producer wait caused by the global byte budget. + public void RecordBudgetWaitTicks(long stopwatchTicks) + { + Interlocked.Increment(ref _budgetWaits); + _budgetWait.RecordStopwatchTicks(stopwatchTicks); + } + + /// Records time spent in WaitForCapacityAsync because Occupied is at Depth. + public void RecordOccupiedFullWait(long startTimestamp) + { + Interlocked.Increment(ref _occupiedFullWaits); + _occupiedFullWait.Record(startTimestamp); + } + + /// Tracks the high-water ingestion-queue depth observed by the worker. + public void ObserveQueueDepth(int depth) + { + if (depth <= 0) + { + return; + } + + var current = Volatile.Read(ref _maxQueueDepth); + while (depth > current) + { + var previous = Interlocked.CompareExchange(ref _maxQueueDepth, depth, current); + if (previous == current) + { + break; + } + + current = previous; + } + } + + /// Tracks peak TAKETHIS Occupied and concurrent ProcessArticleAsync counts. + public void ObserveTakeThis(int occupied, int processing) + { + UpdateMax(ref _maxTakeThisOccupied, occupied); + UpdateMax(ref _maxTakeThisProcessing, processing); + } + + /// Records the configured worker-pool bounds and current worker count. + public void ObserveWorkerPool(int currentWorkers, int minWorkers, int maxWorkers) + { + Volatile.Write(ref _currentWorkers, Math.Max(0, currentWorkers)); + Volatile.Write(ref _minWorkers, Math.Max(0, minWorkers)); + Volatile.Write(ref _maxWorkers, Math.Max(0, maxWorkers)); + } + + /// Records the current number of leased OverviewDB publish slots. + public void ObserveInFlightPublishes(int inFlight) + { + Volatile.Write(ref _inFlightPublishes, Math.Max(0, inFlight)); + UpdateMax(ref _maxInFlightPublishes, inFlight); + } + + /// Records one worker-pool scale-up decision. + public void RecordScaleUp() => Interlocked.Increment(ref _scaleUps); + + /// Records one worker-pool scale-down decision. + public void RecordScaleDown() => Interlocked.Increment(ref _scaleDowns); + + /// Records OverviewDB publisher worker-pool bounds and current count. + public void ObserveOverviewDbPublisherPool(int currentWorkers, int minWorkers, int maxWorkers) + { + Volatile.Write(ref _overviewPublisherWorkers, Math.Max(0, currentWorkers)); + Volatile.Write(ref _overviewPublisherMinWorkers, Math.Max(0, minWorkers)); + Volatile.Write(ref _overviewPublisherMaxWorkers, Math.Max(0, maxWorkers)); + } + + /// Records OverviewDB in-process work-queue depth gauges. + public void ObserveOverviewDbWorkQueue(int count, long queuedBytes, int waitingProducers) + { + Volatile.Write(ref _overviewWorkCount, Math.Max(0, count)); + Volatile.Write(ref _overviewWorkBytes, Math.Max(0, queuedBytes)); + Volatile.Write(ref _overviewWorkWaiting, Math.Max(0, waitingProducers)); + } + + /// Records one OverviewDB publisher scale-up. + public void RecordOverviewDbPublisherScaleUp() => + Interlocked.Increment(ref _overviewPublisherScaleUps); + + /// Records one OverviewDB publisher scale-down. + public void RecordOverviewDbPublisherScaleDown() => + Interlocked.Increment(ref _overviewPublisherScaleDowns); + + /// + /// Captures OverviewDB work-queue / publisher-pool gauges and interval scale counters. + /// + public OverviewDbStageSnapshot CaptureOverviewDbStage() + { + return new OverviewDbStageSnapshot( + Volatile.Read(ref _overviewPublisherWorkers), + Volatile.Read(ref _overviewPublisherMinWorkers), + Volatile.Read(ref _overviewPublisherMaxWorkers), + Volatile.Read(ref _overviewWorkCount), + Volatile.Read(ref _overviewWorkBytes), + Volatile.Read(ref _overviewWorkWaiting), + Interlocked.Exchange(ref _overviewPublisherScaleUps, 0), + Interlocked.Exchange(ref _overviewPublisherScaleDowns, 0)); + } + + /// Captures interval totals and resets histograms and counters. + public IngestionPipelineSnapshot CaptureInterval() + { + return new IngestionPipelineSnapshot( + Interlocked.Exchange(ref _workerItems, 0), + Interlocked.Exchange(ref _confirmFailures, 0), + Interlocked.Exchange(ref _confirmTimeouts, 0), + Interlocked.Exchange(ref _busyTicks, 0), + Interlocked.Exchange(ref _idleTicks, 0), + Interlocked.Exchange(ref _occupiedFullWaits, 0), + Interlocked.Exchange(ref _budgetWaits, 0), + Interlocked.Exchange(ref _maxQueueDepth, 0), + Interlocked.Exchange(ref _maxTakeThisOccupied, 0), + Interlocked.Exchange(ref _maxTakeThisProcessing, 0), + Volatile.Read(ref _currentWorkers), + Volatile.Read(ref _minWorkers), + Volatile.Read(ref _maxWorkers), + Volatile.Read(ref _inFlightPublishes), + Interlocked.Exchange(ref _maxInFlightPublishes, 0), + Interlocked.Exchange(ref _scaleUps, 0), + Interlocked.Exchange(ref _scaleDowns, 0), + _dequeueWait.CaptureAndReset(), + _toPublishStart.CaptureAndReset(), + _encode.CaptureAndReset(), + _gateWait.CaptureAndReset(), + _publish.CaptureAndReset(), + _confirm.CaptureAndReset(), + _handoff.CaptureAndReset(), + _news.CaptureAndReset(), + _workerItem.CaptureAndReset(), + _emitGateWait.CaptureAndReset(), + _takeThisEnqueue.CaptureAndReset(), + _enqueueWait.CaptureAndReset(), + _budgetWait.CaptureAndReset(), + _occupiedFullWait.CaptureAndReset()); + } + + private static void UpdateMax(ref int location, int candidate) + { + if (candidate <= 0) + { + return; + } + + var current = Volatile.Read(ref location); + while (candidate > current) + { + var previous = Interlocked.CompareExchange(ref location, candidate, current); + if (previous == current) + { + return; + } + + current = previous; + } + } + } + + /// One telemetry interval of ingestion-pipeline histograms. + public readonly struct IngestionPipelineSnapshot + { + /// Initializes a captured interval snapshot. + public IngestionPipelineSnapshot( + long workerItems, + long confirmFailures, + long confirmTimeouts, + long busyTicks, + long idleTicks, + long occupiedFullWaits, + long budgetWaits, + int maxQueueDepth, + int maxTakeThisOccupied, + int maxTakeThisProcessing, + int currentWorkers, + int minWorkers, + int maxWorkers, + int inFlightPublishes, + int maxInFlightPublishes, + long scaleUps, + long scaleDowns, + DurationSnapshot dequeueWait, + DurationSnapshot toPublishStart, + DurationSnapshot encode, + DurationSnapshot gateWait, + DurationSnapshot publish, + DurationSnapshot confirm, + DurationSnapshot handoff, + DurationSnapshot news, + DurationSnapshot workerItem, + DurationSnapshot emitGateWait, + DurationSnapshot takeThisEnqueue, + DurationSnapshot enqueueWait, + DurationSnapshot budgetWait, + DurationSnapshot occupiedFullWait) + { + WorkerItems = workerItems; + ConfirmFailures = confirmFailures; + ConfirmTimeouts = confirmTimeouts; + BusyTicks = busyTicks; + IdleTicks = idleTicks; + OccupiedFullWaits = occupiedFullWaits; + BudgetWaits = budgetWaits; + MaxQueueDepth = maxQueueDepth; + MaxTakeThisOccupied = maxTakeThisOccupied; + MaxTakeThisProcessing = maxTakeThisProcessing; + CurrentWorkers = currentWorkers; + MinWorkers = minWorkers; + MaxWorkers = maxWorkers; + InFlightPublishes = inFlightPublishes; + MaxInFlightPublishes = maxInFlightPublishes; + ScaleUps = scaleUps; + ScaleDowns = scaleDowns; + DequeueWait = dequeueWait; + ToPublishStart = toPublishStart; + Encode = encode; + GateWait = gateWait; + Publish = publish; + Confirm = confirm; + Handoff = handoff; + News = news; + WorkerItem = workerItem; + EmitGateWait = emitGateWait; + TakeThisEnqueue = takeThisEnqueue; + EnqueueWait = enqueueWait; + BudgetWait = budgetWait; + OccupiedFullWait = occupiedFullWait; + } + + /// Gets worker items completed in the interval. + public long WorkerItems { get; } + + /// Gets OverviewDB publish/confirm failures in the interval. + public long ConfirmFailures { get; } + + /// Gets OverviewDB confirm timeouts in the interval. + public long ConfirmTimeouts { get; } + + /// Gets Stopwatch ticks the worker spent processing items. + public long BusyTicks { get; } + + /// Gets Stopwatch ticks the worker spent waiting to dequeue. + public long IdleTicks { get; } + + /// Gets TAKETHIS WaitForCapacityAsync waits while Occupied was at Depth. + public long OccupiedFullWaits { get; } + + /// Gets producers that waited on the global byte budget. + public long BudgetWaits { get; } + + /// Gets the interval high-water ingestion-queue depth observed by the worker. + public int MaxQueueDepth { get; } + + /// Gets the interval high-water TAKETHIS Occupied count. + public int MaxTakeThisOccupied { get; } + + /// Gets the interval high-water concurrent ProcessArticleAsync count. + public int MaxTakeThisProcessing { get; } + + /// Gets the worker count observed at capture. + public int CurrentWorkers { get; } + + /// Gets the configured minimum worker count. + public int MinWorkers { get; } + + /// Gets the configured maximum worker count. + public int MaxWorkers { get; } + + /// Gets in-flight OverviewDB publishes at capture. + public int InFlightPublishes { get; } + + /// Gets the interval high-water in-flight OverviewDB publishes. + public int MaxInFlightPublishes { get; } + + /// Gets scale-up events in the interval. + public long ScaleUps { get; } + + /// Gets scale-down events in the interval. + public long ScaleDowns { get; } + + /// Gets dequeue-wait samples. + public DurationSnapshot DequeueWait { get; } + + /// Gets dequeue-to-OverviewDB-publish-start samples. + public DurationSnapshot ToPublishStart { get; } + + /// Gets protobuf encode samples. + public DurationSnapshot Encode { get; } + + /// Gets OverviewDB publish-gate wait samples. + public DurationSnapshot GateWait { get; } + + /// Gets fused publish-call samples (same as with tracking). + public DurationSnapshot Publish { get; } + + /// Gets fused confirm-wait samples (same as with tracking). + public DurationSnapshot Confirm { get; } + + /// Gets total OverviewDB handoff samples including the gate. + public DurationSnapshot Handoff { get; } + + /// Gets news-log write samples. + public DurationSnapshot News { get; } + + /// Gets total worker-item samples. + public DurationSnapshot WorkerItem { get; } + + /// Gets TAKETHIS emit-gate wait samples. + public DurationSnapshot EmitGateWait { get; } + + /// Gets TAKETHIS EnqueueAsync samples taken while holding the emit gate. + public DurationSnapshot TakeThisEnqueue { get; } + + /// Gets all ingestion-queue EnqueueAsync wait samples. + public DurationSnapshot EnqueueWait { get; } + + /// Gets byte-budget wait samples. + public DurationSnapshot BudgetWait { get; } + + /// Gets Occupied==Depth wait samples. + public DurationSnapshot OccupiedFullWait { get; } + + /// Busy time as a percentage of busy+idle, or 0 when the worker was unused. + public double BusyPercent => Percent(BusyTicks, BusyTicks + IdleTicks); + + /// Confirm-wait time as a percentage of worker busy time. + public double ConfirmBusyPercent => Percent(Confirm.SumMs, TicksToMilliseconds(BusyTicks)); + + /// Converts Stopwatch ticks to whole milliseconds. + public static long TicksToMilliseconds(long stopwatchTicks) + { + if (stopwatchTicks <= 0) + { + return 0; + } + + return (long)Stopwatch.GetElapsedTime(0, stopwatchTicks).TotalMilliseconds; + } + + /// Articles per second over . + public double ArticlesPerSecond(TimeSpan period) + { + var seconds = period.TotalSeconds; + if (seconds <= 0 || WorkerItems <= 0) + { + return 0; + } + + return WorkerItems / seconds; + } + + private static double Percent(long numerator, long denominator) + { + if (numerator <= 0 || denominator <= 0) + { + return 0; + } + + return 100d * numerator / denominator; + } + } + + /// Gauge/counter snapshot for the OverviewDB work-queue publisher stage. + public readonly struct OverviewDbStageSnapshot + { + /// Initializes a stage snapshot. + public OverviewDbStageSnapshot( + int publisherWorkers, + int minPublisherWorkers, + int maxPublisherWorkers, + int workQueueCount, + long workQueueBytes, + int workQueueWaiting, + long publisherScaleUps, + long publisherScaleDowns) + { + PublisherWorkers = publisherWorkers; + MinPublisherWorkers = minPublisherWorkers; + MaxPublisherWorkers = maxPublisherWorkers; + WorkQueueCount = workQueueCount; + WorkQueueBytes = workQueueBytes; + WorkQueueWaiting = workQueueWaiting; + PublisherScaleUps = publisherScaleUps; + PublisherScaleDowns = publisherScaleDowns; + } + + /// Gets current OverviewDB publisher workers. + public int PublisherWorkers { get; } + + /// Gets configured minimum publisher workers. + public int MinPublisherWorkers { get; } + + /// Gets configured maximum publisher workers. + public int MaxPublisherWorkers { get; } + + /// Gets OverviewDB work-queue item count. + public int WorkQueueCount { get; } + + /// Gets OverviewDB work-queue reserved bytes. + public long WorkQueueBytes { get; } + + /// Gets producers waiting on the OverviewDB work queue. + public int WorkQueueWaiting { get; } + + /// Gets publisher scale-ups in the interval. + public long PublisherScaleUps { get; } + + /// Gets publisher scale-downs in the interval. + public long PublisherScaleDowns { get; } + } + + /// Bucketed duration summary for one telemetry interval. + public readonly struct DurationSnapshot + { + /// Initializes a duration snapshot. + public DurationSnapshot(long count, double avgMs, long p50Ms, long p95Ms, long maxMs, long sumMs) + { + Count = count; + AvgMs = avgMs; + P50Ms = p50Ms; + P95Ms = p95Ms; + MaxMs = maxMs; + SumMs = sumMs; + } + + /// Gets the number of samples. + public long Count { get; } + + /// Gets the arithmetic mean in milliseconds. + public double AvgMs { get; } + + /// Gets an estimated 50th percentile in milliseconds. + public long P50Ms { get; } + + /// Gets an estimated 95th percentile in milliseconds. + public long P95Ms { get; } + + /// Gets the maximum sample in milliseconds. + public long MaxMs { get; } + + /// Gets the summed sample duration in milliseconds. + public long SumMs { get; } + } + + /// Lock-free millisecond histogram used by . + internal sealed class DurationHistogram + { + private static readonly double[] UpperMs = + [ + 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, + ]; + + private readonly long[] _buckets = new long[UpperMs.Length + 1]; + private long _count; + private long _sumTicks; + private long _maxTicks; + + public void Record(long startTimestamp) => + RecordElapsed(Stopwatch.GetElapsedTime(startTimestamp)); + + /// Records without reading the clock again. + internal void RecordExact(TimeSpan elapsed) => RecordElapsed(elapsed); + + public void RecordStopwatchTicks(long stopwatchTicks) + { + if (stopwatchTicks <= 0) + { + RecordElapsed(TimeSpan.Zero); + return; + } + + RecordElapsed(Stopwatch.GetElapsedTime(0, stopwatchTicks)); + } + + public DurationSnapshot CaptureAndReset() + { + var count = Interlocked.Exchange(ref _count, 0); + var sumTicks = Interlocked.Exchange(ref _sumTicks, 0); + var maxTicks = Interlocked.Exchange(ref _maxTicks, 0); + var buckets = new long[_buckets.Length]; + for (var i = 0; i < _buckets.Length; i++) + { + buckets[i] = Interlocked.Exchange(ref _buckets[i], 0); + } + + if (count <= 0) + { + return default; + } + + var sumMs = (long)Stopwatch.GetElapsedTime(0, sumTicks).TotalMilliseconds; + var maxMs = (long)Math.Ceiling(Stopwatch.GetElapsedTime(0, maxTicks).TotalMilliseconds); + var avgMs = sumMs / (double)count; + return new DurationSnapshot( + count, + avgMs, + PercentileMs(buckets, count, 0.50), + PercentileMs(buckets, count, 0.95), + maxMs, + sumMs); + } + + private void RecordElapsed(TimeSpan elapsed) + { + if (elapsed < TimeSpan.Zero) + { + elapsed = TimeSpan.Zero; + } + + var stopwatchTicks = (long)(elapsed.TotalSeconds * Stopwatch.Frequency); + Interlocked.Increment(ref _count); + Interlocked.Add(ref _sumTicks, stopwatchTicks); + UpdateMax(stopwatchTicks); + var ms = elapsed.TotalMilliseconds; + var index = UpperMs.Length; + for (var i = 0; i < UpperMs.Length; i++) + { + if (ms <= UpperMs[i]) + { + index = i; + break; + } + } + + Interlocked.Increment(ref _buckets[index]); + } + + private void UpdateMax(long stopwatchTicks) + { + var current = Volatile.Read(ref _maxTicks); + while (stopwatchTicks > current) + { + var previous = Interlocked.CompareExchange(ref _maxTicks, stopwatchTicks, current); + if (previous == current) + { + return; + } + + current = previous; + } + } + + private static long PercentileMs(long[] buckets, long count, double percentile) + { + var target = Math.Max(1, (long)Math.Ceiling(count * percentile)); + long cumulative = 0; + for (var i = 0; i < buckets.Length; i++) + { + cumulative += buckets[i]; + if (cumulative >= target) + { + return i < UpperMs.Length ? (long)UpperMs[i] : (long)UpperMs[^1]; + } + } + + return (long)UpperMs[^1]; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Diagnostics/NullFeedDiagnostics.cs b/src/VectorNNTPD.TransitService/Diagnostics/NullFeedDiagnostics.cs new file mode 100644 index 00000000..c657eeca --- /dev/null +++ b/src/VectorNNTPD.TransitService/Diagnostics/NullFeedDiagnostics.cs @@ -0,0 +1,54 @@ +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Diagnostics +{ + /// No-op feed diagnostics used when the reporter is disabled. + public sealed class NullFeedDiagnostics : IFeedDiagnostics + { + /// Shared disabled instance. + public static NullFeedDiagnostics Instance { get; } = new(); + + private NullFeedDiagnostics() + { + } + + /// + public bool IsEnabled => false; + + /// + public void OnRejected(string peerName, string remote) + { + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) => null; + + /// + public void OnReleased(FeedSessionProbe? probe) + { + } + + /// + public void RecordTcpBytes(int bytes) + { + } + + /// + public void BeginSpoolWork() + { + } + + /// + public void EndSpoolWork(int bytes, bool persisted) + { + } + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) => + FeedDiagnosticsSnapshot.Empty; + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailAddress.cs b/src/VectorNNTPD.TransitService/Email/EmailAddress.cs new file mode 100644 index 00000000..09665698 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailAddress.cs @@ -0,0 +1,53 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Email +{ + /// A protocol-independent mailbox with an optional display name. + /// + /// The address is an RFC 5321/5322 mailbox. Display names are sanitized by the + /// message encoder. Neither field may contain CR or LF. + /// + public sealed class EmailAddress + { + /// Initializes a new mailbox. + /// Mailbox (local@domain). + /// Optional display name. + public EmailAddress(string address, string? displayName = null) + { + if (!EmailOptionsValidator.TryValidateMailbox(address, out var mailbox)) + { + throw new ArgumentException("Mailbox address is missing or invalid.", nameof(address)); + } + + if (displayName is not null && ContainsLineBreak(displayName)) + { + throw new ArgumentException("Display name must not contain CR or LF.", nameof(displayName)); + } + + Address = mailbox; + DisplayName = string.IsNullOrWhiteSpace(displayName) ? null : displayName.Trim(); + } + + /// Gets the mailbox. + public string Address { get; } + + /// Gets the optional display name. + public string? DisplayName { get; } + + /// + public override string ToString() => Address; + + internal static bool ContainsLineBreak(ReadOnlySpan value) + { + foreach (var ch in value) + { + if (ch is '\r' or '\n') + { + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailAttachment.cs b/src/VectorNNTPD.TransitService/Email/EmailAttachment.cs new file mode 100644 index 00000000..0524f4fc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailAttachment.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// An optional MIME attachment. + /// Sanitized file name (no path separators, CR, or LF). + /// MIME type (default application/octet-stream). + /// Raw attachment bytes. + public sealed record EmailAttachment( + string FileName, + ReadOnlyMemory Content, + string ContentType = "application/octet-stream") + { + /// Validates that the file name cannot inject headers or paths. + public void Validate() + { + if (string.IsNullOrWhiteSpace(FileName) + || EmailAddress.ContainsLineBreak(FileName) + || FileName.IndexOfAny(['/', '\\', ':', '*', '?', '"', '<', '>', '|']) >= 0) + { + throw new ArgumentException("Attachment file name is invalid.", nameof(FileName)); + } + + if (string.IsNullOrWhiteSpace(ContentType) + || EmailAddress.ContainsLineBreak(ContentType) + || ContentType.Contains('\0')) + { + throw new ArgumentException("Attachment content type is invalid.", nameof(ContentType)); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailDeliveryService.cs b/src/VectorNNTPD.TransitService/Email/EmailDeliveryService.cs new file mode 100644 index 00000000..e249729f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailDeliveryService.cs @@ -0,0 +1,228 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Email.Smtp; + +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Background email worker. Scans the filesystem spool and talks SMTP. + /// + /// + /// Startup creates the spool directory and recovers leftover .wrk claims + /// back to .eml. It does not open an SMTP connection and does not + /// requeue .delivered or failed/ files. Shutdown stops new + /// submissions, allows the current SMTP operation to finish within + /// , then leaves remaining + /// pending .eml files on disk. + /// + public sealed class EmailDeliveryService : IApplicationService + { + private readonly IEmailSpool _spool; + private readonly ISmtpTransport _transport; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private readonly Random _random = new(); + private Task? _execution; + private int _started; + + /// Initializes a new delivery service. + public EmailDeliveryService( + IEmailSpool spool, + ISmtpTransport transport, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(spool); + ArgumentNullException.ThrowIfNull(transport); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _spool = spool; + _transport = transport; + _options = options; + _logger = logger; + } + + /// + public string Name => "EmailDelivery"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + var options = _options.Value; + EmailLogMessages.DeliveryStarting( + _logger, + options.Enabled, + options.Smtp.Security.ToString()); + if (!options.Enabled) + { + return Task.CompletedTask; + } + + _spool.EnsureDirectory(); + _spool.RecoverClaims(); + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _spool.StopAccepting(); + var execution = _execution; + if (execution is null) + { + await _runCts.CancelAsync().ConfigureAwait(false); + return; + } + + using var drainCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var drain = _options.Value.Spool?.ShutdownTimeout ?? TimeSpan.FromSeconds(15); + drainCts.CancelAfter(drain); + try + { + await execution.WaitAsync(drainCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await _runCts.CancelAsync().ConfigureAwait(false); + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + finally + { + EmailLogMessages.DeliveryStopped(_logger); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var smtp = _options.Value.Smtp ?? new SmtpOptions(); + var scan = _options.Value.Spool?.ScanInterval ?? TimeSpan.FromSeconds(1); + while (!cancellationToken.IsCancellationRequested) + { + EmailSpoolClaim? claim; + try + { + claim = _spool.TryClaimNext(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + EmailLogMessages.DeliveryUnexpected(_logger, ex, 0, null); + claim = null; + } + + if (claim is null) + { + try + { + await _spool.WaitAsync(scan, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + continue; + } + + try + { + await DeliverAsync(claim, smtp, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _spool.Release(claim); + break; + } + } + } + + private async Task DeliverAsync(EmailSpoolClaim claim, SmtpOptions smtp, CancellationToken cancellationToken) + { + var item = claim.Item; + while (true) + { + try + { + _ = await _transport.SendAsync(item, cancellationToken).ConfigureAwait(false); + _spool.Complete(claim); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _spool.Release(claim); + throw; + } + catch (SmtpException ex) when (ex.Kind == SmtpFailureKind.Cancelled) + { + _spool.Release(claim); + throw; + } + catch (SmtpException ex) + { + var retry = ex.IsTransient && item.Attempt < smtp.MaxAttempts; + if (!retry) + { + EmailLogMessages.DeliveryFailed( + _logger, + ex.Kind.ToString(), + ex.StatusCode, + item.Recipients.Count, + item.CorrelationId); + if (ex.IsTransient) + { + _spool.Defer(claim); + } + else + { + _spool.Quarantine(claim); + } + + return; + } + + EmailLogMessages.DeliveryRetry( + _logger, + item.Attempt, + smtp.MaxAttempts, + ex.Kind.ToString(), + ex.StatusCode); + var delay = EmailRetryDelay.Compute(smtp, item.Attempt, _random); + item.Attempt++; + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + EmailLogMessages.DeliveryUnexpected( + _logger, + ex, + item.Recipients.Count, + item.CorrelationId); + _spool.Quarantine(claim); + return; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailEnqueueStatus.cs b/src/VectorNNTPD.TransitService/Email/EmailEnqueueStatus.cs new file mode 100644 index 00000000..a75c8902 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailEnqueueStatus.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Result of . This is durable local spool + /// acceptance, not remote SMTP delivery. + /// + public enum EmailEnqueueStatus + { + /// + /// The complete message was atomically written to the local filesystem spool. + /// Not proof of remote SMTP receipt. + /// + Accepted = 0, + + /// The outbound subsystem is disabled. No spool file was written. + Disabled = 1, + + /// The message failed validation or encoding. No spool file was written. + Rejected = 2, + + /// The service is stopping and is not accepting new submissions. + Unavailable = 3, + + /// The spool file could not be created (I/O, permission, disk, or cancellation). + Failed = 4, + } + + /// Outcome of one call. + /// Local spool acceptance status. + /// Internal detail (never a credential). + public readonly record struct EmailEnqueueResult(EmailEnqueueStatus Status, string Detail); +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailHeader.cs b/src/VectorNNTPD.TransitService/Email/EmailHeader.cs new file mode 100644 index 00000000..46604a0c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailHeader.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// A custom RFC 5322 header field. + /// Header field name (token; no colon, CR, or LF). + /// Unfolded field body (no CR or LF). + public sealed record EmailHeader(string Name, string Value) + { + /// Validates that and cannot inject headers. + public void Validate() + { + if (string.IsNullOrWhiteSpace(Name) + || Name.Contains(':') + || EmailAddress.ContainsLineBreak(Name) + || !IsHeaderName(Name)) + { + throw new ArgumentException("Header name is invalid or contains injection characters.", nameof(Name)); + } + + if (Value is null || EmailAddress.ContainsLineBreak(Value)) + { + throw new ArgumentException("Header value must not contain CR or LF.", nameof(Value)); + } + } + + private static bool IsHeaderName(string name) + { + foreach (var ch in name) + { + if (ch is < (char)33 or > (char)126 || ch == ':') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailLogMessages.cs b/src/VectorNNTPD.TransitService/Email/EmailLogMessages.cs new file mode 100644 index 00000000..148ca163 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailLogMessages.cs @@ -0,0 +1,104 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// Source-generated email subsystem log messages. Never includes credentials. + internal static partial class EmailLogMessages + { + [LoggerMessage( + EventId = 2600, + Level = LogLevel.Information, + Message = "Email delivery service starting (enabled {Enabled}, security {SecurityMode})")] + public static partial void DeliveryStarting(ILogger logger, bool Enabled, string SecurityMode); + + [LoggerMessage( + EventId = 2601, + Level = LogLevel.Information, + Message = "Email delivery service stopped")] + public static partial void DeliveryStopped(ILogger logger); + + [LoggerMessage( + EventId = 2602, + Level = LogLevel.Debug, + Message = "Outbound email accepted by local spool (recipients {RecipientCount}, bytes {EncodedBytes}, correlation {CorrelationId})")] + public static partial void Enqueued( + ILogger logger, + int RecipientCount, + int EncodedBytes, + string? CorrelationId); + + [LoggerMessage( + EventId = 2604, + Level = LogLevel.Information, + Message = "SMTP session starting ({Host}:{Port}, {SecurityMode})")] + public static partial void SmtpSessionStarting(ILogger logger, string Host, int Port, string SecurityMode); + + [LoggerMessage( + EventId = 2605, + Level = LogLevel.Information, + Message = "SMTP session completed ({Host}:{Port}, accepted recipients {AcceptedRecipients})")] + public static partial void SmtpSessionCompleted(ILogger logger, string Host, int Port, int AcceptedRecipients); + + [LoggerMessage( + EventId = 2606, + Level = LogLevel.Warning, + Message = "SMTP delivery will retry (attempt {Attempt}/{MaxAttempts}, kind {FailureKind}, status {StatusCode})")] + public static partial void DeliveryRetry( + ILogger logger, + int Attempt, + int MaxAttempts, + string FailureKind, + int StatusCode); + + [LoggerMessage( + EventId = 2607, + Level = LogLevel.Error, + Message = "Email delivery failed (kind {FailureKind}, status {StatusCode}, recipients {RecipientCount}, correlation {CorrelationId})")] + public static partial void DeliveryFailed( + ILogger logger, + string FailureKind, + int StatusCode, + int RecipientCount, + string? CorrelationId); + + [LoggerMessage( + EventId = 2609, + Level = LogLevel.Debug, + Message = "Email message rejected before enqueue ({Reason})")] + public static partial void MessageRejected(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 2610, + Level = LogLevel.Error, + Message = "Unexpected email delivery failure (recipients {RecipientCount}, correlation {CorrelationId})")] + public static partial void DeliveryUnexpected(ILogger logger, Exception exception, int RecipientCount, string? CorrelationId); + + [LoggerMessage( + EventId = 2611, + Level = LogLevel.Error, + Message = "Email spool write failed")] + public static partial void SpoolWriteFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2612, + Level = LogLevel.Warning, + Message = "Email spool claim recovery failed for {FileName}")] + public static partial void SpoolRecoverFailed(ILogger logger, Exception exception, string FileName); + + [LoggerMessage( + EventId = 2613, + Level = LogLevel.Error, + Message = "Email spool file {SpoolId} is malformed and was moved to failed")] + public static partial void SpoolMalformed(ILogger logger, string SpoolId); + + [LoggerMessage( + EventId = 2614, + Level = LogLevel.Error, + Message = "Email spool quarantine failed for {SpoolId}")] + public static partial void SpoolQuarantineFailed(ILogger logger, Exception exception, string SpoolId); + + [LoggerMessage( + EventId = 2615, + Level = LogLevel.Critical, + Message = "SMTP accepted the message but the spool file {SpoolId} could not be deleted; it may be resent after restart")] + public static partial void SpoolDeleteAfterAcceptFailed(ILogger logger, Exception exception, string SpoolId); + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailMessage.cs b/src/VectorNNTPD.TransitService/Email/EmailMessage.cs new file mode 100644 index 00000000..ad7b2ae2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailMessage.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Protocol-independent outbound email. SMTP envelope fields are distinct from headers. + /// + /// + /// Producers construct this model. The email subsystem owns MIME encoding and SMTP. + /// Credentials must never be placed in headers or the body by this type. + /// + public sealed class EmailMessage + { + /// Gets the RFC 5322 From mailbox. + public required EmailAddress From { get; init; } + + /// Gets the visible To recipients. + public required IReadOnlyList To { get; init; } + + /// Gets the Cc recipients. + public IReadOnlyList Cc { get; init; } = []; + + /// Gets the Bcc recipients (envelope only; omitted from headers). + public IReadOnlyList Bcc { get; init; } = []; + + /// Gets the Reply-To mailboxes. + public IReadOnlyList ReplyTo { get; init; } = []; + + /// + /// Gets the SMTP envelope sender. When , configuration + /// Email:EnvelopeSender / Email:DefaultFrom is used. + /// + public EmailAddress? EnvelopeSender { get; init; } + + /// Gets the subject (no CR/LF; encoded by the message encoder when needed). + public required string Subject { get; init; } + + /// Gets the body octets (interpreted using ). + public required ReadOnlyMemory Body { get; init; } + + /// Gets the body media type without parameters. Default text/plain. + public string ContentType { get; init; } = "text/plain"; + + /// Gets the body charset. Default utf-8. + public string Charset { get; init; } = "utf-8"; + + /// Gets extra header fields (not From/To/Cc/Subject/MIME). + public IReadOnlyList Headers { get; init; } = []; + + /// Gets attachments. Non-empty messages become multipart/mixed. + public IReadOnlyList Attachments { get; init; } = []; + + /// Gets an optional correlation id for structured logs (not a secret). + public string? CorrelationId { get; init; } + + /// Enumerates distinct envelope recipients (To, Cc, Bcc). + public IReadOnlyList EnvelopeRecipients() + { + var list = new List(To.Count + Cc.Count + Bcc.Count); + AddDistinct(list, To); + AddDistinct(list, Cc); + AddDistinct(list, Bcc); + return list; + } + + private static void AddDistinct(List target, IReadOnlyList source) + { + foreach (var address in source) + { + var found = false; + for (var i = 0; i < target.Count; i++) + { + if (string.Equals(target[i].Address, address.Address, StringComparison.OrdinalIgnoreCase)) + { + found = true; + break; + } + } + + if (!found) + { + target.Add(address); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailRetryDelay.cs b/src/VectorNNTPD.TransitService/Email/EmailRetryDelay.cs new file mode 100644 index 00000000..2d2f4049 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailRetryDelay.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Email +{ + /// Exponential backoff with full jitter for SMTP retries. + internal static class EmailRetryDelay + { + /// Computes the delay before the next attempt ( is 1-based completed attempt). + public static TimeSpan Compute(SmtpOptions smtp, int attempt, Random random) + { + ArgumentNullException.ThrowIfNull(smtp); + ArgumentNullException.ThrowIfNull(random); + if (attempt < 1) + { + attempt = 1; + } + + var initial = smtp.InitialRetryDelay; + var maximum = smtp.MaximumRetryDelay < initial ? initial : smtp.MaximumRetryDelay; + var multiplier = 1L << Math.Min(attempt - 1, 16); + var rawTicks = initial.Ticks * multiplier; + var capped = rawTicks > maximum.Ticks ? maximum : TimeSpan.FromTicks(rawTicks); + if (capped <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + var jitter = random.NextDouble(); + return TimeSpan.FromTicks((long)(capped.Ticks * jitter)); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailService.cs b/src/VectorNNTPD.TransitService/Email/EmailService.cs new file mode 100644 index 00000000..c70ec2f1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailService.cs @@ -0,0 +1,120 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Public email API: validate, encode, and durably spool. Does not perform SMTP I/O. + /// + public sealed class EmailService : IEmailService + { + private readonly IEmailSpool _spool; + private readonly IEmailMessageEncoder _encoder; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// Initializes a new email service. + public EmailService( + IEmailSpool spool, + IEmailMessageEncoder encoder, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(spool); + ArgumentNullException.ThrowIfNull(encoder); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _spool = spool; + _encoder = encoder; + _options = options; + _logger = logger; + } + + /// + public async ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + + var options = _options.Value; + if (!options.Enabled) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Disabled, "email subsystem is disabled"); + } + + if (!_spool.IsAccepting) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Unavailable, "email spool is not accepting work"); + } + + EmailWorkItem item; + try + { + Rfc5322MessageEncoder.Validate(message); + var encoded = _encoder.Encode(message); + item = new EmailWorkItem + { + Message = message, + EncodedMessage = encoded, + EnvelopeSender = ResolveEnvelopeSender(message, options), + Recipients = message.EnvelopeRecipients(), + CorrelationId = message.CorrelationId, + }; + } + catch (ArgumentException ex) + { + EmailLogMessages.MessageRejected(_logger, ex.Message); + return new EmailEnqueueResult(EmailEnqueueStatus.Rejected, ex.Message); + } + + try + { + _ = await _spool.WriteAsync(item, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Failed, "spool write was canceled"); + } + catch (InvalidOperationException) + { + return new EmailEnqueueResult(EmailEnqueueStatus.Unavailable, "email spool is not accepting work"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolWriteFailed(_logger, ex); + return new EmailEnqueueResult(EmailEnqueueStatus.Failed, "spool write failed"); + } + + EmailLogMessages.Enqueued( + _logger, + item.Recipients.Count, + item.EncodedMessage.Length, + item.CorrelationId); + return new EmailEnqueueResult( + EmailEnqueueStatus.Accepted, + "durably accepted into the local email spool"); + } + + internal static EmailAddress ResolveEnvelopeSender(EmailMessage message, EmailOptions options) + { + if (message.EnvelopeSender is not null) + { + return message.EnvelopeSender; + } + + if (EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out var envelope)) + { + return new EmailAddress(envelope); + } + + if (EmailOptionsValidator.TryValidateMailbox(options.DefaultFrom, out var from)) + { + return new EmailAddress(from); + } + + return message.From; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailSpoolFileName.cs b/src/VectorNNTPD.TransitService/Email/EmailSpoolFileName.cs new file mode 100644 index 00000000..c13f08a3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailSpoolFileName.cs @@ -0,0 +1,35 @@ +using System.Security.Cryptography; + +namespace VectorNNTPD.TransitService.Email +{ + /// Generates opaque spool filenames: lowercase hex MD5 of a new UUID. + internal static class EmailSpoolFileName + { + /// Creates a new 32-character lowercase hex identifier. + public static string NewId() + { + var hash = MD5.HashData(Guid.NewGuid().ToByteArray()); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// Returns whether is a 32-character lowercase hex id. + public static bool IsId(ReadOnlySpan name) + { + if (name.Length != 32) + { + return false; + } + + foreach (var ch in name) + { + var hex = ch is >= '0' and <= '9' or >= 'a' and <= 'f'; + if (!hex) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailSpoolRecord.cs b/src/VectorNNTPD.TransitService/Email/EmailSpoolRecord.cs new file mode 100644 index 00000000..7ecd0cee --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailSpoolRecord.cs @@ -0,0 +1,112 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Deterministic spool file: SMTP envelope preamble plus the RFC 5322/MIME message. + /// + /// + /// Envelope sender/recipients are stored separately because they are not always + /// reconstructible from headers (Bcc, explicit MAIL FROM). + /// Credentials and transport state are never stored. + /// + internal static class EmailSpoolRecord + { + internal const string Magic = "VNNTP-SMTP-SPOOL/1"; + + public static byte[] Serialize(EmailWorkItem item) + { + ArgumentNullException.ThrowIfNull(item); + var builder = new StringBuilder(64 + item.EncodedMessage.Length); + builder.Append(Magic).Append("\r\n"); + builder.Append("MAIL FROM:<").Append(item.EnvelopeSender.Address).Append(">\r\n"); + foreach (var recipient in item.Recipients) + { + builder.Append("RCPT TO:<").Append(recipient.Address).Append(">\r\n"); + } + + builder.Append("\r\n"); + var header = Encoding.ASCII.GetBytes(builder.ToString()); + var output = new byte[header.Length + item.EncodedMessage.Length]; + header.CopyTo(output, 0); + item.EncodedMessage.Span.CopyTo(output.AsSpan(header.Length)); + return output; + } + + public static bool TryParse(ReadOnlySpan data, out EmailWorkItem? item) + { + item = null; + var text = Encoding.ASCII.GetString(data); + var separator = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + if (separator < 0) + { + return false; + } + + var preamble = text[..separator].Split("\r\n", StringSplitOptions.None); + if (preamble.Length < 3 || !string.Equals(preamble[0], Magic, StringComparison.Ordinal)) + { + return false; + } + + if (!TryMailbox(preamble[1], "MAIL FROM:", out var sender)) + { + return false; + } + + var recipients = new List(); + for (var i = 2; i < preamble.Length; i++) + { + if (string.IsNullOrEmpty(preamble[i])) + { + continue; + } + + if (!TryMailbox(preamble[i], "RCPT TO:", out var recipient)) + { + return false; + } + + recipients.Add(recipient); + } + + if (recipients.Count == 0) + { + return false; + } + + var message = data[(separator + 4)..].ToArray(); + item = new EmailWorkItem + { + EncodedMessage = message, + EnvelopeSender = sender, + Recipients = recipients, + }; + return true; + } + + private static bool TryMailbox(string line, string prefix, out EmailAddress address) + { + address = null!; + if (!line.StartsWith(prefix, StringComparison.Ordinal) + || line.Length < prefix.Length + 3 + || line[prefix.Length] != '<' + || !line.EndsWith('>')) + { + return false; + } + + var mailbox = line[(prefix.Length + 1)..^1]; + if (!EmailOptionsValidatorMailbox(mailbox)) + { + return false; + } + + address = new EmailAddress(mailbox); + return true; + } + + private static bool EmailOptionsValidatorMailbox(string mailbox) => + Configuration.EmailOptionsValidator.TryValidateMailbox(mailbox, out _); + } +} diff --git a/src/VectorNNTPD.TransitService/Email/EmailWorkItem.cs b/src/VectorNNTPD.TransitService/Email/EmailWorkItem.cs new file mode 100644 index 00000000..b72f39cf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/EmailWorkItem.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// One outbound message after MIME encoding (in memory or loaded from spool). + public sealed class EmailWorkItem + { + /// Gets the original message when still in the producer process; otherwise . + public EmailMessage? Message { get; init; } + + /// Gets the RFC 5322/MIME wire representation (CRLF, not SMTP-dot-stuffed). + public required ReadOnlyMemory EncodedMessage { get; init; } + + /// Gets the SMTP envelope sender. + public required EmailAddress EnvelopeSender { get; init; } + + /// Gets the SMTP envelope recipients. + public required IReadOnlyList Recipients { get; init; } + + /// Gets an optional correlation id for logs (not a secret). + public string? CorrelationId { get; init; } + + /// Gets the 1-based in-process delivery attempt count. Not persisted on disk. + public int Attempt { get; set; } = 1; + } +} diff --git a/src/VectorNNTPD.TransitService/Email/FilesystemEmailSpool.cs b/src/VectorNNTPD.TransitService/Email/FilesystemEmailSpool.cs new file mode 100644 index 00000000..cdcd8dbe --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/FilesystemEmailSpool.cs @@ -0,0 +1,418 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Email +{ + /// Filesystem spool under spool/smtp (configurable). + /// + /// + /// The delivery scanner enumerates only the spool directory itself + /// (), never failed/: + /// + /// + /// Suffix / pathMeaning + /// .tmpIncomplete atomic write. Never delivered. + /// .emlPending outbound message. The only automatically deliverable state. + /// .wrkIn-flight claim. Crash recovery may rename this back to .eml. + /// failed/*.emlPermanent failure. Retained for operators. Never automatically delivered or moved back. + /// .deliveredSMTP already accepted the message but local delete failed. Retained for operators. Never automatically delivered, claimed, or renamed to .eml (retry would duplicate). + /// + /// + public sealed class FilesystemEmailSpool : IEmailSpool + { + private readonly EmailSpoolOptions _options; + private readonly ILogger _logger; + private readonly SemaphoreSlim _signal = new(0); + private readonly object _claimGate = new(); + private readonly HashSet _claimed = new(StringComparer.Ordinal); + private readonly HashSet _deferred = new(StringComparer.Ordinal); + private readonly string _directory; + private readonly string _failedDirectory; + private int _accepting = 1; + + /// + /// Optional replacement for the work-file delete inside . + /// Production leaves this unset, so runs. + /// Tests assign a delegate that throws or + /// to prove the delivered-marker path. + /// + internal Action? CompleteWorkFileDelete { get; set; } + + /// Initializes a new filesystem spool from . + public FilesystemEmailSpool(IOptions options, ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _options = options.Value.Spool ?? new EmailSpoolOptions(); + _logger = logger; + _directory = ApplicationLocalPath.ResolveApplicationLocalPath( + string.IsNullOrWhiteSpace(_options.Directory) + ? EmailSpoolOptions.DefaultDirectory + : _options.Directory.Trim(), + AppContext.BaseDirectory); + _failedDirectory = Path.Join(_directory, "failed"); + } + + /// + public string Directory => _directory; + + /// + public bool IsAccepting => Volatile.Read(ref _accepting) != 0; + + /// + public void EnsureDirectory() + { + System.IO.Directory.CreateDirectory(_directory); + System.IO.Directory.CreateDirectory(_failedDirectory); + } + + /// + public void StopAccepting() => Volatile.Write(ref _accepting, 0); + + /// + public void RecoverClaims() + { + lock (_claimGate) + { + RecoverClaimsCore(); + } + } + + private void RecoverClaimsCore() + { + EnsureDirectory(); + foreach (var work in System.IO.Directory.EnumerateFiles( + _directory, + "*.wrk", + SearchOption.TopDirectoryOnly)) + { + var id = Path.GetFileNameWithoutExtension(work); + lock (_claimed) + { + if (_claimed.Contains(id)) + { + continue; + } + } + + var eml = Path.ChangeExtension(work, ".eml"); + try + { + if (File.Exists(eml)) + { + File.Delete(work); + continue; + } + + File.Move(work, eml); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolRecoverFailed(_logger, ex, Path.GetFileName(work)); + } + } + } + + /// + public async Task WriteAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + if (!IsAccepting) + { + throw new InvalidOperationException("The email spool is not accepting submissions."); + } + + EnsureDirectory(); + var payload = EmailSpoolRecord.Serialize(item); + for (var attempt = 0; attempt < 8; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + var id = EmailSpoolFileName.NewId(); + var tmp = Path.Join(_directory, id + ".tmp"); + var eml = Path.Join(_directory, id + ".eml"); + if (File.Exists(eml) || File.Exists(tmp) || File.Exists(Path.Join(_directory, id + ".wrk"))) + { + continue; + } + + try + { + await using (var stream = new FileStream( + tmp, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + FileOptions.Asynchronous)) + { + await stream.WriteAsync(payload, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + stream.Flush(flushToDisk: true); + } + + File.Move(tmp, eml); + Signal(); + return eml; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TryDelete(tmp); + if (ex is IOException && attempt < 7) + { + continue; + } + + throw; + } + } + + throw new IOException("Could not allocate a unique email spool filename."); + } + + /// + public EmailSpoolClaim? TryClaimNext() + { + lock (_claimGate) + { + return TryClaimNextCore(); + } + } + + private EmailSpoolClaim? TryClaimNextCore() + { + EnsureDirectory(); + string[] pending; + try + { + pending = System.IO.Directory.GetFiles( + _directory, + "*.eml", + SearchOption.TopDirectoryOnly); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return null; + } + + Array.Sort(pending, StringComparer.Ordinal); + foreach (var eml in pending) + { + var id = Path.GetFileNameWithoutExtension(eml); + if (!EmailSpoolFileName.IsId(id)) + { + continue; + } + + lock (_deferred) + { + if (_deferred.Contains(id)) + { + continue; + } + } + + var wrk = Path.Join(_directory, id + ".wrk"); + try + { + File.Move(eml, wrk); + lock (_claimed) + { + _claimed.Add(id); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + continue; + } + + byte[] bytes; + try + { + bytes = File.ReadAllBytes(wrk); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + ForgetClaim(id); + TryReleasePath(wrk, eml); + continue; + } + + if (!EmailSpoolRecord.TryParse(bytes, out var item) || item is null) + { + var claim = new EmailSpoolClaim(id, wrk, new EmailWorkItem + { + EncodedMessage = bytes, + EnvelopeSender = new EmailAddress("invalid@invalid.invalid"), + Recipients = [new EmailAddress("invalid@invalid.invalid")], + }); + Quarantine(claim); + EmailLogMessages.SpoolMalformed(_logger, id); + continue; + } + + return new EmailSpoolClaim(id, wrk, item); + } + + return null; + } + + /// + public void Release(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + var eml = Path.Join(_directory, claim.Id + ".eml"); + TryReleasePath(claim.WorkPath, eml); + } + + /// + public void Defer(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + lock (_deferred) + { + _deferred.Add(claim.Id); + } + + Release(claim); + } + + /// + public void Quarantine(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + EnsureDirectory(); + var dest = Path.Join(_failedDirectory, claim.Id + ".eml"); + try + { + if (File.Exists(dest)) + { + File.Delete(dest); + } + + File.Move(claim.WorkPath, dest); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolQuarantineFailed(_logger, ex, claim.Id); + } + } + + /// + public void Complete(EmailSpoolClaim claim) + { + ArgumentNullException.ThrowIfNull(claim); + ForgetClaim(claim.Id); + try + { + var delete = CompleteWorkFileDelete ?? File.Delete; + delete(claim.WorkPath); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + var delivered = Path.Join(_directory, claim.Id + ".delivered"); + try + { + if (File.Exists(delivered)) + { + File.Delete(claim.WorkPath); + return; + } + + File.Move(claim.WorkPath, delivered); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception moveEx) when (moveEx is IOException or UnauthorizedAccessException) + { + EmailLogMessages.SpoolDeleteAfterAcceptFailed(_logger, moveEx, claim.Id); + return; + } + + EmailLogMessages.SpoolDeleteAfterAcceptFailed(_logger, ex, claim.Id); + } + } + + /// + public void Signal() + { + try + { + _signal.Release(); + } + catch (SemaphoreFullException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async Task WaitAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + try + { + _ = await _signal.WaitAsync(timeout, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + } + + private void ForgetClaim(string id) + { + lock (_claimed) + { + _claimed.Remove(id); + } + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private static void TryReleasePath(string work, string eml) + { + try + { + if (!File.Exists(work)) + { + return; + } + + if (File.Exists(eml)) + { + File.Delete(work); + return; + } + + File.Move(work, eml); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/IEmailMessageEncoder.cs b/src/VectorNNTPD.TransitService/Email/IEmailMessageEncoder.cs new file mode 100644 index 00000000..177c1a9e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/IEmailMessageEncoder.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// Encodes an to RFC 5322 / MIME wire bytes. + /// + /// Separate from SMTP transport. Output uses CRLF. SMTP DATA dot-stuffing is + /// applied later by the SMTP client, not here. + /// + public interface IEmailMessageEncoder + { + /// Validates and encodes . + ReadOnlyMemory Encode(EmailMessage message); + } +} diff --git a/src/VectorNNTPD.TransitService/Email/IEmailService.cs b/src/VectorNNTPD.TransitService/Email/IEmailService.cs new file mode 100644 index 00000000..72f7dd60 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/IEmailService.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Application-facing outbound email API. Producers spool work; they do not + /// perform SMTP. + /// + /// + /// + /// means the complete RFC 5322/MIME message (plus SMTP + /// envelope) has been durably accepted into the local filesystem spool + /// (spool/smtp by default). It does not mean a remote SMTP server accepted + /// the message. + /// + /// + /// SMTP delivery is asynchronous. Successful remote acceptance deletes the spool + /// file. Undelivered files survive process restart. SMTP acceptance followed by + /// filesystem deletion is at-least-once and cannot be exactly-once. + /// + /// + /// Callers must not depend on TcpClient, SslStream, SMTP commands, + /// STARTTLS, authentication, retries, spool paths, or connection reuse. + /// + /// + public interface IEmailService + { + /// + /// Validates, encodes, and durably writes to the outbound spool. + /// + /// Protocol-independent message. + /// Cancels the spool write. + /// Durable local-spool acceptance, not remote SMTP delivery. + ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Email/IEmailSpool.cs b/src/VectorNNTPD.TransitService/Email/IEmailSpool.cs new file mode 100644 index 00000000..53c74d1d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/IEmailSpool.cs @@ -0,0 +1,92 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Durable filesystem outbound email spool. This is the queue. + /// + public interface IEmailSpool + { + /// Gets the resolved absolute spool directory. + string Directory { get; } + + /// Gets whether new submissions are accepted. + bool IsAccepting { get; } + + /// Creates the spool directory (and failed) if missing. + void EnsureDirectory(); + + /// Stops accepting new submissions. Existing .eml files remain. + void StopAccepting(); + + /// + /// Renames leftover .wrk claims back to .eml so a crash cannot + /// leave messages stuck in-flight. Does not touch .delivered or + /// failed/. + /// + void RecoverClaims(); + + /// + /// Atomically publishes a complete spool file. Returns the final .eml path. + /// + Task WriteAsync(EmailWorkItem item, CancellationToken cancellationToken); + + /// + /// Claims the next pending top-level .eml by renaming it to .wrk. + /// Does not scan failed/, .delivered, .wrk, or .tmp. + /// In-process callers are serialized. Returns when none are available. + /// + EmailSpoolClaim? TryClaimNext(); + + /// Returns a claimed file to pending .eml after a transient failure or cancel. + void Release(EmailSpoolClaim claim); + + /// + /// Returns the file to pending .eml but skips it for the rest of this + /// process lifetime so in-process retries do not hammer SMTP. After restart + /// the file is eligible again. + /// + void Defer(EmailSpoolClaim claim); + + /// + /// Moves a claimed file to failed/ after a permanent failure or parse error. + /// Failed files are never automatically requeued. + /// + void Quarantine(EmailSpoolClaim claim); + + /// + /// Deletes the claimed file after SMTP accepted the message. + /// If delete fails, the file is renamed to .delivered and must never be + /// automatically retransmitted (SMTP already accepted it). + /// + void Complete(EmailSpoolClaim claim); + + /// Wakes a waiting delivery worker after a successful write. + void Signal(); + + /// Waits for a write signal or , then returns. + Task WaitAsync(TimeSpan timeout, CancellationToken cancellationToken); + } + + /// One claimed spool file being delivered. + public sealed class EmailSpoolClaim + { + /// Initializes a claimed spool item. + public EmailSpoolClaim(string id, string workPath, EmailWorkItem item) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + ArgumentException.ThrowIfNullOrWhiteSpace(workPath); + ArgumentNullException.ThrowIfNull(item); + Id = id; + WorkPath = workPath; + Item = item; + } + + /// Gets the opaque filename id (no extension). + public string Id { get; } + + /// Gets the claimed .wrk path. + public string WorkPath { get; } + + /// Gets the loaded message and envelope. + public EmailWorkItem Item { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/NullEmailService.cs b/src/VectorNNTPD.TransitService/Email/NullEmailService.cs new file mode 100644 index 00000000..45b7e532 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/NullEmailService.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Email +{ + /// + /// Transit host stand-in: ninpaths may reference , but + /// the full SMTP spool stack is Reader-owned. Enqueues are reported disabled. + /// + internal sealed class NullEmailService : IEmailService + { + /// Shared instance. + public static NullEmailService Instance { get; } = new(); + + /// + public ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(new EmailEnqueueResult(EmailEnqueueStatus.Disabled, "email disabled on transit host")); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Rfc5322MessageEncoder.cs b/src/VectorNNTPD.TransitService/Email/Rfc5322MessageEncoder.cs new file mode 100644 index 00000000..36391af7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Rfc5322MessageEncoder.cs @@ -0,0 +1,360 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Email +{ + /// RFC 5322 / MIME encoder with header-injection rejection. + public sealed class Rfc5322MessageEncoder : IEmailMessageEncoder + { + private static readonly Encoding Utf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + + /// + public ReadOnlyMemory Encode(EmailMessage message) + { + ArgumentNullException.ThrowIfNull(message); + Validate(message); + + var builder = new StringBuilder(512 + message.Body.Length); + WriteHeader(builder, "From", FormatMailbox(message.From)); + WriteHeader(builder, "To", FormatMailboxList(message.To)); + if (message.Cc.Count > 0) + { + WriteHeader(builder, "Cc", FormatMailboxList(message.Cc)); + } + + if (message.ReplyTo.Count > 0) + { + WriteHeader(builder, "Reply-To", FormatMailboxList(message.ReplyTo)); + } + + WriteHeader(builder, "Subject", EncodeHeaderValue(message.Subject)); + WriteHeader(builder, "Date", DateTimeOffset.UtcNow.ToString("r", CultureInfo.InvariantCulture)); + WriteHeader(builder, "MIME-Version", "1.0"); + WriteHeader(builder, "Message-ID", CreateMessageId(message.From.Address)); + + foreach (var header in message.Headers) + { + header.Validate(); + RejectReservedHeader(header.Name); + WriteHeader(builder, header.Name, EncodeHeaderValue(header.Value)); + } + + if (message.Attachments.Count == 0) + { + WriteSinglePart(builder, message); + } + else + { + WriteMultipart(builder, message); + } + + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + internal static void Validate(EmailMessage message) + { + ArgumentNullException.ThrowIfNull(message.From); + if (message.To is null || message.To.Count == 0) + { + throw new ArgumentException("At least one To recipient is required.", nameof(message)); + } + + if (message.Subject is null || EmailAddress.ContainsLineBreak(message.Subject)) + { + throw new ArgumentException("Subject must not contain CR or LF.", nameof(message)); + } + + if (string.IsNullOrWhiteSpace(message.ContentType) + || EmailAddress.ContainsLineBreak(message.ContentType)) + { + throw new ArgumentException("ContentType must be a media type without CR or LF.", nameof(message)); + } + + if (string.IsNullOrWhiteSpace(message.Charset) || EmailAddress.ContainsLineBreak(message.Charset)) + { + throw new ArgumentException("Charset is invalid.", nameof(message)); + } + + foreach (var header in message.Headers) + { + header.Validate(); + } + + foreach (var attachment in message.Attachments) + { + attachment.Validate(); + } + + if (message.EnvelopeRecipients().Count == 0) + { + throw new ArgumentException("The message has no envelope recipients.", nameof(message)); + } + } + + private static void WriteSinglePart(StringBuilder builder, EmailMessage message) + { + var body = NormalizeBody(message.Body); + var needsQuotedPrintable = RequiresQuotedPrintable(body); + var contentType = message.ContentType; + if ((contentType.Equals("text/plain", StringComparison.OrdinalIgnoreCase) + || contentType.Equals("text/html", StringComparison.OrdinalIgnoreCase)) + && contentType.IndexOf("charset=", StringComparison.OrdinalIgnoreCase) < 0) + { + contentType = $"{contentType}; charset={message.Charset}"; + } + + WriteHeader(builder, "Content-Type", contentType); + WriteHeader(builder, "Content-Transfer-Encoding", needsQuotedPrintable ? "quoted-printable" : "7bit"); + builder.Append("\r\n"); + builder.Append(needsQuotedPrintable ? EncodeQuotedPrintable(body) : Encoding.ASCII.GetString(body)); + if (body.Length == 0 || body[^1] != (byte)'\n') + { + builder.Append("\r\n"); + } + } + + private static void WriteMultipart(StringBuilder builder, EmailMessage message) + { + var boundary = "vnntp-" + Guid.NewGuid().ToString("N"); + WriteHeader(builder, "Content-Type", $"multipart/mixed; boundary=\"{boundary}\""); + builder.Append("\r\n"); + builder.Append("This is a multi-part message in MIME format.\r\n\r\n"); + builder.Append("--").Append(boundary).Append("\r\n"); + + var bodyMessage = new EmailMessage + { + From = message.From, + To = message.To, + Subject = message.Subject, + Body = message.Body, + ContentType = message.ContentType, + Charset = message.Charset, + }; + WriteSinglePart(builder, bodyMessage); + + foreach (var attachment in message.Attachments) + { + builder.Append("--").Append(boundary).Append("\r\n"); + WriteHeader(builder, "Content-Type", attachment.ContentType); + WriteHeader(builder, "Content-Transfer-Encoding", "base64"); + WriteHeader( + builder, + "Content-Disposition", + $"attachment; filename=\"{attachment.FileName}\""); + builder.Append("\r\n"); + builder.Append( + Convert.ToBase64String(attachment.Content.Span, Base64FormattingOptions.InsertLineBreaks) + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace("\n", "\r\n", StringComparison.Ordinal)); + if (!builder.ToString().EndsWith("\r\n", StringComparison.Ordinal)) + { + builder.Append("\r\n"); + } + } + + builder.Append("--").Append(boundary).Append("--\r\n"); + } + + private static void WriteHeader(StringBuilder builder, string name, string value) + { + builder.Append(name).Append(": ").Append(Fold(name, value)).Append("\r\n"); + } + + private static string Fold(string name, string value) + { + if (name.Length + 2 + value.Length <= 78) + { + return value; + } + + var folded = new StringBuilder(); + var budget = 78 - name.Length - 2; + var offset = 0; + while (offset < value.Length) + { + var take = Math.Min(budget, value.Length - offset); + if (offset > 0) + { + folded.Append("\r\n "); + } + + folded.Append(value, offset, take); + offset += take; + budget = 77; + } + + return folded.ToString(); + } + + internal static string FormatMailbox(EmailAddress address) + { + if (string.IsNullOrEmpty(address.DisplayName)) + { + return address.Address; + } + + return $"{QuoteDisplayName(address.DisplayName)} <{address.Address}>"; + } + + private static string FormatMailboxList(IReadOnlyList addresses) + { + var builder = new StringBuilder(); + for (var i = 0; i < addresses.Count; i++) + { + if (i > 0) + { + builder.Append(", "); + } + + builder.Append(FormatMailbox(addresses[i])); + } + + return builder.ToString(); + } + + private static string QuoteDisplayName(string displayName) + { + var escaped = displayName.Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + return "\"" + escaped + "\""; + } + + internal static string EncodeHeaderValue(string value) + { + var ascii = true; + // codeql[cs/linq/missed-all] Foreach over string avoids CharEnumerator allocation from All(). + foreach (var ch in value) + { + if (ch > 127) + { + ascii = false; + break; + } + } + + if (ascii) + { + return value; + } + + var bytes = Utf8.GetBytes(value); + return "=?UTF-8?B?" + Convert.ToBase64String(bytes) + "?="; + } + + private static string CreateMessageId(string from) + { + var at = from.LastIndexOf('@'); + var domain = at >= 0 ? from[(at + 1)..] : "localhost"; + return "<" + Guid.NewGuid().ToString("N") + "@" + domain + ">"; + } + + private static void RejectReservedHeader(string name) + { + if (name.Equals("From", StringComparison.OrdinalIgnoreCase) + || name.Equals("To", StringComparison.OrdinalIgnoreCase) + || name.Equals("Cc", StringComparison.OrdinalIgnoreCase) + || name.Equals("Bcc", StringComparison.OrdinalIgnoreCase) + || name.Equals("Subject", StringComparison.OrdinalIgnoreCase) + || name.Equals("MIME-Version", StringComparison.OrdinalIgnoreCase) + || name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase) + || name.Equals("Content-Transfer-Encoding", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException($"Header '{name}' is reserved and must be set on EmailMessage."); + } + } + + internal static byte[] NormalizeBody(ReadOnlyMemory body) + { + if (body.Length == 0) + { + return []; + } + + var span = body.Span; + var output = new List(span.Length + 8); + for (var i = 0; i < span.Length; i++) + { + var b = span[i]; + if (b == (byte)'\r') + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + if (i + 1 < span.Length && span[i + 1] == (byte)'\n') + { + i++; + } + } + else if (b == (byte)'\n') + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + else + { + output.Add(b); + } + } + + return [.. output]; + } + + internal static bool RequiresQuotedPrintable(ReadOnlySpan body) + { + var lineLength = 0; + foreach (var b in body) + { + if (b > 127 || b == 0) + { + return true; + } + + if (b == (byte)'\n') + { + lineLength = 0; + } + else if (b != (byte)'\r') + { + lineLength++; + if (lineLength > 998) + { + return true; + } + } + } + + return false; + } + + internal static string EncodeQuotedPrintable(ReadOnlySpan body) + { + var builder = new StringBuilder(body.Length + (body.Length / 8)); + var line = 0; + for (var i = 0; i < body.Length; i++) + { + var b = body[i]; + if (b == (byte)'\r' && i + 1 < body.Length && body[i + 1] == (byte)'\n') + { + builder.Append("\r\n"); + line = 0; + i++; + continue; + } + + var encode = b > 127 || b < 32 || b == (byte)'='; + var token = encode + ? "=" + b.ToString("X2", CultureInfo.InvariantCulture) + : ((char)b).ToString(); + if (line + token.Length > 75) + { + builder.Append("=\r\n"); + line = 0; + } + + builder.Append(token); + line += token.Length; + } + + return builder.ToString(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/ISmtpTransport.cs b/src/VectorNNTPD.TransitService/Email/Smtp/ISmtpTransport.cs new file mode 100644 index 00000000..de0e1841 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/ISmtpTransport.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// SMTP delivery transport. Implementations own TCP/TLS/AUTH. + public interface ISmtpTransport + { + /// Delivers one encoded message on a new or reset SMTP session. + Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpCapabilities.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpCapabilities.cs new file mode 100644 index 00000000..883f0a95 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpCapabilities.cs @@ -0,0 +1,101 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Server capabilities advertised by EHLO (RFC 5321). + public sealed class SmtpCapabilities + { + /// Gets whether STARTTLS (RFC 3207) is advertised. + public bool StartTls { get; init; } + + /// Gets whether 8BITMIME is advertised. + public bool EightBitMime { get; init; } + + /// Gets whether SMTPUTF8 (RFC 6531) is advertised. + public bool SmtpUtf8 { get; init; } + + /// Gets the advertised SIZE limit in octets, when present. + public long? MaxSize { get; init; } + + /// Gets AUTH mechanism names in server order (uppercased). + public IReadOnlyList AuthMechanisms { get; init; } = []; + + /// Parses EHLO reply lines after the greeting line. + public static SmtpCapabilities Parse(SmtpResponse ehlo) + { + ArgumentNullException.ThrowIfNull(ehlo); + var startTls = false; + var eightBit = false; + var utf8 = false; + long? size = null; + var auth = new List(); + + foreach (var line in ehlo.Lines) + { + var keyword = Keyword(line); + if (keyword.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + startTls = true; + } + else if (keyword.Equals("8BITMIME", StringComparison.OrdinalIgnoreCase)) + { + eightBit = true; + } + else if (keyword.Equals("SMTPUTF8", StringComparison.OrdinalIgnoreCase)) + { + utf8 = true; + } + else if (keyword.Equals("SIZE", StringComparison.OrdinalIgnoreCase)) + { + var rest = Rest(line); + if (long.TryParse(rest, out var parsed) && parsed > 0) + { + size = parsed; + } + } + else if (keyword.Equals("AUTH", StringComparison.OrdinalIgnoreCase)) + { + foreach (var mechanism in Rest(line).Split(' ', StringSplitOptions.RemoveEmptyEntries)) + { + auth.Add(mechanism.ToUpperInvariant()); + } + } + } + + return new SmtpCapabilities + { + StartTls = startTls, + EightBitMime = eightBit, + SmtpUtf8 = utf8, + MaxSize = size, + AuthMechanisms = auth, + }; + } + + /// Returns whether was advertised. + public bool SupportsAuth(string mechanism) + { + foreach (var advertised in AuthMechanisms) + { + if (advertised.Equals(mechanism, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static string Keyword(string line) + { + var span = line.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space < 0 ? span.ToString() : span[..space].ToString(); + } + + private static string Rest(string line) + { + var span = line.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space < 0 ? string.Empty : span[(space + 1)..].Trim().ToString(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpConnection.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpConnection.cs new file mode 100644 index 00000000..ca7e1cfb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpConnection.cs @@ -0,0 +1,451 @@ +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// + /// One SMTP TCP/TLS session. Supports sequential messages, QUIT, and reconnect. + /// Not a connection pool. + /// + internal sealed class SmtpConnection : IAsyncDisposable + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + + private readonly SmtpOptions _options; + private readonly string _ehloHostname; + private readonly IReadOnlyCollection? _extraTrustedRoots; + private TcpClient? _tcp; + private Stream? _stream; + private SmtpResponseReader? _reader; + private bool _tls; + private bool _quit; + + public SmtpConnection( + SmtpOptions options, + string ehloHostname, + IReadOnlyCollection? extraTrustedRoots = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(ehloHostname); + _options = options; + _ehloHostname = ehloHostname; + _extraTrustedRoots = extraTrustedRoots; + } + + public SmtpCapabilities Capabilities { get; private set; } = new(); + + public bool IsAuthenticated { get; private set; } + + public bool IsTls => _tls; + + public async Task ConnectAsync(CancellationToken cancellationToken) + { + await DisposeSocketAsync().ConfigureAwait(false); + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(_options.ConnectTimeout); + + _tcp = new TcpClient(); + try + { + await _tcp.ConnectAsync(_options.Host.Trim(), _options.Port, connectCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connect timed out."); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is SocketException or IOException) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connect failed.", ex); + } + + _tcp.NoDelay = true; + _stream = _tcp.GetStream(); + _reader = new SmtpResponseReader(_stream); + + if (_options.Security == SmtpSecurityMode.ImplicitTls) + { + await UpgradeTlsAsync(cancellationToken).ConfigureAwait(false); + } + + var greeting = await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + if (!greeting.IsPositiveCompletion) + { + throw Classify(greeting, "SMTP greeting rejected."); + } + } + + public async Task EhloAsync(CancellationToken cancellationToken) + { + var response = await CommandAsync("EHLO", SanitizeEhlo(_ehloHostname), cancellationToken) + .ConfigureAwait(false); + if (!response.IsPositiveCompletion) + { + var helo = await CommandAsync("HELO", SanitizeEhlo(_ehloHostname), cancellationToken) + .ConfigureAwait(false); + if (!helo.IsPositiveCompletion) + { + throw Classify(helo, "EHLO/HELO failed."); + } + + Capabilities = new SmtpCapabilities(); + return; + } + + Capabilities = SmtpCapabilities.Parse(response); + } + + public async Task StartTlsAsync(CancellationToken cancellationToken) + { + if (!Capabilities.StartTls) + { + throw new SmtpException( + SmtpFailureKind.Tls, + "STARTTLS is required but the server did not advertise STARTTLS."); + } + + var response = await CommandAsync("STARTTLS", argument: null, cancellationToken).ConfigureAwait(false); + if (response.Code != 220) + { + throw Classify(response, "STARTTLS was rejected."); + } + + await UpgradeTlsAsync(cancellationToken).ConfigureAwait(false); + await EhloAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task AuthenticateAsync(CancellationToken cancellationToken) + { + var username = _options.Username.Trim(); + if (string.IsNullOrEmpty(username)) + { + return; + } + + if (_options.RequireTlsForAuthentication && !_tls) + { + throw new SmtpException( + SmtpFailureKind.Authentication, + "SMTP AUTH requires TLS and the session is not TLS-protected."); + } + + if (Capabilities.SupportsAuth("PLAIN")) + { + await AuthPlainAsync(username, _options.Password, cancellationToken).ConfigureAwait(false); + IsAuthenticated = true; + return; + } + + if (Capabilities.SupportsAuth("LOGIN")) + { + await AuthLoginAsync(username, _options.Password, cancellationToken).ConfigureAwait(false); + IsAuthenticated = true; + return; + } + + throw new SmtpException( + SmtpFailureKind.Authentication, + "SMTP AUTH is configured but the server advertised neither PLAIN nor LOGIN."); + } + + public Task MailFromAsync(string mailbox, CancellationToken cancellationToken) => + CommandAsync("MAIL", "FROM:<" + mailbox + ">", cancellationToken); + + public Task RcptToAsync(string mailbox, CancellationToken cancellationToken) => + CommandAsync("RCPT", "TO:<" + mailbox + ">", cancellationToken); + + public async Task DataAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + var intro = await CommandAsync("DATA", argument: null, cancellationToken).ConfigureAwait(false); + if (!intro.IsPositiveIntermediate) + { + throw Classify(intro, "DATA was rejected."); + } + + await WriteDataAsync(message, cancellationToken).ConfigureAwait(false); + return await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task QuitAsync(CancellationToken cancellationToken) + { + if (_quit || _stream is null) + { + return; + } + + _quit = true; + try + { + _ = await CommandAsync("QUIT", argument: null, cancellationToken).ConfigureAwait(false); + } + catch (SmtpException) + { + // Best-effort close. + } + } + + public async ValueTask DisposeAsync() + { + await DisposeSocketAsync().ConfigureAwait(false); + } + + private async Task UpgradeTlsAsync(CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP TLS upgrade has no transport stream."); + } + + var ssl = new SslStream(_stream, leaveInnerStreamOpen: false); + try + { + using var tlsCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + tlsCts.CancelAfter(_options.ConnectTimeout); + await ssl.AuthenticateAsClientAsync(CreateClientTlsOptions(), tlsCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw new SmtpException(SmtpFailureKind.Tls, "SMTP TLS handshake timed out."); + } + catch (OperationCanceledException) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is AuthenticationException or IOException) + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw new SmtpException(SmtpFailureKind.Tls, "SMTP TLS handshake failed.", ex); + } + + _stream = ssl; + _reader?.ReturnBuffer(); + _reader = new SmtpResponseReader(ssl); + _tls = true; + } + + private async Task AuthPlainAsync(string username, string password, CancellationToken cancellationToken) + { + var payload = Convert.ToBase64String(Encoding.UTF8.GetBytes("\0" + username + "\0" + password)); + var response = await CommandAsync("AUTH", "PLAIN " + payload, cancellationToken, redact: true) + .ConfigureAwait(false); + if (!response.IsPositiveCompletion) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH PLAIN failed.") + { + StatusCode = response.Code, + }; + } + } + + private async Task AuthLoginAsync(string username, string password, CancellationToken cancellationToken) + { + var intro = await CommandAsync("AUTH", "LOGIN", cancellationToken, redact: true).ConfigureAwait(false); + if (!intro.IsPositiveIntermediate) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN was rejected.") + { + StatusCode = intro.Code, + }; + } + + var userResponse = await CommandAsync( + Convert.ToBase64String(Encoding.UTF8.GetBytes(username)), + argument: null, + cancellationToken, + redact: true) + .ConfigureAwait(false); + if (!userResponse.IsPositiveIntermediate) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN username was rejected.") + { + StatusCode = userResponse.Code, + }; + } + + var passResponse = await CommandAsync( + Convert.ToBase64String(Encoding.UTF8.GetBytes(password)), + argument: null, + cancellationToken, + redact: true) + .ConfigureAwait(false); + if (!passResponse.IsPositiveCompletion) + { + throw new SmtpException(SmtpFailureKind.Authentication, "SMTP AUTH LOGIN failed.") + { + StatusCode = passResponse.Code, + }; + } + } + + private async Task CommandAsync( + string verb, + string? argument, + CancellationToken cancellationToken, + bool redact = false) + { + _ = redact; + var line = argument is null ? verb : verb + " " + argument; + if (line.IndexOf('\r') >= 0 || line.IndexOf('\n') >= 0) + { + throw new SmtpException(SmtpFailureKind.Message, "SMTP command contained CR or LF."); + } + + await WriteLineAsync(line, cancellationToken).ConfigureAwait(false); + return await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task WriteLineAsync(string line, CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + var encoded = Encoding.ASCII.GetBytes(line); + using var writeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + writeCts.CancelAfter(_options.CommandTimeout); + try + { + await _stream.WriteAsync(encoded, writeCts.Token).ConfigureAwait(false); + await _stream.WriteAsync(Crlf, writeCts.Token).ConfigureAwait(false); + await _stream.FlushAsync(writeCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP write timed out."); + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP write failed.", ex); + } + } + + private async Task WriteDataAsync(ReadOnlyMemory message, CancellationToken cancellationToken) + { + if (_stream is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + using var writeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + writeCts.CancelAfter(_options.CommandTimeout); + try + { + await SmtpDataEncoder.WriteStuffedAsync(_stream, message, writeCts.Token).ConfigureAwait(false); + await _stream.FlushAsync(writeCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP DATA write timed out."); + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP DATA write failed.", ex); + } + } + + private async Task ReadResponseAsync(CancellationToken cancellationToken) + { + if (_reader is null) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP connection is closed."); + } + + using var readCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readCts.CancelAfter(_options.CommandTimeout); + try + { + return await _reader.ReadAsync(readCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new SmtpException(SmtpFailureKind.Transient, "SMTP read timed out."); + } + catch (SmtpException) + { + throw; + } + catch (IOException ex) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP read failed.", ex); + } + } + + private async Task DisposeSocketAsync() + { + _reader?.ReturnBuffer(); + _reader = null; + if (_stream is not null) + { + await _stream.DisposeAsync().ConfigureAwait(false); + _stream = null; + } + + _tcp?.Dispose(); + _tcp = null; + _tls = false; + IsAuthenticated = false; + _quit = false; + } + + internal static SmtpException Classify(SmtpResponse response, string message) + { + var kind = response.IsTransientNegative + ? SmtpFailureKind.Transient + : response.IsPermanentNegative + ? SmtpFailureKind.Permanent + : SmtpFailureKind.Protocol; + return new SmtpException(kind, message) { StatusCode = response.Code }; + } + + private SslClientAuthenticationOptions CreateClientTlsOptions() + { + var options = new SslClientAuthenticationOptions + { + TargetHost = _options.Host.Trim(), + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }; + + if (_extraTrustedRoots is { Count: > 0 }) + { + var policy = new X509ChainPolicy + { + TrustMode = X509ChainTrustMode.CustomRootTrust, + RevocationMode = X509RevocationMode.NoCheck, + }; + foreach (var root in _extraTrustedRoots) + { + policy.CustomTrustStore.Add(root); + } + + options.CertificateChainPolicy = policy; + } + + return options; + } + + private static string SanitizeEhlo(string hostname) + { + var trimmed = hostname.Trim(); + if (trimmed.Length == 0 || trimmed.IndexOfAny(['\r', '\n', ' ']) >= 0) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP EHLO hostname is missing or invalid."); + } + + return trimmed; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpDataEncoder.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpDataEncoder.cs new file mode 100644 index 00000000..2122dcd9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpDataEncoder.cs @@ -0,0 +1,80 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Applies RFC 5321 transparency (dot-stuffing) and the DATA terminator. + internal static class SmtpDataEncoder + { + /// Writes with dot-stuffing and a final CRLF.CRLF. + public static async Task WriteStuffedAsync( + Stream stream, + ReadOnlyMemory message, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + var stuffed = Stuff(message.Span); + await stream.WriteAsync(stuffed, cancellationToken).ConfigureAwait(false); + } + + internal static byte[] Stuff(ReadOnlySpan span) + { + if (span.Length == 0) + { + return ".\r\n"u8.ToArray(); + } + + var output = new List(span.Length + 8); + var lineStart = true; + var lastWasCr = false; + var offset = 0; + while (offset < span.Length) + { + var remaining = span[offset..]; + if (lineStart && remaining[0] == (byte)'.') + { + output.Add((byte)'.'); + } + + var cr = remaining.IndexOf((byte)'\r'); + var lf = remaining.IndexOf((byte)'\n'); + int take; + var nextLineStart = false; + if (cr >= 0 && (lf < 0 || cr <= lf)) + { + take = cr + 1; + if (cr + 1 < remaining.Length && remaining[cr + 1] == (byte)'\n') + { + take++; + nextLineStart = true; + } + + lastWasCr = remaining[cr] == (byte)'\r' && !nextLineStart; + } + else if (lf >= 0) + { + take = lf + 1; + nextLineStart = true; + lastWasCr = false; + } + else + { + take = remaining.Length; + lastWasCr = remaining[^1] == (byte)'\r'; + } + + output.AddRange(remaining[..take].ToArray()); + offset += take; + lineStart = nextLineStart; + } + + if (!lastWasCr && !(span.Length >= 2 && span[^2] == (byte)'\r' && span[^1] == (byte)'\n')) + { + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + + output.Add((byte)'.'); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + return [.. output]; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpException.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpException.cs new file mode 100644 index 00000000..6624594f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpException.cs @@ -0,0 +1,22 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// SMTP transport failure with a classified . + public sealed class SmtpException : Exception + { + /// Initializes a new SMTP failure. + public SmtpException(SmtpFailureKind kind, string message, Exception? inner = null) + : base(message, inner) + { + Kind = kind; + } + + /// Gets the failure classification. + public SmtpFailureKind Kind { get; } + + /// Gets the last SMTP status code when one was received; otherwise 0. + public int StatusCode { get; init; } + + /// Gets whether retry is appropriate. + public bool IsTransient => Kind is SmtpFailureKind.Transient or SmtpFailureKind.Network; + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpFailureKind.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpFailureKind.cs new file mode 100644 index 00000000..8179d10e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpFailureKind.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Classifies an SMTP delivery failure for retry policy. + public enum SmtpFailureKind + { + /// Transient SMTP 4xx or temporary network/DNS/TLS issue. May retry. + Transient = 0, + + /// Permanent SMTP 5xx recipient or mailbox rejection. Do not retry. + Permanent = 1, + + /// SMTP protocol violation or malformed response. Do not retry indefinitely. + Protocol = 2, + + /// TCP/DNS connect or disconnect failure. May retry when transient. + Network = 3, + + /// TLS handshake or certificate validation failure. + Tls = 4, + + /// SMTP AUTH failed or is unavailable. Do not retry. + Authentication = 5, + + /// The operation was cancelled. + Cancelled = 6, + + /// Message is malformed or exceeds the server SIZE limit. Do not retry. + Message = 7, + + /// Some recipients were accepted and others rejected. Do not retry (would duplicate). + Partial = 8, + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpRecipientResult.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpRecipientResult.cs new file mode 100644 index 00000000..1b54bed7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpRecipientResult.cs @@ -0,0 +1,13 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Per-recipient SMTP RCPT outcome. + /// Envelope recipient. + /// Whether the server returned a 2xx RCPT reply. + /// SMTP status code. + /// Reply text (never a credential). + public readonly record struct SmtpRecipientResult( + EmailAddress Address, + bool Accepted, + int StatusCode, + string Text); +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponse.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponse.cs new file mode 100644 index 00000000..2dd3ae10 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponse.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// A complete SMTP reply (RFC 5321), including continuation lines. + public sealed class SmtpResponse + { + /// Initializes a parsed SMTP reply. + public SmtpResponse(int code, IReadOnlyList lines, string? enhancedStatus, string text) + { + Code = code; + Lines = lines; + EnhancedStatus = enhancedStatus; + Text = text; + } + + /// Gets the three-digit status code. + public int Code { get; } + + /// Gets the optional enhanced status code (RFC 2034) from the last line. + public string? EnhancedStatus { get; } + + /// Gets concatenated reply text without status prefixes. + public string Text { get; } + + /// Gets raw text portions of each reply line. + public IReadOnlyList Lines { get; } + + /// Gets whether the code is 2xx. + public bool IsPositiveCompletion => Code is >= 200 and <= 299; + + /// Gets whether the code is 3xx. + public bool IsPositiveIntermediate => Code is >= 300 and <= 399; + + /// Gets whether the code is 4xx. + public bool IsTransientNegative => Code is >= 400 and <= 499; + + /// Gets whether the code is 5xx. + public bool IsPermanentNegative => Code is >= 500 and <= 599; + + /// + public override string ToString() => Code + " " + Text; + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponseReader.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponseReader.cs new file mode 100644 index 00000000..26841e36 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpResponseReader.cs @@ -0,0 +1,199 @@ +using System.Buffers; +using System.Text; + +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Reads RFC 5321 multiline SMTP replies from a stream. + internal sealed class SmtpResponseReader + { + internal const int MaxLineLength = 512; + internal const int MaxReplyLines = 64; + + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _buffered; + private int _consumed; + + public SmtpResponseReader(Stream stream, int bufferSize = 1024) + { + ArgumentNullException.ThrowIfNull(stream); + _stream = stream; + _buffer = ArrayPool.Shared.Rent(bufferSize); + } + + public async Task ReadAsync(CancellationToken cancellationToken) + { + var lines = new List(4); + int? code = null; + while (true) + { + var line = await ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (!TryParseReplyLine(line.Span, out var lineCode, out var continuation, out var text)) + { + throw new SmtpException(SmtpFailureKind.Protocol, "Malformed SMTP reply line."); + } + + if (code is null) + { + code = lineCode; + } + else if (code.Value != lineCode) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply code changed during continuation."); + } + + lines.Add(text); + if (lines.Count > MaxReplyLines) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply exceeded the maximum number of lines."); + } + + if (!continuation) + { + var last = lines[^1]; + TrySplitEnhanced(last, out var enhanced, out var remainder); + var joined = string.Join("\n", lines); + return new SmtpResponse(code.Value, lines, enhanced, joined); + } + } + } + + internal static bool TryParseReplyLine( + ReadOnlySpan line, + out int code, + out bool continuation, + out string text) + { + code = 0; + continuation = false; + text = string.Empty; + if (line.Length < 3) + { + return false; + } + + for (var i = 0; i < 3; i++) + { + var digit = line[i]; + if (digit is < (byte)'0' or > (byte)'9') + { + return false; + } + + code = (code * 10) + (digit - (byte)'0'); + } + + if (line.Length == 3) + { + return true; + } + + var separator = line[3]; + if (separator == (byte)'-') + { + continuation = true; + } + else if (separator != (byte)' ') + { + return false; + } + + text = line.Length > 4 + ? Encoding.ASCII.GetString(line[4..]) + : string.Empty; + return true; + } + + internal static bool TrySplitEnhanced(string text, out string? enhanced, out string remainder) + { + enhanced = null; + remainder = text; + var span = text.AsSpan().TrimStart(); + if (span.Length < 5) + { + return false; + } + + var classDigit = span[0]; + if (classDigit is < '2' or > '5' || span[1] != '.') + { + return false; + } + + var end = 2; + while (end < span.Length && (char.IsAsciiDigit(span[end]) || span[end] == '.')) + { + end++; + } + + if (end >= span.Length || span[end] != ' ') + { + return false; + } + + enhanced = span[..end].ToString(); + remainder = span[(end + 1)..].ToString(); + return true; + } + + private async Task> ReadLineAsync(CancellationToken cancellationToken) + { + var line = new ArrayBufferWriter(64); + while (true) + { + if (_consumed >= _buffered) + { + _consumed = 0; + _buffered = await _stream.ReadAsync(_buffer, cancellationToken).ConfigureAwait(false); + if (_buffered == 0) + { + throw new SmtpException(SmtpFailureKind.Network, "SMTP server closed the connection."); + } + } + + var span = _buffer.AsSpan(_consumed, _buffered - _consumed); + var cr = span.IndexOf((byte)'\r'); + var lf = span.IndexOf((byte)'\n'); + if (cr >= 0 && (lf < 0 || cr < lf)) + { + Append(line, span[..cr]); + _consumed += cr + 1; + if (_consumed < _buffered && _buffer[_consumed] == (byte)'\n') + { + _consumed++; + } + + return FinishLine(line); + } + + if (lf >= 0) + { + Append(line, span[..lf]); + _consumed += lf + 1; + return FinishLine(line); + } + + Append(line, span); + _consumed = _buffered; + } + } + + private static void Append(ArrayBufferWriter line, ReadOnlySpan span) + { + if (line.WrittenCount + span.Length > MaxLineLength) + { + throw new SmtpException(SmtpFailureKind.Protocol, "SMTP reply line exceeded 512 octets."); + } + + line.Write(span); + } + + private static ReadOnlyMemory FinishLine(ArrayBufferWriter line) => + line.WrittenMemory.ToArray(); + + public void ReturnBuffer() + { + ArrayPool.Shared.Return(_buffer); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpSendResult.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpSendResult.cs new file mode 100644 index 00000000..faeba684 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpSendResult.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// Outcome of one SMTP transaction (MAIL/RCPT/DATA). + public sealed class SmtpSendResult + { + /// Gets whether every recipient was accepted and DATA completed with 2xx. + public required bool Succeeded { get; init; } + + /// Gets per-recipient RCPT results. + public required IReadOnlyList Recipients { get; init; } + + /// Gets the DATA status code when DATA was issued; otherwise 0. + public int DataStatusCode { get; init; } + } +} diff --git a/src/VectorNNTPD.TransitService/Email/Smtp/SmtpTransport.cs b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpTransport.cs new file mode 100644 index 00000000..83e96492 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Email/Smtp/SmtpTransport.cs @@ -0,0 +1,170 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Email.Smtp +{ + /// + /// Native SMTP client using and + /// . One connection per delivery. + /// + /// + /// Does not use System.Net.Mail.SmtpClient or a third-party SMTP library. + /// Connection reuse across messages is supported by + /// but this transport opens a fresh session per send so the delivery worker is not + /// coupled to a single remote connection. + /// EHLO/HELO uses the application FQDN from . + /// Certificate validation is mandatory. Tests may supply extra trusted roots; + /// there is no production bypass switch. + /// + public sealed class SmtpTransport : ISmtpTransport + { + private readonly IOptions _options; + private readonly IOptions _nntpd; + private readonly ILogger _logger; + private readonly IReadOnlyCollection? _extraTrustedRoots; + + /// Initializes a new SMTP transport. + public SmtpTransport( + IOptions options, + IOptions nntpd, + ILogger logger) + : this(options, nntpd, logger, extraTrustedRoots: null) + { + } + + /// + /// Initializes a transport that trusts additional roots for certificate + /// validation (tests only). Production DI uses the three-argument constructor. + /// + public SmtpTransport( + IOptions options, + IOptions nntpd, + ILogger logger, + IReadOnlyCollection? extraTrustedRoots) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(nntpd); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _nntpd = nntpd; + _logger = logger; + _extraTrustedRoots = extraTrustedRoots; + } + + /// + public async Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + var smtp = _options.Value.Smtp ?? new SmtpOptions(); + var ehlo = _nntpd.Value.Fqdn; + if (string.IsNullOrWhiteSpace(ehlo)) + { + throw new SmtpException(SmtpFailureKind.Protocol, "Application FQDN is required for SMTP EHLO."); + } + + EmailLogMessages.SmtpSessionStarting( + _logger, + smtp.Host, + smtp.Port, + smtp.Security.ToString()); + + await using var connection = new SmtpConnection(smtp, ehlo, _extraTrustedRoots); + await connection.ConnectAsync(cancellationToken).ConfigureAwait(false); + await connection.EhloAsync(cancellationToken).ConfigureAwait(false); + + if (smtp.Security == SmtpSecurityMode.StartTls) + { + await connection.StartTlsAsync(cancellationToken).ConfigureAwait(false); + } + + if (connection.Capabilities.MaxSize is { } maxSize + && item.EncodedMessage.Length > maxSize) + { + throw new SmtpException( + SmtpFailureKind.Message, + "Encoded message exceeds the server SIZE limit."); + } + + await connection.AuthenticateAsync(cancellationToken).ConfigureAwait(false); + + var mail = await connection + .MailFromAsync(item.EnvelopeSender.Address, cancellationToken) + .ConfigureAwait(false); + if (!mail.IsPositiveCompletion) + { + throw SmtpConnection.Classify(mail, "MAIL FROM was rejected."); + } + + var recipients = new List(item.Recipients.Count); + var accepted = 0; + var transient = 0; + var permanent = 0; + foreach (var recipient in item.Recipients) + { + var rcpt = await connection.RcptToAsync(recipient.Address, cancellationToken).ConfigureAwait(false); + var ok = rcpt.IsPositiveCompletion; + if (ok) + { + accepted++; + } + else if (rcpt.IsTransientNegative) + { + transient++; + } + else + { + permanent++; + } + + recipients.Add(new SmtpRecipientResult(recipient, ok, rcpt.Code, rcpt.Text)); + } + + if (accepted == 0) + { + if (transient > 0 && permanent == 0) + { + throw new SmtpException(SmtpFailureKind.Transient, "All recipients were deferred (4xx).") + { + StatusCode = recipients[0].StatusCode, + }; + } + + throw new SmtpException(SmtpFailureKind.Permanent, "All recipients were rejected.") + { + StatusCode = recipients[0].StatusCode, + }; + } + + var data = await connection.DataAsync(item.EncodedMessage, cancellationToken).ConfigureAwait(false); + await connection.QuitAsync(cancellationToken).ConfigureAwait(false); + + if (!data.IsPositiveCompletion) + { + throw SmtpConnection.Classify(data, "DATA was not accepted."); + } + + if (accepted != item.Recipients.Count) + { + throw new SmtpException( + SmtpFailureKind.Partial, + "Some recipients were rejected after others were accepted.") + { + StatusCode = data.Code, + }; + } + + EmailLogMessages.SmtpSessionCompleted( + _logger, + smtp.Host, + smtp.Port, + accepted); + return new SmtpSendResult + { + Succeeded = true, + Recipients = recipients, + DataStatusCode = data.Code, + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/GlobalUsings.cs b/src/VectorNNTPD.TransitService/GlobalUsings.cs new file mode 100644 index 00000000..ca57c645 --- /dev/null +++ b/src/VectorNNTPD.TransitService/GlobalUsings.cs @@ -0,0 +1,5 @@ +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; diff --git a/src/VectorNNTPD.TransitService/History/HistoryDb.cs b/src/VectorNNTPD.TransitService/History/HistoryDb.cs new file mode 100644 index 00000000..5ad9c488 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryDb.cs @@ -0,0 +1,227 @@ +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.History +{ + /// + /// HistoryDB consumer of : local memory first, then Redis. + /// + public sealed class HistoryDb : IHistoryDb, IHistoryLookupMetrics + { + private readonly LocalHistoryStore _local; + private readonly IRedisService _redis; + private readonly HistoryWriteQueue _writes; + private readonly TimeSpan _retention; + private readonly ILogger _logger; + private long _lookups; + private long _hits; + private long _misses; + private long _errors; + private long _waitTicks; + + /// Initializes a new instance of the class. + public HistoryDb( + IRedisService redis, + IOptions options, + ILogger logger) + : this( + redis, + options?.Value.HistoryTime ?? throw new ArgumentNullException(nameof(options)), + logger, + TimeProvider.System, + new HistoryWriteQueue()) + { + } + + /// Initializes a new instance with explicit retention (tests). + internal HistoryDb( + IRedisService redis, + TimeSpan retention, + ILogger logger, + TimeProvider timeProvider, + HistoryWriteQueue writeQueue, + int maxEntries = LocalHistoryStore.MaxEntries) + { + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(writeQueue); + _redis = redis; + _retention = retention; + _logger = logger; + _local = new LocalHistoryStore(retention, timeProvider, maxEntries); + _writes = writeQueue; + } + + /// Gets the Redis write queue drained by . + internal HistoryWriteQueue Writes => _writes; + + /// Gets the configured retention period. + internal TimeSpan Retention => _retention; + + /// Gets the local store (tests and maintenance). + internal LocalHistoryStore Local => _local; + + /// + public bool ContainsLocal(in HistoryDigest digest) => _local.Contains(digest); + + /// Removes expired local entries. Invoked by only. + internal int Maintain() => _local.Maintain(); + + /// + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + var started = Stopwatch.GetTimestamp(); + var digest = HistoryDigest.FromMessageId(messageId.Span); + if (_local.Contains(digest)) + { + RecordLookup(HistoryLookupResult.Seen, started); + return new ValueTask(HistoryLookupResult.Seen); + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + RecordLookup(HistoryLookupResult.Unavailable, started); + return new ValueTask(HistoryLookupResult.Unavailable); + } + + return AwaitLookupAsync( + LookupRedisAsync(digest, isRecoveryProbe, recordOnMiss: true, cancellationToken), + started); + } + + /// + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + var started = Stopwatch.GetTimestamp(); + var digest = HistoryDigest.FromMessageId(messageId.Span); + if (_local.Contains(digest)) + { + RecordLookup(HistoryLookupResult.Seen, started); + return new ValueTask(HistoryLookupResult.Seen); + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + RecordLookup(HistoryLookupResult.Unavailable, started); + return new ValueTask(HistoryLookupResult.Unavailable); + } + + return AwaitLookupAsync( + LookupRedisAsync(digest, isRecoveryProbe, recordOnMiss: false, cancellationToken), + started); + } + + /// + public HistoryLookupMetricsSnapshot Capture() => + new( + Volatile.Read(ref _lookups), + Volatile.Read(ref _hits), + Volatile.Read(ref _misses), + Volatile.Read(ref _errors), + Volatile.Read(ref _waitTicks)); + + /// + public void Remember(ReadOnlyMemory messageId) + { + var digest = HistoryDigest.FromMessageId(messageId.Span); + _local.Add(digest); + if (!_writes.TryEnqueue(digest)) + { + HistoryLogMessages.WriteQueueFull(_logger); + } + } + + private async ValueTask LookupRedisAsync( + HistoryDigest digest, + bool isRecoveryProbe, + bool recordOnMiss, + CancellationToken cancellationToken) + { + try + { + var key = HistoryRedisKeys.Create(digest); + var exists = await _redis.KeyExistsAsync(key, cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + + if (exists) + { + _local.Add(digest); + return HistoryLookupResult.Seen; + } + + if (recordOnMiss) + { + _local.Add(digest); + if (!_writes.TryEnqueue(digest)) + { + HistoryLogMessages.WriteQueueFull(_logger); + } + } + + return HistoryLookupResult.Unseen; + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + throw; + } + } + + private async ValueTask AwaitLookupAsync( + ValueTask pending, + long started) + { + try + { + var result = await pending.ConfigureAwait(false); + RecordLookup(result, started); + return result; + } + catch (OperationCanceledException) + { + throw; + } + } + + private void RecordLookup(HistoryLookupResult result, long started) + { + Interlocked.Increment(ref _lookups); + switch (result) + { + case HistoryLookupResult.Seen: + Interlocked.Increment(ref _hits); + break; + case HistoryLookupResult.Unseen: + Interlocked.Increment(ref _misses); + break; + default: + Interlocked.Increment(ref _errors); + break; + } + + var ticks = Stopwatch.GetTimestamp() - started; + if (ticks > 0) + { + Interlocked.Add(ref _waitTicks, ticks); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryDigest.cs b/src/VectorNNTPD.TransitService/History/HistoryDigest.cs new file mode 100644 index 00000000..b73caab8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryDigest.cs @@ -0,0 +1,63 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.History +{ + /// 32-byte BLAKE3 digest of a Message-ID's protocol bytes. + /// + /// Delegates to Common so History and ArticleRecord share + /// the same BLAKE3(Message-ID) identity bytes. + /// + public readonly struct HistoryDigest : IEquatable + { + /// BLAKE3 output length in bytes. + public const int Length = ArticleId.Length; + + private readonly ArticleId _id; + + private HistoryDigest(ArticleId id) + { + _id = id; + } + + /// Computes the HistoryDB digest from Message-ID wire octets. + public static HistoryDigest FromMessageId(ReadOnlySpan messageId) + => new(ArticleId.FromMessageId(messageId)); + + /// Creates a digest from an existing 32-byte BLAKE3 output. + public static HistoryDigest FromSpan(ReadOnlySpan digest) + { + if (digest.Length != Length) + { + throw new ArgumentException($"History digest must be {Length} bytes.", nameof(digest)); + } + + return new HistoryDigest(ArticleId.FromSpan(digest)); + } + + /// Copies the digest bytes into . + public void CopyTo(Span destination) + { + if (destination.Length < Length) + { + throw new ArgumentException("Destination is shorter than the digest.", nameof(destination)); + } + + _id.CopyTo(destination); + } + + /// + public bool Equals(HistoryDigest other) => _id.Equals(other._id); + + /// + public override bool Equals(object? obj) => obj is HistoryDigest other && Equals(other); + + /// + public override int GetHashCode() => _id.GetHashCode(); + + /// Equality operator. + public static bool operator ==(HistoryDigest left, HistoryDigest right) => left.Equals(right); + + /// Inequality operator. + public static bool operator !=(HistoryDigest left, HistoryDigest right) => !left.Equals(right); + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryLogMessages.cs b/src/VectorNNTPD.TransitService/History/HistoryLogMessages.cs new file mode 100644 index 00000000..89c3b8d9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryLogMessages.cs @@ -0,0 +1,54 @@ +namespace VectorNNTPD.TransitService.History +{ + /// Source-generated HistoryDB log messages. + internal static partial class HistoryLogMessages + { + [LoggerMessage( + EventId = 2200, + Level = LogLevel.Warning, + Message = "HistoryDB Redis write queue is full; local history was recorded and the Redis write was dropped")] + public static partial void WriteQueueFull(ILogger logger); + + [LoggerMessage( + EventId = 2201, + Level = LogLevel.Error, + Message = "HistoryDB Redis write failed")] + public static partial void RedisWriteFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2202, + Level = LogLevel.Warning, + Message = "HistoryDB Redis lookup failed; returning 431")] + public static partial void RedisLookupFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2203, + Level = LogLevel.Information, + Message = "HistoryDB write worker started (queue capacity {Capacity}, retention {Retention})")] + public static partial void WriterStarted(ILogger logger, int Capacity, TimeSpan Retention); + + [LoggerMessage( + EventId = 2204, + Level = LogLevel.Information, + Message = "HistoryDB write worker stopped")] + public static partial void WriterStopped(ILogger logger); + + [LoggerMessage( + EventId = 2205, + Level = LogLevel.Warning, + Message = "HistoryDB write worker stopped with {Queued} queued write(s) remaining")] + public static partial void WriterStoppedWithQueued(ILogger logger, int Queued); + + [LoggerMessage( + EventId = 2206, + Level = LogLevel.Information, + Message = "HistoryDB maintenance worker started (interval {Interval})")] + public static partial void MaintenanceStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2207, + Level = LogLevel.Information, + Message = "HistoryDB maintenance worker stopped")] + public static partial void MaintenanceStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryLookupMetrics.cs b/src/VectorNNTPD.TransitService/History/HistoryLookupMetrics.cs new file mode 100644 index 00000000..f7299549 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryLookupMetrics.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.History +{ + /// Lifetime HistoryDB lookup counters (hits, misses, errors, wait). + public readonly struct HistoryLookupMetricsSnapshot + { + /// Initializes a snapshot. + public HistoryLookupMetricsSnapshot(long lookups, long hits, long misses, long errors, long waitTicks) + { + Lookups = lookups; + Hits = hits; + Misses = misses; + Errors = errors; + WaitTicks = waitTicks; + } + + /// Gets completed lookups. + public long Lookups { get; } + + /// Gets outcomes. + public long Hits { get; } + + /// Gets outcomes. + public long Misses { get; } + + /// Gets outcomes. + public long Errors { get; } + + /// Gets accumulated lookup wait ticks (). + public long WaitTicks { get; } + } + + /// Authoritative process-wide HistoryDB lookup metrics. + public interface IHistoryLookupMetrics + { + /// Captures lifetime lookup counters. + HistoryLookupMetricsSnapshot Capture(); + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryLookupResult.cs b/src/VectorNNTPD.TransitService/History/HistoryLookupResult.cs new file mode 100644 index 00000000..aade0f4f --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryLookupResult.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.TransitService.History +{ + /// Outcome of a HistoryDB lookup for CHECK. + public enum HistoryLookupResult : byte + { + /// Message-ID has not been seen; CHECK should return 238. + Unseen = 0, + + /// Message-ID is already in history. CHECK and IHAVE map this result to a reply. + Seen = 1, + + /// Redis infrastructure failed; CHECK should return 431, not a false 238. + Unavailable = 2, + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryMaintenanceService.cs b/src/VectorNNTPD.TransitService/History/HistoryMaintenanceService.cs new file mode 100644 index 00000000..392ad0c3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryMaintenanceService.cs @@ -0,0 +1,103 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.History +{ + /// + /// Background worker that removes expired local HistoryDB entries off the CHECK path. + /// + public sealed class HistoryMaintenanceService : IApplicationService + { + /// Default interval between dictionary maintenance passes. + internal static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(30); + + private readonly HistoryDb _history; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public HistoryMaintenanceService( + HistoryDb history, + ILogger logger) + : this(history, logger, TimeProvider.System, DefaultInterval) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal HistoryMaintenanceService( + HistoryDb history, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _history = history; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "HistoryMaintenance"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + HistoryLogMessages.MaintenanceStopped(_logger); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + HistoryLogMessages.MaintenanceStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + _ = _history.Maintain(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + HistoryLogMessages.MaintenanceStopped(_logger); + } + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryRedisKeys.cs b/src/VectorNNTPD.TransitService/History/HistoryRedisKeys.cs new file mode 100644 index 00000000..9cd79a3b --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryRedisKeys.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.History +{ + /// + /// Binary Redis keys for HistoryDB: ASCII namespace prefix plus the 32-byte BLAKE3 digest. + /// + /// + /// Format: nntpd:hist: (11 bytes) concatenated with the digest. The digest is not + /// hex- or base64-encoded. Conversion to byte[] happens at the Redis API boundary. + /// + public static class HistoryRedisKeys + { + /// ASCII namespace isolating HistoryDB keys from other Redis consumers. + public static ReadOnlySpan NamespacePrefix => "nntpd:hist:"u8; + + /// Total Redis key length in bytes. + public static int KeyLength => NamespacePrefix.Length + HistoryDigest.Length; + + /// Builds the deterministic HistoryDB Redis key for . + public static byte[] Create(in HistoryDigest digest) + { + var key = new byte[KeyLength]; + NamespacePrefix.CopyTo(key); + digest.CopyTo(key.AsSpan(NamespacePrefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryWriteQueue.cs b/src/VectorNNTPD.TransitService/History/HistoryWriteQueue.cs new file mode 100644 index 00000000..aa1172f9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryWriteQueue.cs @@ -0,0 +1,64 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.TransitService.History +{ + /// Bounded queue of HistoryDB Redis writes. Full writes are dropped, not unbounded. + internal sealed class HistoryWriteQueue + { + /// Default bound for pending Redis history writes. + public const int DefaultCapacity = 4096; + + private readonly Channel _channel; + private int _count; + + /// Initializes a new instance of the class. + public HistoryWriteQueue(int capacity = DefaultCapacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + Capacity = capacity; + _channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// Gets the queue capacity. + public int Capacity { get; } + + /// Gets the approximate number of queued writes. + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// Attempts to enqueue a Redis write. Returns when full or completed. + public bool TryEnqueue(in HistoryDigest digest) + { + if (!_channel.Writer.TryWrite(digest)) + { + return false; + } + + Interlocked.Increment(ref _count); + return true; + } + + /// Completes the writer so the consumer can drain and exit. + public void Complete() => _channel.Writer.TryComplete(); + + /// Dequeues the next write, or when completed and empty. + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var digest)) + { + Interlocked.Decrement(ref _count); + return digest; + } + } + + return null; + } + } +} diff --git a/src/VectorNNTPD.TransitService/History/HistoryWriteService.cs b/src/VectorNNTPD.TransitService/History/HistoryWriteService.cs new file mode 100644 index 00000000..7187cb8a --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/HistoryWriteService.cs @@ -0,0 +1,125 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.History +{ + /// Background consumer that persists HistoryDB markers to Redis. + public sealed class HistoryWriteService : IApplicationService + { + private readonly HistoryDb _history; + private readonly IRedisService _redis; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public HistoryWriteService( + HistoryDb history, + IRedisService redis, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(redis); + ArgumentNullException.ThrowIfNull(logger); + _history = history; + _redis = redis; + _logger = logger; + } + + /// + public string Name => "HistoryWrite"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _history.Writes.Complete(); + await _runCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + HistoryLogMessages.WriterStoppedWithQueued(_logger, _history.Writes.Count); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + HistoryLogMessages.WriterStarted(_logger, _history.Writes.Capacity, _history.Retention); + while (true) + { + HistoryDigest? digest; + try + { + digest = await _history.Writes.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (digest is null) + { + break; + } + + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + continue; + } + + try + { + var key = HistoryRedisKeys.Create(digest.Value); + await _redis.SetAsync(key, HistoryDbMarker.Value, _history.Retention, CancellationToken.None) + .ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + HistoryLogMessages.RedisWriteFailed(_logger, ex); + } + + if (cancellationToken.IsCancellationRequested && _history.Writes.Count == 0) + { + break; + } + } + + HistoryLogMessages.WriterStopped(_logger); + } + } + + /// Single-byte Redis value stored for a HistoryDB marker. + internal static class HistoryDbMarker + { + /// Presence marker. Existence, not payload, is the history signal. + internal static readonly byte[] Value = [1]; + } +} diff --git a/src/VectorNNTPD.TransitService/History/IHistoryDb.cs b/src/VectorNNTPD.TransitService/History/IHistoryDb.cs new file mode 100644 index 00000000..9a3b8aac --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/IHistoryDb.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.TransitService.History +{ + /// + /// Shared HistoryDB used by CHECK (peek), IHAVE, and TAKETHIS (peek + remember). + /// + public interface IHistoryDb + { + /// + /// Looks up wire octets. Memory hits do not touch Redis. + /// A miss records the identifier locally and queues a Redis write. + /// CHECK uses instead so a query does not make the identifier known. + /// + ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default); + + /// + /// Looks up without recording a miss. + /// Used by CHECK, IHAVE admission, and TAKETHIS so a query or failed transfer + /// does not make the identifier known. TAKETHIS starts this lookup as soon as + /// the Message-ID is parsed so it overlaps article receive. + /// + ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default); + + /// + /// Records locally and queues a Redis write after a + /// successful IHAVE or TAKETHIS accept. Does not perform a Redis EXISTS. + /// + void Remember(ReadOnlyMemory messageId); + + /// Returns whether the digest is currently present in the local store (tests). + bool ContainsLocal(in HistoryDigest digest); + } +} diff --git a/src/VectorNNTPD.TransitService/History/LocalHistoryStore.cs b/src/VectorNNTPD.TransitService/History/LocalHistoryStore.cs new file mode 100644 index 00000000..af54a2fa --- /dev/null +++ b/src/VectorNNTPD.TransitService/History/LocalHistoryStore.cs @@ -0,0 +1,150 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.TransitService.History +{ + /// + /// Hard-capped expiring in-memory HistoryDB. + /// + /// + /// CHECK never walks the dictionary. A local hit is one bucket lookup plus a tick + /// comparison. Insert reserves a slot with an interlocked compare-exchange + /// then — never + /// . Expired keys are removed + /// only when the observed expiry still matches (compare/remove), on that key's + /// lookup or by on . + /// When the cap is reached, new digests are not inserted (no false hits); Redis + /// remains the authority for those IDs until maintenance frees a slot. Unexpired + /// entries are not evicted to make room. + /// + internal sealed class LocalHistoryStore + { + /// Hard cap on live local entries. Not a CHECK-path compaction trigger. + internal const int MaxEntries = 1_048_576; + + private readonly ConcurrentDictionary _entries = new(); + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _retention; + private readonly int _maxEntries; + private int _reservedCount; + + /// Initializes a new instance of the class. + public LocalHistoryStore(TimeSpan retention, TimeProvider timeProvider, int maxEntries = MaxEntries) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(retention, TimeSpan.Zero); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThan(maxEntries, 1); + _retention = retention; + _timeProvider = timeProvider; + _maxEntries = maxEntries; + } + + /// Gets the reserved slot count (includes not-yet-scanned expired keys). + public int ApproximateCount => Math.Max(0, Volatile.Read(ref _reservedCount)); + + /// Gets the dictionary entry count (tests). Not used on the CHECK path. + internal int EntryCountForTests => _entries.Count; + + /// Gets how many times has walked the dictionary. + internal int MaintenanceScanCount { get; private set; } + + /// Returns whether is present and unexpired. + public bool Contains(in HistoryDigest digest) + { + if (!_entries.TryGetValue(digest, out var expiryTicks)) + { + return false; + } + + if (_timeProvider.GetUtcNow().UtcTicks <= expiryTicks) + { + return true; + } + + TryRemoveObservedExpiry(digest, expiryTicks); + return false; + } + + /// + /// Records with the configured retention when a slot is available. + /// + public void Add(in HistoryDigest digest) + { + var expiry = _timeProvider.GetUtcNow().Add(_retention).UtcTicks; + while (true) + { + if (_entries.TryGetValue(digest, out var observed) + && (_entries.TryUpdate(digest, expiry, observed) || _entries.ContainsKey(digest))) + { + return; + } + + if (!TryReserveSlot()) + { + return; + } + + if (_entries.TryAdd(digest, expiry)) + { + return; + } + + ReleaseSlot(); + } + } + + /// + /// Removes expired entries. Must run off the CHECK thread (maintenance worker). + /// + /// The number of entries removed. + public int Maintain() + { + MaintenanceScanCount++; + var now = _timeProvider.GetUtcNow().UtcTicks; + var removed = 0; + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _entries) + { + if (pair.Value < now && TryRemoveObservedExpiry(pair.Key, pair.Value)) + { + removed++; + } + } + + return removed; + } + + /// + /// Removes only if its stored expiry still equals + /// . Used by lookup, maintenance, and tests. + /// + internal bool TryRemoveObservedExpiry(in HistoryDigest digest, long observedExpiryTicks) + { + if (_entries.TryRemove(new KeyValuePair(digest, observedExpiryTicks))) + { + ReleaseSlot(); + return true; + } + + return false; + } + + private bool TryReserveSlot() + { + while (true) + { + var current = Volatile.Read(ref _reservedCount); + if (current >= _maxEntries) + { + return false; + } + + if (Interlocked.CompareExchange(ref _reservedCount, current + 1, current) == current) + { + return true; + } + } + } + + private void ReleaseSlot() => Interlocked.Decrement(ref _reservedCount); + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/NntpServerRole.cs b/src/VectorNNTPD.TransitService/Hosting/NntpServerRole.cs new file mode 100644 index 00000000..e529d48b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/NntpServerRole.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.TransitService.Hosting +{ + /// Operational role of this NNTP host process (intrinsic; not a runtime switch). + public enum NntpServerRole + { + /// NNTP reader-server role (MODE READER, article retrieval, POST). + Reader = 1, + + /// NNTP transit-server role (streaming peers, CHECK/TAKETHIS/IHAVE). + Transit = 2, + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/NntpdHostLifetime.cs b/src/VectorNNTPD.TransitService/Hosting/NntpdHostLifetime.cs new file mode 100644 index 00000000..0f5e6b53 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/NntpdHostLifetime.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting +{ + /// + /// Ensures application shutdown is initiated exactly once and coordinates with the generic host lifetime. + /// + public sealed class NntpdHostLifetime + { + private readonly ApplicationLifecycle _lifecycle; + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly object _sync = new(); + private Task? _shutdownTask; + private int _shutdownRequested; + + /// + /// Initializes a new instance of the class. + /// + public NntpdHostLifetime( + ApplicationLifecycle lifecycle, + IHostApplicationLifetime hostApplicationLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(hostApplicationLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _hostApplicationLifetime = hostApplicationLifetime; + _options = options; + _logger = logger; + } + + /// + /// Requests a single graceful application shutdown. Subsequent calls await the same operation. + /// + /// Host-provided shutdown token. + public Task RequestShutdownAsync(CancellationToken cancellationToken) + { + lock (_sync) + { + if (_shutdownTask is not null) + { + return _shutdownTask; + } + + if (Interlocked.Exchange(ref _shutdownRequested, 1) == 0) + { + HostingLogMessages.ShutdownRequestedOnce(_logger, _options.Value.ApplicationName); + } + + _shutdownTask = ShutdownCoreAsync(cancellationToken); + return _shutdownTask; + } + } + + /// + /// Stops the generic host after an unexpected application-service termination. + /// + public void NotifyUnexpectedTermination() + { + if (!_options.Value.StopHostOnUnexpectedServiceTermination) + { + HostingLogMessages.UnexpectedTerminationOptionDisabled( + _logger, + nameof(NntpdOptions.StopHostOnUnexpectedServiceTermination)); + return; + } + + HostingLogMessages.RequestingHostStop(_logger); + + _hostApplicationLifetime.StopApplication(); + } + + private async Task ShutdownCoreAsync(CancellationToken cancellationToken) + { + try + { + await _lifecycle.StopAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + HostingLogMessages.ShutdownCompletedWithFailure(_logger, ex); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/NntpdHostedService.cs b/src/VectorNNTPD.TransitService/Hosting/NntpdHostedService.cs new file mode 100644 index 00000000..83c2a8cc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/NntpdHostedService.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting +{ + /// + /// Integrates with the .NET Generic Host. + /// + /// + /// Startup runs during so a failed initialization prevents the host from + /// reporting successful start. While running, the background execution waits for host shutdown or + /// unexpected service termination. Fatal failures stop the host via + /// . + /// + public sealed class NntpdHostedService : BackgroundService + { + private readonly ApplicationLifecycle _lifecycle; + private readonly NntpdHostLifetime _hostLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public NntpdHostedService( + ApplicationLifecycle lifecycle, + NntpdHostLifetime hostLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(hostLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _hostLifetime = hostLifetime; + _options = options; + _logger = logger; + } + + /// + public override async Task StartAsync(CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + HostingLogMessages.HostStartingApplication(_logger, _options.Value.ApplicationName); + + await _lifecycle.StartAsync(cancellationToken).ConfigureAwait(false); + + HostingLogMessages.ApplicationEnteredRunning(_logger, sw.ElapsedMilliseconds); + + await base.StartAsync(cancellationToken).ConfigureAwait(false); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await _lifecycle.WaitAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + HostingLogMessages.HostedServiceExecutionCanceled(_logger); + } + catch (InvalidOperationException ex) + { + HostingLogMessages.UnexpectedApplicationServiceTermination(_logger, ex); + _hostLifetime.NotifyUnexpectedTermination(); + throw; + } + } + + /// + public override async Task StopAsync(CancellationToken cancellationToken) + { + HostingLogMessages.HostStoppingApplication(_logger, _options.Value.ApplicationName); + + try + { + await _hostLifetime.RequestShutdownAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await base.StopAsync(cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/NntpdServiceCollectionExtensions.cs b/src/VectorNNTPD.TransitService/Hosting/NntpdServiceCollectionExtensions.cs new file mode 100644 index 00000000..8e569ecb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/NntpdServiceCollectionExtensions.cs @@ -0,0 +1,780 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Hosting; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Acme; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.RabbitMq.Management; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting.Systemd; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState.RateLimiting; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.PostFilter.Quota; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Email.Smtp; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Ninpaths; +using VectorNNTPD.TransitService.Telemetry; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Transport.Vatp; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Hosting +{ + /// + /// Extension methods for registering VectorNNTPD.TransitService hosting and lifecycle services. + /// + public static class NntpdServiceCollectionExtensions + { + + /// + /// Registers the Reader-role NNTP server host (Core + reader authentication/POST stack). + /// + /// The service collection. + /// Optional additional options configuration. + /// When true, registers a no-op placeholder application service. + /// The same instance. + public static IServiceCollection AddNntpReaderHosting( + this IServiceCollection services, + Action? configure = null, + bool includePlaceholderService = false) => throw new NotSupportedException("Reader hosting is not registered in TransitService."); + + /// + /// Registers the Transit-role NNTP server host (Core + transit peer/streaming stack). + /// + /// The service collection. + /// Optional additional options configuration. + /// When true, registers a no-op placeholder application service. + /// The same instance. + public static IServiceCollection AddNntpTransitHosting( + this IServiceCollection services, + Action? configure = null, + bool includePlaceholderService = false) => + AddNntpdHosting(services, configure, includePlaceholderService); + + /// + /// Adds NNTP server lifecycle, options, and hosted service integration for this host role. + /// + /// The service collection. + /// Optional additional options configuration. + /// + /// When , registers a no-op placeholder . + /// Defaults to now that NNTP listeners are registered. + /// + /// The same instance. + public static IServiceCollection AddNntpdHosting(this IServiceCollection services, Action? configure = null, bool includePlaceholderService = false) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(typeof(NntpServerRole), NntpServerRole.Transit); + var applicationPrefix = NntpdOptions.TransitApplicationPrefix; + bool includeReader = false; + bool includeTransit = true; + + services.TryAddSingleton(); + services.TryAddSingleton(static _ => SocketListenBinder.Instance); + + if (includeReader) + { + // Newsmaster (when configured) then MySQL nntpusers. Reader-authority only. + // Transit AUTHINFO is MODE STREAM / Transit authority and never enters this provider. + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + new CachedNntpUserRecordStore( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new DistributedSessionStateTracker( + sp.GetRequiredService(), + sp.GetRequiredService>(), + nodeId, + TimeProvider.System, + bytes: sp.GetRequiredService(), + rates: sp.GetRequiredService()); + }); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new CompositeNntpAuthenticationProvider( + NewsmasterNntpAuthenticationProvider.Create(options), + options.NewsmasterUser, + sp.GetRequiredService()); + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + } + else + { + // Transit-only host: deny reader AUTHINFO; peer identity comes from AllowFrom + Transit AUTHINFO. + services.TryAddSingleton(_ => + DenyAllNntpAuthenticationProvider.Instance); + } + + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new DistributedTransitPeerStateTracker( + sp.GetRequiredService(), + sp.GetRequiredService>(), + nodeId); + }); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return FeedDiagnosticsOptions.ResolveEnabled(options) + ? ActivatorUtilities.CreateInstance(sp) + : NullFeedDiagnostics.Instance; + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + } + else + { + // Reader-only hosts never grant transit/streaming peer privileges on connect. + services.TryAddSingleton(_ => TransitPeerAuthorization.Disabled); + services.TryAddSingleton(_ => TransitInboundConnectionLimiter.Disabled); + services.TryAddSingleton(_ => NullFeedDiagnostics.Instance); + } + + var optionsBuilder = services + .AddOptions() + .BindConfiguration(NntpdOptions.SectionName) + .Configure(static (options, configuration) => + { + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + }) + .ValidateDataAnnotations() + .ValidateOnStart() + .PostConfigure(options => + { + options.HostApplicationPrefix = applicationPrefix; + options.Systemd ??= new SystemdOptions(); + options.ArticleIngestion ??= new ArticleIngestionOptions(); + options.Transit ??= new TransitOptions(); + options.SpeedTest ??= new SpeedTestOptions(); + options.FeedDiagnostics ??= new FeedDiagnosticsOptions(); + NormalizeBindAddresses(options); + NormalizeProxyHosts(options); + options.Transit ??= new TransitOptions(); + options.AcmeStateDir = ApplicationLocalPath.ResolveApplicationLocalPath( + options.AcmeStateDir, + AppContext.BaseDirectory); + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + options.CloudFlareApiKey = string.Empty; + }); + + services.AddSingleton, NntpdOptionsValidator>(); + services.AddSingleton, PostFilterLeftoverConfigurationValidator>(); + services.AddSingleton, TransitLeftoverConfigurationValidator>(); + services.AddSingleton>(static sp => + Options.Create(sp.GetRequiredService>().Value)); + services.AddSingleton(); + services.AddAcmeCloudflareInfrastructure(); + + services + .AddOptions() + .BindConfiguration(RedisOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, RedisOptionsValidator>(); + + services + .AddOptions() + .BindConfiguration(RabbitMqOptions.SectionName) + .PostConfigure(static options => + { + options.Management ??= new RabbitMqManagementOptions(); + options.Username = null; + options.Password = null; + }) + .ValidateOnStart(); + services.AddSingleton, NntpdRabbitMqOptionsValidator>(); + + services.AddHttpClient(RabbitMqManagementHttpClient.HttpClientName, static (sp, client) => + { + var options = sp.GetRequiredService>().Value; + var baseUrl = options.Management?.BaseUrl?.Trim().TrimEnd('/'); + if (string.IsNullOrWhiteSpace(baseUrl)) + { + throw new InvalidOperationException("RabbitMQ:Management:BaseUrl is required."); + } + + client.BaseAddress = new Uri(baseUrl + "/", UriKind.Absolute); + // Stall protection comes from per-request CancelAfter on the Management client. + client.Timeout = Timeout.InfiniteTimeSpan; + }); + services.TryAddSingleton(static sp => + { + var http = sp.GetRequiredService() + .CreateClient(RabbitMqManagementHttpClient.HttpClientName); + return new RabbitMqManagementHttpClient( + http, + sp.GetRequiredService>()); + }); + + services + .AddOptions() + .BindConfiguration(TransitPeersOptions.SectionName); + // Validation is applied when building the immutable snapshot (startup throw / + // reload ignore). IValidateOptions is not registered so an invalid reload + // cannot throw from IOptionsMonitor before the last valid snapshot is kept. + + services + .AddOptions() + .BindConfiguration(ControlOptions.SectionName) + .PostConfigure(static options => + { + options.PgpAuthorities ??= new PgpAuthoritiesOptions(); + options.PgpAuthorities.Source ??= new PgpAuthoritySourceOptions(); + options.PgpAuthorities.Authorities ??= []; + foreach (var authority in options.PgpAuthorities.Authorities) + { + if (authority is null) + { + continue; + } + + authority.Authorizations ??= []; + if (PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out var normalized)) + { + authority.KeyFingerprint = normalized; + } + } + }); + services.AddSingleton, ControlOptionsValidator>(); + // Catalogue validation is available for tests and explicit IOptions resolution. + // ValidateOnStart is not used: an empty or omitted Control section must not + // prevent NNTPD startup, and this catalogue is not a runtime dependency. + + if (includeReader) + { + services + .AddOptions() + .BindConfiguration(ModerationOptions.SectionName) + .PostConfigure(static options => + { + options.Source ??= new ModerationSourceOptions(); + options.Moderators ??= []; + }) + .ValidateOnStart(); + services.AddSingleton, ModerationOptionsValidator>(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services + .AddOptions() + .BindConfiguration(EmailOptions.SectionName) + .PostConfigure(static options => + { + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + }) + .ValidateOnStart(); + services.AddSingleton, EmailOptionsValidator>(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + } + else + { + // Ninpaths depends on IEmailService + EmailOptions; Transit does not host SMTP. + services + .AddOptions() + .BindConfiguration(EmailOptions.SectionName) + .PostConfigure(static options => + { + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + }); + services.TryAddSingleton(_ => NullEmailService.Instance); + } + + services.AddNntpDbOptions(); + + if (configure is not null) + { + optionsBuilder.Configure(configure); + } + + // Startup order (sequential ApplicationServiceManager): + // NntpDB (hard dep; MySqlConnector pool) → + // nntpsharedconfig (initial snapshot, including RabbitMQ, ACME, and Cloudflare credentials, before those consumers) → + // Cloudflare DNS → Redis → RabbitMQ (hard dep; connection lifecycle only) → + // RabbitMQ topology (hard dep; cache.requests fanout + cache.broadcast + overviewdb.queue) → + // StorageServer fleet consumer (hard dep; cache. + in-memory registry) → + // StorageServer article lookup (hard dep; cache.requests publish + reply consumer) → + // RabbitMQ article-work RPC (hard dep; reply consumer + request orchestration) → + // newsgroup catalogue (initial snapshot before RUNNING) → + // moderator catalogue (nntpmoderators snapshot before RUNNING) → + // PostFilter policy → PostFilter rejection evidence → + // HistoryDB writer → HistoryDB maintenance → incoming spool writer → + // Email delivery (lazy SMTP) → ninpaths (completed inpaths; Top1000) → + // Transit AllowFrom DNS refresh → SessionState lease renewal → + // Transit inbound-ownership renewal → + // plain NNTP listener → ACME → TLS NNTP listener → + // optional feed-diagnostics reporter → always-on application telemetry. + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(static sp => + new DelegateRabbitMqConnectionNameProvider(() => + RabbitMqRuntimeOptions.GetDefaultConnectionName( + ApplicationJsonConfiguration.EntryAssemblyName, + sp.GetRequiredService>().Value.Fqdn))); + services.AddRabbitMqInfrastructure(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton( + static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static provider => + { + var ingestion = provider.GetRequiredService>().Value.ArticleIngestion + ?? new ArticleIngestionOptions(); + return new ArticlePlacementClient( + provider.GetRequiredService>(), + ingestion.MaxOutstandingStoresPerConnection); + }); + + services.TryAddSingleton(); + services.TryAddSingleton( + static sp => sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton( + static sp => sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + if (includeReader) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var prefix = options.HostApplicationPrefix; + var nodeId = options.ServerId is { } id + ? ApplicationFqdn.FormatHostLabel(prefix, id) + : prefix; + return new PostFilterReservationIdentity(nodeId); + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + new SpamdCheckClient( + sp.GetRequiredService(), + sp.GetService>())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(static sp => + AesGcmPostingTraceProtector.Create(sp.GetRequiredService>())); + } + + services.TryAddSingleton(); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddSingleton(static sp => sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddSingleton(IngestionPipelineMetrics.Shared); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(static sp => + sp.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includeReader) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + { + _ = sp.GetRequiredService(); + return sp.GetRequiredService(); + })); + } + + // SessionState lease renewal follows Transit DNS refresh when both roles are hosted + // (Combined tests). Reader-only registers it after ninpaths; Transit-only omits it. + if (includeReader) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + if (includeTransit) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + if (includeTransit) + { + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + } + + services.TryAddSingleton(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton(static sp => + sp.GetRequiredService())); + + if (includePlaceholderService) + { + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + } + + services.AddSingleton(static sp => + sp.GetRequiredService>().Value); + services.AddSingleton(); + services.AddSingleton(static sp => new ApplicationLifecycle( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + services.AddSingleton(); + services.AddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + // Register systemd lifecycle notifier before the main hosted service so it is + // constructed and subscribed before application startup transitions occur. + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddHostedService(); + services.AddHostedService(); + + return services; + } + + /// + /// Configures platform hosting integrations for Windows Service and Linux systemd without + /// preventing interactive console execution on other platforms. + /// + /// The host application builder. + /// The same instance. + /// + /// + /// AddSystemd() is context-aware: it activates systemd lifetime and notify support only + /// when the process is detected as a systemd service or NOTIFY_SOCKET is set. + /// Interactive console and Windows Service execution remain supported. + /// + /// + /// Console log formatting is owned exclusively by Serilog. Any Microsoft console formatter + /// options registered by AddSystemd() are removed immediately afterward because they + /// are unused under the Serilog-only logging pipeline. + /// + /// + /// Signal handling (SIGTERM/SIGINT) is provided by the Generic Host / + /// SystemdLifetime. This application does not register competing signal handlers. + /// + /// + public static HostApplicationBuilder ConfigureNntpdPlatformHosting(this HostApplicationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Configuration.AddVectorEnvironmentVariables(); + + builder.Services.AddWindowsService(options => + { + options.ServiceName = ApplicationJsonConfiguration.EntryAssemblyName; + }); + + builder.Services.AddSystemd(); + RemoveObsoleteMicrosoftConsoleFormatterConfiguration(builder.Services); + + builder.Services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + + var shutdown = builder.Configuration.GetSection(NntpdOptions.SectionName) + .GetValue("GracefulShutdownTimeout"); + if (shutdown is { } timeout) + { + options.ShutdownTimeout = timeout; + } + }); + + return builder; + } + + /// + /// Adds the canonical VECTOR__ environment-variable source. + /// + /// The configuration builder. + /// The same instance. + public static IConfigurationBuilder AddNntpdPrefixedEnvironmentVariables(this IConfigurationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + return builder.AddVectorEnvironmentVariables(); + } + + /// + /// Removes Microsoft console formatter options that AddSystemd() may register. + /// + /// + /// Microsoft.Extensions.Hosting.Systemd configures + /// to the systemd MEL formatter when it + /// detects a systemd service. That configuration is obsolete after Phase 0.2 because Serilog + /// owns console output. Stripping these registrations prevents a misleading unused MEL + /// formatter from remaining in DI while preserving systemd lifetime and notify registration. + /// + public static void RemoveObsoleteMicrosoftConsoleFormatterConfiguration(IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + for (var i = services.Count - 1; i >= 0; i--) + { + if (IsMicrosoftConsoleLoggerOptionsConfiguration(services[i])) + { + services.RemoveAt(i); + } + } + } + + /// + /// Returns whether a service descriptor configures Microsoft console logger options. + /// + public static bool IsMicrosoftConsoleLoggerOptionsConfiguration(ServiceDescriptor descriptor) + { + ArgumentNullException.ThrowIfNull(descriptor); + + var serviceType = descriptor.ServiceType; + if (!serviceType.IsGenericType) + { + return false; + } + + var definition = serviceType.GetGenericTypeDefinition(); + if (definition != typeof(IConfigureOptions<>) + && definition != typeof(IPostConfigureOptions<>) + && definition != typeof(IValidateOptions<>)) + { + return false; + } + + return serviceType.GenericTypeArguments[0] == typeof(ConsoleLoggerOptions); + } + + private static void NormalizeBindAddresses(NntpdOptions options) + { + if (options.BindAddress is null || options.BindAddress.Length == 0) + { + // Code default when configuration omits BindAddress: listen on all interfaces. + options.BindAddress = ["*"]; + return; + } + + for (var i = 0; i < options.BindAddress.Length; i++) + { + options.BindAddress[i] = options.BindAddress[i]?.Trim() ?? string.Empty; + } + } + + private static void NormalizeProxyHosts(NntpdOptions options) + { + if (options.ProxyHosts is null) + { + options.ProxyHosts = []; + return; + } + + for (var i = 0; i < options.ProxyHosts.Length; i++) + { + options.ProxyHosts[i] = options.ProxyHosts[i]?.Trim() ?? string.Empty; + } + } + + } + + /// + /// Phase-0 placeholder application service so the host has a deterministic no-op service to manage. + /// Replaced by real NNTP services in later phases. + /// + internal sealed class PlaceholderApplicationService : IApplicationService + { + public string Name => "Placeholder"; + + public Task? Execution => null; + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/ApplicationHealth.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/ApplicationHealth.cs new file mode 100644 index 00000000..051d2e36 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/ApplicationHealth.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Lifecycle-based application health used for systemd watchdog decisions. + /// + public sealed class ApplicationHealth : IApplicationHealth + { + private readonly ApplicationLifecycle _lifecycle; + + /// + /// Initializes a new instance of the class. + /// + public ApplicationHealth(ApplicationLifecycle lifecycle) + { + ArgumentNullException.ThrowIfNull(lifecycle); + _lifecycle = lifecycle; + } + + /// + public bool IsHealthyForWatchdog => + _lifecycle is { State: ApplicationState.Running, UnexpectedTermination.IsCompleted: false, ShutdownRequested: false }; + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/IApplicationHealth.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/IApplicationHealth.cs new file mode 100644 index 00000000..3cb8a57b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/IApplicationHealth.cs @@ -0,0 +1,20 @@ +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Application health view used for systemd watchdog keep-alives. + /// + /// + /// Phase 0.1 health is lifecycle-based only. NNTP/data-plane checks are intentionally deferred. + /// + public interface IApplicationHealth + { + /// + /// Gets a value indicating whether the application is healthy enough to send watchdog keep-alives. + /// + /// + /// Healthy means: lifecycle state is Running, required services have initialized, + /// no fatal supervised service failure has been observed, and shutdown has not started. + /// + bool IsHealthyForWatchdog { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdNotifyBridge.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdNotifyBridge.cs new file mode 100644 index 00000000..25b1ad8a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdNotifyBridge.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Testable abstraction over systemd service notifications. + /// + public interface ISystemdNotifyBridge + { + /// Gets a value indicating whether notifications are delivered to systemd. + bool IsEnabled { get; } + + /// Sends READY=1. + void NotifyReady(); + + /// Sends STOPPING=1. + void NotifyStopping(); + + /// Sends STATUS= with the specified status text. + /// Human-readable status (must not contain newlines). + void NotifyStatus(string status); + + /// Sends WATCHDOG=1. + void NotifyWatchdog(); + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdRuntime.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdRuntime.cs new file mode 100644 index 00000000..eaef0e79 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/ISystemdRuntime.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Describes the process relationship to systemd for hosting decisions. + /// + public interface ISystemdRuntime + { + /// Gets a value indicating whether the process is running on Linux. + bool IsLinux { get; } + + /// + /// Gets a value indicating whether the process appears to be managed by systemd + /// (direct systemd service / compatible notify environment). + /// + bool IsSystemdService { get; } + + /// + /// Gets a value indicating whether systemd notify is available for this process + /// (typically NOTIFY_SOCKET was present when the host notifier was constructed). + /// + bool IsNotifyEnabled { get; } + + /// + /// Gets the systemd watchdog deadline when configured and applicable to this process; + /// otherwise . + /// + TimeSpan? WatchdogTimeout { get; } + + /// + /// Gets a value indicating whether systemd expects watchdog keep-alives from this process. + /// + bool IsWatchdogConfigured { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdLifecycleNotifier.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdLifecycleNotifier.cs new file mode 100644 index 00000000..f612b567 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdLifecycleNotifier.cs @@ -0,0 +1,200 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Publishes systemd readiness and status notifications from application lifecycle transitions. + /// + /// + /// Readiness is reported at most once, and only after a successful transition to + /// . Startup failure never reports ready. Once shutdown + /// begins, further readiness notifications are suppressed. + /// + public sealed class SystemdLifecycleNotifier : IHostedService, IDisposable + { + private readonly ApplicationLifecycle _lifecycle; + private readonly ISystemdNotifyBridge _notify; + private readonly ISystemdRuntime _runtime; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly object _sync = new(); + private int _readySent; + private int _stoppingSent; + private string? _lastStatus; + private bool _subscribed; + + /// + /// Initializes a new instance of the class. + /// + public SystemdLifecycleNotifier( + ApplicationLifecycle lifecycle, + ISystemdNotifyBridge notify, + ISystemdRuntime runtime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(lifecycle); + ArgumentNullException.ThrowIfNull(notify); + ArgumentNullException.ThrowIfNull(runtime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _lifecycle = lifecycle; + _notify = notify; + _runtime = runtime; + _options = options; + _logger = logger; + + // Subscribe in the constructor so transitions are observed even if this hosted + // service's StartAsync runs after ApplicationLifecycle has begun starting. + _lifecycle.StateChanged += OnStateChanged; + _subscribed = true; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (_runtime.IsLinux && (_runtime.IsSystemdService || _notify.IsEnabled)) + { + SystemdLogMessages.LifecycleNotificationsActivated( + _logger, + _notify.IsEnabled, + _runtime.IsSystemdService); + } + + // Synchronize with current state in case transitions already occurred. + var current = _lifecycle.State; + OnStateChanged(_lifecycle, new ApplicationStateChangedEventArgs(current, current)); + return Task.CompletedTask; + } + + /// + public Task StopAsync(CancellationToken cancellationToken) + { + Unsubscribe(); + return Task.CompletedTask; + } + + /// + public void Dispose() => Unsubscribe(); + + private void Unsubscribe() + { + if (!_subscribed) + { + return; + } + + _lifecycle.StateChanged -= OnStateChanged; + _subscribed = false; + } + + private void OnStateChanged(object? sender, ApplicationStateChangedEventArgs e) + { + // When called for synchronization, FromState == ToState == current. + var state = e.ToState; + switch (state) + { + case ApplicationState.Starting: + TryNotifyStatus("Starting: initializing application services"); + break; + + case ApplicationState.Running: + TryNotifyStatus("Running: application initialization completed"); + TryNotifyReady(); + break; + + case ApplicationState.Stopping: + TryNotifyStopping(); + TryNotifyStatus("Stopping: graceful shutdown in progress"); + break; + + case ApplicationState.Stopped: + TryNotifyStopping(); + TryNotifyStatus("Stopped"); + break; + + case ApplicationState.Created: + break; + + default: + TryNotifyStatus($"State={state}"); + break; + } + } + + private void TryNotifyReady() + { + if (!_options.Value.Systemd.NotifyReadyOnApplicationRunning) + { + return; + } + + if (!_notify.IsEnabled) + { + return; + } + + if (_lifecycle.ShutdownRequested || _lifecycle.State != ApplicationState.Running) + { + return; + } + + if (Interlocked.Exchange(ref _readySent, 1) != 0) + { + return; + } + + _notify.NotifyReady(); + SystemdLogMessages.ReadinessReported(_logger); + } + + private void TryNotifyStopping() + { + if (!_options.Value.Systemd.NotifyStoppingOnApplicationShutdown) + { + return; + } + + if (!_notify.IsEnabled) + { + return; + } + + if (Interlocked.Exchange(ref _stoppingSent, 1) != 0) + { + return; + } + + // Once stopping, readiness must not be reported later. + Interlocked.Exchange(ref _readySent, 1); + + _notify.NotifyStopping(); + SystemdLogMessages.StoppingReported(_logger); + } + + private void TryNotifyStatus(string status) + { + if (!_options.Value.Systemd.ReportLifecycleStatus || !_notify.IsEnabled) + { + return; + } + + lock (_sync) + { + if (string.Equals(_lastStatus, status, StringComparison.Ordinal)) + { + return; + } + + _lastStatus = status; + } + + _notify.NotifyStatus(status); + SystemdLogMessages.StatusReported(_logger, status); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdNotifyBridge.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdNotifyBridge.cs new file mode 100644 index 00000000..0120273d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdNotifyBridge.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.Hosting.Systemd; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Forwards notifications to the official when registered and enabled. + /// + public sealed class SystemdNotifyBridge : ISystemdNotifyBridge + { + private static readonly ServiceState Ready = ServiceState.Ready; + private static readonly ServiceState Stopping = ServiceState.Stopping; + private static readonly ServiceState Watchdog = new("WATCHDOG=1"); + + private readonly ISystemdNotifier? _notifier; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public SystemdNotifyBridge( + IEnumerable systemdNotifiers, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(systemdNotifiers); + ArgumentNullException.ThrowIfNull(logger); + + _notifier = systemdNotifiers.FirstOrDefault(); + _logger = logger; + } + + /// + public bool IsEnabled => _notifier?.IsEnabled == true; + + /// + public void NotifyReady() => Notify(Ready, "READY=1"); + + /// + public void NotifyStopping() => Notify(Stopping, "STOPPING=1"); + + /// + public void NotifyStatus(string status) + { + ArgumentException.ThrowIfNullOrWhiteSpace(status); + if (status.Contains('\n', StringComparison.Ordinal) || status.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException("Status must not contain newlines or NUL characters.", nameof(status)); + } + + Notify(new ServiceState("STATUS=" + status), "STATUS"); + } + + /// + public void NotifyWatchdog() => Notify(Watchdog, "WATCHDOG=1"); + + private void Notify(ServiceState state, string label) + { + if (_notifier is null || !_notifier.IsEnabled) + { + return; + } + + try + { + _notifier.Notify(state); + SystemdLogMessages.NotificationSent(_logger, label); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.NotificationFailed(_logger, ex, label); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdRuntime.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdRuntime.cs new file mode 100644 index 00000000..88256edc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdRuntime.cs @@ -0,0 +1,133 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Hosting.Systemd; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Default systemd environment detection based on OS platform checks and systemd variables. + /// + /// + /// + /// Detection distinguishes Linux from other OSes, and systemd-managed execution from ordinary + /// Linux console sessions. is preferred for + /// service detection; notify enablement uses the registered when + /// available (the hosting package clears NOTIFY_SOCKET after constructing the notifier). + /// + /// + /// Watchdog configuration is read from WATCHDOG_USEC and optionally WATCHDOG_PID. + /// Missing or malformed values disable watchdog behaviour rather than inventing an interval. + /// + /// + public sealed class SystemdRuntime : ISystemdRuntime + { + /// Environment variable containing the watchdog timeout in microseconds. + public const string WatchdogUsecVariable = "WATCHDOG_USEC"; + + /// Environment variable containing the PID expected to send watchdog keep-alives. + public const string WatchdogPidVariable = "WATCHDOG_PID"; + + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public SystemdRuntime( + IEnumerable systemdNotifiers, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(systemdNotifiers); + ArgumentNullException.ThrowIfNull(logger); + + _logger = logger; + + IsLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + IsSystemdService = IsLinux && SystemdHelpers.IsSystemdService(); + + var notifier = systemdNotifiers.FirstOrDefault(); + IsNotifyEnabled = notifier?.IsEnabled == true; + + WatchdogTimeout = TryReadWatchdogTimeout(out var reason); + IsWatchdogConfigured = WatchdogTimeout is not null; + + if (IsLinux) + { + SystemdLogMessages.RuntimeDetection( + _logger, + IsSystemdService, + IsNotifyEnabled, + IsWatchdogConfigured, + WatchdogTimeout, + reason); + } + } + + /// + public bool IsLinux { get; } + + /// + public bool IsSystemdService { get; } + + /// + public bool IsNotifyEnabled { get; } + + /// + public TimeSpan? WatchdogTimeout { get; } + + /// + public bool IsWatchdogConfigured { get; } + + private TimeSpan? TryReadWatchdogTimeout(out string reason) + { + if (!IsLinux) + { + reason = "non-Linux platform"; + return null; + } + + var usecText = Environment.GetEnvironmentVariable(WatchdogUsecVariable); + if (string.IsNullOrWhiteSpace(usecText)) + { + reason = "WATCHDOG_USEC not set"; + return null; + } + + if (!ulong.TryParse(usecText, NumberStyles.None, CultureInfo.InvariantCulture, out var usec) || usec == 0) + { + reason = "WATCHDOG_USEC missing or invalid"; + SystemdLogMessages.MalformedWatchdogUsec(_logger); + return null; + } + + var watchdogPidText = Environment.GetEnvironmentVariable(WatchdogPidVariable); + if (!string.IsNullOrWhiteSpace(watchdogPidText)) + { + if (!int.TryParse(watchdogPidText, NumberStyles.None, CultureInfo.InvariantCulture, out var watchdogPid)) + { + reason = "WATCHDOG_PID malformed"; + SystemdLogMessages.MalformedWatchdogPid(_logger); + return null; + } + + if (watchdogPid != Environment.ProcessId) + { + reason = "WATCHDOG_PID does not match current process"; + SystemdLogMessages.WatchdogConfiguredForDifferentPid(_logger); + return null; + } + } + + // TimeSpan ticks are 100ns; microseconds * 10 = ticks. + if (usec > (ulong)(TimeSpan.MaxValue.Ticks / 10)) + { + reason = "WATCHDOG_USEC out of range"; + SystemdLogMessages.WatchdogUsecOutOfRange(_logger); + return null; + } + + reason = "WATCHDOG_USEC accepted"; + return TimeSpan.FromTicks((long)usec * 10); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogInterval.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogInterval.cs new file mode 100644 index 00000000..4106b98b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogInterval.cs @@ -0,0 +1,43 @@ +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Calculates systemd watchdog heartbeat intervals from the systemd-provided deadline. + /// + public static class SystemdWatchdogInterval + { + /// + /// Derives a heartbeat interval that leaves margin before the systemd watchdog deadline. + /// + /// Deadline supplied by systemd (WATCHDOG_USEC). + /// Fraction of the deadline between heartbeats; must be in (0, 1). + /// The heartbeat interval. + /// Thrown when inputs are invalid. + public static TimeSpan Calculate(TimeSpan watchdogTimeout, double fraction = 0.5) + { + if (watchdogTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(watchdogTimeout), watchdogTimeout, "Watchdog timeout must be positive."); + } + + if (double.IsNaN(fraction) || fraction is <= 0 or >= 1) + { + throw new ArgumentOutOfRangeException(nameof(fraction), fraction, "Fraction must be in the open interval (0, 1)."); + } + + var ticks = (long)Math.Floor(watchdogTimeout.Ticks * fraction); + if (ticks < TimeSpan.TicksPerMillisecond) + { + ticks = TimeSpan.TicksPerMillisecond; + } + + // Always leave at least 1ms margin before the deadline. + var maxTicks = Math.Max(TimeSpan.TicksPerMillisecond, watchdogTimeout.Ticks - TimeSpan.TicksPerMillisecond); + if (ticks > maxTicks) + { + ticks = maxTicks; + } + + return TimeSpan.FromTicks(ticks); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogService.cs b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogService.cs new file mode 100644 index 00000000..24434a4f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/Systemd/SystemdWatchdogService.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Hosting.Systemd +{ + /// + /// Sends systemd watchdog keep-alives while the application is healthy. + /// + /// + /// The watchdog is disabled unless systemd configured WATCHDOG_USEC for this process and + /// is . Heartbeats stop when + /// shutdown begins or the application becomes unhealthy. Unexpected failures stop the host. + /// + public sealed class SystemdWatchdogService : BackgroundService + { + private readonly ISystemdRuntime _runtime; + private readonly ISystemdNotifyBridge _notify; + private readonly IApplicationHealth _health; + private readonly IHostApplicationLifetime _hostLifetime; + private readonly IOptions _options; + private readonly ILogger _logger; + private TimeSpan _interval; + private int _active; + + /// + /// Initializes a new instance of the class. + /// + public SystemdWatchdogService( + ISystemdRuntime runtime, + ISystemdNotifyBridge notify, + IApplicationHealth health, + IHostApplicationLifetime hostLifetime, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(runtime); + ArgumentNullException.ThrowIfNull(notify); + ArgumentNullException.ThrowIfNull(health); + ArgumentNullException.ThrowIfNull(hostLifetime); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _runtime = runtime; + _notify = notify; + _health = health; + _hostLifetime = hostLifetime; + _options = options; + _logger = logger; + } + + /// Gets a value indicating whether the watchdog loop is active. + public bool IsActive => Volatile.Read(ref _active) != 0; + + /// Gets the heartbeat interval when active; otherwise . + public TimeSpan HeartbeatInterval => _interval; + + /// + public override Task StartAsync(CancellationToken cancellationToken) + { + if (!ShouldActivate(out var reason)) + { + SystemdLogMessages.WatchdogRemainDisabled(_logger, reason); + return Task.CompletedTask; + } + + var timeout = _runtime.WatchdogTimeout + ?? throw new InvalidOperationException("Watchdog timeout unexpectedly unavailable."); + + _interval = SystemdWatchdogInterval.Calculate( + timeout, + _options.Value.Systemd.WatchdogIntervalFraction); + + Interlocked.Exchange(ref _active, 1); + SystemdLogMessages.WatchdogActivated(_logger, timeout, _interval); + + return base.StartAsync(cancellationToken); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (Volatile.Read(ref _active) == 0) + { + return; + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + stoppingToken, + _hostLifetime.ApplicationStopping); + + try + { + using var timer = new PeriodicTimer(_interval); + while (await timer.WaitForNextTickAsync(linked.Token).ConfigureAwait(false)) + { + if (!_health.IsHealthyForWatchdog) + { + SystemdLogMessages.WatchdogKeepAliveSkippedUnhealthy(_logger); + continue; + } + + try + { + _notify.NotifyWatchdog(); + SystemdLogMessages.WatchdogKeepAliveSent(_logger); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.WatchdogNotificationFailed(_logger, ex); + throw; + } + } + } + catch (OperationCanceledException) when (linked.IsCancellationRequested) + { + SystemdLogMessages.WatchdogDeactivated(_logger); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SystemdLogMessages.WatchdogLoopFailed(_logger, ex); + Interlocked.Exchange(ref _active, 0); + _hostLifetime.StopApplication(); + throw; + } + finally + { + Interlocked.Exchange(ref _active, 0); + } + } + + /// + public override async Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Exchange(ref _active, 0); + SystemdLogMessages.WatchdogStopping(_logger); + await base.StopAsync(cancellationToken).ConfigureAwait(false); + } + + private bool ShouldActivate(out string reason) + { + if (!_options.Value.Systemd.EnableWatchdog) + { + reason = "EnableWatchdog is false"; + return false; + } + + if (!_runtime.IsLinux) + { + reason = "non-Linux platform"; + return false; + } + + if (!_runtime.IsWatchdogConfigured || _runtime.WatchdogTimeout is null) + { + reason = "systemd did not configure WATCHDOG_USEC for this process"; + return false; + } + + if (!_notify.IsEnabled) + { + reason = "systemd notify is not enabled"; + return false; + } + + reason = "ok"; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Hosting/TransitStartupSmoke.cs b/src/VectorNNTPD.TransitService/Hosting/TransitStartupSmoke.cs new file mode 100644 index 00000000..967171a3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Hosting/TransitStartupSmoke.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Hosting +{ + /// + /// Builds the real Transit host and stops after configuration binding, without starting hosted services. + /// + /// + /// Set to and + /// to an isolated JSON file. Does not call + /// IHost.RunAsync, so Redis, MySQL, RabbitMQ, DNS, ACME, and listeners are not started. + /// + internal static class TransitStartupSmoke + { + /// Environment variable that selects startup smoke. The value must be . + internal const string EnvironmentVariable = "VECTORNNTP_TRANSIT_STARTUP_SMOKE"; + + /// The only value of that selects startup smoke. + internal const string EnabledValue = "1"; + + /// Environment variable naming the isolated JSON file that replaces application file configuration. + internal const string ConfigurationPathVariable = "VECTORNNTP_TRANSIT_STARTUP_SMOKE_CONFIG"; + + /// Stdout prefix written after configuration binding succeeds. + private const string CheckpointPrefix = "TransitService startup smoke: configuration"; + + /// Returns whether requests startup smoke. + internal static bool IsRequested(string? value) => + string.Equals(value, EnabledValue, StringComparison.Ordinal); + + /// + /// Removes file configuration sources and appends the JSON file named by . + /// + internal static void UseIsolatedConfiguration(IConfigurationBuilder configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var path = Environment.GetEnvironmentVariable(ConfigurationPathVariable); + if (string.IsNullOrWhiteSpace(path)) + { + throw new InvalidOperationException( + ConfigurationPathVariable + " must name the isolated startup smoke configuration file."); + } + + if (!File.Exists(path)) + { + throw new FileNotFoundException("Startup smoke configuration file was not found.", path); + } + + for (var index = configuration.Sources.Count - 1; index >= 0; index--) + { + if (configuration.Sources[index] is FileConfigurationSource) + { + configuration.Sources.RemoveAt(index); + } + } + + configuration.AddJsonFile(Path.GetFullPath(path), optional: false, reloadOnChange: false); + } + + /// Resolves bound Nntpd options and writes the checkpoint. + internal static int Complete(IHost host, TextWriter output) + { + ArgumentNullException.ThrowIfNull(host); + ArgumentNullException.ThrowIfNull(output); + using (host) + { + var options = host.Services.GetRequiredService>().Value; + if (options.ServerId is not { } serverId || string.IsNullOrWhiteSpace(options.Fqdn)) + { + throw new InvalidOperationException( + "Startup smoke did not bind a usable Nntpd ServerId and Fqdn."); + } + + if (!string.Equals(options.HostApplicationPrefix, NntpdOptions.TransitApplicationPrefix, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "Startup smoke expected HostApplicationPrefix '" + NntpdOptions.TransitApplicationPrefix + + "' and found '" + options.HostApplicationPrefix + "'."); + } + + output.WriteLine( + "{0} serverId={1} fqdn={2}", + CheckpointPrefix, + serverId, + options.Fqdn); + output.Flush(); + return 0; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/HostingLogMessages.cs b/src/VectorNNTPD.TransitService/Logging/HostingLogMessages.cs new file mode 100644 index 00000000..6c8e29ff --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/HostingLogMessages.cs @@ -0,0 +1,74 @@ +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Source-generated structured log messages for Generic Host integration events. + /// + internal static partial class HostingLogMessages + { + [LoggerMessage( + EventId = 1200, + Level = LogLevel.Information, + Message = "Host starting application {ApplicationName}")] + public static partial void HostStartingApplication(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1201, + Level = LogLevel.Information, + Message = "Application entered Running state after {ElapsedMs} ms. Host startup continuing")] + public static partial void ApplicationEnteredRunning(ILogger logger, long ElapsedMs); + + [LoggerMessage( + EventId = 1202, + Level = LogLevel.Information, + Message = "Hosted service execution canceled due to host shutdown")] + public static partial void HostedServiceExecutionCanceled(ILogger logger); + + [LoggerMessage( + EventId = 1203, + Level = LogLevel.Critical, + Message = "Background hosted service observed unexpected application-service termination")] + public static partial void UnexpectedApplicationServiceTermination(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1204, + Level = LogLevel.Information, + Message = "Host stopping application {ApplicationName}")] + public static partial void HostStoppingApplication(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1205, + Level = LogLevel.Information, + Message = "Application shutdown requested exactly once for {ApplicationName}")] + public static partial void ShutdownRequestedOnce(ILogger logger, string ApplicationName); + + [LoggerMessage( + EventId = 1206, + Level = LogLevel.Warning, + Message = "Unexpected service termination observed, but {Option} is disabled")] + public static partial void UnexpectedTerminationOptionDisabled(ILogger logger, string Option); + + [LoggerMessage( + EventId = 1207, + Level = LogLevel.Critical, + Message = "Requesting host stop due to unexpected application-service termination")] + public static partial void RequestingHostStop(ILogger logger); + + [LoggerMessage( + EventId = 1208, + Level = LogLevel.Error, + Message = "Application shutdown completed with failure")] + public static partial void ShutdownCompletedWithFailure(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1209, + Level = LogLevel.Information, + Message = "Application logging initialized Application={Application} Provider={Provider} Category={Category} Environment={Environment} ContentRoot={ContentRoot}")] + public static partial void LoggingInitialized( + ILogger logger, + string Application, + string Provider, + string Category, + string Environment, + string ContentRoot); + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/ICompletedPathSurveyFileHandler.cs b/src/VectorNNTPD.TransitService/Logging/ICompletedPathSurveyFileHandler.cs new file mode 100644 index 00000000..4051f52e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/ICompletedPathSurveyFileHandler.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Receives a completed uncompressed Path-survey file before gzip and deletion. + /// + /// + /// Serilog's File sink invokes file-lifecycle delete before gzip-archiving a + /// rolled uncompressed file. The Path-survey archive chain calls this handler + /// first, then gzip-archives the same path, then returns so Serilog can delete + /// the uncompressed original. + /// Production registers NinpathsCompletedFileHandler, which opens the + /// file and enqueues it for background ninpaths processing. The handler must + /// return without parsing Path records. + /// + public interface ICompletedPathSurveyFileHandler + { + /// + /// Called with the still-uncompressed completed Path-survey file. + /// + /// Absolute or sink-supplied path of the rolled inpaths file. + /// + /// The file is readable and must not be deleted by the handler. Gzip and + /// Serilog deletion run after this method returns. Implementations should + /// not retain Path observations in memory beyond processing the file. + /// + void OnCompletedFile(string completedFilePath); + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/InnNewsTextFormatter.cs b/src/VectorNNTPD.TransitService/Logging/InnNewsTextFormatter.cs new file mode 100644 index 00000000..bde6d419 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/InnNewsTextFormatter.cs @@ -0,0 +1,107 @@ +using System.Text; +using Serilog.Events; +using Serilog.Formatting; +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Application-owned Serilog formatter for the INN news line. + /// + /// + /// The INN format is an application invariant. This type ignores the log + /// event message template and any configured outputTemplate. Operators + /// cannot change field order, timestamp representation, or delimiters through + /// appsettings. + /// + public sealed class InnNewsTextFormatter : ITextFormatter + { + /// + public void Format(LogEvent logEvent, TextWriter output) + { + ArgumentNullException.ThrowIfNull(logEvent); + ArgumentNullException.ThrowIfNull(output); + + var disposition = ReadDisposition(logEvent); + var messageId = ReadUtf8Property(logEvent, NntpdNewsLogging.MessageIdProperty); + var feed = ReadUtf8Property(logEvent, NntpdNewsLogging.FeedProperty); + var sites = ReadUtf8Property(logEvent, NntpdNewsLogging.SitesProperty); + var reason = ReadUtf8Property(logEvent, NntpdNewsLogging.ReasonProperty); + var evt = new NewsLogEvent( + disposition, + messageId, + feed, + sites, + logEvent.Timestamp, + ReadResponseCode(logEvent), + reason, + ReadSize(logEvent)); + var required = NewsLogLineFormatter.RequiredLength(in evt); + Span buffer = required <= 512 ? stackalloc byte[required] : new byte[required]; + var written = NewsLogLineFormatter.Write(buffer, in evt, logEvent.Timestamp); + output.Write(Encoding.ASCII.GetString(buffer[..written])); + } + + private static NewsLogDisposition ReadDisposition(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.DispositionProperty, out var value) + || value is not ScalarValue { Value: char ch }) + { + return NewsLogDisposition.Accepted; + } + + return ch switch + { + (char)NewsLogDisposition.Junk => NewsLogDisposition.Junk, + (char)NewsLogDisposition.Rejected => NewsLogDisposition.Rejected, + (char)NewsLogDisposition.Moderated => NewsLogDisposition.Moderated, + (char)NewsLogDisposition.Accepted => NewsLogDisposition.Accepted, + _ => NewsLogDisposition.Accepted, + }; + } + + private static int ReadResponseCode(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.ResponseCodeProperty, out var value) + || value is not ScalarValue scalar) + { + return 0; + } + + return scalar.Value switch + { + int code => code, + long code => (int)code, + _ => 0, + }; + } + + private static int ReadSize(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdNewsLogging.SizeProperty, out var value) + || value is not ScalarValue scalar) + { + return 0; + } + + return scalar.Value switch + { + int size => size, + long size => (int)size, + _ => 0, + }; + } + + private static ReadOnlyMemory ReadUtf8Property(LogEvent logEvent, string name) + { + if (!logEvent.Properties.TryGetValue(name, out var value) + || value is not ScalarValue { Value: string text } + || text.Length == 0) + { + return default; + } + + return Encoding.ASCII.GetBytes(text); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/InnPathSurveyTextFormatter.cs b/src/VectorNNTPD.TransitService/Logging/InnPathSurveyTextFormatter.cs new file mode 100644 index 00000000..695517b3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/InnPathSurveyTextFormatter.cs @@ -0,0 +1,52 @@ +using System.Text; +using Serilog.Events; +using Serilog.Formatting; +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Application-owned Serilog formatter for the INN-style Path-survey line. + /// + /// + /// The Path-survey format is an application invariant. This type ignores the + /// log event message template and any configured outputTemplate. + /// Operators cannot change the Path: prefix, field contents, or + /// terminator through appsettings. + /// + public sealed class InnPathSurveyTextFormatter : ITextFormatter + { + /// + public void Format(LogEvent logEvent, TextWriter output) + { + ArgumentNullException.ThrowIfNull(logEvent); + ArgumentNullException.ThrowIfNull(output); + + var path = ReadPath(logEvent); + var required = PathSurveyLineFormatter.RequiredLength(path.Span); + Span buffer = required <= 1024 ? stackalloc byte[required] : new byte[required]; + var written = PathSurveyLineFormatter.Write(buffer, path.Span); + output.Write(Encoding.Latin1.GetString(buffer[..written])); + } + + private static ReadOnlyMemory ReadPath(LogEvent logEvent) + { + if (!logEvent.Properties.TryGetValue(NntpdPathSurveyLogging.PathProperty, out var value) + || value is not ScalarValue scalar + || scalar.Value is null) + { + return default; + } + + switch (scalar.Value) + { + case string text: + return text.Length == 0 ? default : Encoding.Latin1.GetBytes(text); + case byte[] bytes: + return bytes; + default: + return default; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/NntpdConfiguredSinks.cs b/src/VectorNNTPD.TransitService/Logging/NntpdConfiguredSinks.cs new file mode 100644 index 00000000..ded820a9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NntpdConfiguredSinks.cs @@ -0,0 +1,421 @@ +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Configuration; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Registers the application Console, File, and Async File sinks with direct Serilog calls. + /// + /// + /// ReadFrom.Configuration discovers sink extension methods by reflection. Native AOT does + /// not retain that metadata, so a configured + /// Serilog:WriteTo section produced a logger with no sinks. Argument values still come from + /// configuration. Omitted arguments use the same defaults as the Serilog extension methods. + /// A sink this type cannot represent exactly stays on ReadFrom.Configuration. + /// + internal static class NntpdConfiguredSinks + { + /// + /// Default File template used by Serilog when outputTemplate is omitted. + /// + private const string DefaultFileTemplate = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}"; + + /// + /// Default Console template used by Serilog when outputTemplate is omitted. + /// + private const string DefaultConsoleTemplate = + "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}"; + + /// + /// Serilog File default when fileSizeLimitBytes is absent. An explicit JSON null stays unlimited. + /// + private const long DefaultFileSizeLimitBytes = 1L * 1024 * 1024 * 1024; + + private static readonly string[] ConsoleArgumentNames = ["restrictedToMinimumLevel", "outputTemplate"]; + + private static readonly string[] AsyncArgumentNames = ["bufferSize", "blockWhenFull", "configure"]; + + private static readonly string[] FileArgumentNames = + [ + "path", + "restrictedToMinimumLevel", + "outputTemplate", + "fileSizeLimitBytes", + "buffered", + "flushToDiskInterval", + "rollingInterval", + "rollOnFileSizeLimit", + "retainedFileCountLimit" + ]; + + /// + /// Adds each Serilog:WriteTo entry. An empty section keeps the direct console fallback. + /// + /// The host logger being built. + /// The application configuration after the File path has been resolved. + public static void Apply(LoggerConfiguration logger, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(configuration); + + var children = configuration.GetSection("Serilog:WriteTo").GetChildren().ToList(); + if (children.Count == 0) + { + logger.WriteTo.Console( + restrictedToMinimumLevel: NntpdFileLogging.ConsoleMinimumLevel, + outputTemplate: NntpdLoggingExtensions.ConsoleOutputTemplate); + return; + } + + // codeql[cs/linq/missed-where] TryApply predicate mutates logger; not a pure filter. + foreach (var sink in children) + { + if (!TryApply(logger, sink)) + ApplyRemainingSink(logger, configuration, sink); + } + } + + /// + /// Returns the Serilog sections that do not construct sinks, so + /// ReadFrom.Configuration can still apply minimum level and enrichment. + /// + /// The application configuration. + /// A configuration whose Serilog section has no WriteTo entries. + public static IConfiguration WithoutWriteTo(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + CopySection(configuration.GetSection("Serilog:MinimumLevel"), values); + CopySection(configuration.GetSection("Serilog:Properties"), values); + CopySection(configuration.GetSection("Serilog:LevelSwitches"), values); + CopySection(configuration.GetSection("Serilog:FilterSwitches"), values); + CopySection(configuration.GetSection("Serilog:Enrich"), values); + CopySection(configuration.GetSection("Serilog:Filter"), values); + CopySection(configuration.GetSection("Serilog:Destructure"), values); + CopySection(configuration.GetSection("Serilog:AuditTo"), values); + if (RequiresSinkAssemblies(configuration)) + CopySection(configuration.GetSection("Serilog:Using"), values); + + return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + } + + /// + /// Applies one sink by name. Returns false when the entry needs the configuration binder. + /// + private static bool TryApply(LoggerConfiguration logger, IConfigurationSection sink) + { + var name = sink["Name"]; + var args = sink.GetSection("Args"); + if (name is null || !HasOnlyArguments(args, name.Equals("Console", StringComparison.OrdinalIgnoreCase) + ? ConsoleArgumentNames + : name.Equals("File", StringComparison.OrdinalIgnoreCase) + ? FileArgumentNames + : name.Equals("Async", StringComparison.OrdinalIgnoreCase) + ? AsyncArgumentNames + : [])) + { + return false; + } + + if (name.Equals("Console", StringComparison.OrdinalIgnoreCase)) + { + logger.WriteTo.Console( + restrictedToMinimumLevel: ReadLevel(args, LogEventLevel.Verbose), + outputTemplate: ReadText(args, "outputTemplate", DefaultConsoleTemplate)); + return true; + } + + if (name.Equals("File", StringComparison.OrdinalIgnoreCase)) + { + WriteFile(logger.WriteTo, args); + return true; + } + + if (name.Equals("Async", StringComparison.OrdinalIgnoreCase) + && TryReadSingleFile(args, out var fileArgs)) + { + logger.WriteTo.Async( + wrapped => WriteFile(wrapped, fileArgs), + bufferSize: ReadInt(args, "bufferSize", 10_000), + blockWhenFull: ReadBool(args, "blockWhenFull", false)); + return true; + } + + return false; + } + + /// + /// Passes one unsupported sink through the configuration binder without the sinks already applied. + /// + private static void ApplyRemainingSink( + LoggerConfiguration logger, + IConfiguration configuration, + IConfigurationSection sink) + { + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + CopySection(configuration.GetSection("Serilog:Using"), values); + CopySection(sink, values, "Serilog:WriteTo:0"); + var remaining = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + logger.ReadFrom.Configuration(remaining); + } + + /// + /// Reads the single nested File sink inside an Async wrapper. + /// + private static bool TryReadSingleFile( + IConfigurationSection args, + [NotNullWhen(true)] out IConfigurationSection? fileArgs) + { + fileArgs = null; + var nested = args.GetSection("configure").GetChildren().ToList(); + if (nested.Count != 1 + || !string.Equals(nested[0]["Name"], "File", StringComparison.OrdinalIgnoreCase) + || !HasOnlyArguments(nested[0].GetSection("Args"), FileArgumentNames)) + { + return false; + } + + fileArgs = nested[0].GetSection("Args"); + return true; + } + + /// + /// Adds a File sink from the same argument names Serilog.Settings.Configuration binds. + /// + private static void WriteFile(LoggerSinkConfiguration sink, IConfigurationSection args) + { + var path = args["path"]; + if (string.IsNullOrWhiteSpace(path)) + throw new InvalidOperationException("The Serilog File sink requires Args:path."); + + sink.DailyGzipFile( + path, + restrictedToMinimumLevel: ReadLevel(args, LogEventLevel.Verbose), + outputTemplate: ReadText(args, "outputTemplate", DefaultFileTemplate), + fileSizeLimitBytes: ReadNullableLong(args, "fileSizeLimitBytes", DefaultFileSizeLimitBytes), + buffered: ReadBool(args, "buffered", false), + flushToDiskInterval: ReadTimeSpan(args, "flushToDiskInterval"), + rollingInterval: ReadRollingInterval(args), + rollOnFileSizeLimit: ReadBool(args, "rollOnFileSizeLimit", true), + retainedFileCountLimit: ReadNullableInt(args, "retainedFileCountLimit", 31)); + } + + /// + /// True when a copied section discovers methods and therefore needs Serilog:Using. + /// + private static bool RequiresSinkAssemblies(IConfiguration configuration) + { + return configuration.GetSection("Serilog:Enrich").GetChildren().Any() + || configuration.GetSection("Serilog:Filter").GetChildren().Any() + || configuration.GetSection("Serilog:Destructure").GetChildren().Any() + || configuration.GetSection("Serilog:AuditTo").GetChildren().Any(); + } + + /// + /// True when every argument name is one this type binds itself. + /// + private static bool HasOnlyArguments(IConfigurationSection args, string[] allowed) + { + if (allowed.Length == 0 && args.GetChildren().Any()) + return false; + + foreach (var child in args.GetChildren()) + { + if (!allowed.Contains(child.Key, StringComparer.OrdinalIgnoreCase)) + return false; + } + + return true; + } + + /// + /// Copies a configuration section, optionally placing it at a different key. + /// + private static void CopySection( + IConfigurationSection section, + Dictionary values, + string? destinationPath = null) + { + if (!section.Exists()) + return; + + var path = destinationPath ?? section.Path; + if (section.Value is not null || !section.GetChildren().Any()) + values[path] = section.Value; + + foreach (var child in section.GetChildren()) + CopySection(child, values, path + ":" + child.Key); + } + + /// + /// Reads a minimum level, using when the argument is absent. + /// + private static LogEventLevel ReadLevel(IConfigurationSection args, LogEventLevel fallback) + { + var text = args["restrictedToMinimumLevel"]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!Enum.TryParse(text, ignoreCase: true, out LogEventLevel level)) + { + throw new InvalidOperationException( + $"Serilog level '{text}' at '{args.Path}:restrictedToMinimumLevel' is not a log event level."); + } + + return level; + } + + /// + /// Reads text, using when the argument is absent. + /// + private static string ReadText(IConfigurationSection args, string name, string fallback) + { + var text = args[name]; + return string.IsNullOrWhiteSpace(text) ? fallback : text; + } + + /// + /// Reads a boolean, using when the argument is absent. + /// + private static bool ReadBool(IConfigurationSection args, string name, bool fallback) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!bool.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not a boolean."); + } + + return value; + } + + /// + /// Reads an integer, using when the argument is absent. + /// + private static int ReadInt(IConfigurationSection args, string name, int fallback) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return fallback; + + if (!int.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not an integer."); + } + + return value; + } + + /// + /// Reads a nullable long. A missing key uses ; an explicit null is unlimited. + /// + /// + /// is false for a JSON null, + /// so presence is taken from . + /// + private static long? ReadNullableLong(IConfigurationSection args, string name, long missing) + { + if (!TryGetArgument(args, name, out var argument)) + return missing; + + if (string.IsNullOrWhiteSpace(argument.Value)) + return null; + + if (!long.TryParse(argument.Value, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{argument.Value}' at '{argument.Path}' is not an integer."); + } + + return value; + } + + /// + /// Reads a nullable integer. A missing key uses ; an explicit null stays null. + /// + private static int? ReadNullableInt(IConfigurationSection args, string name, int missing) + { + if (!TryGetArgument(args, name, out var argument)) + return missing; + + if (string.IsNullOrWhiteSpace(argument.Value)) + return null; + + if (!int.TryParse(argument.Value, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{argument.Value}' at '{argument.Path}' is not an integer."); + } + + return value; + } + + /// + /// Finds an argument that is present even when its value is JSON null. + /// + private static bool TryGetArgument( + IConfigurationSection args, + string name, + [NotNullWhen(true)] out IConfigurationSection? argument) + { + foreach (var child in args.GetChildren()) + { + if (child.Key.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + argument = child; + return true; + } + } + + argument = null; + return false; + } + + /// + /// Reads a flush interval. A missing key leaves flushing to the File sink default. + /// + private static TimeSpan? ReadTimeSpan(IConfigurationSection args, string name) + { + var text = args[name]; + if (string.IsNullOrWhiteSpace(text)) + return null; + + if (!TimeSpan.TryParse(text, out var value)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:{name}' is not a time span."); + } + + return value; + } + + /// + /// Reads the rolling interval. A missing key uses . + /// + private static RollingInterval ReadRollingInterval(IConfigurationSection args) + { + var text = args["rollingInterval"]; + if (string.IsNullOrWhiteSpace(text)) + return RollingInterval.Infinite; + + if (!Enum.TryParse(text, ignoreCase: true, out RollingInterval interval)) + { + throw new InvalidOperationException( + $"Serilog value '{text}' at '{args.Path}:rollingInterval' is not a rolling interval."); + } + + return interval; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/NntpdFileLogging.cs b/src/VectorNNTPD.TransitService/Logging/NntpdFileLogging.cs new file mode 100644 index 00000000..34e05644 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NntpdFileLogging.cs @@ -0,0 +1,130 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Resolves into the Serilog File path before the File sink + /// is constructed. Operational File/Async settings live in VectorNNTPD.TransitService.json. + /// + /// + /// Serilog.Settings.Configuration cannot expand Nntpd:LogDir into path. + /// This type creates the directory and overwrites the File sink path so the JSON + /// placeholder is never the runtime directory. It does not add a second File sink. + /// + internal static class NntpdFileLogging + { + /// Serilog rolling path token: {ApplicationName}-yyyyMMdd.log. + public const string RollingPathSuffix = "-.log"; + + /// + /// Suffix appended when a completed log is gzip-compressed: {original-file-name}.gz. + /// + public const string GzipArchiveSuffix = ".gz"; + + /// Console-sink fallback minimum when Serilog:WriteTo is omitted. + public const Serilog.Events.LogEventLevel ConsoleMinimumLevel = Serilog.Events.LogEventLevel.Information; + + /// + /// Resolves through Common + /// against + /// (default + /// ). Absolute paths stay absolute. + /// + public static string ResolveDirectory(string? logDir, string? applicationBaseDirectory = null) + { + var configured = string.IsNullOrWhiteSpace(logDir) + ? NntpdOptions.DefaultLogDir + : logDir.Trim(); + return ApplicationLocalPath.ResolveApplicationLocalPath( + configured, + applicationBaseDirectory ?? AppContext.BaseDirectory); + } + + /// + /// Builds the Serilog rolling path {logDirectory}/{applicationName}-.log. + /// + public static string RollingFilePath(string logDirectory, string applicationName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + ArgumentException.ThrowIfNullOrWhiteSpace(applicationName); + return Path.Join(logDirectory, applicationName.Trim() + RollingPathSuffix); + } + + /// + /// Archive file name produced when a completed log is gzip-compressed. + /// + public static string GzipArchiveFileName(string rolledLogPath) => + Path.GetFileName(rolledLogPath) + GzipArchiveSuffix; + + /// + /// Creates and binds the resolved File path over the JSON placeholder. + /// + public static void BindResolvedFilePath( + ConfigurationManager configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + + var logDir = ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + + var path = RollingFilePath(logDir, ApplicationJsonConfiguration.EntryAssemblyName); + var pathKey = FindConfiguredFilePathKey(configuration); + if (pathKey is null) + { + return; + } + + configuration.AddInMemoryCollection(new Dictionary { [pathKey] = path }); + NntpdNewsLogging.BindResolvedNewsPath(configuration, applicationBaseDirectory); + NntpdPathSurveyLogging.BindResolvedPathSurveyPath(configuration, applicationBaseDirectory); + } + + private static string? FindConfiguredFilePathKey(IConfiguration configuration) + { + foreach (var writeTo in configuration.GetSection("Serilog:WriteTo").GetChildren()) + { + if (!string.Equals(writeTo["Name"], "Async", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + foreach (var inner in writeTo.GetSection("Args:configure").GetChildren()) + { + if (string.Equals(inner["Name"], "File", StringComparison.OrdinalIgnoreCase)) + { + return $"{inner.Path}:Args:path"; + } + } + } + + return null; + } + + /// + /// In-memory Serilog File/Async keys matching production VectorNNTPD.TransitService.json (path is a placeholder). + /// + public static Dictionary AsyncFileWriteToKeys(int writeToIndex = 1) => new() + { + [$"Serilog:WriteTo:{writeToIndex}:Name"] = "Async", + [$"Serilog:WriteTo:{writeToIndex}:Args:bufferSize"] = "50000", + [$"Serilog:WriteTo:{writeToIndex}:Args:blockWhenFull"] = "true", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Name"] = "File", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:path"] = + "logs/" + ApplicationJsonConfiguration.EntryAssemblyName + "-.log", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:restrictedToMinimumLevel"] = "Debug", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:outputTemplate"] = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:fileSizeLimitBytes"] = null, + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:buffered"] = "true", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:flushToDiskInterval"] = "00:00:01", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:rollingInterval"] = "Day", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:rollOnFileSizeLimit"] = "false", + [$"Serilog:WriteTo:{writeToIndex}:Args:configure:0:Args:retainedFileCountLimit"] = "14", + }; + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/NntpdLoggingExtensions.cs b/src/VectorNNTPD.TransitService/Logging/NntpdLoggingExtensions.cs new file mode 100644 index 00000000..97173b0d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NntpdLoggingExtensions.cs @@ -0,0 +1,157 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using Serilog; +using Serilog.Events; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Configures Serilog as the exclusive logging implementation for VectorNNTPD.TransitService. + /// + public static class NntpdLoggingExtensions + { + /// + /// Single-line console template suitable for interactive terminals and journald collection. + /// + public const string ConsoleOutputTemplate = + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}"; + + /// + /// Creates an early bootstrap logger used before the Generic Host is built. + /// + /// A bootstrap logger assigned to . + public static Serilog.ILogger CreateBootstrapLogger() + { + return new LoggerConfiguration() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", ApplicationJsonConfiguration.EntryAssemblyName) + .WriteTo.Console(outputTemplate: ConsoleOutputTemplate) + .CreateBootstrapLogger(); + } + + /// + /// Makes flush every write. + /// + /// + /// When stdout is a console, the runtime already flushes often enough that Information + /// lines appear immediately. When stdout is a pipe (IDE, redirected capture), + /// is block-buffered: a startup burst fills the buffer and + /// becomes visible, then later Information events sit unseen until the buffer fills + /// again or the process exits. Serilog's Console sink writes to + /// and does not Flush. This must run before the first Serilog Console sink is created. + /// + public static void UseAutoFlushConsoleOutput() + { + // codeql[cs/local-not-disposed] Owning Console.Out; disposing would close process stdout. + var writer = new StreamWriter(Console.OpenStandardOutput(), Console.OutputEncoding) + { + AutoFlush = true, + }; + Console.SetOut(writer); + } + + /// + /// Removes Microsoft default logging providers and registers Serilog as the sole provider. + /// + /// The host application builder. + /// + /// Optional additional Serilog configuration applied after reading appsettings. + /// Used by tests to attach in-memory sinks without reintroducing MEL providers. + /// + /// The same instance. + /// + /// + /// Call order: clear MEL providers, remove default registrations, + /// then AddSerilog with writeToProviders: false so framework and application + /// traffic flows only through Serilog sinks. + /// + /// + /// Console formatting is owned by Serilog. journald collects Serilog console stdout under + /// systemd. Microsoft console formatter options that AddSystemd() may register are + /// stripped by ConfigureNntpdPlatformHosting and are not part of this logging pipeline. + /// + /// + /// Console, File, and Async File sinks are registered with direct Serilog calls from the + /// Serilog:WriteTo argument values. ReadFrom.Configuration still applies + /// minimum level, enrichment, and any sink entry those direct calls do not represent. + /// Native AOT does not retain the reflection metadata sink discovery requires. + /// + /// + public static HostApplicationBuilder ConfigureNntpdLogging( + this HostApplicationBuilder builder, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Logging.ClearProviders(); + + // Host.CreateApplicationBuilder registers a default LoggerFactory; remove it so + // SerilogLoggerFactory is the only ILoggerFactory in the container. + builder.Services.RemoveAll(); + builder.Services.RemoveAll(); + + NntpdFileLogging.BindResolvedFilePath(builder.Configuration); + + builder.Services.AddSerilog( + (services, loggerConfiguration) => + { + loggerConfiguration + .ReadFrom.Configuration(NntpdConfiguredSinks.WithoutWriteTo(builder.Configuration)) + .ReadFrom.Services(services) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", ApplicationJsonConfiguration.EntryAssemblyName); + + NntpdConfiguredSinks.Apply(loggerConfiguration, builder.Configuration); + configure?.Invoke(loggerConfiguration); + }, + preserveStaticLogger: false, + writeToProviders: false); + + return builder; + } + + /// + /// Emits one Information event through the host after + /// Serilog has replaced the bootstrap logger. + /// + /// The built host service provider. + /// Host environment name (no secrets). + /// Resolved content root used for VectorNNTPD.TransitService.json. + public static void WriteLoggingInitialized( + IServiceProvider services, + string environmentName, + string contentRootPath) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(environmentName); + ArgumentException.ThrowIfNullOrWhiteSpace(contentRootPath); + + var factory = services.GetRequiredService(); + var logger = factory.CreateLogger(NntpdLogCategories.Hosting); + HostingLogMessages.LoggingInitialized( + logger, + ApplicationJsonConfiguration.EntryAssemblyName, + factory.GetType().Name, + NntpdLogCategories.Hosting, + environmentName, + contentRootPath); + } + + /// + /// Returns registered implementations for diagnostics and tests. + /// + /// + /// When Serilog is configured via AddSerilog, logging is provided by replacing + /// with SerilogLoggerFactory. In that mode the provider + /// list is typically empty, which confirms Microsoft default providers were not retained. + /// + public static IReadOnlyList GetLoggerProviders(this IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetServices().ToArray(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/NntpdNewsLogging.cs b/src/VectorNNTPD.TransitService/Logging/NntpdNewsLogging.cs new file mode 100644 index 00000000..847ddb50 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NntpdNewsLogging.cs @@ -0,0 +1,180 @@ +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Dedicated Serilog pipeline for the INN news file. + /// + /// + /// + /// Operational File/Async settings bind from Serilog:News using the same + /// argument names as the application File sink (path, rolling, retention, + /// buffering, flushToDiskInterval). The INN line format is not read from configuration; the + /// sink is always constructed with . + /// + /// + /// This small bridge exists because Serilog.Settings.Configuration cannot attach + /// a code-owned formatter without also exposing a user-editable + /// outputTemplate or formatter string. Path resolution still uses + /// via . + /// + /// + public static class NntpdNewsLogging + { + /// Serilog source context for news events. Not used by the application File/Console sinks. + public const string SourceContext = "VectorNNTPD.TransitService.News"; + + /// Configuration section that holds news File/Async operational arguments. + public const string SectionName = "Serilog:News"; + + /// Canonical rolling path token: news-yyyyMMdd.log. + public const string RollingFileName = "news"; + + /// Log event property for the INN disposition character. + public const string DispositionProperty = "NewsDisposition"; + + /// Log event property for the INN feed token. + public const string FeedProperty = "NewsFeed"; + + /// Log event property for the article Message-ID. + public const string MessageIdProperty = "NewsMessageId"; + + /// Log event property for optional future SITE tokens. + public const string SitesProperty = "NewsSites"; + + /// Log event property for the INN article size in bytes. + public const string SizeProperty = "NewsSize"; + + /// Log event property for the NNTP rejection response code. + public const string ResponseCodeProperty = "NewsResponseCode"; + + /// Log event property for the already-decided rejection reason. + public const string ReasonProperty = "NewsReason"; + + /// + /// Overwrites Serilog:News:path from + /// so the JSON placeholder is never the runtime path. + /// + public static void BindResolvedNewsPath( + IConfiguration configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration[SectionName + ":path"] is null) + { + return; + } + + var logDir = NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + var path = NewsRollingFilePath(logDir); + if (configuration is IConfigurationBuilder builder) + { + builder.AddInMemoryCollection(new Dictionary { [SectionName + ":path"] = path }); + } + } + + /// Builds {logDirectory}/news-.log for Serilog daily rolling. + public static string NewsRollingFilePath(string logDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + return Path.Join(logDirectory, RollingFileName + NntpdFileLogging.RollingPathSuffix); + } + + /// + /// Reads File/Async operational settings from Serilog:News. + /// The INN formatter is never taken from configuration. + /// + public static NewsFileSinkSettings ReadSettings(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var news = configuration.GetSection(SectionName); + var path = news["path"]; + if (string.IsNullOrWhiteSpace(path)) + { + path = NewsRollingFilePath( + NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"])); + } + + return new NewsFileSinkSettings( + path, + ParseRolling(news["rollingInterval"]), + news.GetValue("retainedFileCountLimit") ?? 1, + news.GetValue("buffered", true), + news.GetValue("rollOnFileSizeLimit", false), + news.GetValue("fileSizeLimitBytes"), + news.GetValue("bufferSize", 50000), + news.GetValue("blockWhenFull", true), + news.GetValue("flushToDiskInterval")); + } + + /// + /// Creates a dedicated Serilog logger that writes only INN news lines. + /// + public static Serilog.Core.Logger CreateLogger(IConfiguration configuration) + { + var settings = ReadSettings(configuration); + Directory.CreateDirectory( + Path.GetDirectoryName(Path.GetFullPath(settings.Path)) ?? NntpdOptions.DefaultLogDir); + + return new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.Async( + a => a.DailyGzipFile( + new InnNewsTextFormatter(), + settings.Path, + restrictedToMinimumLevel: LogEventLevel.Information, + fileSizeLimitBytes: settings.FileSizeLimitBytes, + buffered: settings.Buffered, + flushToDiskInterval: settings.FlushToDiskInterval, + rollingInterval: settings.RollingInterval, + rollOnFileSizeLimit: settings.RollOnFileSizeLimit, + retainedFileCountLimit: settings.RetainedFileCountLimit), + bufferSize: settings.BufferSize, + blockWhenFull: settings.BlockWhenFull) + .CreateLogger(); + } + + private static RollingInterval ParseRolling(string? value) => + Enum.TryParse(value, ignoreCase: true, out var interval) + ? interval + : RollingInterval.Day; + } + + /// + /// Operational File/Async settings for the dedicated news sink. + /// The INN line format is not represented here. + /// + /// Resolved news file path, including Serilog rolling token. + /// Serilog File rolling interval. + /// + /// Daily files to keep, including the active file. .log and .log.gz for the same day count as one. + /// + /// Whether the File sink buffers writes. + /// Whether size-based rolling is enabled. + /// Serilog File size cap; is unlimited. + /// Serilog.Sinks.Async buffer size. + /// Whether Async blocks when the buffer is full. + /// + /// Serilog File periodic flush. leaves buffering until dispose. + /// Production Serilog:News sets one second. + /// + public readonly record struct NewsFileSinkSettings( + string Path, + RollingInterval RollingInterval, + int RetainedFileCountLimit, + bool Buffered, + bool RollOnFileSizeLimit, + long? FileSizeLimitBytes, + int BufferSize, + bool BlockWhenFull, + TimeSpan? FlushToDiskInterval); +} diff --git a/src/VectorNNTPD.TransitService/Logging/NntpdPathSurveyLogging.cs b/src/VectorNNTPD.TransitService/Logging/NntpdPathSurveyLogging.cs new file mode 100644 index 00000000..2da9ee52 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NntpdPathSurveyLogging.cs @@ -0,0 +1,197 @@ +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Events; +using Serilog.Sinks.File; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Dedicated Serilog pipeline for the INN-style Path-survey (inpaths) file. + /// + /// + /// + /// Operational File/Async settings bind from Serilog:Inpaths using the same + /// argument names as the application File sink (path, rolling, retention, + /// buffering, flushToDiskInterval). The Path-survey line format is not read from configuration; + /// the sink is always constructed with . + /// + /// + /// This small bridge exists because Serilog.Settings.Configuration cannot attach + /// a code-owned formatter without also exposing a user-editable + /// outputTemplate or formatter string. Path resolution still uses + /// via . + /// + /// + /// Production is 1, + /// so only the active day is kept. When the day rolls, the completed file is handed + /// to and then gzip-compressed. + /// Retention then deletes that compressed day because it is outside the one-file limit. + /// The handler opens the file with share-read/write/delete before compression, so it + /// can keep reading after the uncompressed file is replaced. Handler failure is logged + /// and does not skip gzip. + /// + /// + public static class NntpdPathSurveyLogging + { + /// Serilog source context for Path-survey events. Not used by the application File/Console sinks. + public const string SourceContext = "VectorNNTPD.TransitService.Inpaths"; + + /// Configuration section that holds Path-survey File/Async operational arguments. + public const string SectionName = "Serilog:Inpaths"; + + /// Canonical rolling path token: inpaths-yyyyMMdd.log. + public const string RollingFileName = "inpaths"; + + /// Log event property for the canonical Path value. + public const string PathProperty = "PathSurveyPath"; + + /// + /// Overwrites Serilog:Inpaths:path from + /// so the JSON placeholder is never the runtime path. + /// + public static void BindResolvedPathSurveyPath( + IConfiguration configuration, + string? applicationBaseDirectory = null) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration[SectionName + ":path"] is null) + { + return; + } + + var logDir = NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"], + applicationBaseDirectory); + Directory.CreateDirectory(logDir); + var path = PathSurveyRollingFilePath(logDir); + if (configuration is IConfigurationBuilder builder) + { + builder.AddInMemoryCollection(new Dictionary { [SectionName + ":path"] = path }); + } + } + + /// Builds {logDirectory}/inpaths-.log for Serilog daily rolling. + public static string PathSurveyRollingFilePath(string logDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logDirectory); + return Path.Join(logDirectory, RollingFileName + NntpdFileLogging.RollingPathSuffix); + } + + /// + /// Reads File/Async operational settings from Serilog:Inpaths. + /// The Path-survey formatter is never taken from configuration. + /// + public static PathSurveyFileSinkSettings ReadSettings(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + var inpaths = configuration.GetSection(SectionName); + var path = inpaths["path"]; + if (string.IsNullOrWhiteSpace(path)) + { + path = PathSurveyRollingFilePath( + NntpdFileLogging.ResolveDirectory( + configuration[$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"])); + } + + return new PathSurveyFileSinkSettings( + path, + ParseRolling(inpaths["rollingInterval"]), + inpaths.GetValue("retainedFileCountLimit") ?? 1, + inpaths.GetValue("buffered", true), + inpaths.GetValue("rollOnFileSizeLimit", false), + inpaths.GetValue("fileSizeLimitBytes"), + inpaths.GetValue("bufferSize", 50000), + inpaths.GetValue("blockWhenFull", true), + inpaths.GetValue("flushToDiskInterval")); + } + + /// + /// Creates a dedicated Serilog logger that writes only Path-survey lines. + /// + /// Application configuration containing Serilog:Inpaths. + /// + /// Invoked with the uncompressed rolled file before gzip. Production + /// enqueues the file for background ninpaths processing. + /// + /// Logger for completed-file handler failures; must not be the Path-survey logger. + public static Serilog.Core.Logger CreateLogger( + IConfiguration configuration, + ICompletedPathSurveyFileHandler completedFileHandler, + Microsoft.Extensions.Logging.ILogger hookLogger) + { + ArgumentNullException.ThrowIfNull(completedFileHandler); + ArgumentNullException.ThrowIfNull(hookLogger); + var settings = ReadSettings(configuration); + Directory.CreateDirectory( + Path.GetDirectoryName(Path.GetFullPath(settings.Path)) ?? NntpdOptions.DefaultLogDir); + + return new LoggerConfiguration() + .MinimumLevel.Information() + .Enrich.WithProperty(Serilog.Core.Constants.SourceContextPropertyName, SourceContext) + .Filter.ByIncludingOnly(static e => e.Properties.ContainsKey(PathProperty)) + .WriteTo.Async( + a => a.DailyGzipFile( + new InnPathSurveyTextFormatter(), + settings.Path, + restrictedToMinimumLevel: LogEventLevel.Information, + fileSizeLimitBytes: settings.FileSizeLimitBytes, + buffered: settings.Buffered, + flushToDiskInterval: settings.FlushToDiskInterval, + rollingInterval: settings.RollingInterval, + rollOnFileSizeLimit: settings.RollOnFileSizeLimit, + retainedFileCountLimit: settings.RetainedFileCountLimit, + onCompletedFile: path => + { + try + { + completedFileHandler.OnCompletedFile(path); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PathSurveyLogMessages.CompletedFileHandlerFailed(hookLogger, ex, path); + } + }), + bufferSize: settings.BufferSize, + blockWhenFull: settings.BlockWhenFull) + .CreateLogger(); + } + + private static RollingInterval ParseRolling(string? value) => + Enum.TryParse(value, ignoreCase: true, out var interval) + ? interval + : RollingInterval.Day; + } + + /// + /// Operational File/Async settings for the dedicated Path-survey sink. + /// The Path-survey line format is not represented here. + /// + /// Resolved Path-survey file path, including Serilog rolling token. + /// Serilog File rolling interval. + /// + /// Daily files to keep, including the active file. .log and .log.gz for the same day count as one. + /// Production uses 1 so a completed day is removed after the ninpaths handoff and gzip. + /// + /// Whether the File sink buffers writes. + /// Whether size-based rolling is enabled. + /// Serilog File size cap; is unlimited. + /// Serilog.Sinks.Async buffer size. + /// Whether Async blocks when the buffer is full. + /// + /// Serilog File periodic flush. leaves buffering until dispose. + /// Production Serilog:Inpaths sets one second. + /// + public readonly record struct PathSurveyFileSinkSettings( + string Path, + RollingInterval RollingInterval, + int RetainedFileCountLimit, + bool Buffered, + bool RollOnFileSizeLimit, + long? FileSizeLimitBytes, + int BufferSize, + bool BlockWhenFull, + TimeSpan? FlushToDiskInterval); +} diff --git a/src/VectorNNTPD.TransitService/Logging/NullCompletedPathSurveyFileHandler.cs b/src/VectorNNTPD.TransitService/Logging/NullCompletedPathSurveyFileHandler.cs new file mode 100644 index 00000000..2943d94e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/NullCompletedPathSurveyFileHandler.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// No-op completed-file handler. Path-survey files are still gzip-archived. + /// Production DI registers NinpathsCompletedFileHandler instead; this + /// type remains for tests that opt out of ninpaths mail. + /// + public sealed class NullCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler + { + /// Gets the shared no-op instance. + public static NullCompletedPathSurveyFileHandler Instance { get; } = new(); + + /// + public void OnCompletedFile(string completedFilePath) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/PathSurveyLogMessages.cs b/src/VectorNNTPD.TransitService/Logging/PathSurveyLogMessages.cs new file mode 100644 index 00000000..5c3c3b4c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/PathSurveyLogMessages.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.TransitService.Logging +{ + /// Source-generated Path-survey lifecycle log messages. + internal static partial class PathSurveyLogMessages + { + [LoggerMessage( + EventId = 2010, + Level = LogLevel.Error, + Message = "Path-survey completed-file handler failed for {CompletedFile}; gzip archive will still run")] + public static partial void CompletedFileHandlerFailed(ILogger logger, Exception exception, string CompletedFile); + } +} diff --git a/src/VectorNNTPD.TransitService/Logging/SystemdLogMessages.cs b/src/VectorNNTPD.TransitService/Logging/SystemdLogMessages.cs new file mode 100644 index 00000000..800f218c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Logging/SystemdLogMessages.cs @@ -0,0 +1,134 @@ +namespace VectorNNTPD.TransitService.Logging +{ + /// + /// Source-generated structured log messages for systemd notify and watchdog events. + /// + internal static partial class SystemdLogMessages + { + [LoggerMessage( + EventId = 1300, + Level = LogLevel.Information, + Message = "systemd lifecycle notifications activated (notifyEnabled={NotifyEnabled}, isSystemdService={IsSystemdService}")] + public static partial void LifecycleNotificationsActivated( + ILogger logger, + bool NotifyEnabled, + bool IsSystemdService); + + [LoggerMessage( + EventId = 1301, + Level = LogLevel.Information, + Message = "Reported systemd readiness after successful application initialization")] + public static partial void ReadinessReported(ILogger logger); + + [LoggerMessage( + EventId = 1302, + Level = LogLevel.Information, + Message = "Reported systemd STOPPING notification for graceful shutdown")] + public static partial void StoppingReported(ILogger logger); + + [LoggerMessage( + EventId = 1303, + Level = LogLevel.Information, + Message = "Reported systemd status: {Status}")] + public static partial void StatusReported(ILogger logger, string Status); + + [LoggerMessage( + EventId = 1304, + Level = LogLevel.Debug, + Message = "Sent systemd notification {Notification}")] + public static partial void NotificationSent(ILogger logger, string Notification); + + [LoggerMessage( + EventId = 1305, + Level = LogLevel.Error, + Message = "Failed to send systemd notification {Notification}")] + public static partial void NotificationFailed(ILogger logger, Exception exception, string Notification); + + [LoggerMessage( + EventId = 1306, + Level = LogLevel.Information, + Message = "systemd runtime detection: isSystemdService={IsSystemdService}, notifyEnabled={NotifyEnabled}, watchdogConfigured={WatchdogConfigured}, watchdogTimeout={WatchdogTimeout}, detail={Detail}")] + public static partial void RuntimeDetection( + ILogger logger, + bool IsSystemdService, + bool NotifyEnabled, + bool WatchdogConfigured, + TimeSpan? WatchdogTimeout, + string Detail); + + [LoggerMessage( + EventId = 1307, + Level = LogLevel.Warning, + Message = "Ignoring malformed systemd watchdog configuration (WATCHDOG_USEC is not a positive integer)")] + public static partial void MalformedWatchdogUsec(ILogger logger); + + [LoggerMessage( + EventId = 1308, + Level = LogLevel.Warning, + Message = "Ignoring systemd watchdog configuration because WATCHDOG_PID is malformed")] + public static partial void MalformedWatchdogPid(ILogger logger); + + [LoggerMessage( + EventId = 1309, + Level = LogLevel.Information, + Message = "systemd watchdog is configured for a different PID; watchdog keep-alives will remain disabled")] + public static partial void WatchdogConfiguredForDifferentPid(ILogger logger); + + [LoggerMessage( + EventId = 1310, + Level = LogLevel.Warning, + Message = "Ignoring systemd watchdog configuration because WATCHDOG_USEC is out of range")] + public static partial void WatchdogUsecOutOfRange(ILogger logger); + + [LoggerMessage( + EventId = 1311, + Level = LogLevel.Information, + Message = "systemd watchdog remain disabled ({Reason})")] + public static partial void WatchdogRemainDisabled(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 1312, + Level = LogLevel.Information, + Message = "systemd watchdog activated. Deadline={WatchdogTimeout}, heartbeatInterval={HeartbeatInterval}")] + public static partial void WatchdogActivated( + ILogger logger, + TimeSpan WatchdogTimeout, + TimeSpan HeartbeatInterval); + + [LoggerMessage( + EventId = 1313, + Level = LogLevel.Debug, + Message = "Skipping systemd watchdog keep-alive because the application is unhealthy")] + public static partial void WatchdogKeepAliveSkippedUnhealthy(ILogger logger); + + [LoggerMessage( + EventId = 1314, + Level = LogLevel.Trace, + Message = "Sent systemd watchdog keep-alive")] + public static partial void WatchdogKeepAliveSent(ILogger logger); + + [LoggerMessage( + EventId = 1315, + Level = LogLevel.Error, + Message = "systemd watchdog notification failed")] + public static partial void WatchdogNotificationFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1316, + Level = LogLevel.Information, + Message = "systemd watchdog deactivated due to shutdown")] + public static partial void WatchdogDeactivated(ILogger logger); + + [LoggerMessage( + EventId = 1317, + Level = LogLevel.Critical, + Message = "systemd watchdog loop failed unexpectedly; requesting host stop")] + public static partial void WatchdogLoopFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 1318, + Level = LogLevel.Information, + Message = "systemd watchdog stopping")] + public static partial void WatchdogStopping(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/EmailModerationSubmissionService.cs b/src/VectorNNTPD.TransitService/Moderation/EmailModerationSubmissionService.cs new file mode 100644 index 00000000..d52988ae --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/EmailModerationSubmissionService.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Forwards a proto-article to the moderator by enqueueing outbound email. + /// + /// + /// + /// This type knows email messages, not SMTP. + /// acceptance means the complete message was durably written to the local + /// filesystem spool. It is not proof that a remote SMTP server or moderator + /// mailbox received the message. SMTP delivery is asynchronous. + /// + /// + /// POST maps to 240 and + /// other statuses to 441. + /// + /// + public sealed class EmailModerationSubmissionService : IModerationSubmissionService + { + private readonly IEmailService _email; + private readonly ModerationEmailComposer _composer; + private readonly IOptions _emailOptions; + + /// Initializes a new email-backed submission service. + public EmailModerationSubmissionService( + IEmailService email, + ModerationEmailComposer composer, + IOptions emailOptions) + { + ArgumentNullException.ThrowIfNull(email); + ArgumentNullException.ThrowIfNull(composer); + ArgumentNullException.ThrowIfNull(emailOptions); + _email = email; + _composer = composer; + _emailOptions = emailOptions; + } + + /// + public async ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + if (!_emailOptions.Value.Enabled) + { + return new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + "email subsystem is disabled"); + } + + EmailMessage message; + try + { + message = _composer.Compose(submission); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + "moderation email could not be constructed"); + } + + var enqueue = await _email.SendAsync(message, cancellationToken).ConfigureAwait(false); + return enqueue.Status switch + { + EmailEnqueueStatus.Accepted => new ModerationSubmissionResult( + ModerationSubmissionStatus.Accepted, + "durably accepted into the local email spool"), + EmailEnqueueStatus.Disabled or EmailEnqueueStatus.Unavailable => new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + enqueue.Detail), + EmailEnqueueStatus.Rejected => new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + enqueue.Detail), + _ => new ModerationSubmissionResult( + ModerationSubmissionStatus.Failed, + enqueue.Detail), + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/EmptyModeratorAuthorization.cs b/src/VectorNNTPD.TransitService/Moderation/EmptyModeratorAuthorization.cs new file mode 100644 index 00000000..6af96301 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/EmptyModeratorAuthorization.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Moderator authorization with no configured routes. Every resolve and approval fails. + /// + public sealed class EmptyModeratorAuthorization : IModeratorAuthorization + { + /// Shared instance. + public static EmptyModeratorAuthorization Instance { get; } = new(); + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) + { + _ = authenticatedUsername; + return false; + } + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) + { + _ = newsgroup; + identity = default; + return false; + } + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) + { + _ = authenticatedUsername; + _ = approvedIdentity; + _ = newsgroup; + return false; + } + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) + { + _ = authenticatedUsername; + _ = approvedIdentities; + failureDetail = moderatedGroups.Length == 0 + ? "no moderated groups" + : "unresolved moderator"; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/IModerationSubmissionService.cs b/src/VectorNNTPD.TransitService/Moderation/IModerationSubmissionService.cs new file mode 100644 index 00000000..4b62d238 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/IModerationSubmissionService.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Durable boundary that accepts a proto-article for moderator forwarding. + /// + /// + /// POST policy calls this abstraction; it does not send email or speak SMTP. + /// The production implementation composes a moderator email and calls + /// IEmailService.SendAsync. Queue acceptance is not remote SMTP delivery. + /// + public interface IModerationSubmissionService + { + /// + /// Offers for moderator delivery. + /// + /// Proto-article and resolved moderator route. + /// Cancels the attempt. Must not be ignored by implementations that wait. + /// Accepted, unavailable, or failed. Does not inject the article. + ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/IModeratorAuthorization.cs b/src/VectorNNTPD.TransitService/Moderation/IModeratorAuthorization.cs new file mode 100644 index 00000000..3c25e7a5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/IModeratorAuthorization.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Authorizes moderated-newsgroup approval from the authenticated NNTP principal, + /// the configured moderator route, and the claimed Approved: identity. + /// + /// + /// An Approved: header is an assertion. It is trusted only when the + /// authenticated AUTHINFO principal is configured as the moderator for every + /// moderated target. Presence of Approved: alone is never sufficient. + /// This contract is not Control:PgpAuthorities. + /// + public interface IModeratorAuthorization + { + /// + /// Returns whether is configured as a + /// moderator for at least one mapping. This is not group authorization. + /// + bool IsAuthenticatedModerator(string? authenticatedUsername); + + /// + /// Resolves the first-match moderator route for . + /// + bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity); + + /// + /// Returns whether the authenticated principal may approve + /// using . + /// + bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup); + + /// + /// Authorizes every moderated target against the collected Approved: identities. + /// + /// AUTHINFO principal, or when unauthenticated. + /// Distinct mailbox identities parsed from Approved:. + /// Moderated Newsgroups: targets in header order. + /// Internal reject detail when authorization fails. + /// only when every moderated target is authorized. + bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/IModeratorCatalogue.cs b/src/VectorNNTPD.TransitService/Moderation/IModeratorCatalogue.cs new file mode 100644 index 00000000..6ae39314 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/IModeratorCatalogue.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Process-wide moderator catalogue. Command handlers capture once. + /// + public interface IModeratorCatalogue + { + /// + /// Gets the last successfully published immutable snapshot. + /// + /// + /// After a successful start this is never null. Readers need no lock. + /// Capture the reference once per POST; a concurrent refresh may replace + /// the published snapshot while that command continues on the captured instance. + /// + ModeratorSnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/INntpModeratorRepository.cs b/src/VectorNNTPD.TransitService/Moderation/INntpModeratorRepository.cs new file mode 100644 index 00000000..67783ba5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/INntpModeratorRepository.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// Loads enabled moderator rows from the NntpDB catalogue source. + public interface INntpModeratorRepository + { + /// + /// Returns enabled rows in moderator_id order. Callers must not query this + /// from the POST hot path. + /// + ValueTask> GetEnabledAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModerationEmailComposer.cs b/src/VectorNNTPD.TransitService/Moderation/ModerationEmailComposer.cs new file mode 100644 index 00000000..4331cc35 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModerationEmailComposer.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Builds a standards-defined moderator email from a proto-article. + /// + /// + /// RFC 5537 §3.5.1 / §4.1: the proto-article is encapsulated as + /// application/news-transmission; usage=moderate. Injection-Info, + /// Injection-Date, Path, and X-Trace are not added here — forwarding happens + /// before those headers are written. + /// + public sealed class ModerationEmailComposer + { + private readonly IOptions _emailOptions; + + /// Initializes a new composer. + public ModerationEmailComposer(IOptions emailOptions) + { + ArgumentNullException.ThrowIfNull(emailOptions); + _emailOptions = emailOptions; + } + + /// Creates the outbound moderation email. Does not send it. + public EmailMessage Compose(ModerationSubmission submission) + { + ArgumentNullException.ThrowIfNull(submission); + if (!EmailOptionsValidator.TryValidateMailbox(submission.ModeratorAddress, out var moderator)) + { + throw new ArgumentException("Moderator address is not a mailbox.", nameof(submission)); + } + + var options = _emailOptions.Value; + if (!EmailOptionsValidator.TryValidateMailbox(options.DefaultFrom, out var fromMailbox) + && !EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out fromMailbox)) + { + throw new InvalidOperationException("Email:DefaultFrom must be a mailbox when composing moderation mail."); + } + + EmailAddress? envelope = null; + if (EmailOptionsValidator.TryValidateMailbox(options.EnvelopeSender, out var envelopeMailbox)) + { + envelope = new EmailAddress(envelopeMailbox); + } + + var proto = DestuffProtoArticle(submission.ProtoArticle.Span); + var subject = submission.TargetModeratedGroup + " " + submission.MessageId; + return new EmailMessage + { + From = new EmailAddress(fromMailbox), + To = [new EmailAddress(moderator)], + EnvelopeSender = envelope, + Subject = subject, + Body = proto, + ContentType = "application/news-transmission; usage=moderate", + Charset = "us-ascii", + CorrelationId = submission.MessageId, + Headers = + [ + new EmailHeader("X-Moderated-Newsgroup", submission.TargetModeratedGroup), + new EmailHeader("X-Original-Message-ID", submission.MessageId), + new EmailHeader( + "X-Submitting-Sender", + string.IsNullOrEmpty(submission.AuthenticatedUsername) + ? "unauthenticated" + : submission.AuthenticatedUsername), + ], + }; + } + + /// Destuffs NNTP proto-article wire into a complete article (CRLF lines). + internal static byte[] DestuffProtoArticle(ReadOnlySpan stuffed) + { + if (stuffed.IsEmpty) + { + return []; + } + + var output = new List(stuffed.Length); + var offset = 0; + while (offset < stuffed.Length) + { + var remaining = stuffed[offset..]; + var crlf = remaining.IndexOf("\r\n"u8); + ReadOnlySpan line; + if (crlf < 0) + { + line = remaining; + offset = stuffed.Length; + } + else + { + line = remaining[..crlf]; + offset += crlf + 2; + } + + if (line.Length > 0 && line[0] == (byte)'.') + { + line = line[1..]; + } + + output.AddRange(line.ToArray()); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + } + + return [.. output]; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModerationSubmission.cs b/src/VectorNNTPD.TransitService/Moderation/ModerationSubmission.cs new file mode 100644 index 00000000..dd20f98b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModerationSubmission.cs @@ -0,0 +1,58 @@ +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Proto-article offered to a moderator before Injection-Info / Injection-Date are added. + /// + /// + /// RFC 5537 §3.5 step 7 and §3.5.1: forwarding happens after Message-ID/Date are present + /// and before injection trace headers. The payload is the stuffed proto-article (IHAVE + /// wire representation without the terminator), not a finalized injected article. + /// + public sealed class ModerationSubmission + { + /// Gets the stuffed proto-article (terminator omitted). + public required ReadOnlyMemory ProtoArticle { get; init; } + + /// Gets the article Message-ID (supplied or synthesized). + public required string MessageId { get; init; } + + /// Gets the syntax-validated Newsgroups: targets. + public required string[] Newsgroups { get; init; } + + /// + /// Gets the leftmost moderated group — the RFC 5537 §3.5.1 forwarding target. + /// + public required string TargetModeratedGroup { get; init; } + + /// Gets the resolved moderator mailbox for . + public required string ModeratorAddress { get; init; } + + /// Gets the submitting AUTHINFO principal, or when unauthenticated. + public string? AuthenticatedUsername { get; init; } + + /// Gets the session client identity of the submitting poster. + public required ConnectionClientIdentity Sender { get; init; } + } + + /// Outcome of . + public enum ModerationSubmissionStatus + { + /// The proto-article was accepted for moderator delivery. + Accepted = 0, + + /// No delivery mechanism is configured or available. + Unavailable = 1, + + /// Delivery was attempted and failed. + Failed = 2, + } + + /// Result of one moderation submission attempt. + /// Acceptance / availability / failure. + /// Internal detail (not sent on the NNTP wire). + public readonly record struct ModerationSubmissionResult( + ModerationSubmissionStatus Status, + string Detail); +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModeratorAddressTemplate.cs b/src/VectorNNTPD.TransitService/Moderation/ModeratorAddressTemplate.cs new file mode 100644 index 00000000..429ba7bf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModeratorAddressTemplate.cs @@ -0,0 +1,102 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// INN samples/moderators address-template expansion. + /// + /// + /// %s is replaced with the matched newsgroup name after . is + /// changed to -. The result is a routing mailbox for unapproved + /// submission. Expansion does not authorize an NNTP principal. + /// + public static class ModeratorAddressTemplate + { + /// INN substitution token. + public const string Token = "%s"; + + /// + /// Returns whether contains the INN %s token. + /// + public static bool IsTemplate(ReadOnlySpan address) => + address.IndexOf(Token, StringComparison.Ordinal) >= 0; + + /// + /// Expands a static mailbox or an INN %s template against + /// . + /// + public static string Expand(string address, ReadOnlySpan newsgroup) + { + ArgumentException.ThrowIfNullOrWhiteSpace(address); + if (!IsTemplate(address)) + { + return address; + } + + var name = Encoding.ASCII.GetString(newsgroup); + var substituted = name.Replace('.', '-'); + return address.Replace(Token, substituted, StringComparison.Ordinal); + } + + /// + /// Validates a static mailbox or a single-%s INN template. + /// + public static bool TryValidate(string address) + { + if (string.IsNullOrWhiteSpace(address) || !IsAscii(address)) + { + return false; + } + + var tokenCount = 0; + var i = 0; + while (i < address.Length) + { + if (address[i] == '%') + { + if (i + 1 >= address.Length || address[i + 1] != 's') + { + return false; + } + + tokenCount++; + i += 2; + continue; + } + + i++; + } + + if (tokenCount > 1) + { + return false; + } + + var candidate = tokenCount == 0 + ? address + : address.Replace(Token, "x", StringComparison.Ordinal); + return IsMailboxIdentity(candidate); + } + + private static bool IsMailboxIdentity(string value) + { + var at = value.IndexOf('@'); + return at > 0 + && at == value.LastIndexOf('@') + && at < value.Length - 1; + } + + private static bool IsAscii(string value) + { + foreach (var c in value) + { + if (c > 127) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueLogMessages.cs b/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueLogMessages.cs new file mode 100644 index 00000000..6f9dfcf8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// Source-generated moderator catalogue lifecycle log messages. + internal static partial class ModeratorCatalogueLogMessages + { + [LoggerMessage( + EventId = 2510, + Level = LogLevel.Information, + Message = "Loading initial moderator catalogue")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2511, + Level = LogLevel.Information, + Message = "Published moderator catalogue snapshot with {RuleCount} rules")] + public static partial void SnapshotPublished(ILogger logger, int RuleCount); + + [LoggerMessage( + EventId = 2512, + Level = LogLevel.Error, + Message = "Initial moderator catalogue load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2513, + Level = LogLevel.Error, + Message = "Moderator catalogue refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2514, + Level = LogLevel.Information, + Message = "Moderator catalogue refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2515, + Level = LogLevel.Information, + Message = "Moderator catalogue refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + + [LoggerMessage( + EventId = 2516, + Level = LogLevel.Error, + Message = "nntpmoderators lookup failed")] + public static partial void RepositoryFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueService.cs b/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueService.cs new file mode 100644 index 00000000..e157ddf1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModeratorCatalogueService.cs @@ -0,0 +1,237 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Loads the moderator catalogue from NntpDB, publishes an immutable snapshot, + /// and refreshes it on the same interval as the newsgroup catalogue. + /// + /// + /// Initial population runs during and fails startup when + /// the query cannot be completed. Subsequent refresh failures keep the last + /// known-good snapshot. POST captures once and never queries MySQL. + /// + public sealed class ModeratorCatalogueService : IModeratorCatalogue, IModeratorAuthorization, IApplicationService, IAsyncDisposable + { + /// Catalogue refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5); + + private readonly INntpModeratorRepository _repository; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private ModeratorSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + public ModeratorCatalogueService( + INntpModeratorRepository repository, + ILogger logger) + : this(repository, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + internal ModeratorCatalogueService( + INntpModeratorRepository repository, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(repository); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _repository = repository; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "ModeratorCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public ModeratorSnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + throw new InvalidOperationException("Moderator catalogue has not been published."); + } + + return snapshot; + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) => + Current.IsAuthenticatedModerator(authenticatedUsername); + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) => + Current.TryResolve(newsgroup, out identity); + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) => + Current.CanApprove(authenticatedUsername, approvedIdentity, newsgroup); + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) => + Current.TryAuthorizeApproval(authenticatedUsername, approvedIdentities, moderatedGroups, out failureDetail); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + ModeratorCatalogueLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + ModeratorCatalogueLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + ModeratorCatalogueLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + /// Replaces the published snapshot (tests). + internal void PublishForTests(ModeratorSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Publish(snapshot); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + + ModeratorCatalogueLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ModeratorCatalogueLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + var rows = await _repository.GetEnabledAsync(cancellationToken).ConfigureAwait(false); + return ModeratorSnapshot.Create(rows); + } + + private void Publish(ModeratorSnapshot snapshot) + { + Interlocked.Exchange(ref _current, snapshot); + ModeratorCatalogueLogMessages.SnapshotPublished(_logger, snapshot.Count); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModeratorIdentity.cs b/src/VectorNNTPD.TransitService/Moderation/ModeratorIdentity.cs new file mode 100644 index 00000000..ca38d625 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModeratorIdentity.cs @@ -0,0 +1,11 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Resolved moderator route for one newsgroup: expected Approved: identity and + /// authorized AUTHINFO principal. + /// + /// Wildmat that matched the newsgroup. + /// Resolved routing mailbox (static or INN %s expansion). + /// AUTHINFO username authorized to inject that approval, or empty for routing-only. + public readonly record struct ModeratorIdentity(string Pattern, string Address, string Username); +} diff --git a/src/VectorNNTPD.TransitService/Moderation/ModeratorSnapshot.cs b/src/VectorNNTPD.TransitService/Moderation/ModeratorSnapshot.cs new file mode 100644 index 00000000..03175378 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/ModeratorSnapshot.cs @@ -0,0 +1,314 @@ +using System.Text; +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Immutable first-match moderator catalogue compiled from nntpmoderators. + /// + /// + /// Lookups do not query MySQL. The compiled table is safe for concurrent POST use. + /// Username comparison is ordinal (same as AUTHINFO). Approved identities compare + /// ASCII case-insensitively after mailbox extraction. + /// + public sealed class ModeratorSnapshot : IModeratorAuthorization + { + /// Empty catalogue. Every resolve and approval fails. + public static ModeratorSnapshot Empty { get; } = new([]); + + private readonly ModeratorRule[] _rules; + + private ModeratorSnapshot(ModeratorRule[] rules) + { + _rules = rules; + } + + /// Gets the number of compiled rules (tests). + public int Count => _rules.Length; + + /// + public bool IsAuthenticatedModerator(string? authenticatedUsername) + { + if (string.IsNullOrEmpty(authenticatedUsername)) + { + return false; + } + + for (var i = 0; i < _rules.Length; i++) + { + if (!string.IsNullOrEmpty(_rules[i].Username) + && string.Equals(_rules[i].Username, authenticatedUsername, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + /// + public bool TryResolve(ReadOnlySpan newsgroup, out ModeratorIdentity identity) + { + for (var i = 0; i < _rules.Length; i++) + { + var rule = _rules[i]; + if (NntpWildmat.IsMatchValidated(newsgroup, rule.PatternBytes)) + { + identity = new ModeratorIdentity( + rule.Pattern, + ModeratorAddressTemplate.Expand(rule.Address, newsgroup), + rule.Username); + return true; + } + } + + identity = default; + return false; + } + + /// + public bool CanApprove( + string? authenticatedUsername, + ReadOnlySpan approvedIdentity, + ReadOnlySpan newsgroup) + { + if (string.IsNullOrEmpty(authenticatedUsername) + || !TryResolve(newsgroup, out var identity) + || string.IsNullOrEmpty(identity.Username) + || !string.Equals(authenticatedUsername, identity.Username, StringComparison.Ordinal)) + { + return false; + } + + return MailboxesEqual(approvedIdentity, identity.Address); + } + + /// + public bool TryAuthorizeApproval( + string? authenticatedUsername, + ReadOnlySpan approvedIdentities, + ReadOnlySpan moderatedGroups, + out string? failureDetail) + { + if (string.IsNullOrEmpty(authenticatedUsername)) + { + failureDetail = "unauthenticated approval"; + return false; + } + + if (approvedIdentities.IsEmpty) + { + failureDetail = "missing Approved"; + return false; + } + + if (moderatedGroups.IsEmpty) + { + failureDetail = "no moderated groups"; + return false; + } + + Span groupBytes = stackalloc byte[128]; + for (var i = 0; i < moderatedGroups.Length; i++) + { + var group = moderatedGroups[i]; + if (group.Length is < 1 or > 128) + { + failureDetail = "malformed newsgroup name"; + return false; + } + + var written = Encoding.ASCII.GetBytes(group, groupBytes); + if (!TryResolve(groupBytes[..written], out var identity)) + { + failureDetail = "unresolved moderator"; + return false; + } + + if (string.IsNullOrEmpty(identity.Username) + || !string.Equals(authenticatedUsername, identity.Username, StringComparison.Ordinal)) + { + failureDetail = "unauthorized moderator principal"; + return false; + } + + if (!ContainsIdentity(approvedIdentities, identity.Address)) + { + failureDetail = "Approved identity not authorized for group"; + return false; + } + } + + for (var i = 0; i < approvedIdentities.Length; i++) + { + if (!IdentityCoversAModeratedGroup(approvedIdentities[i], moderatedGroups)) + { + failureDetail = "conflicting Approved"; + return false; + } + } + + failureDetail = null; + return true; + } + + /// Compiles enabled rows in the supplied order (must already be moderator_id ASC). + public static ModeratorSnapshot Create(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + if (rows.Count == 0) + { + return Empty; + } + + var rules = new List(rows.Count); + for (var i = 0; i < rows.Count; i++) + { + var row = rows[i]; + if (row is null) + { + continue; + } + + var pattern = row.GroupPattern.Trim(); + var address = row.ModeratorAddress.Trim(); + var username = row.AccountName.Trim(); + if (pattern.Length == 0 || address.Length == 0) + { + continue; + } + + var patternBytes = Encoding.ASCII.GetBytes(pattern); + if (!NntpWildmat.TryValidate(patternBytes) || !ModeratorAddressTemplate.TryValidate(address)) + { + continue; + } + + rules.Add(new ModeratorRule(pattern, patternBytes, address, username)); + } + + return rules.Count == 0 ? Empty : new ModeratorSnapshot([.. rules]); + } + + private bool IdentityCoversAModeratedGroup(string approvedIdentity, ReadOnlySpan moderatedGroups) + { + Span groupBytes = stackalloc byte[128]; + for (var i = 0; i < moderatedGroups.Length; i++) + { + var group = moderatedGroups[i]; + if (group.Length is < 1 or > 128) + { + continue; + } + + var written = Encoding.ASCII.GetBytes(group, groupBytes); + if (TryResolve(groupBytes[..written], out var identity) + && MailboxesEqual(approvedIdentity, identity.Address)) + { + return true; + } + } + + return false; + } + + private static bool ContainsIdentity(ReadOnlySpan identities, string expected) + { + for (var i = 0; i < identities.Length; i++) + { + if (MailboxesEqual(identities[i], expected)) + { + return true; + } + } + + return false; + } + + private static bool MailboxesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + var a = Trim(left); + var b = Trim(right); + if (a.Length != b.Length) + { + return false; + } + + for (var i = 0; i < a.Length; i++) + { + if (FoldChar(a[i]) != FoldChar(b[i])) + { + return false; + } + } + + return true; + } + + private static bool MailboxesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + var a = Trim(left); + var b = Trim(right); + if (a.Length != b.Length) + { + return false; + } + + for (var i = 0; i < a.Length; i++) + { + if (FoldByte(a[i]) != FoldChar(b[i])) + { + return false; + } + } + + return true; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && char.IsWhiteSpace(value[start])) + { + start++; + } + + while (end > start && char.IsWhiteSpace(value[end - 1])) + { + end--; + } + + return value[start..end]; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && value[start] is (byte)' ' or (byte)'\t') + { + start++; + } + + while (end > start && value[end - 1] is (byte)' ' or (byte)'\t') + { + end--; + } + + return value[start..end]; + } + + private static char FoldChar(char c) => + c is >= 'a' and <= 'z' ? (char)(c - 32) : c; + + private static char FoldByte(byte b) => + (char)(b is >= (byte)'a' and <= (byte)'z' ? b - 32 : b); + + private readonly record struct ModeratorRule( + string Pattern, + byte[] PatternBytes, + string Address, + string Username); + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/MySqlNntpModeratorRepository.cs b/src/VectorNNTPD.TransitService/Moderation/MySqlNntpModeratorRepository.cs new file mode 100644 index 00000000..3b9b70a3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/MySqlNntpModeratorRepository.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.Moderation +{ + /// Loads nntpmoderators through the existing NntpDB connection pool. + public sealed class MySqlNntpModeratorRepository : INntpModeratorRepository + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlNntpModeratorRepository(NntpDbService nntpDb, ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask> GetEnabledAsync( + CancellationToken cancellationToken = default) + { + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryEnabledModeratorsAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ModeratorCatalogueLogMessages.RepositoryFailed(_logger, ex); + throw new NntpDbUnavailableException("MySQL nntpmoderators lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/NntpModeratorRow.cs b/src/VectorNNTPD.TransitService/Moderation/NntpModeratorRow.cs new file mode 100644 index 00000000..e1628fae --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/NntpModeratorRow.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// One enabled nntpmoderators row after mapping. + public sealed class NntpModeratorRow + { + /// Initializes a mapped moderator row. + public NntpModeratorRow(long moderatorId, string groupPattern, string moderatorAddress, string accountName) + { + ArgumentNullException.ThrowIfNull(groupPattern); + ArgumentNullException.ThrowIfNull(moderatorAddress); + ModeratorId = moderatorId; + GroupPattern = groupPattern; + ModeratorAddress = moderatorAddress; + AccountName = accountName ?? string.Empty; + } + + /// Gets moderator_id. Snapshot order follows this value ascending. + public long ModeratorId { get; } + + /// Gets the RFC 3977 wildmat matched against the newsgroup name. + public string GroupPattern { get; } + + /// Gets the routing / Approved mailbox or INN %s template. + public string ModeratorAddress { get; } + + /// Gets the AUTHINFO username principal, or empty for routing-only. + public string AccountName { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Moderation/UnavailableModerationSubmissionService.cs b/src/VectorNNTPD.TransitService/Moderation/UnavailableModerationSubmissionService.cs new file mode 100644 index 00000000..6f4ad4ab --- /dev/null +++ b/src/VectorNNTPD.TransitService/Moderation/UnavailableModerationSubmissionService.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.Moderation +{ + /// + /// Production default when no mail or durable moderator-delivery backend is registered. + /// + /// + /// RFC 5537 §3.5 step 7 requires the injecting agent to reject the proto-article when + /// forwarding is not possible. POST maps + /// to 441. + /// + public sealed class UnavailableModerationSubmissionService : IModerationSubmissionService + { + /// Shared instance. + public static UnavailableModerationSubmissionService Instance { get; } = new(); + + /// + public ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + _ = cancellationToken; + return ValueTask.FromResult( + new ModerationSubmissionResult( + ModerationSubmissionStatus.Unavailable, + "moderation delivery is not configured")); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Listeners/ConnectionAcceptanceLogging.cs b/src/VectorNNTPD.TransitService/Networking/Listeners/ConnectionAcceptanceLogging.cs new file mode 100644 index 00000000..3cae0664 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Listeners/ConnectionAcceptanceLogging.cs @@ -0,0 +1,94 @@ +using System.Net; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Networking.Listeners +{ + /// Formats connection-acceptance Information logs (TCP peer vs PROXY effective client). + internal static class ConnectionAcceptanceLogging + { + /// Logs a successful plain (or plain+PROXY) connection acceptance. + public static void LogPlainAccepted(ILogger logger, ConnectionClientIdentity identity) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(identity); + + if (identity.UsedProxyHeader) + { + NetworkingLogMessages.ProxyConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + FormatEndpoint(identity.Client)); + } + else + { + NetworkingLogMessages.PlainConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer)); + } + } + + /// Logs a successful TLS (or TLS+PROXY) connection acceptance after handshake. + public static void LogTlsAccepted( + ILogger logger, + ConnectionClientIdentity identity, + string tlsVersion, + string cipher) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(identity); + ArgumentException.ThrowIfNullOrWhiteSpace(tlsVersion); + ArgumentException.ThrowIfNullOrWhiteSpace(cipher); + + if (identity.UsedProxyHeader) + { + NetworkingLogMessages.TlsProxyConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + FormatEndpoint(identity.Client), + tlsVersion, + cipher); + } + else + { + NetworkingLogMessages.TlsConnectionAccepted( + logger, + FormatEndpoint(identity.TcpPeer), + tlsVersion, + cipher); + } + } + + /// Formats an endpoint as ip:port for connection acceptance logs. + public static string FormatEndpoint(IPEndPoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + return $"{endpoint.Address}:{endpoint.Port}"; + } + + /// Formats a TCP endpoint for accept/disconnect logs, or unknown. + public static string FormatEndpoint(EndPoint? endpoint) => + endpoint switch + { + IPEndPoint ip => FormatEndpoint(ip), + not null => endpoint.ToString() ?? "unknown", + _ => "unknown", + }; + + /// Logs the single Information disconnect summary for an established TCP connection. + public static void LogDisconnected( + ILogger logger, + EndPoint? remote, + EndPoint? local, + string reason) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + NetworkingLogMessages.TcpConnectionDisconnected( + logger, + FormatEndpoint(remote), + FormatEndpoint(local), + reason); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Listeners/IListenSocketBinder.cs b/src/VectorNNTPD.TransitService/Networking/Listeners/IListenSocketBinder.cs new file mode 100644 index 00000000..db4b9b92 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Listeners/IListenSocketBinder.cs @@ -0,0 +1,41 @@ +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.TransitService.Networking.Listeners +{ + /// + /// Binds and listens on an already-created TCP socket. + /// + /// + /// This is the startup ownership seam for a listening endpoint. A successful + /// means the socket accepted bind/listen under the + /// application's socket options. Accept-loop failures after that point are not + /// startup failures. + /// + public interface IListenSocketBinder + { + /// + /// Binds to and starts listening. + /// + /// The listen socket created for . + /// The local endpoint that must be owned after this call. + /// The listen backlog. + void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog); + } + + /// Production / implementation. + internal sealed class SocketListenBinder : IListenSocketBinder + { + /// Gets the shared production binder. + public static SocketListenBinder Instance { get; } = new(); + + /// + public void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(endpoint); + socket.Bind(endpoint); + socket.Listen(backlog); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Listeners/NntpPlainListenerService.cs b/src/VectorNNTPD.TransitService/Networking/Listeners/NntpPlainListenerService.cs new file mode 100644 index 00000000..fb87b712 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Listeners/NntpPlainListenerService.cs @@ -0,0 +1,506 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState.RateLimiting; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Transport.Vatp; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.TransitService.Networking.Listeners +{ + /// + /// Application service that binds cleartext NNTP TCP listeners and owns accepted plain connections. + /// + /// + /// Starts after Cloudflare DNS reconciliation and before ACME. Does not wait for certificates. + /// succeeds only after every planned endpoint has bound; a bind + /// failure disposes already-bound endpoints and fails startup. Accept-loop failures after a + /// successful bind do not fail startup. + /// + public sealed class NntpPlainListenerService : IApplicationService, IAsyncDisposable + { + private readonly IOptions _options; + private readonly ITrustedProxyHosts _trustedProxyHosts; + private readonly ITlsCertificateContextProvider _certificateProvider; + private readonly INntpAuthenticationProvider _authenticationProvider; + private readonly IArticleIngestionQueue _articleIngestion; + private readonly IHistoryDb? _historyDb; + private readonly ISpeedTestCoordinator? _speedTest; + private readonly ITransitPeerAuthorization _transitPeerAuthorization; + private readonly ITransitInboundConnectionLimiter _inboundConnectionLimiter; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly INntpSessionCensus? _sessionCensus; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IPostingTraceProtector? _postingTraceProtector; + private readonly INewsgroupCatalogue? _newsgroupCatalogue; + private readonly INntpArticlePolicySource? _sharedConfiguration; + private readonly IModeratorCatalogue? _moderatorCatalogue; + private readonly IModeratorAuthorization _moderatorAuthorization; + private readonly IModerationSubmissionService _moderationSubmission; + private readonly ISessionStateTracker? _sessionAdmission; + private readonly NntpSaslService? _saslService; + private readonly IAccountByteAccountant _accountBytes; + private readonly IAccountRateAllocator _accountRates; + private readonly IArticleWorkRpcClient? _articleWorkRpc; + private readonly IVatpArticleClient? _vatpArticleClient; + private readonly IStorageArticleLookupClient? _storageArticleLookup; + private readonly IPostFilter _postFilter; + private readonly PostFilterMetrics _postFilterMetrics; + private readonly IPostFilterRejectionEvidenceQueue _postFilterEvidence; + private readonly INewsLogWriter _newsLog; + private readonly IListenSocketBinder _listenBinder; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly NntpServerRole _serverRole; + + /// + /// Publishes DNS after every plain socket is bound, and only when the TLS listener is disabled. + /// When TLS is enabled, publishes after its own bind. + /// Null skips publication. A failure rolls back the sockets bound by this start. + /// + private readonly IListenerDnsPublication? _publication; + private readonly ConcurrentDictionary _connections = new(); + private readonly List _listeners = []; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public NntpPlainListenerService( + IOptions options, + ITrustedProxyHosts trustedProxyHosts, + ITlsCertificateContextProvider certificateProvider, + INntpAuthenticationProvider authenticationProvider, + IArticleIngestionQueue articleIngestion, + ITransitPeerAuthorization transitPeerAuthorization, + ILoggerFactory loggerFactory, + ILogger logger, + ITransitInboundConnectionLimiter? inboundConnectionLimiter = null, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + IFeedDiagnostics? feedDiagnostics = null, + IListenSocketBinder? listenBinder = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + INntpArticlePolicySource? sharedConfiguration = null, + IListenerDnsPublication? publication = null, + NntpServerRole serverRole = NntpServerRole.Transit) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(authenticationProvider); + ArgumentNullException.ThrowIfNull(articleIngestion); + ArgumentNullException.ThrowIfNull(transitPeerAuthorization); + ArgumentNullException.ThrowIfNull(loggerFactory); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _trustedProxyHosts = trustedProxyHosts; + _certificateProvider = certificateProvider; + _authenticationProvider = authenticationProvider; + _articleIngestion = articleIngestion; + _transitPeerAuthorization = transitPeerAuthorization; + _inboundConnectionLimiter = inboundConnectionLimiter ?? TransitInboundConnectionLimiter.Disabled; + _historyDb = historyDb; + _speedTest = speedTest; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _sessionCensus = sessionCensus; + _peerMetrics = peerMetrics; + _postingTraceProtector = postingTraceProtector; + _newsgroupCatalogue = newsgroupCatalogue; + _sharedConfiguration = sharedConfiguration; + _moderatorCatalogue = moderatorCatalogue; + _moderatorAuthorization = moderatorAuthorization ?? EmptyModeratorAuthorization.Instance; + _moderationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + _sessionAdmission = sessionAdmission; + _saslService = saslService; + _accountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + _accountRates = accountRates ?? NullAccountRateAllocator.Instance; + _articleWorkRpc = articleWorkRpc; + _vatpArticleClient = vatpArticleClient; + _storageArticleLookup = storageArticleLookup; + _postFilter = postFilter ?? DisabledPostFilter.Instance; + _postFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + _postFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _listenBinder = listenBinder ?? SocketListenBinder.Instance; + _loggerFactory = loggerFactory; + _logger = logger; + _publication = publication; + _serverRole = serverRole; + } + + /// + public string Name => "NntpPlainListener"; + + /// + public Task? Execution => _execution; + + /// Gets the number of active plain connections (tests). + internal int ActiveConnectionCount => _connections.Count; + + /// Gets bound local endpoints after start (tests). + internal IReadOnlyList LocalEndPoints => + [.. _listeners.Select(static l => l.LocalEndPoint)]; + + /// Gets whether any accept loop is still running (tests). + internal bool HasActiveAcceptLoops => _listeners.Exists(static l => l.AcceptLoopActive); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.Value; + var bindings = ListenEndpointPlanner.Plan(options.BindAddress, options.BindPort); + if (bindings.Count == 0) + { + throw new InvalidOperationException("No listen bindings were produced for the plain NNTP port."); + } + + foreach (var binding in bindings) + { + cancellationToken.ThrowIfCancellationRequested(); + var listener = new SocketAcceptListener( + binding, + OnAcceptedAsync, + _loggerFactory.CreateLogger($"{nameof(SocketAcceptListener)}.Plain"), + _listenBinder); + try + { + listener.Start(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.ListenerBindFailed( + _logger, + ex, + "Plain", + binding.EndPoint.ToString(), + options.BindPort); + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + + _listeners.Add(listener); + } + + if (_publication is not null && !_options.Value.IsTlsListenerEnabled) + { + await _publication.PublishAfterListenerBoundAsync(cancellationToken).ConfigureAwait(false); + } + + _execution = WaitUntilStoppedAsync(_runCts.Token); + NetworkingLogMessages.PlainListenersStarted(_logger, _listeners.Count, options.BindPort); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await RollbackPartialStartupAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + + foreach (var listener in _listeners) + { + await listener.StopAsync().ConfigureAwait(false); + } + + var connections = _connections.Keys.ToArray(); + foreach (var connection in connections) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(TcpDisconnectReason.Shutdown); + } + + await connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.PlainConnectionCompleteError(_logger, ex); + } + } + + foreach (var connection in connections) + { + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _connections.TryRemove(connection, out _); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort dispose. + } + + foreach (var listener in _listeners) + { + await listener.DisposeAsync().ConfigureAwait(false); + } + + _listeners.Clear(); + _runCts.Dispose(); + } + + private async Task RollbackPartialStartupAsync() + { + foreach (var listener in _listeners) + { + try + { + await listener.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort rollback of already-bound endpoints. + } + } + + _listeners.Clear(); + Interlocked.Exchange(ref _started, 0); + } + + // codeql[cs/missed-using-statement] Accepted socket ownership transfers to NntpConnection on success; disposed in the method finally only when no connection was started. A using on the parameter would dispose under the live session. + private async ValueTask OnAcceptedAsync(Socket socket, CancellationToken cancellationToken) + { + NntpConnection? connection = null; + try + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + NetworkingLogMessages.PlainAcceptDiscarded(_logger); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, _trustedProxyHosts, cancellationToken) + .ConfigureAwait(false); + } + catch (ProxyProtocolException ex) + { + NetworkingLogMessages.PlainProxyPreambleRejected(_logger, ex, tcpPeer); + return; + } + + connection = NntpConnection.StartPlain( + socket, + preamble.Identity, + _loggerFactory.CreateLogger(), + preamble.Leftover, + _feedDiagnostics); + _connections[connection] = 0; + + // Session owns NNTP greeting/command loop; transport owns the socket lifecycle. + var session = new NntpSession( + connection, + _loggerFactory.CreateLogger(), + certificateProvider: _certificateProvider, + authenticationProvider: _authenticationProvider, + allowCleartextAuth: _options.Value.AllowCleartextAuth, + loggerFactory: _loggerFactory, + articleIngestion: _articleIngestion, + transitPeerAuthorization: _transitPeerAuthorization, + streamOutstandingArticleDepth: _options.Value.Transit.StreamOutstandingArticleDepth, + historyDb: _historyDb, + speedTest: _speedTest, + sessionCensus: _sessionCensus, + peerMetrics: _peerMetrics, + commandIdleTimeout: TimeSpan.FromSeconds(_options.Value.IdleTime), + maxArticleSize: _options.Value.MaxArticleSize, + injectionIdentity: _options.Value.Fqdn, + mailComplaintsTo: _options.Value.MailComplaintsTo, + postingTraceProtector: _postingTraceProtector, + newsgroupCatalogue: _newsgroupCatalogue, + moderatorCatalogue: _moderatorCatalogue, + moderatorAuthorization: _moderatorAuthorization, + moderationSubmission: _moderationSubmission, + sessionAdmission: _sessionAdmission, + saslService: _saslService, + accountBytes: _accountBytes, + accountRates: _accountRates, + articleWorkRpc: _articleWorkRpc, + vatpArticleClient: _vatpArticleClient, + storageArticleLookup: _storageArticleLookup, + postFilter: _postFilter, + postFilterMetrics: _postFilterMetrics, + postFilterEvidence: _postFilterEvidence, + newsLog: _newsLog, + transit: _options.Value.Transit, + serverRole: _serverRole) + { + SharedConfiguration = _sharedConfiguration, + }; + ConnectionAcceptanceLogging.LogPlainAccepted(_logger, connection.ClientIdentity); + + TransitInboundAdmitResult admission; + try + { + admission = await TransitConnectionAdmission + .TryAdmitAsync(_inboundConnectionLimiter, session, _runCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (_runCts.IsCancellationRequested) + { + return; + } + + if (!admission.Admitted) + { + session.RecordPeerRejected(); + _feedDiagnostics.OnRejected( + session.Authorization.TransitPeerName ?? "none", + ConnectionAcceptanceLogging.FormatEndpoint(session.ClientIdentity.Client)); + await TransitConnectionAdmission + .WriteUnavailableAsync(connection, _runCts.Token) + .ConfigureAwait(false); + return; + } + + session.AttachTransitAdmission(admission.Lease); + session.RecordPeerAccepted(); + session.FeedProbe = _feedDiagnostics.OnAccepted(session); + try + { + await session.RunAsync(_runCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_runCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Listener stopping. + } + finally + { + _feedDiagnostics.OnReleased(session.FeedProbe); + await admission.Lease.DisposeAsync().ConfigureAwait(false); + } + } + finally + { + if (connection is not null) + { + _connections.TryRemove(connection, out _); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + else + { + socket.Dispose(); + } + } + } + + private static async Task WaitUntilStoppedAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Listeners/NntpTlsListenerService.cs b/src/VectorNNTPD.TransitService/Networking/Listeners/NntpTlsListenerService.cs new file mode 100644 index 00000000..4727bac3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Listeners/NntpTlsListenerService.cs @@ -0,0 +1,549 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState.RateLimiting; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Transport.Vatp; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.TransitService.Networking.Listeners +{ + /// + /// Application service that binds implicit-TLS NNTP listeners after a usable certificate context exists. + /// + /// + /// Idle when is . + /// Does not rebind or restart on certificate rotation; new handshakes observe the latest context. + /// PROXY preamble (when required) is consumed on the cleartext socket before TLS. + /// When TLS is enabled, succeeds only after every planned endpoint has + /// bound; a bind failure disposes already-bound endpoints and fails startup. Accept-loop + /// failures after a successful bind do not fail startup. + /// + public sealed class NntpTlsListenerService : IApplicationService, IAsyncDisposable + { + private readonly IOptions _options; + private readonly ITlsCertificateContextProvider _certificateProvider; + private readonly ITrustedProxyHosts _trustedProxyHosts; + private readonly INntpAuthenticationProvider _authenticationProvider; + private readonly IArticleIngestionQueue _articleIngestion; + private readonly IHistoryDb? _historyDb; + private readonly ISpeedTestCoordinator? _speedTest; + private readonly ITransitPeerAuthorization _transitPeerAuthorization; + private readonly ITransitInboundConnectionLimiter _inboundConnectionLimiter; + private readonly IFeedDiagnostics _feedDiagnostics; + private readonly INntpSessionCensus? _sessionCensus; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IPostingTraceProtector? _postingTraceProtector; + private readonly INewsgroupCatalogue? _newsgroupCatalogue; + private readonly INntpArticlePolicySource? _sharedConfiguration; + private readonly IModeratorCatalogue? _moderatorCatalogue; + private readonly IModeratorAuthorization _moderatorAuthorization; + private readonly IModerationSubmissionService _moderationSubmission; + private readonly ISessionStateTracker? _sessionAdmission; + private readonly NntpSaslService? _saslService; + private readonly IAccountByteAccountant _accountBytes; + private readonly IAccountRateAllocator _accountRates; + private readonly IArticleWorkRpcClient? _articleWorkRpc; + private readonly IVatpArticleClient? _vatpArticleClient; + private readonly IStorageArticleLookupClient? _storageArticleLookup; + private readonly IPostFilter _postFilter; + private readonly PostFilterMetrics _postFilterMetrics; + private readonly IPostFilterRejectionEvidenceQueue _postFilterEvidence; + private readonly INewsLogWriter _newsLog; + private readonly IListenSocketBinder _listenBinder; + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + private readonly NntpServerRole _serverRole; + + /// + /// Publishes DNS after every TLS socket is bound. Null skips publication. + /// A failure rolls back the sockets bound by this start. The plain listener publishes instead when TLS is disabled. + /// + private readonly IListenerDnsPublication? _publication; + private readonly ConcurrentDictionary _connections = new(); + private readonly List _listeners = []; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public NntpTlsListenerService( + IOptions options, + ITlsCertificateContextProvider certificateProvider, + ITrustedProxyHosts trustedProxyHosts, + INntpAuthenticationProvider authenticationProvider, + IArticleIngestionQueue articleIngestion, + ITransitPeerAuthorization transitPeerAuthorization, + ILoggerFactory loggerFactory, + ILogger logger, + ITransitInboundConnectionLimiter? inboundConnectionLimiter = null, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + IFeedDiagnostics? feedDiagnostics = null, + IListenSocketBinder? listenBinder = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + INntpArticlePolicySource? sharedConfiguration = null, + IListenerDnsPublication? publication = null, + NntpServerRole serverRole = NntpServerRole.Transit) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + ArgumentNullException.ThrowIfNull(authenticationProvider); + ArgumentNullException.ThrowIfNull(articleIngestion); + ArgumentNullException.ThrowIfNull(transitPeerAuthorization); + ArgumentNullException.ThrowIfNull(loggerFactory); + ArgumentNullException.ThrowIfNull(logger); + _options = options; + _authenticationProvider = authenticationProvider; + _certificateProvider = certificateProvider; + _trustedProxyHosts = trustedProxyHosts; + _articleIngestion = articleIngestion; + _transitPeerAuthorization = transitPeerAuthorization; + _inboundConnectionLimiter = inboundConnectionLimiter ?? TransitInboundConnectionLimiter.Disabled; + _historyDb = historyDb; + _speedTest = speedTest; + _feedDiagnostics = feedDiagnostics ?? NullFeedDiagnostics.Instance; + _sessionCensus = sessionCensus; + _peerMetrics = peerMetrics; + _postingTraceProtector = postingTraceProtector; + _newsgroupCatalogue = newsgroupCatalogue; + _sharedConfiguration = sharedConfiguration; + _moderatorCatalogue = moderatorCatalogue; + _moderatorAuthorization = moderatorAuthorization ?? EmptyModeratorAuthorization.Instance; + _moderationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + _sessionAdmission = sessionAdmission; + _saslService = saslService; + _accountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + _accountRates = accountRates ?? NullAccountRateAllocator.Instance; + _articleWorkRpc = articleWorkRpc; + _vatpArticleClient = vatpArticleClient; + _storageArticleLookup = storageArticleLookup; + _postFilter = postFilter ?? DisabledPostFilter.Instance; + _postFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + _postFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + _newsLog = newsLog ?? NullNewsLogWriter.Instance; + _listenBinder = listenBinder ?? SocketListenBinder.Instance; + _loggerFactory = loggerFactory; + _logger = logger; + _publication = publication; + _serverRole = serverRole; + } + + /// + public string Name => "NntpTlsListener"; + + /// + public Task? Execution => _execution; + + /// Gets whether TLS listeners were bound (tests). + internal bool ListenersBound => _listeners.Count > 0; + + /// Gets the number of active TLS connections (tests). + internal int ActiveConnectionCount => _connections.Count; + + /// Gets bound local endpoints after start (tests). + internal IReadOnlyList LocalEndPoints => + [.. _listeners.Select(static l => l.LocalEndPoint)]; + + /// Gets whether any accept loop is still running (tests). + internal bool HasActiveAcceptLoops => _listeners.Exists(static l => l.AcceptLoopActive); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var options = _options.Value; + if (!options.IsTlsListenerEnabled) + { + NetworkingLogMessages.TlsListenerIdle(_logger); + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!_certificateProvider.IsAvailable) + { + throw new InvalidOperationException( + "TLS listener cannot start: no usable TLS certificate context is published."); + } + + var bindings = ListenEndpointPlanner.Plan(options.BindAddress, options.BindPortTls); + if (bindings.Count == 0) + { + throw new InvalidOperationException("No listen bindings were produced for the TLS NNTP port."); + } + + foreach (var binding in bindings) + { + cancellationToken.ThrowIfCancellationRequested(); + var listener = new SocketAcceptListener( + binding, + OnAcceptedAsync, + _loggerFactory.CreateLogger($"{nameof(SocketAcceptListener)}.Tls"), + _listenBinder); + try + { + listener.Start(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.ListenerBindFailed( + _logger, + ex, + "TLS", + binding.EndPoint.ToString(), + options.BindPortTls); + await listener.DisposeAsync().ConfigureAwait(false); + throw; + } + + _listeners.Add(listener); + } + + if (_publication is not null) + { + await _publication.PublishAfterListenerBoundAsync(cancellationToken).ConfigureAwait(false); + } + + _execution = WaitUntilStoppedAsync(_runCts.Token); + NetworkingLogMessages.TlsListenersStarted(_logger, _listeners.Count, options.BindPortTls); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await RollbackPartialStartupAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + + foreach (var listener in _listeners) + { + await listener.StopAsync().ConfigureAwait(false); + } + + var connections = _connections.Keys.ToArray(); + foreach (var connection in connections) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(TcpDisconnectReason.Shutdown); + } + + await connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.TlsConnectionCompleteError(_logger, ex); + } + } + + foreach (var connection in connections) + { + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _connections.TryRemove(connection, out _); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort dispose. + } + + foreach (var listener in _listeners) + { + await listener.DisposeAsync().ConfigureAwait(false); + } + + _listeners.Clear(); + _runCts.Dispose(); + } + + private async Task RollbackPartialStartupAsync() + { + foreach (var listener in _listeners) + { + try + { + await listener.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort rollback of already-bound endpoints. + } + } + + _listeners.Clear(); + Interlocked.Exchange(ref _started, 0); + } + + // codeql[cs/missed-using-statement] Accepted socket ownership transfers to NntpConnection on success; disposed in the method finally only when no connection was started. A using on the parameter would dispose under the live session. + private async ValueTask OnAcceptedAsync(Socket socket, CancellationToken cancellationToken) + { + NntpConnection? connection = null; + try + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + NetworkingLogMessages.TlsAcceptDiscarded(_logger); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, _trustedProxyHosts, cancellationToken) + .ConfigureAwait(false); + } + catch (ProxyProtocolException ex) + { + NetworkingLogMessages.TlsProxyPreambleRejected(_logger, ex, tcpPeer); + return; + } + + connection = await NntpConnection.StartTlsAsync( + socket, + _certificateProvider, + preamble.Identity, + _loggerFactory.CreateLogger(), + cancellationToken, + preamble.Leftover, + _feedDiagnostics) + .ConfigureAwait(false); + _connections[connection] = 0; + + var session = new NntpSession( + connection, + _loggerFactory.CreateLogger(), + certificateProvider: _certificateProvider, + authenticationProvider: _authenticationProvider, + allowCleartextAuth: _options.Value.AllowCleartextAuth, + loggerFactory: _loggerFactory, + articleIngestion: _articleIngestion, + transitPeerAuthorization: _transitPeerAuthorization, + streamOutstandingArticleDepth: _options.Value.Transit.StreamOutstandingArticleDepth, + historyDb: _historyDb, + speedTest: _speedTest, + sessionCensus: _sessionCensus, + peerMetrics: _peerMetrics, + commandIdleTimeout: TimeSpan.FromSeconds(_options.Value.IdleTime), + maxArticleSize: _options.Value.MaxArticleSize, + injectionIdentity: _options.Value.Fqdn, + mailComplaintsTo: _options.Value.MailComplaintsTo, + postingTraceProtector: _postingTraceProtector, + newsgroupCatalogue: _newsgroupCatalogue, + moderatorCatalogue: _moderatorCatalogue, + moderatorAuthorization: _moderatorAuthorization, + moderationSubmission: _moderationSubmission, + sessionAdmission: _sessionAdmission, + saslService: _saslService, + accountBytes: _accountBytes, + accountRates: _accountRates, + articleWorkRpc: _articleWorkRpc, + vatpArticleClient: _vatpArticleClient, + storageArticleLookup: _storageArticleLookup, + postFilter: _postFilter, + postFilterMetrics: _postFilterMetrics, + postFilterEvidence: _postFilterEvidence, + newsLog: _newsLog, + transit: _options.Value.Transit, + serverRole: _serverRole) + { + SharedConfiguration = _sharedConfiguration, + }; + + if (!connection.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)) + { + throw new InvalidOperationException("TLS connection missing negotiated parameters after handshake."); + } + + ConnectionAcceptanceLogging.LogTlsAccepted( + _logger, + connection.ClientIdentity, + tlsVersion, + cipher); + + TransitInboundAdmitResult admission; + try + { + admission = await TransitConnectionAdmission + .TryAdmitAsync(_inboundConnectionLimiter, session, _runCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (_runCts.IsCancellationRequested) + { + return; + } + + if (!admission.Admitted) + { + session.RecordPeerRejected(); + _feedDiagnostics.OnRejected( + session.Authorization.TransitPeerName ?? "none", + ConnectionAcceptanceLogging.FormatEndpoint(session.ClientIdentity.Client)); + await TransitConnectionAdmission + .WriteUnavailableAsync(connection, _runCts.Token) + .ConfigureAwait(false); + return; + } + + session.AttachTransitAdmission(admission.Lease); + session.RecordPeerAccepted(); + session.FeedProbe = _feedDiagnostics.OnAccepted(session); + try + { + await session.RunAsync(_runCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_runCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Listener stopping. + } + finally + { + _feedDiagnostics.OnReleased(session.FeedProbe); + await admission.Lease.DisposeAsync().ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + NetworkingLogMessages.TlsHandshakeOrSetupFailed(_logger, ex); + } + finally + { + if (connection is not null) + { + _connections.TryRemove(connection, out _); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + else + { + try + { + socket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + } + + private static async Task WaitUntilStoppedAsync(CancellationToken cancellationToken) + { + try + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Listeners/SocketAcceptListener.cs b/src/VectorNNTPD.TransitService/Networking/Listeners/SocketAcceptListener.cs new file mode 100644 index 00000000..1d233dc8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Listeners/SocketAcceptListener.cs @@ -0,0 +1,217 @@ +using System.Net; +using System.Net.Sockets; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Listeners; + +namespace VectorNNTPD.TransitService.Networking.Listeners +{ + using VectorNNTPD.Common.Core; + /// + /// Owns one listening and runs an allocation-efficient asynchronous accept loop. + /// + /// + /// A single failed accepted connection must not stop this listener. The listen socket remains open + /// until / stop. + /// + public sealed class SocketAcceptListener : IAsyncDisposable + { + private readonly ListenBinding _binding; + private readonly Func _onAccepted; + private readonly ILogger _logger; + private readonly IListenSocketBinder _binder; + private readonly Socket _listenSocket; + private readonly CancellationTokenSource _cts = new(); + private Task? _acceptLoop; + private int _stopped; + private int _disposed; + + /// Initializes a new instance of the class. + public SocketAcceptListener( + ListenBinding binding, + Func onAccepted, + ILogger logger, + IListenSocketBinder? listenBinder = null) + { + ArgumentNullException.ThrowIfNull(onAccepted); + ArgumentNullException.ThrowIfNull(logger); + _binding = binding; + _onAccepted = onAccepted; + _logger = logger; + _binder = listenBinder ?? SocketListenBinder.Instance; + + var family = binding.Address.AddressFamily; + _listenSocket = new Socket(family, SocketType.Stream, ProtocolType.Tcp); + if (family == AddressFamily.InterNetworkV6) + { + _listenSocket.DualMode = binding.DualMode; + } + + // SO_REUSEADDR is retained: Bind success is ownership under these socket options. + // On Windows it can allow another process to bind the same endpoint; that is not a + // Bind failure and is not treated as exclusive port ownership. + try + { + _listenSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Best-effort; some platforms differ. + } + } + + /// Gets the local endpoint after (port may be ephemeral in tests). + public IPEndPoint LocalEndPoint => + _listenSocket.LocalEndPoint as IPEndPoint ?? _binding.EndPoint; + + /// Gets the planned binding. + public ListenBinding Binding => _binding; + + /// Gets whether the accept loop is still running (tests). + internal bool AcceptLoopActive => _acceptLoop is { IsCompleted: false }; + + /// Binds, listens, and starts the accept loop. + /// + /// Bind/listen exceptions propagate to the caller and are startup failures. + /// The accept loop is started only after bind/listen succeed; later accept + /// failures are logged and the loop continues. + /// + public void Start(int backlog = 512) + { + _binder.BindAndListen(_listenSocket, _binding.EndPoint, backlog); + _acceptLoop = AcceptLoopAsync(_cts.Token); + NetworkingLogMessages.ListenerStarted(_logger, LocalEndPoint, _binding.DualMode); + } + + /// Stops accepting and closes the listen socket. + public async Task StopAsync() + { + if (Interlocked.Exchange(ref _stopped, 1) == 1) + { + return; + } + + try + { + await _cts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Already disposed. + } + + try + { + _listenSocket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + if (_acceptLoop is not null) + { + try + { + await _acceptLoop.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptLoopStopError(_logger, ex); + } + } + + NetworkingLogMessages.ListenerStopped(_logger, _binding.Address, _binding.Port); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await StopAsync().ConfigureAwait(false); + _cts.Dispose(); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + Socket accepted; + try + { + accepted = await _listenSocket.AcceptAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested || _stopped != 0) + { + break; + } + catch (SocketException ex) when (cancellationToken.IsCancellationRequested || _stopped != 0) + { + NetworkingLogMessages.AcceptInterruptedDuringShutdown(_logger, ex); + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptFailed(_logger, ex, _binding.EndPoint); + continue; + } + + _ = HandleAcceptedAsync(accepted, cancellationToken); + } + } + + private async Task HandleAcceptedAsync(Socket accepted, CancellationToken cancellationToken) + { + try + { + await _onAccepted(accepted, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + try + { + accepted.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NetworkingLogMessages.AcceptedHandlerFailed(_logger, ex); + try + { + accepted.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/ConnectionClientIdentity.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/ConnectionClientIdentity.cs new file mode 100644 index 00000000..d165a0c9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/ConnectionClientIdentity.cs @@ -0,0 +1,94 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// + /// Immutable connection-level client identity established once at accept time. + /// + /// + /// + /// is always the accepted socket's remote endpoint. + /// is the effective client endpoint exposed to the session layer + /// (equal to for direct and untrusted peers; the PROXY-reported + /// source for trusted HAProxy peers that supplied a proxied address). + /// + /// + /// Trust and PROXY parsing are completed before this value is published; the session layer + /// must not reparse PROXY headers or inspect the socket to obtain the client endpoint. + /// + /// + public sealed class ConnectionClientIdentity + { + private ConnectionClientIdentity( + IPEndPoint tcpPeer, + IPEndPoint client, + bool isTrustedProxy, + bool usedProxyHeader, + int? proxyProtocolVersion) + { + TcpPeer = tcpPeer; + Client = client; + IsTrustedProxy = isTrustedProxy; + UsedProxyHeader = usedProxyHeader; + ProxyProtocolVersion = proxyProtocolVersion; + } + + /// Gets the actual TCP peer endpoint of the accepted socket. + public IPEndPoint TcpPeer { get; } + + /// Gets the effective client endpoint for the session layer. + public IPEndPoint Client { get; } + + /// Gets the effective client IP address. + public IPAddress ClientAddress => Client.Address; + + /// Gets the effective client TCP source port. + public int ClientPort => Client.Port; + + /// Gets whether the TCP peer matched a configured trusted PROXY host. + public bool IsTrustedProxy { get; } + + /// Gets whether a PROXY protocol header was consumed for this connection. + public bool UsedProxyHeader { get; } + + /// Gets the PROXY protocol version when a header was consumed; otherwise . + public int? ProxyProtocolVersion { get; } + + /// Creates a direct (non-proxy) identity where client equals TCP peer. + /// + /// Publishes independent instances so later mutation of the + /// caller's endpoint (or of one published property) cannot alias into the other. + /// + public static ConnectionClientIdentity Direct(IPEndPoint tcpPeer) + { + ArgumentNullException.ThrowIfNull(tcpPeer); + var peer = CloneEndpoint(tcpPeer); + var client = CloneEndpoint(tcpPeer); + return new ConnectionClientIdentity( + peer, + client, + isTrustedProxy: false, + usedProxyHeader: false, + proxyProtocolVersion: null); + } + + /// Creates an identity for a trusted proxy peer with an optional PROXY-reported client. + public static ConnectionClientIdentity FromTrustedProxy( + IPEndPoint tcpPeer, + IPEndPoint client, + int proxyProtocolVersion) + { + ArgumentNullException.ThrowIfNull(tcpPeer); + ArgumentNullException.ThrowIfNull(client); + return new ConnectionClientIdentity( + CloneEndpoint(tcpPeer), + CloneEndpoint(client), + isTrustedProxy: true, + usedProxyHeader: true, + proxyProtocolVersion); + } + + private static IPEndPoint CloneEndpoint(IPEndPoint endpoint) => + new(endpoint.Address, endpoint.Port); + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/Crc32C.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/Crc32C.cs new file mode 100644 index 00000000..d01ccdf6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/Crc32C.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// + /// CRC32c (Castagnoli) for PROXY v2 PP2_TYPE_CRC32C (RFC 4960 Appendix B). + /// + internal static class Crc32C + { + // Reflected Castagnoli polynomial 0x1EDC6F41 → 0x82F63B78. + private const uint Polynomial = 0x82F63B78u; + private static readonly uint[] Table = BuildTable(); + + /// Computes CRC32c of (final XOR applied). + public static uint Compute(ReadOnlySpan data) + { + var crc = 0xFFFFFFFFu; + foreach (var b in data) + { + crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8); + } + + return crc ^ 0xFFFFFFFFu; + } + + private static uint[] BuildTable() + { + var table = new uint[256]; + for (uint i = 0; i < 256; i++) + { + var crc = i; + for (var bit = 0; bit < 8; bit++) + { + crc = (crc & 1) != 0 ? (crc >> 1) ^ Polynomial : crc >> 1; + } + + table[i] = crc; + } + + return table; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/ITrustedProxyHosts.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/ITrustedProxyHosts.cs new file mode 100644 index 00000000..9b3ad953 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/ITrustedProxyHosts.cs @@ -0,0 +1,21 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// + /// Canonical set of trusted HAProxy PROXY-protocol peer addresses from configuration. + /// + public interface ITrustedProxyHosts + { + /// Gets whether any trusted PROXY peers are configured. + bool IsEnabled { get; } + + /// + /// Returns whether matches a configured trusted proxy host. + /// + /// + /// Comparison uses canonical forms: IPv4-mapped IPv6 addresses match their IPv4 equivalents. + /// + bool IsTrusted(IPAddress peerAddress); + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/PrefixedStream.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/PrefixedStream.cs new file mode 100644 index 00000000..b4f3a40e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/PrefixedStream.cs @@ -0,0 +1,137 @@ +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// + /// Stream that yields a fixed prefix before reading from an inner stream (TLS leftover after PROXY). + /// + internal sealed class PrefixedStream : Stream + { + private readonly Stream _inner; + private readonly bool _leaveInnerOpen; + private byte[]? _prefix; + private int _prefixOffset; + + public PrefixedStream(Stream inner, ReadOnlyMemory prefix, bool leaveInnerOpen = false) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + _leaveInnerOpen = leaveInnerOpen; + if (!prefix.IsEmpty) + { + _prefix = prefix.ToArray(); + } + } + + /// + public override bool CanRead => true; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => _inner.CanWrite; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public override void Flush() => _inner.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _inner.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) + { + ArgumentNullException.ThrowIfNull(buffer); + return Read(buffer.AsSpan(offset, count)); + } + + /// + public override int Read(Span buffer) + { + if (_prefix is { } prefix && _prefixOffset < prefix.Length) + { + var available = prefix.Length - _prefixOffset; + var take = Math.Min(available, buffer.Length); + prefix.AsSpan(_prefixOffset, take).CopyTo(buffer); + _prefixOffset += take; + if (_prefixOffset >= prefix.Length) + { + _prefix = null; + } + + return take; + } + + return _inner.Read(buffer); + } + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (_prefix is { } prefix && _prefixOffset < prefix.Length) + { + var available = prefix.Length - _prefixOffset; + var take = Math.Min(available, buffer.Length); + prefix.AsSpan(_prefixOffset, take).CopyTo(buffer.Span); + _prefixOffset += take; + if (_prefixOffset >= prefix.Length) + { + _prefix = null; + } + + return ValueTask.FromResult(take); + } + + return _inner.ReadAsync(buffer, cancellationToken); + } + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + /// + public override void Write(byte[] buffer, int offset, int count) => + _inner.Write(buffer, offset, count); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => + _inner.WriteAsync(buffer, cancellationToken); + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (disposing && !_leaveInnerOpen) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (!_leaveInnerOpen) + { + await _inner.DisposeAsync().ConfigureAwait(false); + } + + await base.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyPreambleResolver.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyPreambleResolver.cs new file mode 100644 index 00000000..b4633814 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyPreambleResolver.cs @@ -0,0 +1,146 @@ +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// Result of resolving connection identity and optional PROXY leftover octets. + public readonly struct ProxyPreambleResolution + { + /// Initializes a new instance of the struct. + public ProxyPreambleResolution(ConnectionClientIdentity identity, ReadOnlyMemory leftover) + { + Identity = identity; + Leftover = leftover; + } + + /// Gets the established client identity. + public ConnectionClientIdentity Identity { get; } + + /// + /// Gets octets that followed a consumed PROXY header and must be processed as application/TLS data. + /// + public ReadOnlyMemory Leftover { get; } + } + + /// + /// Resolves effective client identity from the TCP peer and, when trusted, a PROXY preamble. + /// + public static class ProxyPreambleResolver + { + /// + /// Establishes connection identity for an accepted socket according to trusted PROXY hosts. + /// + /// + /// + /// Empty trusted set: PROXY disabled; client = TCP peer; no socket reads. + /// Untrusted peer: client = TCP peer; PROXY bytes are not interpreted. + /// Trusted peer: PROXY v1/v2 required; malformed/timeout fails the connection. + /// + /// + public static async ValueTask ResolveAsync( + Socket socket, + IPEndPoint tcpPeer, + ITrustedProxyHosts trustedProxyHosts, + CancellationToken cancellationToken, + TimeSpan? preambleTimeout = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(tcpPeer); + ArgumentNullException.ThrowIfNull(trustedProxyHosts); + + if (!trustedProxyHosts.IsEnabled || !trustedProxyHosts.IsTrusted(tcpPeer.Address)) + { + return new ProxyPreambleResolution(ConnectionClientIdentity.Direct(tcpPeer), ReadOnlyMemory.Empty); + } + + var timeout = preambleTimeout ?? ProxyProtocolParser.DefaultPreambleTimeout; + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + + try + { + return await ReadRequiredPreambleAsync(socket, tcpPeer, timeoutCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new ProxyProtocolTimeoutException("PROXY preamble timed out"); + } + } + + private static async ValueTask ReadRequiredPreambleAsync( + Socket socket, + IPEndPoint tcpPeer, + CancellationToken cancellationToken) + { + var buffer = new byte[Math.Min(4096, ProxyProtocolParser.MaxV2HeaderOctets)]; + var gathered = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + var gatheredLength = 0; + + while (true) + { + var status = ProxyProtocolParser.TryParse( + gathered.AsSpan(0, gatheredLength), + tcpPeer, + out var parsed, + out var error); + + if (status == ProxyParseStatus.Ok) + { + var leftoverLength = gatheredLength - parsed.HeaderLength; + ReadOnlyMemory leftover = leftoverLength > 0 + ? gathered.AsMemory(parsed.HeaderLength, leftoverLength).ToArray() + : ReadOnlyMemory.Empty; + return new ProxyPreambleResolution(parsed.Identity, leftover); + } + + if (status == ProxyParseStatus.Error) + { + throw new ProxyProtocolMalformedException(error); + } + + if (gatheredLength >= ProxyProtocolParser.MaxV2HeaderOctets) + { + throw new ProxyProtocolMalformedException("PROXY preamble exceeds maximum size"); + } + + var room = Math.Min(buffer.Length, ProxyProtocolParser.MaxV2HeaderOctets - gatheredLength); + var read = await socket + .ReceiveAsync(buffer.AsMemory(0, room), SocketFlags.None, cancellationToken) + .ConfigureAwait(false); + if (read == 0) + { + throw new ProxyProtocolMalformedException( + gatheredLength == 0 ? "PROXY header required" : "EOF during PROXY preamble"); + } + + buffer.AsSpan(0, read).CopyTo(gathered.AsSpan(gatheredLength)); + gatheredLength += read; + } + } + + /// Extracts an from a socket remote endpoint. + public static bool TryGetTcpPeer(Socket socket, out IPEndPoint tcpPeer) + { + ArgumentNullException.ThrowIfNull(socket); + try + { + if (socket.RemoteEndPoint is IPEndPoint ep) + { + tcpPeer = ep; + return true; + } + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + tcpPeer = null!; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolException.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolException.cs new file mode 100644 index 00000000..d2d55a03 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolException.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// Base failure while consuming a required PROXY protocol preamble. + public class ProxyProtocolException : IOException + { + /// Initializes a new instance of the class. + public ProxyProtocolException(string message) + : base(message) + { + } + + /// Initializes a new instance of the class. + public ProxyProtocolException(string message, Exception innerException) + : base(message, innerException) + { + } + } + + /// Malformed, incomplete, or otherwise invalid PROXY protocol header. + public sealed class ProxyProtocolMalformedException : ProxyProtocolException + { + /// Initializes a new instance of the class. + public ProxyProtocolMalformedException(string message) + : base(message) + { + } + } + + /// Timed out waiting for a complete PROXY protocol preamble. + public sealed class ProxyProtocolTimeoutException : ProxyProtocolException + { + /// Initializes a new instance of the class. + public ProxyProtocolTimeoutException(string message) + : base(message) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolParser.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolParser.cs new file mode 100644 index 00000000..4f38b23d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/ProxyProtocolParser.cs @@ -0,0 +1,610 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// Incremental parse status for a PROXY v1/v2 preamble buffer. + internal enum ProxyParseStatus + { + NeedMore, + Ok, + Error, + } + + /// Successful PROXY preamble parse result (before leftover slicing). + internal readonly struct ProxyParseResult + { + public ProxyParseResult( + ConnectionClientIdentity identity, + int headerLength, + int version, + bool localCommand) + { + Identity = identity; + HeaderLength = headerLength; + Version = version; + LocalCommand = localCommand; + } + + public ConnectionClientIdentity Identity { get; } + + public int HeaderLength { get; } + + public int Version { get; } + + public bool LocalCommand { get; } + } + + /// + /// Byte-oriented PROXY protocol v1/v2 parser (HAProxy PROXY protocol specification). + /// + /// + /// Normative: docs/standards/haproxy/proxy-protocol.txt. Does not perform trust checks; + /// callers must only invoke this for TCP peers already classified as trusted proxies. + /// + internal static class ProxyProtocolParser + { + /// Spec §2.1: worst-case v1 line including CRLF. + public const int MaxV1LineOctets = 107; + + /// Spec §2.2: address length is uint16 — full protocol maximum. + public const int MaxV2AddressOctets = 65_535; + + /// Maximum gather buffer for a v2 header (signature + length). + public const int MaxV2HeaderOctets = 16 + MaxV2AddressOctets; + + /// Default preamble timeout (spec recommends at least 3 seconds). + public static readonly TimeSpan DefaultPreambleTimeout = TimeSpan.FromSeconds(5); + + private static readonly byte[] V2Signature = + [ + 0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A, + ]; + + private static readonly byte[] V1Prefix = "PROXY "u8.ToArray(); + + private const byte Pp2TypeCrc32C = 0x03; + + /// + /// Attempts to parse as a complete PROXY preamble. + /// + public static ProxyParseStatus TryParse( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.IsEmpty) + { + return ProxyParseStatus.NeedMore; + } + + if (buffer[0] == 0x0D) + { + return TryParseV2(buffer, tcpPeer, out result, out error); + } + + if (buffer[0] == (byte)'P') + { + return TryParseV1(buffer, tcpPeer, out result, out error); + } + + error = "PROXY header required"; + return ProxyParseStatus.Error; + } + + private static ProxyParseStatus TryParseV1( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.Length < V1Prefix.Length) + { + if (V1Prefix.AsSpan().StartsWith(buffer)) + { + return ProxyParseStatus.NeedMore; + } + + error = "PROXY v1 invalid signature"; + return ProxyParseStatus.Error; + } + + if (!buffer.StartsWith(V1Prefix)) + { + error = "PROXY v1 invalid signature"; + return ProxyParseStatus.Error; + } + + var crlf = buffer.IndexOf("\r\n"u8); + if (crlf < 0) + { + if (buffer.IndexOf((byte)'\n') >= 0) + { + error = "PROXY v1 requires CRLF terminator"; + return ProxyParseStatus.Error; + } + + var cr = buffer.IndexOf((byte)'\r'); + if (cr >= 0 && cr + 1 < buffer.Length) + { + error = "PROXY v1 requires CRLF terminator"; + return ProxyParseStatus.Error; + } + + if (buffer.Length >= MaxV1LineOctets) + { + error = "PROXY v1 missing CRLF within 107 octets"; + return ProxyParseStatus.Error; + } + + return ProxyParseStatus.NeedMore; + } + + if (crlf + 2 > MaxV1LineOctets) + { + error = "PROXY v1 line exceeds 107 octets"; + return ProxyParseStatus.Error; + } + + var line = buffer[..crlf]; + var headerLength = crlf + 2; + var parts = SplitAsciiSpaces(line); + if (parts.Count < 2 || !parts[0].Span.SequenceEqual("PROXY"u8) || parts[1].IsEmpty) + { + error = "PROXY v1 malformed signature"; + return ProxyParseStatus.Error; + } + + var family = parts[1].Span; + if (family.SequenceEqual("UNKNOWN"u8)) + { + // Spec §2.1: ignore remainder before CRLF; use real connection endpoints. + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 1), + headerLength, + version: 1, + localCommand: false); + return ProxyParseStatus.Ok; + } + + foreach (var part in parts) + { + if (part.IsEmpty) + { + error = "PROXY v1 malformed spacing"; + return ProxyParseStatus.Error; + } + } + + if (!family.SequenceEqual("TCP4"u8) && !family.SequenceEqual("TCP6"u8)) + { + error = "PROXY v1 unsupported family"; + return ProxyParseStatus.Error; + } + + if (parts.Count != 6) + { + error = "PROXY v1 expected 6 fields"; + return ProxyParseStatus.Error; + } + + var isV4 = family.SequenceEqual("TCP4"u8); + if (!TryParseV1Address(parts[2].Span, isV4, out var src) || !TryParseV1Address(parts[3].Span, isV4, out _)) + { + error = "PROXY v1 invalid address"; + return ProxyParseStatus.Error; + } + + if (!TryParseV1Port(parts[4].Span, out var sport) || !TryParseV1Port(parts[5].Span, out _)) + { + error = "PROXY v1 invalid port"; + return ProxyParseStatus.Error; + } + + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 1), + headerLength, + version: 1, + localCommand: false); + return ProxyParseStatus.Ok; + } + + private static ProxyParseStatus TryParseV2( + ReadOnlySpan buffer, + IPEndPoint tcpPeer, + out ProxyParseResult result, + out string error) + { + result = default; + error = string.Empty; + + if (buffer.Length < V2Signature.Length) + { + if (V2Signature.AsSpan().StartsWith(buffer)) + { + return ProxyParseStatus.NeedMore; + } + + error = "PROXY v2 invalid signature"; + return ProxyParseStatus.Error; + } + + if (!buffer.StartsWith(V2Signature)) + { + error = "PROXY v2 invalid signature"; + return ProxyParseStatus.Error; + } + + if (buffer.Length < 16) + { + return ProxyParseStatus.NeedMore; + } + + var verCmd = buffer[12]; + var version = verCmd >> 4; + var command = verCmd & 0x0F; + if (version != 0x2) + { + error = "PROXY v2 invalid version"; + return ProxyParseStatus.Error; + } + + var famProto = buffer[13]; + var family = famProto >> 4; + var protocol = famProto & 0x0F; + var addrLen = BinaryPrimitives.ReadUInt16BigEndian(buffer.Slice(14, 2)); + var total = 16 + addrLen; + if (buffer.Length < total) + { + return ProxyParseStatus.NeedMore; + } + + var fullHeader = buffer[..total]; + var payload = buffer.Slice(16, addrLen); + + if (command == 0x0) + { + if (family is >= 0 and <= 3 && protocol is >= 0 and <= 2) + { + var localMin = AddressBlockMinimum(famProto); + var tlvOff = 16 + (payload.Length >= localMin ? localMin : payload.Length); + if (!TryValidateV2Tlvs(fullHeader, tlvOff, out error)) + { + return ProxyParseStatus.Error; + } + } + + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: true); + return ProxyParseStatus.Ok; + } + + if (command != 0x1) + { + error = "PROXY v2 unsupported command"; + return ProxyParseStatus.Error; + } + + if (family is < 0 or > 3 || protocol is < 0 or > 2) + { + error = "PROXY v2 unsupported address family/protocol"; + return ProxyParseStatus.Error; + } + + var listedMin = AddressBlockMinimumOrNull(famProto); + if (listedMin is null) + { + // Valid nibbles but unlisted combination — UNSPEC fallback. + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + var tlvOffset = 16 + (payload.Length >= listedMin.Value ? listedMin.Value : payload.Length); + if (!TryValidateV2Tlvs(fullHeader, tlvOffset, out error)) + { + return ProxyParseStatus.Error; + } + + if (famProto == 0x00) + { + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + if (famProto == 0x11) + { + if (payload.Length < 12) + { + error = "PROXY v2 address block truncated"; + return ProxyParseStatus.Error; + } + + var src = new IPAddress(payload[..4]); + var sport = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(8, 2)); + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + if (famProto == 0x21) + { + if (payload.Length < 36) + { + error = "PROXY v2 address block truncated"; + return ProxyParseStatus.Error; + } + + var src = new IPAddress(payload[..16]); + var sport = BinaryPrimitives.ReadUInt16BigEndian(payload.Slice(32, 2)); + var client = new IPEndPoint(src, sport); + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, client, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + // UDP/UNIX listed combinations: fall back to real endpoints (spec §2.2). + result = new ProxyParseResult( + ConnectionClientIdentity.FromTrustedProxy(tcpPeer, tcpPeer, proxyProtocolVersion: 2), + total, + version: 2, + localCommand: false); + return ProxyParseStatus.Ok; + } + + private static int AddressBlockMinimum(int famProto) => AddressBlockMinimumOrNull(famProto) ?? 0; + + private static int? AddressBlockMinimumOrNull(int famProto) => famProto switch + { + 0x00 => 0, + 0x11 => 12, + 0x12 => 12, + 0x21 => 36, + 0x22 => 36, + 0x31 => 216, + 0x32 => 216, + _ => null, + }; + + private static bool TryValidateV2Tlvs(ReadOnlySpan fullHeader, int tlvOffset, out string error) + { + error = string.Empty; + var end = fullHeader.Length; + var pos = tlvOffset; + uint? crcValue = null; + int? crcValueOffset = null; + + while (pos < end) + { + if (end - pos < 3) + { + error = "PROXY v2 truncated TLV header"; + return false; + } + + var tlvType = fullHeader[pos]; + var tlvLen = (fullHeader[pos + 1] << 8) | fullHeader[pos + 2]; + var valueOff = pos + 3; + var valueEnd = valueOff + tlvLen; + if (valueEnd > end) + { + error = "PROXY v2 TLV length exceeds header"; + return false; + } + + if (tlvType == Pp2TypeCrc32C) + { + if (tlvLen != 4) + { + error = "PROXY v2 CRC32C TLV must be 4 octets"; + return false; + } + + if (crcValue is not null) + { + error = "PROXY v2 duplicate CRC32C TLV"; + return false; + } + + crcValue = BinaryPrimitives.ReadUInt32BigEndian(fullHeader.Slice(valueOff, 4)); + crcValueOffset = valueOff; + } + + pos = valueEnd; + } + + if (crcValue is not null && crcValueOffset is not null) + { + var mutable = fullHeader.ToArray(); + mutable.AsSpan(crcValueOffset.Value, 4).Clear(); + var calculated = Crc32C.Compute(mutable); + if (calculated != crcValue.Value) + { + error = "PROXY v2 CRC32C mismatch"; + return false; + } + } + + return true; + } + + private static bool TryParseV1Address(ReadOnlySpan token, bool ipv4, out IPAddress address) + { + address = IPAddress.None; + if (ipv4) + { + return TryParseV1Ipv4(token, out address); + } + + return TryParseV1Ipv6(token, out address); + } + + private static bool TryParseV1Ipv4(ReadOnlySpan token, out IPAddress address) + { + address = IPAddress.None; + Span octets = stackalloc byte[4]; + var octetIndex = 0; + var start = 0; + for (var i = 0; i <= token.Length; i++) + { + if (i == token.Length || token[i] == (byte)'.') + { + if (octetIndex >= 4) + { + return false; + } + + if (!TryParseDecimalNoLeadingZero(token[start..i], out var value) || value > 255) + { + return false; + } + + octets[octetIndex++] = (byte)value; + start = i + 1; + } + } + + if (octetIndex != 4) + { + return false; + } + + address = new IPAddress(octets); + return true; + } + + private static bool TryParseV1Ipv6(ReadOnlySpan token, out IPAddress address) + { + address = IPAddress.None; + if (!Ascii.IsValid(token)) + { + return false; + } + + var text = Encoding.ASCII.GetString(token); + if (text.Contains('.') || text.Split("::").Length > 2) + { + return false; + } + + string[] groups; + if (text.Contains("::", StringComparison.Ordinal)) + { + var parts = text.Split("::", 2); + var left = parts[0].Length == 0 ? [] : parts[0].Split(':', StringSplitOptions.RemoveEmptyEntries); + var right = parts[1].Length == 0 ? [] : parts[1].Split(':', StringSplitOptions.RemoveEmptyEntries); + groups = [.. left, .. right]; + } + else + { + groups = text.Split(':'); + if (groups.Length != 8) + { + return false; + } + } + + foreach (var group in groups) + { + if (group.Length is < 1 or > 4) + { + return false; + } + + foreach (var ch in group) + { + if (!Uri.IsHexDigit(ch)) + { + return false; + } + } + } + + return IPAddress.TryParse(text, out address!) + && address.AddressFamily == AddressFamily.InterNetworkV6; + } + + private static bool TryParseV1Port(ReadOnlySpan token, out int port) + { + port = 0; + if (!TryParseDecimalNoLeadingZero(token, out var value) || value > 65535) + { + return false; + } + + port = value; + return true; + } + + private static bool TryParseDecimalNoLeadingZero(ReadOnlySpan token, out int value) + { + value = 0; + if (token.IsEmpty) + { + return false; + } + + if (token.Length > 1 && token[0] == (byte)'0') + { + return false; + } + + foreach (var b in token) + { + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + + value = (value * 10) + (b - '0'); + } + + return true; + } + + private static List> SplitAsciiSpaces(ReadOnlySpan line) + { + // Materialize into owned segments for field indexing (v1 lines are ≤107 octets). + var owned = line.ToArray(); + var parts = new List>(6); + var start = 0; + for (var i = 0; i <= owned.Length; i++) + { + if (i == owned.Length || owned[i] == (byte)' ') + { + parts.Add(owned.AsMemory(start, i - start)); + start = i + 1; + } + } + + return parts; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Proxy/TrustedProxyHosts.cs b/src/VectorNNTPD.TransitService/Networking/Proxy/TrustedProxyHosts.cs new file mode 100644 index 00000000..011bdb76 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Proxy/TrustedProxyHosts.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Networking.Proxy +{ + /// + /// Parses and caches once for hot-path peer matching. + /// + public sealed class TrustedProxyHosts : ITrustedProxyHosts + { + private readonly HashSet _hosts; + + /// Initializes a new instance of the class. + public TrustedProxyHosts(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _hosts = BuildSet(options.Value.ProxyHosts); + } + + /// Creates an instance from an already-validated address list (tests). + internal TrustedProxyHosts(IEnumerable hosts) + { + ArgumentNullException.ThrowIfNull(hosts); + _hosts = new HashSet(hosts.Select(Canonicalize), IPAddressEqualityComparer.Instance); + } + + /// + public bool IsEnabled => _hosts.Count > 0; + + /// + public bool IsTrusted(IPAddress peerAddress) + { + ArgumentNullException.ThrowIfNull(peerAddress); + if (_hosts.Count == 0) + { + return false; + } + + return _hosts.Contains(Canonicalize(peerAddress)); + } + + /// Normalizes an address for PROXY peer trust comparison. + public static IPAddress Canonicalize(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return address is { AddressFamily: AddressFamily.InterNetworkV6, IsIPv4MappedToIPv6: true } ? address.MapToIPv4() : address; + } + + private static HashSet BuildSet(string[]? entries) + { + var set = new HashSet(IPAddressEqualityComparer.Instance); + if (entries is null) + { + return set; + } + + foreach (var entry in entries) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + if (!IPAddress.TryParse(entry.Trim(), out var address)) + { + continue; + } + + set.Add(Canonicalize(address)); + } + + return set; + } + + private sealed class IPAddressEqualityComparer : IEqualityComparer + { + public static readonly IPAddressEqualityComparer Instance = new(); + + public bool Equals(IPAddress? x, IPAddress? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return x.Equals(y); + } + + public int GetHashCode(IPAddress obj) => obj.GetHashCode(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/ConnectionByteTransport.cs b/src/VectorNNTPD.TransitService/Networking/Transport/ConnectionByteTransport.cs new file mode 100644 index 00000000..3197f4c7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/ConnectionByteTransport.cs @@ -0,0 +1,748 @@ +using System.Diagnostics; +using System.Net.Security; +using VectorNNTPD.TransitService.SessionState.RateLimiting; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Single owner of byte-stream I/O under pumps. + /// + /// + /// + /// Pumps call / only. They never touch the socket + /// or directly. The active stream may be replaced during an in-place TLS + /// upgrade after guarantees no outstanding stream I/O. + /// + /// + /// Quiescence blocks new application reads/writes at a gate, cancels in-flight reads + /// (freeing the socket for TLS handshake or DEFLATE wrap), and waits for in-flight writes + /// to finish under the caller token. Admission of an I/O operation and registration of that + /// operation as outstanding are atomic with respect to the quiescence state transition under + /// _gate. + /// + /// + internal sealed class ConnectionByteTransport : IAsyncDisposable + { + private const int StateActive = 0; + private const int StateReadsPaused = 1; + private const int StateQuiesced = 2; + private const int StateAbandoned = 3; + private const int StateDisposed = 4; + + private readonly object _gate = new(); + private Stream _stream; + private CancellationTokenSource _readCts = new(); + private TaskCompletionSource _resumeTcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource? _idleTcs; + private TaskCompletionSource? _readIdleTcs; + private int _activeReads; + private int _activeWrites; + private int _state; + private bool _isTls; + private bool _isCompressed; + private byte[]? _pendingUpgradePrefix; + private readonly OutboundRateLimiter _rateLimiter; + + public ConnectionByteTransport(Stream stream, bool isTls, OutboundRateLimiter? rateLimiter = null) + { + ArgumentNullException.ThrowIfNull(stream); + if (rateLimiter is not null) + { + _rateLimiter = rateLimiter; + _stream = stream; + } + else if (stream is OutboundRateLimiter existing) + { + _rateLimiter = existing; + _stream = existing; + } + else + { + _rateLimiter = new OutboundRateLimiter(stream, initialMaxSendBytesPerSecond: 0, leaveInnerOpen: false); + _stream = _rateLimiter; + } + + _isTls = isTls; + _resumeTcs.SetResult(); + } + + /// + /// Wire-level outbound cap. Sits under TLS/DEFLATE so throttled bytes are + /// the octets written toward the socket. + /// + internal OutboundRateLimiter RateLimiter => _rateLimiter; + + public bool IsTls + { + get + { + lock (_gate) + { + return _isTls; + } + } + } + + public bool IsCompressed + { + get + { + lock (_gate) + { + return _isCompressed; + } + } + } + + /// + /// Test-only: awaited after / writable gate returns and before admission under + /// _gate. Used to reproduce the historical Active-observed / not-yet-counted window. + /// + internal Func? AfterActiveBeforeAdmitProbe { get; set; } + + /// + /// Test-only: awaited inside after write admission and before the stream write. + /// Used to hold SendAsync between ReadAsync and AdvanceTo. + /// + internal Func? BeforeStreamWriteProbe { get; set; } + + /// + /// Test-only: awaited inside after read admission and before the stream read. + /// + internal Func? BeforeStreamReadProbe { get; set; } + + /// + /// Test-only: awaited after a successful stream read returns and before the count is + /// handed back to ReceiveAsync for Advance. + /// + internal Func? AfterStreamReadProbe { get; set; } + + /// DIAGNOSTIC-ONLY session attached when is enabled. + internal TransportIoSession? Io { get; set; } + + public async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilReadableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + CancellationToken readToken; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateActive) + { + continue; + } + + _activeReads++; + stream = _stream; + readToken = _readCts.Token; + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(readToken, cancellationToken); + try + { + if (BeforeStreamReadProbe is { } readProbe) + { + await readProbe().ConfigureAwait(false); + } + + var received = await ReadStreamAsync(stream, buffer, linked.Token).ConfigureAwait(false); + if (AfterStreamReadProbe is { } afterRead) + { + await afterRead().ConfigureAwait(false); + } + + return received; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + continue; + } + finally + { + CompleteOperation(isRead: true); + } + } + } + + public async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilWritableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateActive or StateReadsPaused)) + { + continue; + } + + _activeWrites++; + stream = _stream; + } + + try + { + if (BeforeStreamWriteProbe is { } writeProbe) + { + await writeProbe().ConfigureAwait(false); + } + + await WriteStreamAsync(stream, buffer, cancellationToken).ConfigureAwait(false); + return; + } + finally + { + CompleteOperation(isRead: false); + } + } + } + + /// + /// Flushes the active stream (required for DEFLATE sync-flush so compressed bytes reach the peer). + /// + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + await WaitUntilWritableAsync(cancellationToken).ConfigureAwait(false); + + if (AfterActiveBeforeAdmitProbe is { } probe) + { + await probe().ConfigureAwait(false); + } + + Stream stream; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateActive or StateReadsPaused)) + { + continue; + } + + _activeWrites++; + stream = _stream; + } + + try + { + await FlushStreamAsync(stream, cancellationToken).ConfigureAwait(false); + return; + } + finally + { + CompleteOperation(isRead: false); + } + } + } + + /// + /// Gets a value indicating whether the transport is in the reads-paused upgrade drain state. + /// + public bool IsReadsPaused + { + get + { + lock (_gate) + { + return _state is StateReadsPaused or StateQuiesced; + } + } + } + + /// + /// After a successful , decides whether bytes may enter the application input pipe. + /// + /// + /// When quiesced for TLS/DEFLATE upgrade, socket octets already read must not be injected into + /// — they are retained as a prefix for the upgraded stream + /// (ClientHello race after a STARTTLS 382 response). + /// + /// + /// if the caller should write to the application pipe; + /// if the bytes were claimed for the pending upgrade prefix. + /// + public bool TryCommitReadToApplication(ReadOnlySpan data) + { + lock (_gate) + { + if (_state is not (StateReadsPaused or StateQuiesced or StateAbandoned)) + { + return true; + } + + if (data.IsEmpty) + { + return false; + } + + if (_pendingUpgradePrefix is null) + { + _pendingUpgradePrefix = data.ToArray(); + } + else + { + var combined = new byte[_pendingUpgradePrefix.Length + data.Length]; + _pendingUpgradePrefix.CopyTo(combined, 0); + data.CopyTo(combined.AsSpan(_pendingUpgradePrefix.Length)); + _pendingUpgradePrefix = combined; + } + + return false; + } + } + + /// Takes any socket octets retained during quiescence for the upgraded stream wrap. + public ReadOnlyMemory TakePendingUpgradePrefix() + { + lock (_gate) + { + var prefix = _pendingUpgradePrefix; + _pendingUpgradePrefix = null; + return prefix ?? ReadOnlyMemory.Empty; + } + } + + /// + /// Restores Active I/O after a post-quiescence precondition failure (no stream replacement). + /// + public void ResumeFromQuiesceWithoutUpgrade() + { + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is not (StateQuiesced or StateReadsPaused)) + { + throw new InvalidOperationException("Transport is not quiesced."); + } + + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + _pendingUpgradePrefix = null; + } + + resume.TrySetResult(); + } + + /// + /// Cancels in-flight reads and blocks new reads while still allowing writes (outbound drain). + /// + public async Task PauseReadsAsync(CancellationToken cancellationToken) + { + CancellationTokenSource oldReadCts; + TaskCompletionSource? readIdle; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is StateReadsPaused or StateQuiesced) + { + throw new InvalidOperationException("Transport reads are already paused."); + } + + _state = StateReadsPaused; + _resumeTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + oldReadCts = _readCts; + _readCts = new CancellationTokenSource(); + if (_activeReads == 0) + { + readIdle = null; + } + else + { + readIdle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _readIdleTcs = readIdle; + } + } + + await oldReadCts.CancelAsync().ConfigureAwait(false); + oldReadCts.Dispose(); + + if (readIdle is not null) + { + await readIdle.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// After , blocks writes and waits until no stream I/O is outstanding. + /// + public async Task QuiesceWritesAsync(CancellationToken cancellationToken) + { + TaskCompletionSource idle; + bool alreadyIdle; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state == StateQuiesced) + { + throw new InvalidOperationException("Transport is already quiesced."); + } + + if (_state != StateReadsPaused) + { + throw new InvalidOperationException("Pause reads before quiescing writes."); + } + + _state = StateQuiesced; + idle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _idleTcs = idle; + alreadyIdle = _activeReads == 0 && _activeWrites == 0; + } + + if (alreadyIdle) + { + idle.TrySetResult(); + } + + await idle.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Stops application stream I/O and waits until the underlying stream has no outstanding operations. + /// + public async Task QuiesceAsync(CancellationToken cancellationToken) + { + await PauseReadsAsync(cancellationToken).ConfigureAwait(false); + await QuiesceWritesAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Returns the quiesced plain stream so the caller can wrap it in . + /// + public Stream TakeQuiescedStreamForTlsWrap() + { + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before TLS wrap."); + } + + if (_isTls) + { + throw new InvalidOperationException("Transport is already TLS-protected."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + return _stream; + } + } + + /// + /// Returns the quiesced stream (plain or TLS) so the caller can wrap it in . + /// + public Stream TakeQuiescedStreamForDeflateWrap() + { + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before DEFLATE wrap."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Transport is already DEFLATE-compressed."); + } + + return _stream; + } + } + + /// + /// Publishes an authenticated and resumes pump I/O. + /// + public void PublishTlsAndResume(SslStream sslStream) + { + ArgumentNullException.ThrowIfNull(sslStream); + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before publishing TLS."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Cannot publish TLS after DEFLATE is active."); + } + + _stream = sslStream; + _isTls = true; + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + } + + resume.TrySetResult(); + } + + /// + /// Publishes a bidirectional raw-DEFLATE stream and resumes pump I/O. + /// + public void PublishDeflateAndResume(NntpDeflateStream deflateStream) + { + ArgumentNullException.ThrowIfNull(deflateStream); + TaskCompletionSource resume; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state != StateQuiesced) + { + throw new InvalidOperationException("Transport must be quiesced before publishing DEFLATE."); + } + + if (_isCompressed) + { + throw new InvalidOperationException("Transport is already DEFLATE-compressed."); + } + + _stream = deflateStream; + _isCompressed = true; + _state = StateActive; + _idleTcs = null; + resume = _resumeTcs; + } + + resume.TrySetResult(); + } + + /// + /// Marks the transport abandoned after a failed upgrade and wakes waiters for connection teardown. + /// Does not restore plaintext I/O. + /// + public void AbortQuiesceForConnectionTeardown() + { + TaskCompletionSource? resume; + TaskCompletionSource? idle; + lock (_gate) + { + if (_state != StateQuiesced) + { + return; + } + + _state = StateAbandoned; + resume = _resumeTcs; + idle = _idleTcs; + _idleTcs = null; + } + + idle?.TrySetResult(); + resume.TrySetResult(); + } + + public async ValueTask DisposeAsync() + { + Stream stream; + CancellationTokenSource readCts; + TaskCompletionSource resume; + TaskCompletionSource? idle; + lock (_gate) + { + if (_state == StateDisposed) + { + return; + } + + _state = StateDisposed; + stream = _stream; + readCts = _readCts; + resume = _resumeTcs; + idle = _idleTcs; + _idleTcs = null; + } + + idle?.TrySetResult(); + resume.TrySetResult(); + await readCts.CancelAsync().ConfigureAwait(false); + readCts.Dispose(); + await stream.DisposeAsync().ConfigureAwait(false); + } + + private void ThrowIfUnavailable() + { + if (_state == StateAbandoned) + { + throw new ObjectDisposedException(nameof(ConnectionByteTransport), "Transport abandoned after failed TLS upgrade."); + } + + ObjectDisposedException.ThrowIf(_state == StateDisposed, this); + } + + private void CompleteOperation(bool isRead) + { + TaskCompletionSource? idle = null; + TaskCompletionSource? readIdle = null; + lock (_gate) + { + if (isRead) + { + _activeReads--; + } + else + { + _activeWrites--; + } + + if (isRead + && _state == StateReadsPaused + && _readIdleTcs is not null + && _activeReads == 0) + { + readIdle = _readIdleTcs; + _readIdleTcs = null; + } + + if (_state == StateQuiesced + && _idleTcs is not null + && _activeReads == 0 + && _activeWrites == 0) + { + idle = _idleTcs; + } + } + + readIdle?.TrySetResult(); + idle?.TrySetResult(); + } + + private async ValueTask ReadStreamAsync(Stream stream, Memory buffer, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + return await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.ReadAsync(buffer, cancellationToken); + if (vt.IsCompletedSuccessfully) + { + var n = vt.Result; + io.RecordReceive(buffer.Length, n, started, started, sync: true); + return n; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var received = await vt.ConfigureAwait(false); + io.RecordReceive(buffer.Length, received, started, awaitStart, sync: false); + return received; + } + + private async ValueTask WriteStreamAsync(Stream stream, ReadOnlyMemory buffer, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + await stream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + return; + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.WriteAsync(buffer, cancellationToken); + if (vt.IsCompletedSuccessfully) + { + io.RecordSend(buffer.Length, buffer.Length, started, started, sync: true); + return; + } + + var awaitStart = Stopwatch.GetTimestamp(); + await vt.ConfigureAwait(false); + io.RecordSend(buffer.Length, buffer.Length, started, awaitStart, sync: false); + } + + private async ValueTask FlushStreamAsync(Stream stream, CancellationToken cancellationToken) + { + var io = Io; + if (io is null) + { + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + return; + } + + var started = Stopwatch.GetTimestamp(); + var vt = stream.FlushAsync(cancellationToken); + if (vt.IsCompletedSuccessfully) + { + io.RecordFlush(started, started, sync: true); + return; + } + + var awaitStart = Stopwatch.GetTimestamp(); + await vt.ConfigureAwait(false); + io.RecordFlush(started, awaitStart, sync: false); + } + + private async Task WaitUntilReadableAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state == StateActive) + { + return; + } + + wait = _resumeTcs.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private async Task WaitUntilWritableAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfUnavailable(); + if (_state is StateActive or StateReadsPaused) + { + return; + } + + wait = _resumeTcs.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/INntpConnection.cs b/src/VectorNNTPD.TransitService/Networking/Transport/INntpConnection.cs new file mode 100644 index 00000000..b0cfc048 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/INntpConnection.cs @@ -0,0 +1,187 @@ +using System.IO.Pipelines; +using System.Net; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.SessionState.RateLimiting; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Byte-oriented NNTP connection transport exposed to a future session/protocol layer. + /// + /// + /// + /// carries octets received from the peer (after TLS decryption when applicable). + /// carries octets to send to the peer (before TLS encryption when applicable). + /// + /// + /// No NNTP greeting or command semantics are implemented. The transport does not keep an unused + /// connection alive beyond normal TCP/TLS lifetime and pipeline backpressure. + /// + /// + /// upgrades an established plaintext connection to TLS on the same + /// TCP socket. activates bidirectional raw DEFLATE (RFC 8054) + /// above the current byte stream (plain or TLS). The NNTP STARTTLS and COMPRESS commands live in the + /// session layer and invoke these transport upgrades when appropriate. + /// + /// + public interface INntpConnection : IAsyncDisposable + { + /// + /// Wire-level outbound rate cap, or when the connection + /// does not expose a limiter (test doubles). + /// + IOutboundRateCap? OutboundRate => null; + + /// Gets octets received from the peer (after TLS decryption / DEFLATE inflate when applicable). + PipeReader Input { get; } + + /// Gets the writer used by the application to send octets to the network. + PipeWriter Output { get; } + + /// Gets the remote TCP peer endpoint when known. + EndPoint? RemoteEndPoint { get; } + + /// Gets the local endpoint when known. + EndPoint? LocalEndPoint { get; } + + /// + /// Gets the immutable effective client identity established at connection initialization. + /// + ConnectionClientIdentity ClientIdentity { get; } + + /// Gets a value indicating whether the connection is TLS-protected. + bool IsTls { get; } + + /// + /// When TLS is active, returns the negotiated protocol and cipher suite labels for logging. + /// + /// Operational TLS version label (for example TLSv1.3). + /// Negotiated cipher suite name from . + /// when TLS parameters are available. + bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher); + + /// Gets a value indicating whether bidirectional raw DEFLATE is active. + bool IsCompressed { get; } + + /// Gets a token that is cancelled when the connection transport is shutting down. + CancellationToken ConnectionClosed { get; } + + /// + /// Gets a value indicating whether has been entered (transport is terminal). + /// + bool IsCompleted { get; } + + /// + /// Completes the transport (stops pumps, closes the socket). Safe to call multiple times. + /// + /// Optional exception indicating abortive completion. + Task CompleteAsync(Exception? exception = null); + + /// + /// Gets a monotonic generation bumped each time the send pump becomes idle (awaiting more Output). + /// + /// + /// For STARTTLS: first, capture this version, write 382, then + /// until the generation advances. + /// + long OutboundIdleVersion { get; } + + /// + /// Pauses application socket reads while still allowing writes (STARTTLS handoff ownership). + /// + /// + /// After this returns, successful receive completions are retained for the TLS upgrade prefix + /// instead of being committed to . Writes remain admitted so a 382 + /// response can still reach the peer. Must be called before writing 382. + /// + /// Cancellation token. + Task PauseReadsAsync(CancellationToken cancellationToken = default); + + /// + /// Waits until the send pump has become idle after . + /// + /// + /// captured before flushing pre-upgrade plaintext to . + /// + /// Cancellation token. + Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default); + + /// + /// Pauses application reads, then waits until the send pump has become idle after + /// . + /// + /// + /// Prefer before writing 382, then + /// . This combined helper is for callers that already + /// wrote the pre-upgrade response under an existing pause, or that pause immediately after write + /// when no peer ClientHello can race (test helpers). STARTTLS must pause before 382. + /// + Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default); + + /// + /// Performs an in-place server TLS handshake on the existing TCP socket. + /// + /// Provider used to acquire a connection-lifetime certificate lease. + /// Token that cancels the handshake (and aborts the connection on failure). + /// + /// + /// Preconditions: the connection must be plaintext and not DEFLATE-compressed; the caller must not + /// hold outstanding reads or writes; any plaintext that must + /// be visible to the peer before TLS (for example a STARTTLS 382 response) must already be + /// flushed to (STARTTLS pauses reads before writing 382). + /// The upgrade quiesces writes before inspecting ; any application plaintext + /// that still remains in is a precondition failure (connection stays plaintext). + /// Pipelined NNTP after STARTTLS must be discarded by the session (RFC 8143) before calling this + /// method. Reads may already be paused by . + /// + /// + /// On success, subsequent / traffic is TLS application data + /// on the same socket. is unchanged. On failure, the connection is + /// completed and does not fall back to plaintext. + /// + /// + /// + /// Already TLS, already compressed, upgrade in progress, or unconsumed plaintext remains in + /// (precondition failure does not complete the connection). + /// + /// The connection is closed or disposed. + Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default); + + /// + /// Activates bidirectional raw DEFLATE on the existing connection byte stream. + /// + /// Token that cancels activation (and aborts the connection on failure). + /// + /// + /// Layering (RFC 8054): DEFLATE sits above TLS when both are active — + /// NNTP → DEFLATE → TLS → TCP. TLS must be negotiated first when both are used; this method + /// rejects activation when DEFLATE is already active, and rejects + /// TLS after DEFLATE. + /// + /// + /// Preconditions: DEFLATE not already active; no outstanding reads / + /// writes; application data that belongs before compression must already be + /// consumed from ; any bytes that must reach the peer uncompressed (for example + /// a future 206 COMPRESS response) must already be flushed to . + /// + /// + /// On success, subsequent pipe traffic is compressed in both directions on the same socket and + /// pipes. is unchanged. Post-quiescence failure completes the + /// connection with no uncompressed fallback. + /// + /// + /// + /// Already compressed, upgrade in progress, or unconsumed application data remains in + /// (precondition failure does not complete the connection). + /// + /// The connection is closed or disposed. + Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/NntpConnection.cs b/src/VectorNNTPD.TransitService/Networking/Transport/NntpConnection.cs new file mode 100644 index 00000000..ab23911a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/NntpConnection.cs @@ -0,0 +1,1522 @@ +using System.Buffers; +using System.Diagnostics; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.SessionState.RateLimiting; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + using VectorNNTPD.Common.Core; + /// + /// Duplex pipe transport over a replaceable byte-stream I/O owner (). + /// + /// + /// + /// Ownership: this type owns the accepted socket and the (plain + /// or authenticated ). The application owns + /// consumption of / production to . + /// + /// + /// Plain connections may call to authenticate TLS on the same socket. + /// Plain or TLS connections may call to activate bidirectional raw + /// DEFLATE above the current byte stream (RFC 8054 layering: NNTP → DEFLATE → TLS → TCP). + /// Pumps keep running across upgrades; exclusive stream ownership is enforced by transport quiescence. + /// The NNTP STARTTLS and COMPRESS commands live in the session layer and invoke these upgrades. + /// + /// + /// NNTPD performs server-side TLS authentication only. Client certificates are never requested or validated. + /// + /// + public sealed class NntpConnection : INntpConnection + { + private const int ModePlain = 0; + private const int ModeUpgrading = 1; + private const int ModeTls = 2; + + private const int CompressionOff = 0; + private const int CompressionUpgrading = 1; + private const int CompressionOn = 2; + + private readonly ILogger _logger; + private readonly Pipe _inputPipe; + private readonly Pipe _outputPipe; + private readonly CancellationTokenSource _connectionCts = new(); + private readonly object _completeGate = new(); + private readonly ConnectionClientIdentity _clientIdentity; + private readonly IFeedDiagnostics? _feedDiagnostics; + private Socket? _socket; + private ConnectionByteTransport? _transport; + private TlsCertificateLease? _certificateLease; + private byte[]? _receivePrefix; + private Task? _receiveTask; + private Task? _sendTask; + private int _mode; + private int _compression; + private int _inplaceUpgradeBusy; + private int _completed; + private int _outputWriterCompleted; + private int _outputReaderCompleted; + private int _inputReaderCompleted; + private int _inputWriterCompleted; + private int _inputReaderConsumer; + private int _disconnectReason; + private int _disconnectedLogged; + private int _disposed; + private int _sendPumpAwaitingOutput; + private long _outboundIdleVersion; + private TaskCompletionSource? _outboundIdleWaiter; + private string? _negotiatedTlsVersion; + private string? _negotiatedCipher; + + private NntpConnection( + Socket socket, + ConnectionByteTransport transport, + EndPoint? remoteEndPoint, + EndPoint? localEndPoint, + bool isTls, + ConnectionClientIdentity clientIdentity, + ILogger logger, + IFeedDiagnostics? feedDiagnostics = null) + { + _socket = socket; + _transport = transport; + RemoteEndPoint = remoteEndPoint; + LocalEndPoint = localEndPoint; + _mode = isTls ? ModeTls : ModePlain; + _clientIdentity = clientIdentity; + _logger = logger; + _feedDiagnostics = feedDiagnostics; + var inputOptions = NntpPipeOptions.Create(); + var outputOptions = NntpPipeOptions.CreateOutput(); + _inputPipe = new Pipe(inputOptions); + _outputPipe = new Pipe(outputOptions); + if (transport.Io is { } io) + { + io.TxPipePauseBytes = outputOptions.PauseWriterThreshold; + io.TxPipeResumeBytes = outputOptions.ResumeWriterThreshold; + } + } + + /// + public PipeReader Input => _inputPipe.Reader; + + /// + public PipeWriter Output => _outputPipe.Writer; + + /// + public IOutboundRateCap? OutboundRate => _transport?.RateLimiter; + + /// + public EndPoint? RemoteEndPoint { get; } + + /// + public EndPoint? LocalEndPoint { get; } + + /// + public ConnectionClientIdentity ClientIdentity => _clientIdentity; + + /// + public bool IsTls => Volatile.Read(ref _mode) == ModeTls; + + /// + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + var version = Volatile.Read(ref _negotiatedTlsVersion); + var suite = Volatile.Read(ref _negotiatedCipher); + if (version is null || suite is null) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + tlsVersion = version; + cipher = suite; + return true; + } + + /// + public bool IsCompressed => Volatile.Read(ref _compression) == CompressionOn; + + /// + public long OutboundIdleVersion => Volatile.Read(ref _outboundIdleVersion); + + /// + public bool IsCompleted => Volatile.Read(ref _completed) == 1; + + /// Test-only: current byte transport (null after teardown closes it). + internal ConnectionByteTransport? ByteTransportForTests => _transport; + + /// Test-only: send pump task started by . + internal Task? SendPumpTaskForTests => _sendTask; + + /// Test-only: whether Output.Writer has been completed. + internal bool OutputWriterCompletedForTests => Volatile.Read(ref _outputWriterCompleted) == 1; + + /// Test-only: whether Output.Reader has been completed exactly once. + internal bool OutputReaderCompletedForTests => Volatile.Read(ref _outputReaderCompleted) == 1; + + /// Test-only: whether Input.Reader has been completed exactly once. + internal bool InputReaderCompletedForTests => Volatile.Read(ref _inputReaderCompleted) == 1; + + /// Test-only: whether Input.Writer has been completed exactly once. + internal bool InputWriterCompletedForTests => Volatile.Read(ref _inputWriterCompleted) == 1; + + /// Test-only: receive pump task started by . + internal Task? ReceivePumpTaskForTests => _receiveTask; + + /// Test-only: 1 when the send pump is awaiting more . + internal int SendPumpAwaitingOutputForTests => Volatile.Read(ref _sendPumpAwaitingOutput); + + /// + public Task PauseReadsAsync(CancellationToken cancellationToken = default) + { + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + return transport.PauseReadsAsync(cancellationToken); + } + + /// + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + WaitForOutboundIdleAfterAsync(outboundIdleVersionBeforeFlush, cancellationToken); + + /// + public async Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) + { + await PauseReadsAsync(cancellationToken).ConfigureAwait(false); + await WaitForOutboundIdleAfterAsync(outboundIdleVersionBeforeFlush, cancellationToken) + .ConfigureAwait(false); + } + + /// + public CancellationToken ConnectionClosed => _connectionCts.Token; + + /// Creates and starts a plain TCP connection transport. + public static NntpConnection StartPlain( + Socket socket, + ConnectionClientIdentity clientIdentity, + ILogger logger, + ReadOnlyMemory receivePrefix = default, + IFeedDiagnostics? feedDiagnostics = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentNullException.ThrowIfNull(logger); + + ConfigureAcceptedSocket(socket); + var network = new NetworkStream(socket, ownsSocket: false); + var transport = new ConnectionByteTransport(network, isTls: false) + { + Io = TransportIoProbe.CreateSession(), + }; + var connection = new NntpConnection( + socket, + transport, + TryGetRemote(socket), + TryGetLocal(socket), + isTls: false, + clientIdentity, + logger, + feedDiagnostics); + if (!receivePrefix.IsEmpty) + { + connection._receivePrefix = receivePrefix.ToArray(); + } + + connection.StartPumps(); + return connection; + } + + /// + /// Performs a server TLS handshake then starts the duplex pipe pumps over the encrypted stream. + /// + public static async Task StartTlsAsync( + Socket socket, + ITlsCertificateContextProvider certificateProvider, + ConnectionClientIdentity clientIdentity, + ILogger logger, + CancellationToken cancellationToken, + ReadOnlyMemory tlsPrefix = default, + IFeedDiagnostics? feedDiagnostics = null) + { + ArgumentNullException.ThrowIfNull(socket); + ArgumentNullException.ThrowIfNull(certificateProvider); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentNullException.ThrowIfNull(logger); + + ConfigureAcceptedSocket(socket); + var remote = TryGetRemote(socket); + var local = TryGetLocal(socket); + var lease = certificateProvider.Acquire(); + + var networkStream = new NetworkStream(socket, ownsSocket: false); + Stream sslInner = tlsPrefix.IsEmpty + ? networkStream + : new PrefixedStream(networkStream, tlsPrefix, leaveInnerOpen: false); + var limiter = new OutboundRateLimiter(sslInner, initialMaxSendBytesPerSecond: 0, leaveInnerOpen: false); + var sslStream = new SslStream(limiter, leaveInnerStreamOpen: false); + + try + { + await sslStream + .AuthenticateAsServerAsync(CreateServerAuthenticationOptions(lease.Context), cancellationToken) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await sslStream.DisposeAsync().ConfigureAwait(false); + lease.Dispose(); + socket.Dispose(); + throw; + } + + TlsNegotiationLogging.Capture(sslStream, out var tlsVersion, out var cipher); + var transport = new ConnectionByteTransport(sslStream, isTls: true, limiter) + { + Io = TransportIoProbe.CreateSession(), + }; + var connection = new NntpConnection(socket, transport, remote, local, isTls: true, clientIdentity, logger, feedDiagnostics) + { + _certificateLease = lease, + _negotiatedTlsVersion = tlsVersion, + _negotiatedCipher = cipher, + }; + + connection.StartPumps(); + TransportLogMessages.TlsHandshakeCompleted(logger, remote); + return connection; + } + + /// + public async Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(certificateProvider); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + if (Volatile.Read(ref _completed) == 1) + { + throw new ObjectDisposedException(nameof(NntpConnection), "Connection is closed."); + } + + if (Volatile.Read(ref _mode) == ModeTls) + { + throw new InvalidOperationException("Connection is already TLS-protected."); + } + + if (Volatile.Read(ref _compression) != CompressionOff) + { + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + if (Interlocked.CompareExchange(ref _inplaceUpgradeBusy, 1, 0) != 0) + { + throw new InvalidOperationException("A transport layer upgrade is already in progress."); + } + + var prior = Interlocked.CompareExchange(ref _mode, ModeUpgrading, ModePlain); + if (prior != ModePlain) + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + if (prior == ModeUpgrading) + { + throw new InvalidOperationException("A TLS upgrade is already in progress."); + } + + if (prior == ModeTls) + { + throw new InvalidOperationException("Connection is already TLS-protected."); + } + + throw new InvalidOperationException("Connection cannot be upgraded in its current state."); + } + + if (Volatile.Read(ref _compression) != CompressionOff) + { + Volatile.Write(ref _mode, ModePlain); + Volatile.Write(ref _inplaceUpgradeBusy, 0); + throw new InvalidOperationException("Cannot negotiate TLS after DEFLATE is active."); + } + + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + Exception? failure = null; + var quiesced = false; + var preconditionFailed = false; + + try + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _connectionCts.Token); + + // STARTTLS pauses reads before writing 382; other callers may still need a pause here. + if (!transport.IsReadsPaused) + { + await transport.PauseReadsAsync(linked.Token).ConfigureAwait(false); + } + + await WaitForOutboundIdleAsync(linked.Token).ConfigureAwait(false); + await transport.QuiesceWritesAsync(linked.Token).ConfigureAwait(false); + quiesced = true; + + try + { + EnsureApplicationInputDrainedForUpgrade(); + } + catch (InvalidOperationException) + { + preconditionFailed = true; + transport.ResumeFromQuiesceWithoutUpgrade(); + quiesced = false; + Volatile.Write(ref _mode, ModePlain); + throw; + } + + var lease = certificateProvider.Acquire(); + var plainStream = transport.TakeQuiescedStreamForTlsWrap(); + var upgradePrefix = transport.TakePendingUpgradePrefix(); + Stream sslInner = upgradePrefix.IsEmpty + ? plainStream + : new PrefixedStream(plainStream, upgradePrefix, leaveInnerOpen: false); + var sslStream = new SslStream(sslInner, leaveInnerStreamOpen: false); + + try + { + await sslStream + .AuthenticateAsServerAsync(CreateServerAuthenticationOptions(lease.Context), linked.Token) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + try + { + await sslStream.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; disposes plain NetworkStream as well. + } + + lease.Dispose(); + failure = ex; + throw; + } + + _certificateLease = lease; + TlsNegotiationLogging.Capture(sslStream, out var tlsVersion, out var cipher); + Volatile.Write(ref _negotiatedTlsVersion, tlsVersion); + Volatile.Write(ref _negotiatedCipher, cipher); + transport.PublishTlsAndResume(sslStream); + Volatile.Write(ref _mode, ModeTls); + TransportLogMessages.InPlaceTlsUpgradeCompleted(_logger, RemoteEndPoint); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (failure is null && !preconditionFailed) + { + failure = ex; + throw; + } + finally + { + if (failure is not null) + { + // Mark terminal and cancel ConnectionClosed before waking pumps so the session + // dispatcher cannot write a secondary NNTP status onto a dying transport. + if (Volatile.Read(ref _completed) == 0) + { + await CompleteAsync(failure).ConfigureAwait(false); + } + + if (quiesced) + { + transport.AbortQuiesceForConnectionTeardown(); + } + else if (Volatile.Read(ref _mode) == ModeUpgrading) + { + Volatile.Write(ref _mode, ModePlain); + } + + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + else + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + } + } + + /// + public async Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + if (Volatile.Read(ref _completed) == 1) + { + throw new ObjectDisposedException(nameof(NntpConnection), "Connection is closed."); + } + + if (Volatile.Read(ref _compression) == CompressionOn) + { + throw new InvalidOperationException("Connection is already DEFLATE-compressed."); + } + + if (Interlocked.CompareExchange(ref _inplaceUpgradeBusy, 1, 0) != 0) + { + throw new InvalidOperationException("A transport layer upgrade is already in progress."); + } + + var prior = Interlocked.CompareExchange(ref _compression, CompressionUpgrading, CompressionOff); + if (prior != CompressionOff) + { + Volatile.Write(ref _inplaceUpgradeBusy, 0); + if (prior == CompressionUpgrading) + { + throw new InvalidOperationException("A DEFLATE upgrade is already in progress."); + } + + throw new InvalidOperationException("Connection is already DEFLATE-compressed."); + } + + if (Volatile.Read(ref _mode) == ModeUpgrading) + { + Volatile.Write(ref _compression, CompressionOff); + Volatile.Write(ref _inplaceUpgradeBusy, 0); + throw new InvalidOperationException("A TLS upgrade is already in progress."); + } + + var transport = _transport ?? throw new ObjectDisposedException(nameof(NntpConnection)); + Exception? failure = null; + var quiesced = false; + var preconditionFailed = false; + + try + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _connectionCts.Token); + + if (!transport.IsReadsPaused) + { + await transport.PauseReadsAsync(linked.Token).ConfigureAwait(false); + } + + await WaitForOutboundIdleAsync(linked.Token).ConfigureAwait(false); + await transport.QuiesceWritesAsync(linked.Token).ConfigureAwait(false); + quiesced = true; + + try + { + EnsureApplicationInputDrainedForUpgrade(); + } + catch (InvalidOperationException) + { + preconditionFailed = true; + transport.ResumeFromQuiesceWithoutUpgrade(); + quiesced = false; + Volatile.Write(ref _compression, CompressionOff); + throw; + } + + var inner = transport.TakeQuiescedStreamForDeflateWrap(); + var upgradePrefix = transport.TakePendingUpgradePrefix(); + Stream deflateInner = upgradePrefix.IsEmpty + ? inner + : new PrefixedStream(inner, upgradePrefix, leaveInnerOpen: false); + var deflate = new NntpDeflateStream(deflateInner); + transport.PublishDeflateAndResume(deflate); + Volatile.Write(ref _compression, CompressionOn); + TransportLogMessages.InPlaceDeflateActivated(_logger, RemoteEndPoint); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (!preconditionFailed) + { + failure = ex; + throw; + } + finally + { + if (failure is not null) + { + if (Volatile.Read(ref _completed) == 0) + { + await CompleteAsync(failure).ConfigureAwait(false); + } + + if (quiesced) + { + transport.AbortQuiesceForConnectionTeardown(); + } + else if (Volatile.Read(ref _compression) == CompressionUpgrading) + { + Volatile.Write(ref _compression, CompressionOff); + } + } + + Volatile.Write(ref _inplaceUpgradeBusy, 0); + } + } + + /// + /// + /// + /// writer completion and connection cancellation may unblock + /// SendAsync. Output.Reader is owned exclusively by the send pump: this + /// method must not complete that reader while SendAsync can still + /// AdvanceTo an examined buffer. After the send pump returns, a best-effort + /// reader complete runs only when the pump did not complete the reader itself. + /// + /// + /// Connection cancellation unblocks the receive pump. Input.Writer is owned + /// exclusively by ReceiveAsync: this method must not complete the writer while + /// GetMemory / Advance can still run. The receive pump completes the writer + /// in finally after it exits. Completing the writer here while the session has + /// already completed the reader would CompletePipe and invalidate the writing head. + /// + /// + /// Input.Reader is owned exclusively by the session RX consumer + /// (NntpSession / NntpContinuousRxReader): this method must not complete + /// the reader while ReadAsync / AdvanceTo can still run. The session + /// completes the reader after the RX loop has stopped. + /// + /// + public async Task CompleteAsync(Exception? exception = null) + { + if (Interlocked.Exchange(ref _completed, 1) == 1) + { + return; + } + + NoteDisconnectReason(ClassifyCompleteReason(exception)); + LogTcpDisconnectedOnce(); + + try + { + await _connectionCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Already disposed. + } + + try + { + await _outputPipe.Writer.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + finally + { + Interlocked.Exchange(ref _outputWriterCompleted, 1); + } + + var receive = _receiveTask; + var send = _sendTask; + if (receive is not null) + { + try + { + await receive.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Observed via pump logging. + } + } + else + { + await CompleteInputWriterAsync(exception).ConfigureAwait(false); + } + + if (send is not null) + { + try + { + await send.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Observed via pump logging. + } + + if (!send.IsCompletedSuccessfully) + { + await CompleteOutputReaderBestEffortAsync(exception).ConfigureAwait(false); + } + } + else + { + await CompleteOutputReaderBestEffortAsync(exception).ConfigureAwait(false); + } + + WriteTransportIoDump(); + await CloseTransportSocketAsync().ConfigureAwait(false); + } + + private void WriteTransportIoDump() + { + var io = _transport?.Io; + if (io is null || !TransportIoProbe.IsEnabled || TransportIoProbe.OutputDirectory is not { } dir) + { + return; + } + + try + { + var label = RemoteEndPoint?.ToString() ?? "connection"; + io.Write(dir, label); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Diagnostic dump must not affect teardown. + } + } + + /// + /// Completes Output.Reader only when the send pump did not already do so. + /// + private async Task CompleteOutputReaderBestEffortAsync(Exception? exception) + { + if (Interlocked.Exchange(ref _outputReaderCompleted, 1) == 1) + { + return; + } + + try + { + await _outputPipe.Reader.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort after a faulted send pump. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await CompleteAsync().ConfigureAwait(false); + if (Volatile.Read(ref _inputReaderConsumer) == 0) + { + await CompleteInputReaderAsync().ConfigureAwait(false); + } + + _connectionCts.Dispose(); + _certificateLease?.Dispose(); + _certificateLease = null; + } + + /// + /// Records the first known disconnect cause. Later notes do not overwrite. + /// + internal void NoteDisconnectReason(TcpDisconnectReason reason) + { + if (reason == TcpDisconnectReason.Unspecified) + { + return; + } + + Interlocked.CompareExchange(ref _disconnectReason, (int)reason, (int)TcpDisconnectReason.Unspecified); + } + + /// Test-only: first-wins disconnect reason recorded for this connection. + internal TcpDisconnectReason DisconnectReasonForTests => + (TcpDisconnectReason)Volatile.Read(ref _disconnectReason); + + private static TcpDisconnectReason ClassifyCompleteReason(Exception? exception) => + exception switch + { + null => TcpDisconnectReason.LocalClose, + OperationCanceledException => TcpDisconnectReason.Cancellation, + _ => TcpDisconnectReason.ConnectionClosed, + }; + + private void LogTcpDisconnectedOnce() + { + if (Interlocked.Exchange(ref _disconnectedLogged, 1) == 1) + { + return; + } + + var reason = (TcpDisconnectReason)Volatile.Read(ref _disconnectReason); + if (reason == TcpDisconnectReason.Unspecified) + { + reason = TcpDisconnectReason.ConnectionClosed; + } + + ConnectionAcceptanceLogging.LogDisconnected( + _logger, + RemoteEndPoint, + LocalEndPoint, + reason.ToString()); + } + + /// + /// Marks the session RX loop as the exclusive reader owner. + /// + internal void AttachInputReaderConsumer() => Interlocked.Exchange(ref _inputReaderConsumer, 1); + + /// + /// Completes after the application consumer has stopped reading. + /// Safe to call multiple times. + /// + internal async Task CompleteInputReaderAsync() + { + Interlocked.Exchange(ref _inputReaderConsumer, 0); + if (Interlocked.Exchange(ref _inputReaderCompleted, 1) == 1) + { + return; + } + + try + { + await _inputPipe.Reader.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; reader may already be completed by a test host. + } + } + + /// + /// Completes Input.Writer exactly once. Owned by ReceiveAsync; + /// calls this only when no receive pump was started. + /// + private async Task CompleteInputWriterAsync(Exception? exception) + { + if (Interlocked.Exchange(ref _inputWriterCompleted, 1) == 1) + { + return; + } + + try + { + await _inputPipe.Writer.CompleteAsync(exception).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + /// Shared server TLS options (server auth only; no client certificates). + internal static SslServerAuthenticationOptions CreateServerAuthenticationOptions( + SslStreamCertificateContext certificateContext) => + new() + { + ServerCertificateContext = certificateContext, + ClientCertificateRequired = false, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }; + + private void StartPumps() + { + _receiveTask = ReceiveAsync(); + _sendTask = SendAsync(); + _ = ObservePumpAsync(_receiveTask, "receive"); + _ = ObservePumpAsync(_sendTask, "send"); + } + + private void EnsureApplicationInputDrainedForUpgrade() + { + if (!_inputPipe.Reader.TryRead(out var result)) + { + return; + } + + try + { + if (!result.Buffer.IsEmpty) + { + throw new InvalidOperationException( + "Cannot upgrade to TLS while unconsumed plaintext remains in Input. " + + "Drain application data before calling UpgradeToTlsAsync."); + } + } + finally + { + // Leave unconsumed bytes in place for the caller (do not examine past start on failure). + _inputPipe.Reader.AdvanceTo(result.Buffer.Start, result.Buffer.Start); + } + } + + private async Task ObservePumpAsync(Task pump, string name) + { + Exception? error = null; + try + { + await pump.ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (_connectionCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + error = ex; + TransportLogMessages.PumpEndedWithError(_logger, ex, name); + NoteDisconnectReason( + string.Equals(name, "send", StringComparison.Ordinal) + ? TcpDisconnectReason.SendError + : TcpDisconnectReason.ReceiveError); + } + finally + { + if (Volatile.Read(ref _completed) == 0) + { + _ = CompleteAsync(error); + } + } + } + + private async Task WaitForOutboundIdleAsync(CancellationToken cancellationToken) => + await WaitForOutboundIdleAfterAsync(Volatile.Read(ref _outboundIdleVersion) - 1, cancellationToken) + .ConfigureAwait(false); + + private async Task WaitForOutboundIdleAfterAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (Volatile.Read(ref _sendPumpAwaitingOutput) == 1 + && Volatile.Read(ref _outboundIdleVersion) > outboundIdleVersionBeforeFlush) + { + return; + } + + var waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Volatile.Write(ref _outboundIdleWaiter, waiter); + + if (Volatile.Read(ref _sendPumpAwaitingOutput) == 1 + && Volatile.Read(ref _outboundIdleVersion) > outboundIdleVersionBeforeFlush) + { + Volatile.Write(ref _outboundIdleWaiter, null); + waiter.TrySetResult(); + return; + } + + await waiter.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private void SignalOutboundIdle() + { + Interlocked.Increment(ref _outboundIdleVersion); + Volatile.Write(ref _sendPumpAwaitingOutput, 1); + var waiter = Interlocked.Exchange(ref _outboundIdleWaiter, null); + waiter?.TrySetResult(); + } + + private void SignalOutboundBusy() => Volatile.Write(ref _sendPumpAwaitingOutput, 0); + + private static async ValueTask FlushInputAsync( + PipeWriter writer, + CancellationToken token, + TransportIoSession? io) + { + if (io is null) + { + return await writer.FlushAsync(token).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = writer.FlushAsync(token); + if (vt.IsCompletedSuccessfully) + { + io.RecordRxPipeFlush(started, started, sync: true); + return vt.Result; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var result = await vt.ConfigureAwait(false); + io.RecordRxPipeFlush(started, awaitStart, sync: false); + return result; + } + + private static async ValueTask ReadOutputAsync( + PipeReader reader, + CancellationToken token, + TransportIoSession? io) + { + if (io is null) + { + return await reader.ReadAsync(token).ConfigureAwait(false); + } + + var started = Stopwatch.GetTimestamp(); + var vt = reader.ReadAsync(token); + if (vt.IsCompletedSuccessfully) + { + var completed = vt.Result; + io.RecordTxPipeWait(started, started, sync: true); + io.RecordTxReadShape(completed.Buffer); + return completed; + } + + var awaitStart = Stopwatch.GetTimestamp(); + var result = await vt.ConfigureAwait(false); + io.RecordTxPipeWait(started, awaitStart, sync: false); + io.RecordTxReadShape(result.Buffer); + return result; + } + + private async Task ReceiveAsync() + { + Exception? error = null; + try + { + var transport = _transport ?? throw new InvalidOperationException("Transport missing."); + var writer = _inputPipe.Writer; + var token = _connectionCts.Token; + + var prefix = Interlocked.Exchange(ref _receivePrefix, null); + if (prefix is { Length: > 0 }) + { + var memory = writer.GetMemory(prefix.Length); + prefix.CopyTo(memory); + writer.Advance(prefix.Length); + RecordReceivedBytes(prefix.Length); + var prefixFlush = await FlushInputAsync(writer, token, transport.Io).ConfigureAwait(false); + if (prefixFlush.IsCompleted || prefixFlush.IsCanceled) + { + return; + } + } + + while (!token.IsCancellationRequested) + { + var memory = writer.GetMemory(NntpPipeOptions.MinimumSegmentSize); + var bytes = await transport.ReadAsync(memory, token).ConfigureAwait(false); + if (bytes == 0) + { + if (!token.IsCancellationRequested) + { + NoteDisconnectReason(TcpDisconnectReason.RemoteClosed); + } + + break; + } + + // If Quiesce completed while this read returned, octets belong to the upgrade wrap + // (e.g. TLS ClientHello), not the application Input pipe. + if (!transport.TryCommitReadToApplication(memory.Span[..bytes])) + { + continue; + } + + RecordReceivedBytes(bytes); + writer.Advance(bytes); + var flush = await FlushInputAsync(writer, token, transport.Io).ConfigureAwait(false); + if (flush.IsCompleted || flush.IsCanceled) + { + break; + } + } + } + catch (OperationCanceledException ex) when (_connectionCts.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown; complete the writer without faulting the reader. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + await CompleteInputWriterAsync(error).ConfigureAwait(false); + } + } + + private void RecordReceivedBytes(int bytes) + { + if (bytes > 0 && _feedDiagnostics is { IsEnabled: true }) + { + _feedDiagnostics.RecordTcpBytes(bytes); + } + } + + private async Task SendAsync() + { + var transport = _transport ?? throw new InvalidOperationException("Transport missing."); + var reader = _outputPipe.Reader; + var token = _connectionCts.Token; + var socket = _socket; + var io = transport.Io; + io?.RecordSendLoopStart(); + var iterationEndTs = 0L; + TxWriteCoalescer? coalescer = null; + + while (!token.IsCancellationRequested) + { + SignalOutboundIdle(); + var readIssuedTs = 0L; + if (io is not null) + { + readIssuedTs = Stopwatch.GetTimestamp(); + if (iterationEndTs != 0) + { + io.RecordAdvanceToNextRead(iterationEndTs, readIssuedTs); + } + } + + var result = await ReadOutputAsync(reader, token, io).ConfigureAwait(false); + var readDoneTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + SignalOutboundBusy(); + var buffer = result.Buffer; + var consumed = buffer.Start; + var firstWriteTs = 0L; + var lastWriteEndTs = 0L; + try + { + if (buffer.IsEmpty && result.IsCompleted) + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + break; + } + + if (TxWriteGranularityExperiment.TryResolveTarget(transport, out var targetBytes)) + { + if (io is not null) + { + io.WriteGranularityTargetBytes = targetBytes; + } + + if (coalescer is null || coalescer.TargetBytes != targetBytes) + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + firstWriteTs = lastWriteEndTs; + } + + coalescer = new TxWriteCoalescer(targetBytes); + } + + lastWriteEndTs = await CopyAndWriteAggregatesAsync( + transport, + coalescer, + buffer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + else + { + if (coalescer is { PendingCount: > 0 }) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + coalescer.Clear(); + coalescer = null; + } + + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var writeStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + if (firstWriteTs == 0) + { + firstWriteTs = writeStartTs; + io.RecordReadToFirstWrite(readDoneTs, firstWriteTs); + } + else + { + io.RecordBetweenWrites(lastWriteEndTs, writeStartTs); + } + } + + await transport.WriteAsync(segment, token).ConfigureAwait(false); + if (io is not null) + { + lastWriteEndTs = Stopwatch.GetTimestamp(); + io.RecordTransportWrite(writeStartTs, lastWriteEndTs); + } + } + + // Required for DEFLATE: sync-flush compressed bytes to the peer without ending the stream. + // No-op / cheap for NetworkStream and SslStream over NetworkStream. + await transport.FlushAsync(token).ConfigureAwait(false); + } + + consumed = buffer.End; + } + finally + { + var advanceStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null && lastWriteEndTs != 0) + { + io.RecordLastWriteToAdvance(lastWriteEndTs, advanceStartTs); + } + + reader.AdvanceTo(consumed); + if (io is not null) + { + var advanceEndTs = Stopwatch.GetTimestamp(); + io.RecordAdvanceTo(advanceStartTs, advanceEndTs); + iterationEndTs = advanceEndTs; + } + } + + if (coalescer is not null && !token.IsCancellationRequested) + { + var drained = await DrainImmediateReadsAsync( + reader, + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + lastWriteEndTs = drained.LastWriteEndTs; + + if (coalescer.PendingCount > 0) + { + _ = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + if (drained.Completed) + { + break; + } + } + + if (result.IsCompleted) + { + break; + } + } + + io?.RecordSendLoopEnd(); + await reader.CompleteAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _outputReaderCompleted, 1); + + // Half-close TCP send only for plaintext end-of-stream; TLS shutdown is via SslStream dispose. + if (socket is not null && !IsTls) + { + try + { + socket.Shutdown(SocketShutdown.Send); + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private static async ValueTask<(long LastWriteEndTs, bool Completed)> DrainImmediateReadsAsync( + PipeReader reader, + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + var completed = false; + while (reader.TryRead(out var extra)) + { + var extraConsumed = extra.Buffer.Start; + try + { + if (extra.Buffer.IsEmpty) + { + completed = extra.IsCompleted; + break; + } + + io?.RecordTxReadShape(extra.Buffer); + lastWriteEndTs = await CopyAndWriteAggregatesAsync( + transport, + coalescer, + extra.Buffer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + extraConsumed = extra.Buffer.End; + completed = extra.IsCompleted; + } + finally + { + reader.AdvanceTo(extraConsumed); + } + + if (completed) + { + break; + } + } + + return (lastWriteEndTs, completed); + } + + private static async ValueTask CopyAndWriteAggregatesAsync( + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + ReadOnlySequence buffer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var offset = 0; + while (offset < segment.Length) + { + if (coalescer.RemainingCapacity == 0) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + firstWriteTs = firstWriteTs == 0 ? lastWriteEndTs : firstWriteTs; + } + + offset += coalescer.Copy(segment.Span[offset..], io); + } + } + + if (coalescer.IsFull) + { + lastWriteEndTs = await WriteCoalescedPendingAsync( + transport, + coalescer, + io, + readDoneTs, + firstWriteTs, + lastWriteEndTs, + token).ConfigureAwait(false); + } + + return lastWriteEndTs; + } + + private static async ValueTask WriteCoalescedPendingAsync( + ConnectionByteTransport transport, + TxWriteCoalescer coalescer, + TransportIoSession? io, + long readDoneTs, + long firstWriteTs, + long lastWriteEndTs, + CancellationToken token) + { + if (coalescer.PendingCount == 0) + { + return lastWriteEndTs; + } + + var writeStartTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + if (firstWriteTs == 0 && lastWriteEndTs == 0) + { + io.RecordReadToFirstWrite(readDoneTs, writeStartTs); + } + else + { + io.RecordBetweenWrites(lastWriteEndTs, writeStartTs); + } + } + + await transport.WriteAsync(coalescer.PendingMemory, token).ConfigureAwait(false); + var writeEndTs = io is not null ? Stopwatch.GetTimestamp() : 0L; + if (io is not null) + { + io.RecordTransportWrite(writeStartTs, writeEndTs); + } + + await transport.FlushAsync(token).ConfigureAwait(false); + coalescer.Clear(); + return writeEndTs; + } + + private async Task CloseTransportSocketAsync() + { + ConnectionByteTransport? transport; + Socket? socket; + lock (_completeGate) + { + transport = _transport; + _transport = null; + socket = _socket; + _socket = null; + } + + if (transport is not null) + { + try + { + await transport.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + if (socket is not null) + { + try + { + socket.Dispose(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + } + + private static void ConfigureAcceptedSocket(Socket socket) + { + try + { + socket.NoDelay = true; + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private static EndPoint? TryGetRemote(Socket socket) + { + try + { + return socket.RemoteEndPoint; + } + catch (SocketException) + { + return null; + } + catch (ObjectDisposedException) + { + return null; + } + } + + private static EndPoint? TryGetLocal(Socket socket) + { + try + { + return socket.LocalEndPoint; + } + catch (SocketException) + { + return null; + } + catch (ObjectDisposedException) + { + return null; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/NntpDeflateStream.cs b/src/VectorNNTPD.TransitService/Networking/Transport/NntpDeflateStream.cs new file mode 100644 index 00000000..7de0187a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/NntpDeflateStream.cs @@ -0,0 +1,148 @@ +using System.IO.Compression; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Bidirectional raw DEFLATE transform for an NNTP connection byte stream (RFC 8054 / RFC 1951). + /// + /// + /// + /// Wraps an underlying duplex stream (plain or + /// ) with independent compressor and decompressor state. + /// Application writes are compressed toward the peer; application reads are decompressed from the peer. + /// + /// + /// Wire format is raw DEFLATE (zlib windowBits in the negative range), matching + /// RFC 8054 §4. This uses , which does not emit a zlib (RFC 1950) + /// wrapper or a gzip wrapper. Do not substitute or . + /// + /// + /// forwards to + /// . On .NET 10 that + /// implementation uses DEFLATE sync-flush semantics so pending compressed bytes become visible on + /// the underlying stream while retaining the sliding dictionary. Finalization (end-of-stream + /// trailer) occurs only when this stream is disposed. + /// + /// + /// Concurrent and + /// are supported + /// (full-duplex): the inflater only reads the inner stream; the deflater only writes it. + /// + /// + internal sealed class NntpDeflateStream : Stream + { + private readonly Stream _inner; + private readonly DeflateStream _compressor; + private readonly DeflateStream _decompressor; + private bool _disposed; + + /// + /// Creates a duplex raw-DEFLATE layer over . + /// + /// Underlying duplex byte stream; ownership transfers to this instance. + /// zlib-compatible compression level for the outbound compressor. + public NntpDeflateStream(Stream inner, CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + ArgumentNullException.ThrowIfNull(inner); + if (!inner.CanRead || !inner.CanWrite) + { + throw new ArgumentException("Inner stream must be readable and writable.", nameof(inner)); + } + + _inner = inner; + // leaveOpen: true — this type owns and disposes the inner stream exactly once. + _compressor = new DeflateStream(inner, compressionLevel, leaveOpen: true); + _decompressor = new DeflateStream(inner, CompressionMode.Decompress, leaveOpen: true); + } + + /// + public override bool CanRead => !_disposed; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => !_disposed; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public override void Flush() => _compressor.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _compressor.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) => + _decompressor.Read(buffer, offset, count); + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + _decompressor.ReadAsync(buffer, offset, count, cancellationToken); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + _decompressor.ReadAsync(buffer, cancellationToken); + + /// + public override void Write(byte[] buffer, int offset, int count) => + _compressor.Write(buffer, offset, count); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + _compressor.WriteAsync(buffer, offset, count, cancellationToken); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => + _compressor.WriteAsync(buffer, cancellationToken); + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + // Compressor dispose finalizes the DEFLATE stream (writes remaining bits / end marker). + _compressor.Dispose(); + _decompressor.Dispose(); + _inner.Dispose(); + } + + _disposed = true; + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + await _compressor.DisposeAsync().ConfigureAwait(false); + await _decompressor.DisposeAsync().ConfigureAwait(false); + await _inner.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/NntpPipeOptions.cs b/src/VectorNNTPD.TransitService/Networking/Transport/NntpPipeOptions.cs new file mode 100644 index 00000000..ecbacd3d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/NntpPipeOptions.cs @@ -0,0 +1,57 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Centralized defaults for NNTP connection transports. + /// + /// + /// Per-connection thresholds (not global). Values are engineering defaults for bounded + /// memory under many concurrent clients; they are not a claim about 10/40 Gbps throughput. + /// + public static class NntpPipeOptions + { + /// Pause the pipe writer (and thus receive) when unread bytes reach this size. + public const long PauseWriterThreshold = 64 * 1024; + + /// Resume the pipe writer after unread bytes fall to this size. + public const long ResumeWriterThreshold = 32 * 1024; + + /// Minimum pooled segment size for pipe buffers. + public const int MinimumSegmentSize = 4 * 1024; + + /// Creates reader/writer options for the input (RX) side of a duplex connection transport. + public static PipeOptions Create() => + CreateCore(PauseWriterThreshold, ResumeWriterThreshold, PipeScheduler.ThreadPool, PipeScheduler.ThreadPool); + + /// + /// TX output-pipe options. Same thresholds as unless + /// is configured. Reader and writer schedulers + /// are . Input/RX pipes must keep using + /// . + /// + internal static PipeOptions CreateOutput() + { + if (TxPipePauseExperiment.TryGetEffective(out var pause, out var resume)) + { + return CreateCore(pause, resume, PipeScheduler.Inline, PipeScheduler.Inline); + } + + return CreateCore(PauseWriterThreshold, ResumeWriterThreshold, PipeScheduler.Inline, PipeScheduler.Inline); + } + + private static PipeOptions CreateCore( + long pauseWriterThreshold, + long resumeWriterThreshold, + PipeScheduler readerScheduler, + PipeScheduler writerScheduler) => + new( + pool: null, + readerScheduler: readerScheduler, + writerScheduler: writerScheduler, + pauseWriterThreshold: pauseWriterThreshold, + resumeWriterThreshold: resumeWriterThreshold, + minimumSegmentSize: MinimumSegmentSize, + useSynchronizationContext: false); + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/SocketPayloadSender.cs b/src/VectorNNTPD.TransitService/Networking/Transport/SocketPayloadSender.cs new file mode 100644 index 00000000..c26cd975 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/SocketPayloadSender.cs @@ -0,0 +1,62 @@ +using System.Net.Sockets; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Completes a TCP send of an entire buffer, looping on partial sends. + /// + internal static class SocketPayloadSender + { + /// + /// Sends on until every byte is accepted or the + /// operation fails / is cancelled. + /// + public static ValueTask SendAllAsync( + Socket socket, + ReadOnlyMemory buffer, + CancellationToken cancellationToken) => + SendAllAsync( + static (memory, token, state) => + ((Socket)state!).SendAsync(memory, SocketFlags.None, token), + buffer, + socket, + cancellationToken); + + /// + /// Testable send loop over an arbitrary send delegate (no heap copy of payload). + /// + /// + /// Sends as many bytes as possible from the supplied memory; return value is the count accepted. + /// + /// Payload to transmit completely. + /// Opaque state passed to . + /// Cancellation token. + public static async ValueTask SendAllAsync( + Func, CancellationToken, object?, ValueTask> sendAsync, + ReadOnlyMemory buffer, + object? state, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(sendAsync); + + var remaining = buffer; + while (!remaining.IsEmpty) + { + cancellationToken.ThrowIfCancellationRequested(); + var sent = await sendAsync(remaining, cancellationToken, state).ConfigureAwait(false); + if (sent == 0) + { + throw new IOException("Socket send transferred zero bytes."); + } + + if ((uint)sent > (uint)remaining.Length) + { + throw new IOException( + $"Socket send reported {sent} bytes but only {remaining.Length} were requested."); + } + + remaining = remaining[sent..]; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TcpDisconnectReason.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TcpDisconnectReason.cs new file mode 100644 index 00000000..2e25d38d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TcpDisconnectReason.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// Why an established TCP connection completed. Values are only those the existing + /// teardown paths can set from known cause, not inferred from timing. + /// + internal enum TcpDisconnectReason + { + /// No path has recorded a cause yet. + Unspecified = 0, + + /// Receive pump observed a 0-byte read (peer FIN) before local cancellation. + RemoteClosed = 1, + + /// was invoked without an error or prior cause. + LocalClose = 2, + + /// The receive pump faulted. + ReceiveError = 3, + + /// The send pump faulted. + SendError = 4, + + /// was invoked with . + Cancellation = 5, + + /// Listener stop completed remaining connections. + Shutdown = 6, + + /// Session RequestClose (QUIT and other protocol-initiated close). + ProtocolClose = 7, + + /// Completed with an exception that is not a receive/send pump fault. + ConnectionClosed = 8, + + /// No NNTP command was executed within Nntpd:IdleTime. + IdleTimeout = 9, + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TlsNegotiationLogging.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TlsNegotiationLogging.cs new file mode 100644 index 00000000..07814779 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TlsNegotiationLogging.cs @@ -0,0 +1,33 @@ +using System.Net.Security; +using System.Security.Authentication; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// Formats negotiated parameters for operational logs. + internal static class TlsNegotiationLogging + { + /// Builds the STARTTLS / TLS acceptance detail fragment. + public static string FormatDetail(string tlsVersion, string cipher) => + $"TlsVersion={tlsVersion}, Cipher={cipher}"; + + /// Maps to the operational TLSv1.x label. + public static string FormatSslProtocol(SslProtocols protocol) => + protocol switch + { + SslProtocols.Tls13 => "TLSv1.3", + SslProtocols.Tls12 => "TLSv1.2", + _ => protocol.ToString(), + }; + + /// Formats the negotiated cipher suite name. + public static string FormatCipherSuite(TlsCipherSuite cipherSuite) => cipherSuite.ToString(); + + /// Captures version and cipher labels from an authenticated . + public static void Capture(SslStream sslStream, out string tlsVersion, out string cipher) + { + ArgumentNullException.ThrowIfNull(sslStream); + tlsVersion = FormatSslProtocol(sslStream.SslProtocol); + cipher = FormatCipherSuite(sslStream.NegotiatedCipherSuite); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoProbe.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoProbe.cs new file mode 100644 index 00000000..911055f1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoProbe.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default ConnectionByteTransport socket/Pipe cadence sampler. + /// + /// + /// Disabled unless VECTORNNTP_TRANSPORT_IO is set or is called. + /// When disabled, no session state is allocated and transport I/O is unchanged. + /// This is not a production architecture change. + /// + internal static class TransportIoProbe + { + /// Environment variable that enables transport I/O cadence sampling. + public const string EnvironmentVariableName = "VECTORNNTP_TRANSPORT_IO"; + + private static ProbeState? _state; + + static TransportIoProbe() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var dir = value.Equals("1", StringComparison.OrdinalIgnoreCase) || + value.Equals("true", StringComparison.OrdinalIgnoreCase) + ? DefaultDirectory() + : value; + Enable(dir); + } + + /// Gets whether cadence timestamps should be recorded. + public static bool IsEnabled => Volatile.Read(ref _state) is not null; + + /// Gets the directory that receives connection dumps, or . + public static string? OutputDirectory => Volatile.Read(ref _state)?.Directory; + + /// Enables sampling and writes dumps under . + public static void Enable(string outputDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(outputDirectory); + Directory.CreateDirectory(outputDirectory); + Volatile.Write(ref _state, new ProbeState(outputDirectory)); + } + + /// Disables sampling (tests). + public static void Disable() => Volatile.Write(ref _state, null); + + /// Creates a per-connection log when enabled; otherwise . + public static TransportIoSession? CreateSession() => + IsEnabled ? new TransportIoSession() : null; + + /// + /// Repo-local diagnostic dump directory. + /// + /// + /// Deliberately walks from the process working directory toward a folder that + /// contains VectorNNTPD.TransitService.sln or .artifacts/. This is + /// development-only output, off unless is set. + /// It is not application-local runtime state and must not be used for certs/logs. + /// + internal static string DefaultDirectory() + { + var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.TransitService.sln")) || + Directory.Exists(Path.Join(dir.FullName, ".artifacts"))) + { + return Path.Join(dir.FullName, ".artifacts", "transport-io"); + } + + dir = dir.Parent; + } + + return Path.Join(Directory.GetCurrentDirectory(), ".artifacts", "transport-io"); + } + + private sealed class ProbeState(string directory) + { + public string Directory { get; } = directory; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoReport.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoReport.cs new file mode 100644 index 00000000..8ff32581 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoReport.cs @@ -0,0 +1,294 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// DIAGNOSTIC-ONLY formatter for . + internal static class TransportIoReport + { + /// Formats a human-readable cadence report. + public static string Format(in TransportIoSnapshot snapshot, string sessionLabel) + { + var sb = new StringBuilder(); + sb.AppendLine("CONNECTION BYTE TRANSPORT I/O CADENCE — DIAGNOSTIC"); + sb.AppendLine(); + sb.AppendLine("Off-by-default. Production path is unchanged when VECTORNNTP_TRANSPORT_IO is unset."); + sb.AppendLine("Clock: Stopwatch.GetTimestamp (monotonic). Durations in microseconds."); + sb.AppendLine("Stream ops are NetworkStream/SslStream ReadAsync/WriteAsync/FlushAsync, not raw Socket APIs."); + sb.AppendLine("Sync = ValueTask.IsCompletedSuccessfully before await. Await duration is 0 when sync."); + sb.AppendLine("RX pipe flush is PipeWriter.FlushAsync after Advance (may include backpressure wait)."); + sb.AppendLine("TX pipe wait is PipeReader.ReadAsync and includes waiting for the application producer."); + sb.AppendLine(); + sb.AppendLine($"Session: {sessionLabel}"); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"Connection elapsed: {snapshot.ConnectionElapsed.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"TX pipe pause: {snapshot.TxPipePauseBytes:N0} bytes")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $"TX pipe resume: {snapshot.TxPipeResumeBytes:N0} bytes")); + var experiment = TxPipePauseExperiment.IsConfigured ? "active (VECTORNNTP_TX_PIPE_PAUSE)" : "inactive (production 64 KiB)"; + sb.AppendLine($"TX pipe experiment: {experiment}"); + var coalesce = snapshot.Coalesce.TargetBytes > 0 + ? $"active (VECTORNNTP_TX_WRITE_GRANULARITY {snapshot.Coalesce.TargetBytes:N0} bytes)" + : "inactive (one WriteAsync per Pipe segment)"; + sb.AppendLine($"TX write coalesce: {coalesce}"); + sb.AppendLine(); + + WriteDirection(sb, "TX (stream WriteAsync)", snapshot.Tx, snapshot.TxWindow); + WriteDirection(sb, "RX (stream ReadAsync)", snapshot.Rx, snapshot.RxWindow); + + sb.AppendLine("STREAM FlushAsync (not included in TX send stats)"); + WriteSimple(sb, snapshot.FlushOps, snapshot.FlushSync, snapshot.FlushAsync, snapshot.FlushUs); + sb.AppendLine(); + + sb.AppendLine("RX PipeWriter.FlushAsync (pipe processing / backpressure)"); + WriteSimple(sb, snapshot.RxPipeFlushes, snapshot.RxPipeSync, snapshot.RxPipeAsync, snapshot.RxPipeFlushUs); + sb.AppendLine(); + + sb.AppendLine("TX PipeReader.ReadAsync (includes producer wait; not socket time)"); + WriteSimple(sb, snapshot.TxPipeWaits, snapshot.TxPipeSync, snapshot.TxPipeAsync, snapshot.TxPipeWaitUs); + sb.AppendLine(); + + WriteReadShape(sb, snapshot.TxRead, snapshot.Tx.Ops); + sb.AppendLine(); + + WriteTxLoop(sb, snapshot); + sb.AppendLine(); + WriteWriteCoalesce(sb, snapshot.Coalesce); + sb.AppendLine(); + + if (snapshot.Tx.Ops > 0 && snapshot.Tx.Bytes > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"TX ops/MiB: {snapshot.Tx.Ops / (snapshot.Tx.Bytes / (1024.0 * 1024.0)):F2}")); + } + + if (snapshot.Rx.Ops > 0 && snapshot.Rx.Bytes > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"RX ops/MiB: {snapshot.Rx.Ops / (snapshot.Rx.Bytes / (1024.0 * 1024.0)):F2}")); + } + + return sb.ToString(); + } + + /// Nearest-rank percentile of a sorted sample (microseconds). + internal static double Percentile(int[] sorted, double p) + { + if (sorted.Length == 0) + { + return 0; + } + + if (sorted.Length == 1) + { + return sorted[0]; + } + + var position = p * (sorted.Length - 1); + var lo = (int)position; + var hi = Math.Min(lo + 1, sorted.Length - 1); + var frac = position - lo; + return sorted[lo] + ((sorted[hi] - sorted[lo]) * frac); + } + + internal static (double Mean, double P50, double P95, double P99, int Max) Stats(int[] samples) + { + if (samples.Length == 0) + { + return (0, 0, 0, 0, 0); + } + + var copy = (int[])samples.Clone(); + Array.Sort(copy); + long sum = 0; + for (var i = 0; i < copy.Length; i++) + { + sum += copy[i]; + } + + return (sum / (double)copy.Length, Percentile(copy, 0.50), Percentile(copy, 0.95), Percentile(copy, 0.99), copy[^1]); + } + + private static void WriteDirection(StringBuilder sb, string title, in TransportIoDirection d, TimeSpan window) + { + sb.AppendLine(title); + sb.AppendLine($" operations: {d.Ops:N0}"); + sb.AppendLine($" bytes: {d.Bytes:N0}"); + sb.AppendLine($" requested bytes: {d.Requested:N0}"); + if (d.Ops > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/op: {d.Bytes / (double)d.Ops:F1}")); + sb.AppendLine($" min bytes/op: {d.MinBytes:N0}"); + sb.AppendLine($" max bytes/op: {d.MaxBytes:N0}"); + } + + var gbit = window.TotalSeconds > 0 && d.Bytes > 0 + ? d.Bytes * 8d / window.TotalSeconds / 1_000_000_000d + : 0; + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" I/O window: {window.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" throughput: {gbit:F6} Gbit/s (bytes / I/O window)")); + var total = d.Sync + d.Async; + var syncPct = total > 0 ? 100.0 * d.Sync / total : 0; + var asyncPct = total > 0 ? 100.0 * d.Async / total : 0; + sb.AppendLine($" sync complete: {d.Sync:N0} ({syncPct:F1}%)"); + sb.AppendLine($" async complete: {d.Async:N0} ({asyncPct:F1}%)"); + WriteDuration(sb, " op duration us", d.OpUs); + WriteDuration(sb, " await duration us", d.AwaitUs); + sb.AppendLine(); + } + + private static void WriteSimple(StringBuilder sb, long ops, long sync, long async, int[] samples) + { + sb.AppendLine($" operations: {ops:N0}"); + var total = sync + async; + var syncPct = total > 0 ? 100.0 * sync / total : 0; + var asyncPct = total > 0 ? 100.0 * async / total : 0; + sb.AppendLine($" sync complete: {sync:N0} ({syncPct:F1}%)"); + sb.AppendLine($" async complete: {async:N0} ({asyncPct:F1}%)"); + WriteDuration(sb, " duration us", samples); + } + + private static void WriteReadShape(StringBuilder sb, in TransportIoReadShape shape, long writeOps) + { + sb.AppendLine("TX Pipe ReadResult shape (non-empty; DIAGNOSTIC)"); + sb.AppendLine($" reads: {shape.Reads:N0}"); + sb.AppendLine($" empty reads: {shape.EmptyReads:N0}"); + sb.AppendLine($" bytes: {shape.Bytes:N0}"); + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/read: {shape.Bytes / (double)shape.Reads:F1}")); + sb.AppendLine($" min bytes/read: {shape.MinBytes:N0}"); + sb.AppendLine($" max bytes/read: {shape.MaxBytes:N0}"); + } + + sb.AppendLine($" segments: {shape.Segments:N0} (non-empty)"); + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg segments/read: {shape.Segments / (double)shape.Reads:F2}")); + sb.AppendLine($" min segments/read: {shape.MinSegments:N0}"); + sb.AppendLine($" max segments/read: {shape.MaxSegments:N0}"); + } + + if (shape.Segments > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" avg bytes/segment: {shape.Bytes / (double)shape.Segments:F1}")); + sb.AppendLine($" min bytes/segment: {shape.MinSegmentBytes:N0}"); + sb.AppendLine($" max bytes/segment: {shape.MaxSegmentBytes:N0}"); + } + + if (shape.Reads > 0) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" writes/read: {writeOps / (double)shape.Reads:F2} (stream WriteAsync / non-empty ReadResult)")); + } + + sb.AppendLine($" WriteAsync ops: {writeOps:N0}"); + sb.AppendLine($" segments vs writes: {shape.Segments:N0} segments -> {writeOps:N0} WriteAsync"); + } + + private static void WriteTxLoop(StringBuilder sb, in TransportIoSnapshot snapshot) + { + var loop = snapshot.Loop; + sb.AppendLine("TX SendAsync loop stages (DIAGNOSTIC; NntpConnection.SendAsync)"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" SendAsync elapsed: {loop.SendAsyncElapsed.TotalMilliseconds:F3} ms (pump lifetime, includes idle)")); + sb.AppendLine($" AdvanceTo calls: {loop.AdvanceCount:N0}"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" AdvanceTo total: {loop.AdvanceTo.TotalMilliseconds:F3} ms")); + sb.AppendLine($" Transport writes: {loop.TransportWriteCount:N0} (ConnectionByteTransport.WriteAsync)"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Transport write: {loop.TransportWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine($" Between-write gaps: {loop.BetweenWriteCount:N0}"); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Between writes: {loop.BetweenWrites.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Read→first write: {loop.ReadToFirstWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Last write→Advance: {loop.LastWriteToAdvance.TotalMilliseconds:F3} ms (includes FlushAsync)")); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Advance→next Read: {loop.AdvanceToNextRead.TotalMilliseconds:F3} ms")); + WriteDuration(sb, " read→write us", loop.ReadToFirstWriteUs); + WriteDuration(sb, " between write us", loop.BetweenWriteUs); + WriteDuration(sb, " write→advance us", loop.LastWriteToAdvanceUs); + WriteDuration(sb, " AdvanceTo us", loop.AdvanceUs); + WriteDuration(sb, " advance→read us", loop.AdvanceToNextReadUs); + WriteDuration(sb, " transport wr us", loop.TransportWriteUs); + + var readWait = SumUs(snapshot.TxPipeWaitUs); + var streamWrite = SumUs(snapshot.Tx.OpUs); + var accounted = loop.ReadToFirstWrite + loop.BetweenWrites + loop.LastWriteToAdvance + + loop.AdvanceTo + loop.AdvanceToNextRead + loop.TransportWrite + + TimeSpan.FromMilliseconds(readWait / 1000.0); + sb.AppendLine(); + sb.AppendLine(" Accounting (accumulated stage totals; ReadAsync wait uses sampled us sum)"); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" ReadAsync waiting: {readWait / 1000.0:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Pipe → first Write: {loop.ReadToFirstWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Transport WriteAsync: {loop.TransportWrite.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" NetworkStream write: {streamWrite / 1000.0:F3} ms (subset of transport write)")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Between writes: {loop.BetweenWrites.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Last write → Advance: {loop.LastWriteToAdvance.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" AdvanceTo: {loop.AdvanceTo.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Advance → next Read: {loop.AdvanceToNextRead.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Copy (in read→write): {snapshot.Coalesce.CopyTime.TotalMilliseconds:F3} ms (subset of Pipe → first Write / between writes)")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" Accounted stages: {accounted.TotalMilliseconds:F3} ms")); + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" SendAsync elapsed: {loop.SendAsyncElapsed.TotalMilliseconds:F3} ms")); + if (loop.SendAsyncElapsed > TimeSpan.Zero) + { + var unaccounted = loop.SendAsyncElapsed - accounted; + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" Other/unaccounted: {unaccounted.TotalMilliseconds:F3} ms (idle between commands is expected)")); + } + } + + private static void WriteWriteCoalesce(StringBuilder sb, in TransportIoWriteCoalesce coalesce) + { + sb.AppendLine("TX write granularity (DIAGNOSTIC; VECTORNNTP_TX_WRITE_GRANULARITY)"); + if (coalesce.TargetBytes <= 0) + { + sb.AppendLine(" target: inactive"); + return; + } + + sb.AppendLine($" target: {coalesce.TargetBytes:N0} bytes"); + sb.AppendLine($" copy operations: {coalesce.CopyCount:N0}"); + sb.AppendLine($" copy bytes: {coalesce.CopyBytes:N0}"); + if (coalesce.CopyCount > 0) + { + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" avg copy bytes: {coalesce.CopyBytes / (double)coalesce.CopyCount:F1}")); + } + + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $" copy time: {coalesce.CopyTime.TotalMilliseconds:F3} ms")); + WriteDuration(sb, " copy us", coalesce.CopyUs); + } + + private static long SumUs(int[] samples) + { + long sum = 0; + for (var i = 0; i < samples.Length; i++) + { + sum += samples[i]; + } + + return sum; + } + + private static void WriteDuration(StringBuilder sb, string label, int[] samples) + { + var s = Stats(samples); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"{label,-20} n={samples.Length,-7:N0} mean={s.Mean,10:F1} p50={s.P50,10:F1} p95={s.P95,10:F1} p99={s.P99,10:F1} max={s.Max,10:N0}")); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoSession.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoSession.cs new file mode 100644 index 00000000..5dfa8fa9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TransportIoSession.cs @@ -0,0 +1,680 @@ +using System.Buffers; +using System.Diagnostics; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY per-connection aggregate of stream Read/Write/Flush and pipe wait times. + /// + /// + /// No per-operation objects. Samples are stored in preallocated arrays + /// (microseconds). Excess operations update counters only. + /// + internal sealed class TransportIoSession + { + /// Maximum duration samples retained per series. + public const int SampleCapacity = 32768; + + private readonly int[] _rxOpUs = new int[SampleCapacity]; + private readonly int[] _rxAwaitUs = new int[SampleCapacity]; + private readonly int[] _txOpUs = new int[SampleCapacity]; + private readonly int[] _txAwaitUs = new int[SampleCapacity]; + private readonly int[] _rxPipeFlushUs = new int[SampleCapacity]; + private readonly int[] _txPipeWaitUs = new int[SampleCapacity]; + private readonly int[] _flushUs = new int[SampleCapacity]; + + private int _rxOpSamples; + private int _txOpSamples; + private int _rxPipeSamples; + private int _txPipeSamples; + private int _flushSamples; + + private long _rxOps; + private long _txOps; + private long _flushOps; + private long _rxPipeFlushes; + private long _txPipeWaits; + private long _txReadShapes; + private long _txReadEmpty; + private long _txReadBytes; + private long _txReadSegments; + private int _txReadMinBytes = int.MaxValue; + private int _txReadMaxBytes; + private int _txReadMinSegments = int.MaxValue; + private int _txReadMaxSegments; + private int _txReadMinSegBytes = int.MaxValue; + private int _txReadMaxSegBytes; + + private long _rxBytes; + private long _txBytes; + private long _rxRequested; + private long _txRequested; + + private int _rxMinBytes = int.MaxValue; + private int _rxMaxBytes; + private int _txMinBytes = int.MaxValue; + private int _txMaxBytes; + + private long _rxSync; + private long _rxAsync; + private long _txSync; + private long _txAsync; + private long _flushSync; + private long _flushAsync; + private long _rxPipeSync; + private long _rxPipeAsync; + private long _txPipeSync; + private long _txPipeAsync; + + private long _firstRxTs; + private long _lastRxTs; + private long _firstTxTs; + private long _lastTxTs; + private readonly long _createdTs = Stopwatch.GetTimestamp(); + + private readonly int[] _readToFirstWriteUs = new int[SampleCapacity]; + private readonly int[] _betweenWritesUs = new int[SampleCapacity]; + private readonly int[] _lastWriteToAdvanceUs = new int[SampleCapacity]; + private readonly int[] _advanceUs = new int[SampleCapacity]; + private readonly int[] _advanceToNextReadUs = new int[SampleCapacity]; + private readonly int[] _transportWriteUs = new int[SampleCapacity]; + private readonly int[] _copyUs = new int[SampleCapacity]; + + private int _readToFirstWriteSamples; + private int _betweenWriteSamples; + private int _lastWriteToAdvanceSamples; + private int _advanceSamples; + private int _advanceToNextReadSamples; + private int _transportWriteSamples; + private int _copySamples; + + private long _sendLoopStartTs; + private long _sendLoopEndTs; + private long _readToFirstWriteTicks; + private long _betweenWriteTicks; + private long _lastWriteToAdvanceTicks; + private long _advanceTicks; + private long _advanceToNextReadTicks; + private long _transportWriteTicks; + private long _copyTicks; + private long _advanceCount; + private long _betweenWriteCount; + private long _transportWriteCount; + private long _copyCount; + private long _copyBytes; + + /// TX output-pipe pause threshold actually used by this connection. + public long TxPipePauseBytes { get; set; } = NntpPipeOptions.PauseWriterThreshold; + + /// TX output-pipe resume threshold actually used by this connection. + public long TxPipeResumeBytes { get; set; } = NntpPipeOptions.ResumeWriterThreshold; + + /// DIAGNOSTIC write-aggregate target, or 0 when the experiment is inactive. + public int WriteGranularityTargetBytes { get; set; } + + /// Records one stream ReadAsync. + public void RecordReceive(int requested, int received, long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + var opUs = ToUs(startedTs, endTs); + var awaitUs = sync ? 0 : ToUs(awaitStartTs, endTs); + Interlocked.Increment(ref _rxOps); + Interlocked.Add(ref _rxBytes, received); + Interlocked.Add(ref _rxRequested, requested); + UpdateMin(ref _rxMinBytes, received); + UpdateMax(ref _rxMaxBytes, received); + if (sync) + { + Interlocked.Increment(ref _rxSync); + } + else + { + Interlocked.Increment(ref _rxAsync); + } + + StampWindow(ref _firstRxTs, ref _lastRxTs, startedTs, endTs); + var index = Interlocked.Increment(ref _rxOpSamples) - 1; + if ((uint)index < SampleCapacity) + { + _rxOpUs[index] = opUs; + _rxAwaitUs[index] = awaitUs; + } + } + + /// Records one stream WriteAsync. + public void RecordSend(int requested, int sent, long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + var opUs = ToUs(startedTs, endTs); + var awaitUs = sync ? 0 : ToUs(awaitStartTs, endTs); + Interlocked.Increment(ref _txOps); + Interlocked.Add(ref _txBytes, sent); + Interlocked.Add(ref _txRequested, requested); + UpdateMin(ref _txMinBytes, sent); + UpdateMax(ref _txMaxBytes, sent); + if (sync) + { + Interlocked.Increment(ref _txSync); + } + else + { + Interlocked.Increment(ref _txAsync); + } + + StampWindow(ref _firstTxTs, ref _lastTxTs, startedTs, endTs); + var index = Interlocked.Increment(ref _txOpSamples) - 1; + if ((uint)index < SampleCapacity) + { + _txOpUs[index] = opUs; + _txAwaitUs[index] = awaitUs; + } + } + + /// Records one stream FlushAsync (not mixed into send stats). + public void RecordFlush(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _flushOps); + if (sync) + { + Interlocked.Increment(ref _flushSync); + } + else + { + Interlocked.Increment(ref _flushAsync); + } + + var index = Interlocked.Increment(ref _flushSamples) - 1; + if ((uint)index < SampleCapacity) + { + _flushUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// Records one application-input PipeWriter.FlushAsync (RX backpressure). + public void RecordRxPipeFlush(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _rxPipeFlushes); + if (sync) + { + Interlocked.Increment(ref _rxPipeSync); + } + else + { + Interlocked.Increment(ref _rxPipeAsync); + } + + var index = Interlocked.Increment(ref _rxPipeSamples) - 1; + if ((uint)index < SampleCapacity) + { + _rxPipeFlushUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// + /// Records one application-output PipeReader.ReadAsync. + /// Includes waiting for the producer; this is not socket time. + /// + public void RecordTxPipeWait(long startedTs, long awaitStartTs, bool sync) + { + var endTs = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _txPipeWaits); + if (sync) + { + Interlocked.Increment(ref _txPipeSync); + } + else + { + Interlocked.Increment(ref _txPipeAsync); + } + + var index = Interlocked.Increment(ref _txPipeSamples) - 1; + if ((uint)index < SampleCapacity) + { + _txPipeWaitUs[index] = ToUs(startedTs, endTs); + _ = awaitStartTs; + } + } + + /// + /// Records the shape of one non-empty TX from PipeReader.ReadAsync. + /// Empty buffers are counted separately and do not affect averages. + /// + public void RecordTxReadShape(in ReadOnlySequence buffer) + { + if (buffer.IsEmpty) + { + Interlocked.Increment(ref _txReadEmpty); + return; + } + + var bytesLong = buffer.Length; + var bytes = bytesLong > int.MaxValue ? int.MaxValue : (int)bytesLong; + var segments = 0; + var minSeg = int.MaxValue; + var maxSeg = 0; + foreach (var memory in buffer) + { + var length = memory.Length; + if (length == 0) + { + continue; + } + + segments++; + if (length < minSeg) + { + minSeg = length; + } + + if (length > maxSeg) + { + maxSeg = length; + } + } + + Interlocked.Increment(ref _txReadShapes); + Interlocked.Add(ref _txReadBytes, bytesLong); + Interlocked.Add(ref _txReadSegments, segments); + UpdateMin(ref _txReadMinBytes, bytes); + UpdateMax(ref _txReadMaxBytes, bytes); + UpdateMin(ref _txReadMinSegments, segments); + UpdateMax(ref _txReadMaxSegments, segments); + if (segments > 0) + { + UpdateMin(ref _txReadMinSegBytes, minSeg); + UpdateMax(ref _txReadMaxSegBytes, maxSeg); + } + } + + /// Marks NntpConnection.SendAsync entry (monotonic). + public void RecordSendLoopStart() => + Interlocked.CompareExchange(ref _sendLoopStartTs, Stopwatch.GetTimestamp(), 0); + + /// Marks NntpConnection.SendAsync exit after the read loop. + public void RecordSendLoopEnd() => + Volatile.Write(ref _sendLoopEndTs, Stopwatch.GetTimestamp()); + + /// ReadAsync completion → first transport WriteAsync of that iteration. + public void RecordReadToFirstWrite(long readDoneTs, long firstWriteTs) => + RecordSpan( + readDoneTs, + firstWriteTs, + ref _readToFirstWriteTicks, + ref _readToFirstWriteSamples, + _readToFirstWriteUs); + + /// Gap between consecutive transport WriteAsync calls in one ReadResult. + public void RecordBetweenWrites(long previousWriteEndTs, long nextWriteStartTs) + { + Interlocked.Increment(ref _betweenWriteCount); + RecordSpan( + previousWriteEndTs, + nextWriteStartTs, + ref _betweenWriteTicks, + ref _betweenWriteSamples, + _betweenWritesUs); + } + + /// Last WriteAsync completion → AdvanceTo (includes FlushAsync in the current loop). + public void RecordLastWriteToAdvance(long lastWriteEndTs, long advanceStartTs) => + RecordSpan( + lastWriteEndTs, + advanceStartTs, + ref _lastWriteToAdvanceTicks, + ref _lastWriteToAdvanceSamples, + _lastWriteToAdvanceUs); + + /// Times one PipeReader.AdvanceTo. + public void RecordAdvanceTo(long startedTs, long endedTs) + { + Interlocked.Increment(ref _advanceCount); + RecordSpan(startedTs, endedTs, ref _advanceTicks, ref _advanceSamples, _advanceUs); + } + + /// AdvanceTo return → next ReadAsync issued. + public void RecordAdvanceToNextRead(long advanceEndTs, long nextReadIssuedTs) => + RecordSpan( + advanceEndTs, + nextReadIssuedTs, + ref _advanceToNextReadTicks, + ref _advanceToNextReadSamples, + _advanceToNextReadUs); + + /// Times one ConnectionByteTransport.WriteAsync (admission + stream write). + public void RecordTransportWrite(long startedTs, long endedTs) + { + Interlocked.Increment(ref _transportWriteCount); + RecordSpan(startedTs, endedTs, ref _transportWriteTicks, ref _transportWriteSamples, _transportWriteUs); + } + + /// Times one copy into the diagnostic write-aggregate buffer. + public void RecordCopy(int bytes, long startedTs, long endedTs) + { + if (bytes < 0) + { + return; + } + + Interlocked.Increment(ref _copyCount); + Interlocked.Add(ref _copyBytes, bytes); + RecordSpan(startedTs, endedTs, ref _copyTicks, ref _copySamples, _copyUs); + } + + /// Snapshots counters and duration samples for a report. + public TransportIoSnapshot Snapshot() + { + var now = Stopwatch.GetTimestamp(); + return new TransportIoSnapshot( + ConnectionElapsed: Stopwatch.GetElapsedTime(_createdTs, now), + RxWindow: Window(_firstRxTs, _lastRxTs), + TxWindow: Window(_firstTxTs, _lastTxTs), + Rx: Direction( + Volatile.Read(ref _rxOps), + Volatile.Read(ref _rxBytes), + Volatile.Read(ref _rxRequested), + Volatile.Read(ref _rxMinBytes), + Volatile.Read(ref _rxMaxBytes), + Volatile.Read(ref _rxSync), + Volatile.Read(ref _rxAsync), + CopySamples(_rxOpUs, Volatile.Read(ref _rxOpSamples)), + CopySamples(_rxAwaitUs, Volatile.Read(ref _rxOpSamples))), + Tx: Direction( + Volatile.Read(ref _txOps), + Volatile.Read(ref _txBytes), + Volatile.Read(ref _txRequested), + Volatile.Read(ref _txMinBytes), + Volatile.Read(ref _txMaxBytes), + Volatile.Read(ref _txSync), + Volatile.Read(ref _txAsync), + CopySamples(_txOpUs, Volatile.Read(ref _txOpSamples)), + CopySamples(_txAwaitUs, Volatile.Read(ref _txOpSamples))), + FlushOps: Volatile.Read(ref _flushOps), + FlushSync: Volatile.Read(ref _flushSync), + FlushAsync: Volatile.Read(ref _flushAsync), + FlushUs: CopySamples(_flushUs, Volatile.Read(ref _flushSamples)), + RxPipeFlushes: Volatile.Read(ref _rxPipeFlushes), + RxPipeSync: Volatile.Read(ref _rxPipeSync), + RxPipeAsync: Volatile.Read(ref _rxPipeAsync), + RxPipeFlushUs: CopySamples(_rxPipeFlushUs, Volatile.Read(ref _rxPipeSamples)), + TxPipeWaits: Volatile.Read(ref _txPipeWaits), + TxPipeSync: Volatile.Read(ref _txPipeSync), + TxPipeAsync: Volatile.Read(ref _txPipeAsync), + TxPipeWaitUs: CopySamples(_txPipeWaitUs, Volatile.Read(ref _txPipeSamples)), + TxPipePauseBytes: TxPipePauseBytes, + TxPipeResumeBytes: TxPipeResumeBytes, + TxRead: new TransportIoReadShape( + Volatile.Read(ref _txReadShapes), + Volatile.Read(ref _txReadEmpty), + Volatile.Read(ref _txReadBytes), + Volatile.Read(ref _txReadSegments), + MinOrZero(_txReadMinBytes), + Volatile.Read(ref _txReadMaxBytes), + MinOrZero(_txReadMinSegments), + Volatile.Read(ref _txReadMaxSegments), + MinOrZero(_txReadMinSegBytes), + Volatile.Read(ref _txReadMaxSegBytes)), + Loop: new TransportIoTxLoop( + SendAsyncElapsed: Window( + Volatile.Read(ref _sendLoopStartTs), + EndOrNow(Volatile.Read(ref _sendLoopEndTs), now)), + ReadToFirstWrite: TicksToTimeSpan(Volatile.Read(ref _readToFirstWriteTicks)), + BetweenWrites: TicksToTimeSpan(Volatile.Read(ref _betweenWriteTicks)), + LastWriteToAdvance: TicksToTimeSpan(Volatile.Read(ref _lastWriteToAdvanceTicks)), + AdvanceTo: TicksToTimeSpan(Volatile.Read(ref _advanceTicks)), + AdvanceToNextRead: TicksToTimeSpan(Volatile.Read(ref _advanceToNextReadTicks)), + TransportWrite: TicksToTimeSpan(Volatile.Read(ref _transportWriteTicks)), + AdvanceCount: Volatile.Read(ref _advanceCount), + BetweenWriteCount: Volatile.Read(ref _betweenWriteCount), + TransportWriteCount: Volatile.Read(ref _transportWriteCount), + ReadToFirstWriteUs: CopySamples(_readToFirstWriteUs, Volatile.Read(ref _readToFirstWriteSamples)), + BetweenWriteUs: CopySamples(_betweenWritesUs, Volatile.Read(ref _betweenWriteSamples)), + LastWriteToAdvanceUs: CopySamples(_lastWriteToAdvanceUs, Volatile.Read(ref _lastWriteToAdvanceSamples)), + AdvanceUs: CopySamples(_advanceUs, Volatile.Read(ref _advanceSamples)), + AdvanceToNextReadUs: CopySamples(_advanceToNextReadUs, Volatile.Read(ref _advanceToNextReadSamples)), + TransportWriteUs: CopySamples(_transportWriteUs, Volatile.Read(ref _transportWriteSamples))), + Coalesce: new TransportIoWriteCoalesce( + TargetBytes: WriteGranularityTargetBytes, + CopyCount: Volatile.Read(ref _copyCount), + CopyBytes: Volatile.Read(ref _copyBytes), + CopyTime: TicksToTimeSpan(Volatile.Read(ref _copyTicks)), + CopyUs: CopySamples(_copyUs, Volatile.Read(ref _copySamples)))); + } + + /// Writes the connection dump and returns the path. + public string Write(string directory, string sessionLabel) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionLabel); + Directory.CreateDirectory(directory); + var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", System.Globalization.CultureInfo.InvariantCulture); + var safe = Sanitize(sessionLabel); + var path = Path.Join(directory, $"{stamp}-{safe}.txt"); + File.WriteAllText(path, TransportIoReport.Format(Snapshot(), sessionLabel), System.Text.Encoding.UTF8); + return path; + } + + private static TransportIoDirection Direction( + long ops, + long bytes, + long requested, + int minBytes, + int maxBytes, + long sync, + long async, + int[] opUs, + int[] awaitUs) => + new( + ops, + bytes, + requested, + MinOrZero(minBytes), + maxBytes, + sync, + async, + opUs, + awaitUs); + + private static int MinOrZero(int value) => value == int.MaxValue ? 0 : value; + + private static void RecordSpan(long start, long end, ref long ticks, ref int samples, int[] us) + { + if (start <= 0 || end < start) + { + return; + } + + Interlocked.Add(ref ticks, end - start); + var index = Interlocked.Increment(ref samples) - 1; + if ((uint)index < SampleCapacity) + { + us[index] = ToUs(start, end); + } + } + + private static TimeSpan TicksToTimeSpan(long ticks) => + ticks <= 0 ? TimeSpan.Zero : TimeSpan.FromSeconds(ticks / (double)Stopwatch.Frequency); + + private static long EndOrNow(long end, long now) => end > 0 ? end : now; + + private static TimeSpan Window(long first, long last) => + first > 0 && last >= first ? Stopwatch.GetElapsedTime(first, last) : TimeSpan.Zero; + + private static void StampWindow(ref long first, ref long last, long start, long end) + { + if (Interlocked.CompareExchange(ref first, start, 0) == 0) + { + // first write won + } + + long current; + do + { + current = Volatile.Read(ref last); + if (end <= current) + { + return; + } + } + while (Interlocked.CompareExchange(ref last, end, current) != current); + } + + private static int[] CopySamples(int[] source, int written) + { + var count = Math.Min(Math.Max(written, 0), SampleCapacity); + if (count == 0) + { + return []; + } + + var copy = new int[count]; + Array.Copy(source, copy, count); + return copy; + } + + private static void UpdateMin(ref int field, int value) + { + var current = Volatile.Read(ref field); + while (value < current) + { + var original = Interlocked.CompareExchange(ref field, value, current); + if (original == current) + { + return; + } + + current = original; + } + } + + private static void UpdateMax(ref int field, int value) + { + var current = Volatile.Read(ref field); + while (value > current) + { + var original = Interlocked.CompareExchange(ref field, value, current); + if (original == current) + { + return; + } + + current = original; + } + } + + private static int ToUs(long start, long end) + { + if (start <= 0 || end < start) + { + return 0; + } + + var us = Stopwatch.GetElapsedTime(start, end).TotalMicroseconds; + if (us >= int.MaxValue) + { + return int.MaxValue; + } + + if (us <= 0) + { + return 0; + } + + return (int)us; + } + + private static string Sanitize(string value) + { + var chars = value.ToCharArray(); + for (var i = 0; i < chars.Length; i++) + { + if (!char.IsAsciiLetterOrDigit(chars[i]) && chars[i] is not '-' and not '_') + { + chars[i] = '_'; + } + } + + return new string(chars); + } + } + + /// Immutable counters and duration samples for one connection. + internal readonly record struct TransportIoSnapshot( + TimeSpan ConnectionElapsed, + TimeSpan RxWindow, + TimeSpan TxWindow, + TransportIoDirection Rx, + TransportIoDirection Tx, + long FlushOps, + long FlushSync, + long FlushAsync, + int[] FlushUs, + long RxPipeFlushes, + long RxPipeSync, + long RxPipeAsync, + int[] RxPipeFlushUs, + long TxPipeWaits, + long TxPipeSync, + long TxPipeAsync, + int[] TxPipeWaitUs, + long TxPipePauseBytes, + long TxPipeResumeBytes, + TransportIoReadShape TxRead, + TransportIoTxLoop Loop, + TransportIoWriteCoalesce Coalesce); + + /// DIAGNOSTIC write-granularity copy counters (when the experiment is active). + internal readonly record struct TransportIoWriteCoalesce( + int TargetBytes, + long CopyCount, + long CopyBytes, + TimeSpan CopyTime, + int[] CopyUs); + + /// Accumulated NntpConnection.SendAsync loop-stage timings (when probe enabled). + internal readonly record struct TransportIoTxLoop( + TimeSpan SendAsyncElapsed, + TimeSpan ReadToFirstWrite, + TimeSpan BetweenWrites, + TimeSpan LastWriteToAdvance, + TimeSpan AdvanceTo, + TimeSpan AdvanceToNextRead, + TimeSpan TransportWrite, + long AdvanceCount, + long BetweenWriteCount, + long TransportWriteCount, + int[] ReadToFirstWriteUs, + int[] BetweenWriteUs, + int[] LastWriteToAdvanceUs, + int[] AdvanceUs, + int[] AdvanceToNextReadUs, + int[] TransportWriteUs); + + /// Shape of TX PipeReader.ReadAsync buffers (non-empty reads). + internal readonly record struct TransportIoReadShape( + long Reads, + long EmptyReads, + long Bytes, + long Segments, + int MinBytes, + int MaxBytes, + int MinSegments, + int MaxSegments, + int MinSegmentBytes, + int MaxSegmentBytes); + + /// One direction of stream I/O (RX read or TX write). + internal readonly record struct TransportIoDirection( + long Ops, + long Bytes, + long Requested, + int MinBytes, + int MaxBytes, + long Sync, + long Async, + int[] OpUs, + int[] AwaitUs); +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TransportLogMessages.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TransportLogMessages.cs new file mode 100644 index 00000000..f1a8c679 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TransportLogMessages.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// Source-generated NNTP connection transport log messages. + internal static partial class TransportLogMessages + { + [LoggerMessage( + EventId = 1500, + Level = LogLevel.Debug, + Message = "TLS handshake completed for {Remote}.")] + public static partial void TlsHandshakeCompleted(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1501, + Level = LogLevel.Debug, + Message = "In-place TLS upgrade completed for {Remote}.")] + public static partial void InPlaceTlsUpgradeCompleted(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1502, + Level = LogLevel.Debug, + Message = "In-place DEFLATE compression activated for {Remote}.")] + public static partial void InPlaceDeflateActivated(ILogger logger, EndPoint? Remote); + + [LoggerMessage( + EventId = 1503, + Level = LogLevel.Debug, + Message = "NNTP connection {Pump} pump ended with an error")] + public static partial void PumpEndedWithError(ILogger logger, Exception exception, string Pump); + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TxPipePauseExperiment.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TxPipePauseExperiment.cs new file mode 100644 index 00000000..0b46c3cf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TxPipePauseExperiment.cs @@ -0,0 +1,207 @@ +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default TX output-pipe pause-threshold override. + /// + /// + /// + /// Production default remains (64 KiB). + /// This type never changes the input (RX) pipe. It exists only so a SPEEDTEST sweep can + /// vary how far the TX producer may run ahead before FlushAsync backpressures. + /// + /// + /// Activate only with VECTORNNTP_TX_PIPE_PAUSE set to an allowlisted size + /// (64KiB … 8MiB), or via in tests. Unset / empty / 0 + /// / false leave production options unchanged. Any other value is rejected so a typo + /// cannot silently apply a large buffer. + /// + /// + internal static class TxPipePauseExperiment + { + /// Environment variable that selects an experimental TX pipe pause size. + public const string EnvironmentVariableName = "VECTORNNTP_TX_PIPE_PAUSE"; + + /// Allowlisted pause sizes (bytes). 64 KiB is the production control. + public static readonly long[] AllowedPauseBytes = + [ + 64 * 1024, + 128 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + ]; + + private static OverrideState? _state; + + static TxPipePauseExperiment() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryParseAllowed(value, out var pause)) + { + throw new InvalidOperationException( + EnvironmentVariableName + + " must be an allowlisted size (64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 2MiB, 4MiB, 8MiB) or the equivalent byte count."); + } + + Volatile.Write(ref _state, new OverrideState(pause)); + } + + /// Gets whether an experimental pause size is configured. + public static bool IsConfigured => Volatile.Read(ref _state) is not null; + + /// Gets the configured pause bytes, or when inactive. + public static long? ConfiguredPauseBytes => Volatile.Read(ref _state)?.PauseBytes; + + /// Test/experiment hook. must be allowlisted. + public static void Set(long pauseBytes) + { + if (!IsAllowed(pauseBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(pauseBytes), + pauseBytes, + "TX pipe pause experiment accepts only 64KiB…8MiB allowlisted sizes."); + } + + Volatile.Write(ref _state, new OverrideState(pauseBytes)); + } + + /// Clears the override (tests). Production default applies. + public static void Clear() => Volatile.Write(ref _state, null); + + /// + /// Resolves the TX output-pipe thresholds. When inactive, returns production 64 KiB / 32 KiB + /// and . + /// + public static bool TryGetEffective(out long pauseBytes, out long resumeBytes) + { + var configured = Volatile.Read(ref _state); + if (configured is null) + { + pauseBytes = NntpPipeOptions.PauseWriterThreshold; + resumeBytes = NntpPipeOptions.ResumeWriterThreshold; + return false; + } + + pauseBytes = configured.PauseBytes; + resumeBytes = ResumeFor(configured.PauseBytes); + return true; + } + + /// Resume is half of pause, never below the production 32 KiB resume. + public static long ResumeFor(long pauseBytes) + { + var half = pauseBytes / 2; + return half < NntpPipeOptions.ResumeWriterThreshold + ? NntpPipeOptions.ResumeWriterThreshold + : half; + } + + /// Parses an allowlisted size token or byte count. + public static bool TryParseAllowed(string value, out long pauseBytes) + { + pauseBytes = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (long.TryParse(trimmed, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var raw) && + IsAllowed(raw)) + { + pauseBytes = raw; + return true; + } + + if (TryParseSizeToken(trimmed, out var parsed) && IsAllowed(parsed)) + { + pauseBytes = parsed; + return true; + } + + return false; + } + + /// Returns whether is on the experiment allowlist. + public static bool IsAllowed(long pauseBytes) + { + for (var i = 0; i < AllowedPauseBytes.Length; i++) + { + if (AllowedPauseBytes[i] == pauseBytes) + { + return true; + } + } + + return false; + } + + private static bool TryParseSizeToken(string value, out long bytes) + { + bytes = 0; + var span = value.AsSpan().Trim(); + var suffixLen = 0; + long multiplier; + if (span.EndsWith("KiB", StringComparison.OrdinalIgnoreCase) || + span.EndsWith("kib", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024; + } + else if (span.EndsWith("MiB", StringComparison.OrdinalIgnoreCase) || + span.EndsWith("mib", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024 * 1024; + } + else if (span.Length > 0 && (span[^1] is 'k' or 'K')) + { + suffixLen = 1; + multiplier = 1024; + } + else if (span.Length > 0 && (span[^1] is 'm' or 'M')) + { + suffixLen = 1; + multiplier = 1024 * 1024; + } + else + { + return false; + } + + var number = span[..^suffixLen]; + if (!long.TryParse(number, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var count) || + count <= 0) + { + return false; + } + + try + { + bytes = checked(count * multiplier); + } + catch (OverflowException) + { + return false; + } + + return true; + } + + private sealed class OverrideState(long pauseBytes) + { + public long PauseBytes { get; } = pauseBytes; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteCoalescer.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteCoalescer.cs new file mode 100644 index 00000000..81930bf5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteCoalescer.cs @@ -0,0 +1,70 @@ +using System.Diagnostics; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY reusable contiguous buffer for experimental TX write aggregation. + /// + /// + /// One buffer is allocated for the connection lifetime. Copies are measured when a + /// is supplied. The buffer remains valid until + /// after the transport write has completed. + /// + internal sealed class TxWriteCoalescer + { + private readonly byte[] _buffer; + private int _count; + + /// Creates a coalescer whose buffer is exactly . + public TxWriteCoalescer(int targetBytes) + { + if (!TxWriteGranularityExperiment.IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException(nameof(targetBytes), targetBytes, "Target must be an allowlisted write-granularity size."); + } + + TargetBytes = targetBytes; + _buffer = GC.AllocateUninitializedArray(targetBytes); + } + + /// Configured aggregate size in bytes. + public int TargetBytes { get; } + + /// Bytes currently held in the reusable buffer. + public int PendingCount => _count; + + /// Unused bytes before the next transport write must be issued. + public int RemainingCapacity => TargetBytes - _count; + + /// Gets whether the buffer is full and must be written before more copies. + public bool IsFull => _count >= TargetBytes; + + /// Pending bytes. Valid until . + public ReadOnlyMemory PendingMemory => _buffer.AsMemory(0, _count); + + /// + /// Copies as many bytes as fit from . Returns the number copied. + /// + public int Copy(ReadOnlySpan source, TransportIoSession? io) + { + var take = Math.Min(source.Length, RemainingCapacity); + if (take == 0) + { + return 0; + } + + var started = io is null ? 0L : Stopwatch.GetTimestamp(); + source[..take].CopyTo(_buffer.AsSpan(_count, take)); + _count += take; + if (io is not null) + { + io.RecordCopy(take, started, Stopwatch.GetTimestamp()); + } + + return take; + } + + /// Drops pending bytes after a successful transport write (or a terminal failure). + public void Clear() => _count = 0; + } +} diff --git a/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteGranularityExperiment.cs b/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteGranularityExperiment.cs new file mode 100644 index 00000000..94cdb3f2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Networking/Transport/TxWriteGranularityExperiment.cs @@ -0,0 +1,234 @@ +using System.Runtime.CompilerServices; + +namespace VectorNNTPD.TransitService.Networking.Transport +{ + /// + /// DIAGNOSTIC-ONLY. Off-by-default TX write-granularity override. + /// + /// + /// + /// Production default remains one ConnectionByteTransport.WriteAsync per Pipe + /// segment. This type never changes Pipe thresholds, schedulers, Channel, TCS, + /// NetworkStream, or socket options. It exists only so a SPEEDTEST sweep can + /// copy consecutive Pipe segments into one larger transport write. + /// + /// + /// Activate only with VECTORNNTP_TX_WRITE_GRANULARITY set to an allowlisted + /// size (64KiB … 4MiB), or via / + /// in tests. Unset / empty / 0 / false leave the production path + /// unchanged. Any other value is rejected so a typo cannot silently apply a large + /// aggregate. + /// + /// + internal static class TxWriteGranularityExperiment + { + /// Environment variable that selects an experimental TX write aggregate size. + public const string EnvironmentVariableName = "VECTORNNTP_TX_WRITE_GRANULARITY"; + + /// Allowlisted aggregate sizes (bytes). 64 KiB is the instrumented control. + public static readonly int[] AllowedTargetBytes = + [ + 64 * 1024, + 128 * 1024, + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 4 * 1024 * 1024, + ]; + + private static readonly ConditionalWeakTable> PerTransport = []; + + private static OverrideState? _state; + + static TxWriteGranularityExperiment() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryParseAllowed(value, out var target)) + { + throw new InvalidOperationException( + EnvironmentVariableName + + " must be an allowlisted size (64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 4MiB) or the equivalent byte count."); + } + + Volatile.Write(ref _state, new OverrideState(target)); + } + + /// Gets whether a process-wide experimental aggregate size is configured. + public static bool IsConfigured => Volatile.Read(ref _state) is not null; + + /// Gets the process-wide target bytes, or when inactive. + public static int? ConfiguredTargetBytes => Volatile.Read(ref _state)?.TargetBytes; + + /// Test/experiment hook. must be allowlisted. + public static void Set(int targetBytes) + { + if (!IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(targetBytes), + targetBytes, + "TX write-granularity experiment accepts only 64KiB…4MiB allowlisted sizes."); + } + + Volatile.Write(ref _state, new OverrideState(targetBytes)); + } + + /// Clears the process-wide override (tests). Production default applies. + public static void Clear() => Volatile.Write(ref _state, null); + + /// + /// Test-only per-transport override. Does not affect other connections. + /// + public static void SetFor(ConnectionByteTransport transport, int targetBytes) + { + ArgumentNullException.ThrowIfNull(transport); + if (!IsAllowed(targetBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(targetBytes), + targetBytes, + "TX write-granularity experiment accepts only 64KiB…4MiB allowlisted sizes."); + } + + PerTransport.AddOrUpdate(transport, new StrongBox(targetBytes)); + } + + /// Removes a per-transport override (tests). + public static void ClearFor(ConnectionByteTransport transport) + { + ArgumentNullException.ThrowIfNull(transport); + PerTransport.Remove(transport); + } + + /// + /// Resolves the aggregate target for this send pump. Per-transport overrides win + /// over the process-wide setting. Returns when inactive. + /// + public static bool TryResolveTarget(ConnectionByteTransport? transport, out int targetBytes) + { + if (transport is not null && PerTransport.TryGetValue(transport, out var box)) + { + targetBytes = box.Value; + return true; + } + + return TryGetTarget(out targetBytes); + } + + /// Gets the process-wide target, or when inactive. + public static bool TryGetTarget(out int targetBytes) + { + var configured = Volatile.Read(ref _state); + if (configured is null) + { + targetBytes = 0; + return false; + } + + targetBytes = configured.TargetBytes; + return true; + } + + /// Parses an allowlisted size token or byte count. + public static bool TryParseAllowed(string value, out int targetBytes) + { + targetBytes = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (int.TryParse(trimmed, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var raw) && + IsAllowed(raw)) + { + targetBytes = raw; + return true; + } + + if (TryParseSizeToken(trimmed, out var parsed) && IsAllowed(parsed)) + { + targetBytes = parsed; + return true; + } + + return false; + } + + /// Returns whether is on the experiment allowlist. + public static bool IsAllowed(int targetBytes) + { + for (var i = 0; i < AllowedTargetBytes.Length; i++) + { + if (AllowedTargetBytes[i] == targetBytes) + { + return true; + } + } + + return false; + } + + private static bool TryParseSizeToken(string value, out int bytes) + { + bytes = 0; + var span = value.AsSpan().Trim(); + var suffixLen = 0; + int multiplier; + if (span.EndsWith("KiB", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024; + } + else if (span.EndsWith("MiB", StringComparison.OrdinalIgnoreCase)) + { + suffixLen = 3; + multiplier = 1024 * 1024; + } + else if (span.Length > 0 && (span[^1] is 'k' or 'K')) + { + suffixLen = 1; + multiplier = 1024; + } + else if (span.Length > 0 && (span[^1] is 'm' or 'M')) + { + suffixLen = 1; + multiplier = 1024 * 1024; + } + else + { + return false; + } + + var number = span[..^suffixLen]; + if (!int.TryParse(number, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var count) || + count <= 0) + { + return false; + } + + try + { + bytes = checked(count * multiplier); + } + catch (OverflowException) + { + return false; + } + + return true; + } + + private sealed class OverrideState(int targetBytes) + { + public int TargetBytes { get; } = targetBytes; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/INewsgroupCatalogue.cs b/src/VectorNNTPD.TransitService/Newsgroups/INewsgroupCatalogue.cs new file mode 100644 index 00000000..6b668058 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/INewsgroupCatalogue.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// Process-wide newsgroup catalogue. Command handlers capture once. + /// + public interface INewsgroupCatalogue + { + /// + /// Gets the last successfully published immutable snapshot. + /// + /// + /// After a successful start this is never null. Readers need no lock. + /// Capture the reference once per command; a concurrent refresh may replace + /// the published snapshot while that command continues on the captured instance. + /// + NewsgroupSnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueException.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueException.cs new file mode 100644 index 00000000..0fb8620f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueException.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// Malformed or incomplete newsgroup catalogue data that must not be published. + public sealed class NewsgroupCatalogueException : Exception + { + /// Initializes a new instance of the class. + public NewsgroupCatalogueException(string message) + : base(message) + { + } + + /// Initializes a new instance of the class. + public NewsgroupCatalogueException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueLogMessages.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueLogMessages.cs new file mode 100644 index 00000000..74d54e02 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// Source-generated newsgroup catalogue lifecycle log messages. + internal static partial class NewsgroupCatalogueLogMessages + { + [LoggerMessage( + EventId = 2500, + Level = LogLevel.Information, + Message = "Loading initial newsgroup catalogue")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2501, + Level = LogLevel.Information, + Message = "Published newsgroup catalogue snapshot with {GroupCount} groups")] + public static partial void SnapshotPublished(ILogger logger, int GroupCount); + + [LoggerMessage( + EventId = 2502, + Level = LogLevel.Error, + Message = "Initial newsgroup catalogue load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2503, + Level = LogLevel.Error, + Message = "Newsgroup catalogue refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2504, + Level = LogLevel.Information, + Message = "Newsgroup catalogue refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2505, + Level = LogLevel.Information, + Message = "Newsgroup catalogue refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueService.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueService.cs new file mode 100644 index 00000000..3a064c28 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupCatalogueService.cs @@ -0,0 +1,221 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// Loads the newsgroup catalogue from NntpDB, publishes an immutable snapshot, + /// and refreshes it every five minutes. + /// + /// + /// + /// MySqlConnector owns physical connection pooling. Each refresh opens one logical + /// connection through , consumes the result set, and + /// disposes the connection before the snapshot is published. The snapshot retains + /// only managed immutable data. + /// + /// + /// Initial population runs during and fails startup when + /// the query cannot be completed. Subsequent refresh failures keep the last + /// known-good snapshot. A refresh does not start while the previous refresh is + /// still running (single loop). + /// + /// + public sealed class NewsgroupCatalogueService : INewsgroupCatalogue, IApplicationService, IAsyncDisposable + { + /// Catalogue refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(5); + + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private NewsgroupSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + public NewsgroupCatalogueService( + NntpDbService nntpDb, + ILogger logger) + : this(nntpDb, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + internal NewsgroupCatalogueService( + NntpDbService nntpDb, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "NewsgroupCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public NewsgroupSnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + return snapshot ?? throw new InvalidOperationException("Newsgroup catalogue has not been published."); + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + NewsgroupCatalogueLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + NewsgroupCatalogueLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + NewsgroupCatalogueLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown. + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + /// Replaces the published snapshot (tests). + internal void PublishForTests(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Publish(snapshot); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + + NewsgroupCatalogueLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NewsgroupCatalogueLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryNewsgroupsAsync(cancellationToken).ConfigureAwait(false); + return NewsgroupSnapshot.Create(rows); + } + + private void Publish(NewsgroupSnapshot snapshot) + { + Interlocked.Exchange(ref _current, snapshot); + NewsgroupCatalogueLogMessages.SnapshotPublished(_logger, snapshot.Groups.Count); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupPostingStatus.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupPostingStatus.cs new file mode 100644 index 00000000..193f65f1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupPostingStatus.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// LIST ACTIVE posting-status octet as stored in nntpgroups.posting_status. + /// + /// + /// + /// Supported values are the fixed MySQL enum y, n, m, x, + /// j. Meanings follow RFC 3977 §7.6.3 and RFC 6048 §3.1: + /// + /// + /// y — posting is permitted. + /// n — posting is not permitted. + /// m — postings are forwarded to the newsgroup moderator. + /// x — postings and articles from peers are not permitted. + /// j — articles from peers are permitted, but nothing is locally filed. + /// + /// + /// RFC 6048's =<newsgroup> alias form is intentionally unsupported. + /// There is no enum member for =. Unknown octets are rejected; they are + /// never coerced. + /// + /// + public enum NewsgroupPostingStatus : byte + { + /// Posting is permitted (y). RFC 3977 §7.6.3. + Allowed = (byte)'y', + + /// Posting is prohibited (n). RFC 3977 §7.6.3. + Prohibited = (byte)'n', + + /// Postings are forwarded to the moderator (m). RFC 3977 §7.6.3. + Moderated = (byte)'m', + + /// + /// Posting and articles from peers are not permitted (x). RFC 6048 §3.1. + /// Distinct from ; not equivalent to n. + /// + NoPostingOrPeerArticles = (byte)'x', + + /// + /// Articles from peers are permitted, but nothing is locally filed (j). RFC 6048 §3.1. + /// Distinct from and . + /// This is not a catch-all junk group; the octet is the RFC 6048 j status. + /// + PeerOnly = (byte)'j', + } + + /// Validation helpers for octets. + public static class NewsgroupPostingStatusOctets + { + /// + /// Returns whether is one of the five supported status octets. + /// + /// Candidate status octet. + /// for y/n/m/x/j. + public static bool IsSupported(byte value) => + value is (byte)'y' or (byte)'n' or (byte)'m' or (byte)'x' or (byte)'j'; + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupRecord.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupRecord.cs new file mode 100644 index 00000000..54b1cc33 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupRecord.cs @@ -0,0 +1,77 @@ +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// One immutable newsgroup entry owned by a . + /// + /// + /// Precomputed UTF-8 wire lines include CRLF and, when required, multiline + /// dot-stuffing. Command handlers must not re-encode these buffers. + /// + public sealed class NewsgroupRecord + { + internal NewsgroupRecord( + string groupName, + byte[] groupNameBytes, + string groupDescription, + ulong countHigh, + ulong countLow, + NewsgroupPostingStatus postingStatus, + ulong estimatedCount, + byte[] listActiveLine, + byte[] listCountsLine, + byte[] listNewsgroupsLine, + byte[] groupSelectedLine) + { + GroupName = groupName; + GroupNameBytes = groupNameBytes; + GroupDescription = groupDescription; + CountHigh = countHigh; + CountLow = countLow; + PostingStatus = postingStatus; + EstimatedCount = estimatedCount; + ListActiveLine = listActiveLine; + ListCountsLine = listCountsLine; + ListNewsgroupsLine = listNewsgroupsLine; + GroupSelectedLine = groupSelectedLine; + } + + /// Gets the original group name as stored in the database. + public string GroupName { get; } + + /// Gets the original group name as UTF-8 octets used for lookup and wire output. + public ReadOnlyMemory GroupNameBytes { get; } + + /// Gets the original group description (may be empty). + public string GroupDescription { get; } + + /// Gets the unsigned high water mark. + public ulong CountHigh { get; } + + /// Gets the unsigned low water mark. + public ulong CountLow { get; } + + /// Gets the database posting status (y/n/m/x/j). + public NewsgroupPostingStatus PostingStatus { get; } + + /// + /// Gets the GROUP article-count estimate derived from the water marks + /// (RFC 3977 §6.1.1: high − low + 1, or 0 when the group is empty). + /// + public ulong EstimatedCount { get; } + + /// Gets the precomputed LIST ACTIVE line including CRLF. + public ReadOnlyMemory ListActiveLine { get; } + + /// + /// Gets the precomputed LIST COUNTS line including CRLF + /// (group high low estimated status). + /// + public ReadOnlyMemory ListCountsLine { get; } + + /// Gets the precomputed LIST NEWSGROUPS line including CRLF. + public ReadOnlyMemory ListNewsgroupsLine { get; } + + /// Gets the precomputed 211 number low high group line including CRLF. + public ReadOnlyMemory GroupSelectedLine { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupSnapshot.cs b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupSnapshot.cs new file mode 100644 index 00000000..4129c35d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NewsgroupSnapshot.cs @@ -0,0 +1,430 @@ +using System.Buffers.Text; +using System.Text; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// Immutable, fully constructed newsgroup catalogue snapshot. + /// + /// + /// Readers observe either the previous complete snapshot or this complete snapshot. + /// No lock is required. Command handlers must capture + /// once and use only that instance. + /// + public sealed class NewsgroupSnapshot + { + private static readonly NewsgroupNameComparer NameComparer = new(); + + /// Empty published snapshot (zero groups; valid LIST body is empty). + public static NewsgroupSnapshot Empty { get; } = Create(Array.Empty()); + + private readonly NewsgroupRecord[] _groups; + + private NewsgroupSnapshot( + NewsgroupRecord[] groups, + byte[] listActiveBody, + byte[] listActiveComplete, + byte[] listCountsBody, + byte[] listCountsComplete, + byte[] listNewsgroupsBody, + byte[] listNewsgroupsComplete) + { + _groups = groups; + Groups = groups; + ListActiveBody = listActiveBody; + ListActiveComplete = listActiveComplete; + ListCountsBody = listCountsBody; + ListCountsComplete = listCountsComplete; + ListNewsgroupsBody = listNewsgroupsBody; + ListNewsgroupsComplete = listNewsgroupsComplete; + } + + /// Gets groups in deterministic ordinal-ignore-case name order. + public IReadOnlyList Groups { get; } + + /// Gets concatenated precomputed LIST ACTIVE lines (no status or terminator). + public ReadOnlyMemory ListActiveBody { get; } + + /// Gets the complete LIST / LIST ACTIVE multiline wire (status + body + terminator). + public ReadOnlyMemory ListActiveComplete { get; } + + /// Gets concatenated precomputed LIST COUNTS lines (no status or terminator). + public ReadOnlyMemory ListCountsBody { get; } + + /// Gets the complete LIST COUNTS multiline wire (status + body + terminator). + public ReadOnlyMemory ListCountsComplete { get; } + + /// Gets concatenated precomputed LIST NEWSGROUPS lines (no status or terminator). + public ReadOnlyMemory ListNewsgroupsBody { get; } + + /// Gets the complete LIST NEWSGROUPS multiline wire (status + body + terminator). + public ReadOnlyMemory ListNewsgroupsComplete { get; } + + /// Builds a snapshot from raw database rows. Fails the entire build on any malformed row. + public static NewsgroupSnapshot Create(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + var records = new NewsgroupRecord[rows.Count]; + for (var i = 0; i < rows.Count; i++) + { + records[i] = CreateRecord(rows[i], i); + } + + return Publish(records); + } + + /// Builds a snapshot from already-typed definitions (tests and in-process construction). + public static NewsgroupSnapshot Create(IReadOnlyList definitions) + { + ArgumentNullException.ThrowIfNull(definitions); + var records = new NewsgroupRecord[definitions.Count]; + for (var i = 0; i < definitions.Count; i++) + { + var definition = definitions[i]; + records[i] = CreateRecord( + new NntpGroupRow( + definition.GroupName, + definition.GroupDescription, + definition.CountHigh, + definition.CountLow, + (byte)definition.PostingStatus), + i); + } + + return Publish(records); + } + + /// + /// Looks up a group by name using ASCII ordinal case-insensitive comparison. + /// The original stored name is retained on the returned record. + /// + public bool TryGet(ReadOnlySpan groupName, out NewsgroupRecord group) + { + var lo = 0; + var hi = _groups.Length - 1; + while (lo <= hi) + { + var mid = lo + ((hi - lo) / 2); + var cmp = CompareOrdinalIgnoreCase(_groups[mid].GroupNameBytes.Span, groupName); + if (cmp == 0) + { + group = _groups[mid]; + return true; + } + + if (cmp < 0) + { + lo = mid + 1; + } + else + { + hi = mid - 1; + } + } + + group = null!; + return false; + } + + private static NewsgroupSnapshot Publish(NewsgroupRecord[] records) + { + Array.Sort(records, NameComparer); + for (var i = 1; i < records.Length; i++) + { + if (CompareOrdinalIgnoreCase(records[i - 1].GroupNameBytes.Span, records[i].GroupNameBytes.Span) == 0) + { + throw new NewsgroupCatalogueException( + $"Duplicate newsgroup name '{records[i].GroupName}' (case-insensitive)."); + } + } + + var activeBody = ConcatenateLines(records, static r => r.ListActiveLine); + var countsBody = ConcatenateLines(records, static r => r.ListCountsLine); + var newsgroupsBody = ConcatenateLines(records, static r => r.ListNewsgroupsLine); + return new NewsgroupSnapshot( + records, + activeBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, activeBody), + countsBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, countsBody), + newsgroupsBody, + ConcatenateComplete(NntpResponses.ListOfNewsgroupsFollows, newsgroupsBody)); + } + + private static NewsgroupRecord CreateRecord(NntpGroupRow row, int index) + { + if (string.IsNullOrEmpty(row.GroupName)) + { + throw new NewsgroupCatalogueException($"nntpgroups row {index} has an empty group_name."); + } + + if (!NewsgroupPostingStatusOctets.IsSupported(row.PostingStatus)) + { + throw new NewsgroupCatalogueException( + $"nntpgroups row '{row.GroupName}' has invalid posting_status 0x{row.PostingStatus:X2}."); + } + + var nameBytes = Encoding.UTF8.GetBytes(row.GroupName); + var description = row.GroupDescription ?? string.Empty; + var descriptionBytes = Encoding.UTF8.GetBytes(description); + var status = (NewsgroupPostingStatus)row.PostingStatus; + var estimate = EstimateCount(row.CountLow, row.CountHigh); + return new NewsgroupRecord( + row.GroupName, + nameBytes, + description, + row.CountHigh, + row.CountLow, + status, + estimate, + EncodeActiveLine(nameBytes, row.CountHigh, row.CountLow, row.PostingStatus), + EncodeCountsLine(nameBytes, row.CountHigh, row.CountLow, estimate, row.PostingStatus), + EncodeNewsgroupsLine(nameBytes, descriptionBytes), + EncodeGroupSelectedLine(nameBytes, estimate, row.CountLow, row.CountHigh)); + } + + /// RFC 3977 §6.1.1 estimate: 0 when empty; otherwise high − low + 1 without wrapping. + internal static ulong EstimateCount(ulong low, ulong high) + { + if (high < low) + { + return 0; + } + + if (low == 0 && high == 0) + { + return 0; + } + + var span = high - low; + return span == ulong.MaxValue ? ulong.MaxValue : span + 1; + } + + private static byte[] EncodeActiveLine( + ReadOnlySpan name, + ulong high, + ulong low, + byte status) + { + Span highDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(high, highDigits, out var highLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen)) + { + throw new NewsgroupCatalogueException("Failed to format unsigned water marks."); + } + + var length = name.Length + 1 + highLen + 1 + lowLen + 1 + 1 + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + line[offset++] = status; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeCountsLine( + ReadOnlySpan name, + ulong high, + ulong low, + ulong estimate, + byte status) + { + Span highDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + Span estimateDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(high, highDigits, out var highLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen) + || !Utf8Formatter.TryFormat(estimate, estimateDigits, out var estimateLen)) + { + throw new NewsgroupCatalogueException("Failed to format LIST COUNTS water marks."); + } + + var length = name.Length + 1 + highLen + 1 + lowLen + 1 + estimateLen + 1 + 1 + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + estimateDigits[..estimateLen].CopyTo(line.AsSpan(offset)); + offset += estimateLen; + line[offset++] = (byte)' '; + line[offset++] = status; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeNewsgroupsLine(ReadOnlySpan name, ReadOnlySpan description) + { + var length = name.Length + 1 + description.Length + 2; + var line = new byte[length]; + var offset = 0; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)' '; + description.CopyTo(line.AsSpan(offset)); + offset += description.Length; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return StuffIfNeeded(line); + } + + private static byte[] EncodeGroupSelectedLine( + ReadOnlySpan name, + ulong estimate, + ulong low, + ulong high) + { + Span estimateDigits = stackalloc byte[20]; + Span lowDigits = stackalloc byte[20]; + Span highDigits = stackalloc byte[20]; + if (!Utf8Formatter.TryFormat(estimate, estimateDigits, out var estimateLen) + || !Utf8Formatter.TryFormat(low, lowDigits, out var lowLen) + || !Utf8Formatter.TryFormat(high, highDigits, out var highLen)) + { + throw new NewsgroupCatalogueException("Failed to format GROUP water marks."); + } + + // "211 " + estimate + " " + low + " " + high + " " + name + CRLF + var length = 4 + estimateLen + 1 + lowLen + 1 + highLen + 1 + name.Length + 2; + var line = new byte[length]; + line[0] = (byte)'2'; + line[1] = (byte)'1'; + line[2] = (byte)'1'; + line[3] = (byte)' '; + var offset = 4; + estimateDigits[..estimateLen].CopyTo(line.AsSpan(offset)); + offset += estimateLen; + line[offset++] = (byte)' '; + lowDigits[..lowLen].CopyTo(line.AsSpan(offset)); + offset += lowLen; + line[offset++] = (byte)' '; + highDigits[..highLen].CopyTo(line.AsSpan(offset)); + offset += highLen; + line[offset++] = (byte)' '; + name.CopyTo(line.AsSpan(offset)); + offset += name.Length; + line[offset++] = (byte)'\r'; + line[offset] = (byte)'\n'; + return line; + } + + private static byte[] StuffIfNeeded(byte[] line) + { + if (line.Length == 0 || line[0] != (byte)'.') + { + return line; + } + + var stuffed = new byte[line.Length + 1]; + stuffed[0] = (byte)'.'; + line.CopyTo(stuffed, 1); + return stuffed; + } + + private static byte[] ConcatenateLines( + NewsgroupRecord[] records, + Func> selector) + { + var total = 0; + for (var i = 0; i < records.Length; i++) + { + total = checked(total + selector(records[i]).Length); + } + + var body = new byte[total]; + var offset = 0; + for (var i = 0; i < records.Length; i++) + { + var line = selector(records[i]).Span; + line.CopyTo(body.AsSpan(offset)); + offset += line.Length; + } + + return body; + } + + private static byte[] ConcatenateComplete(ReadOnlyMemory status, ReadOnlySpan body) + { + var total = checked(status.Length + body.Length + NntpResponses.MultilineTerminator.Length); + var complete = new byte[total]; + status.Span.CopyTo(complete); + body.CopyTo(complete.AsSpan(status.Length)); + NntpResponses.MultilineTerminator.Span.CopyTo(complete.AsSpan(status.Length + body.Length)); + return complete; + } + + internal static int CompareOrdinalIgnoreCase(ReadOnlySpan left, ReadOnlySpan right) + { + var n = Math.Min(left.Length, right.Length); + for (var i = 0; i < n; i++) + { + var a = NntpAscii.FoldUpper(left[i]); + var b = NntpAscii.FoldUpper(right[i]); + if (a != b) + { + return a.CompareTo(b); + } + } + + return left.Length.CompareTo(right.Length); + } + + private sealed class NewsgroupNameComparer : IComparer + { + public int Compare(NewsgroupRecord? x, NewsgroupRecord? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var ignoreCase = CompareOrdinalIgnoreCase(x.GroupNameBytes.Span, y.GroupNameBytes.Span); + return ignoreCase != 0 + ? ignoreCase + : x.GroupNameBytes.Span.SequenceCompareTo(y.GroupNameBytes.Span); + } + } + } + + /// Typed newsgroup fields used to construct a snapshot without a database row. + /// Original group name retained for wire output. + /// Group description (UTF-8 on the wire). + /// Unsigned high water mark. + /// Unsigned low water mark. + /// Exactly y, n, m, x, or j. The RFC 6048 =<newsgroup> form is not supported. + public readonly record struct NewsgroupDefinition( + string GroupName, + string GroupDescription, + ulong CountHigh, + ulong CountLow, + NewsgroupPostingStatus PostingStatus); +} diff --git a/src/VectorNNTPD.TransitService/Newsgroups/NntpWildmat.cs b/src/VectorNNTPD.TransitService/Newsgroups/NntpWildmat.cs new file mode 100644 index 00000000..a0722c7c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Newsgroups/NntpWildmat.cs @@ -0,0 +1,242 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Newsgroups +{ + /// + /// RFC 3977 §4 wildmat matcher for newsgroup names. + /// + /// + /// Operates on UTF-8 octets. Literal matching is ASCII ordinal case-insensitive + /// (NNTP newsgroup names). Comma-separated patterns and rightmost ! negation + /// follow §4.2. Characters \, [, and ] are rejected as syntax + /// errors (RFC 3977 §4.1 / §3.2.1.1). + /// + public static class NntpWildmat + { + /// Returns whether is a syntactically valid RFC 3977 wildmat. + public static bool TryValidate(ReadOnlySpan wildmat) + { + if (wildmat.IsEmpty) + { + return false; + } + + var index = 0; + var first = true; + while (index < wildmat.Length) + { + if (!first) + { + if (wildmat[index] != (byte)',') + { + return false; + } + + index++; + if (index < wildmat.Length && wildmat[index] == (byte)'!') + { + index++; + } + } + + first = false; + if (index >= wildmat.Length) + { + return false; + } + + var patternStart = index; + while (index < wildmat.Length && wildmat[index] != (byte)',') + { + var b = wildmat[index]; + if (b is (byte)'\\' or (byte)'[' or (byte)']') + { + return false; + } + + if (b == (byte)'!' && index == patternStart) + { + return false; + } + + index++; + } + + if (patternStart == index) + { + return false; + } + } + + return true; + } + + /// + /// Returns whether matches + /// using ASCII ordinal case-insensitive comparison. + /// + public static bool IsMatch(ReadOnlySpan text, ReadOnlySpan wildmat) + { + if (!TryValidate(wildmat)) + { + return false; + } + + return IsMatchValidated(text, wildmat); + } + + /// + /// Matches against an already-validated wildmat. + /// + /// + /// LIST command handlers use this after has already + /// succeeded at parse time, so each group is not revalidated. + /// + public static bool IsMatchValidated(ReadOnlySpan text, ReadOnlySpan wildmat) + { + var rightmostMatchNegated = false; + var anyMatch = false; + var index = 0; + var first = true; + while (index < wildmat.Length) + { + var negated = false; + if (!first) + { + index++; + if (index < wildmat.Length && wildmat[index] == (byte)'!') + { + negated = true; + index++; + } + } + + first = false; + var start = index; + while (index < wildmat.Length && wildmat[index] != (byte)',') + { + index++; + } + + if (MatchPattern(text, wildmat[start..index])) + { + anyMatch = true; + rightmostMatchNegated = negated; + } + } + + return anyMatch && !rightmostMatchNegated; + } + + private static bool MatchPattern(ReadOnlySpan text, ReadOnlySpan pattern) + { + return Match(text, pattern); + } + + private static bool Match(ReadOnlySpan text, ReadOnlySpan pattern) + { + while (!pattern.IsEmpty) + { + if (pattern[0] == (byte)'*') + { + while (pattern.Length > 1 && pattern[1] == (byte)'*') + { + pattern = pattern[1..]; + } + + pattern = pattern[1..]; + if (pattern.IsEmpty) + { + return true; + } + + var remaining = text; + while (true) + { + if (Match(remaining, pattern)) + { + return true; + } + + if (remaining.IsEmpty) + { + return false; + } + + remaining = ConsumeCharacter(remaining); + } + } + + if (text.IsEmpty) + { + return false; + } + + if (pattern[0] == (byte)'?') + { + text = ConsumeCharacter(text); + pattern = pattern[1..]; + continue; + } + + if (!TryConsumeLiteral(ref text, pattern[0])) + { + return false; + } + + pattern = pattern[1..]; + } + + return text.IsEmpty; + } + + private static bool TryConsumeLiteral(ref ReadOnlySpan text, byte literal) + { + if (text.IsEmpty) + { + return false; + } + + var foldedText = NntpAscii.FoldUpper(text[0]); + var foldedLiteral = NntpAscii.FoldUpper(literal); + if (foldedText != foldedLiteral) + { + return false; + } + + text = text[1..]; + return true; + } + + private static ReadOnlySpan ConsumeCharacter(ReadOnlySpan text) + { + if (text.IsEmpty) + { + return text; + } + + var width = Utf8CharacterWidth(text[0]); + return width >= text.Length ? ReadOnlySpan.Empty : text[width..]; + } + + private static int Utf8CharacterWidth(byte lead) + { + if (lead < 0x80) + { + return 1; + } + + if (lead < 0xE0) + { + return 2; + } + + if (lead < 0xF0) + { + return 3; + } + + return 4; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFile.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFile.cs new file mode 100644 index 00000000..c3291ebb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFile.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// An opened completed Path-survey file held across gzip/deletion. + internal sealed class NinpathsCompletedFile : IDisposable + { + /// Initializes a handoff of with an already-open . + public NinpathsCompletedFile(string path, FileStream stream) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(stream); + Path = path; + Stream = stream; + } + + /// Gets the completed file path supplied by Serilog. + public string Path { get; } + + /// Gets the sequential reader. Opened with share-read/write/delete. + public FileStream Stream { get; } + + /// + public void Dispose() => Stream.Dispose(); + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFileHandler.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFileHandler.cs new file mode 100644 index 00000000..3fa619e6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsCompletedFileHandler.cs @@ -0,0 +1,84 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Rotation-hook handoff: opens the completed Path-survey file and enqueues it + /// for background ninpaths processing. Does not parse the file. + /// + /// + /// The stream is opened with and + /// so gzip and deletion of the completed file can proceed + /// while the worker still reads. Failures are logged and never thrown to rotation. + /// + public sealed class NinpathsCompletedFileHandler : ICompletedPathSurveyFileHandler + { + private readonly NinpathsProcessingService _processor; + private readonly IOptions _options; + private readonly ILogger _logger; + + /// Initializes a handler that enqueues completed files onto . + public NinpathsCompletedFileHandler( + NinpathsProcessingService processor, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(processor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _processor = processor; + _options = options; + _logger = logger; + } + + /// + public void OnCompletedFile(string completedFilePath) + { + if (string.IsNullOrWhiteSpace(completedFilePath)) + { + return; + } + + if (!NinpathsTop1000.IsEnabled(_options.Value)) + { + NinpathsLogMessages.Disabled(_logger, completedFilePath); + return; + } + + FileStream stream; + try + { + stream = new FileStream( + completedFilePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + NinpathsConstants.ReadBufferSize, + FileOptions.SequentialScan); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NinpathsLogMessages.OpenFailed(_logger, ex, completedFilePath); + return; + } + + // codeql[cs/missed-using-statement] On successful TryEnqueue ownership transfers to NinpathsProcessingService; using would dispose an enqueued file. finally disposes only when enqueue fails or throws. + var file = new NinpathsCompletedFile(completedFilePath, stream); + var enqueued = false; + try + { + enqueued = _processor.TryEnqueue(file); + } + finally + { + if (!enqueued) + { + file.Dispose(); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsConstants.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsConstants.cs new file mode 100644 index 00000000..0e1c13b3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsConstants.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Constants matching current INN backends/ninpaths.c (version 3.1.1). + /// + internal static class NinpathsConstants + { + /// Dump format version written in the !!NINP header. + public const string Version = "3.1.1"; + + /// INN MAXLINE: fgets payload is this size including the terminator slot. + public const int MaxLine = 1024; + + /// INN HASH_TBL. + public const int HashTableSize = 65536; + + /// INN MAXHOST including the C string terminator. + public const int MaxHost = 128; + + /// INN RECLINE dump line soft wrap. + public const int RecordLineSoftLimit = 120; + + /// FNV prime used by INN hash(). + public const ulong HashPrime = 16777619; + + /// Maximum host token length (MAXHOST - 1). + public const int MaxHostChars = MaxHost - 1; + + /// Maximum bytes fgets(buf, MAXLINE) stores before the NUL. + public const int MaxFgetsChars = MaxLine - 1; + + /// Bounded pending completed-file queue (daily files, not Path records). + public const int PendingFileCapacity = 8; + + /// Sequential read buffer for completed Path-survey files. + public const int ReadBufferSize = 64 * 1024; + + /// sendinpaths subject prefix including the trailing space before the pathhost. + public const string EmailSubjectPrefix = "inpaths "; + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogMessages.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogMessages.cs new file mode 100644 index 00000000..07fcd77a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogMessages.cs @@ -0,0 +1,90 @@ +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// Source-generated ninpaths processing log messages. + internal static partial class NinpathsLogMessages + { + [LoggerMessage( + EventId = 2900, + Level = LogLevel.Debug, + Message = "Ninpaths skipped completed Path-survey file {CompletedFile}; Nntpd:Top1000 has no recipients")] + public static partial void Disabled(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2901, + Level = LogLevel.Error, + Message = "Ninpaths could not open completed Path-survey file {CompletedFile}")] + public static partial void OpenFailed(ILogger logger, Exception exception, string CompletedFile); + + [LoggerMessage( + EventId = 2902, + Level = LogLevel.Debug, + Message = "Ninpaths ignored duplicate completed Path-survey file {CompletedFile}")] + public static partial void Duplicate(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2903, + Level = LogLevel.Error, + Message = "Ninpaths pending-file queue is full; dropping handoff for {CompletedFile}")] + public static partial void QueueFull(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2904, + Level = LogLevel.Information, + Message = "Ninpaths processing completed Path-survey file {CompletedFile}")] + public static partial void Processing(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2905, + Level = LogLevel.Information, + Message = "Ninpaths produced no dump for {CompletedFile}; total articles is zero")] + public static partial void EmptyDump(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2906, + Level = LogLevel.Error, + Message = "Ninpaths cannot compose mail for {CompletedFile}; Email:DefaultFrom is missing or invalid")] + public static partial void MissingFrom(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2907, + Level = LogLevel.Information, + Message = "Ninpaths report for {CompletedFile} not mailed; Email:Enabled is false")] + public static partial void EmailDisabled(ILogger logger, string CompletedFile); + + [LoggerMessage( + EventId = 2908, + Level = LogLevel.Error, + Message = "Ninpaths failed to spool report for {CompletedFile} status={Status} detail={Detail}")] + public static partial void EmailFailed(ILogger logger, string CompletedFile, string Status, string Detail); + + [LoggerMessage( + EventId = 2909, + Level = LogLevel.Information, + Message = "Ninpaths spooled report for {CompletedFile} recipients={RecipientCount} bytes={ReportBytes}")] + public static partial void EmailAccepted(ILogger logger, string CompletedFile, int RecipientCount, int ReportBytes); + + [LoggerMessage( + EventId = 2910, + Level = LogLevel.Error, + Message = "Ninpaths processing failed for {CompletedFile}")] + public static partial void ProcessingFailed(ILogger logger, Exception exception, string CompletedFile); + + [LoggerMessage( + EventId = 2911, + Level = LogLevel.Error, + Message = "Ninpaths background worker failed")] + public static partial void WorkerFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2912, + Level = LogLevel.Information, + Message = "Ninpaths processing service started")] + public static partial void Started(ILogger logger); + + [LoggerMessage( + EventId = 2913, + Level = LogLevel.Information, + Message = "Ninpaths processing service stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogReader.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogReader.cs new file mode 100644 index 00000000..f7b6c133 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsLogReader.cs @@ -0,0 +1,160 @@ +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Streaming Path-survey reader that applies INN procpaths / fgets line semantics. + /// + /// + /// Reads sequentially with a fixed buffer. Does not load the file, does not + /// retain Path strings, and does not allocate per article beyond interned + /// unique sites. Matches INN fgets(buf, MAXLINE), isspace, + /// case-insensitive Path: , last-element chop, over-length token abort, + /// and the one-window bogus-line skip (not skip-until-newline). + /// + public static class NinpathsLogReader + { + private static ReadOnlySpan PathPrefix => "Path: "u8; + + /// + /// Aggregates every valid Path line from into + /// a new instance. + /// + public static NinpathsStatistics Read(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + var stats = new NinpathsStatistics(); + Read(stream, stats); + return stats; + } + + /// Aggregates Path lines from into . + public static void Read(Stream stream, NinpathsStatistics stats) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(stats); + + var window = new byte[NinpathsConstants.MaxFgetsChars]; + var block = new byte[NinpathsConstants.ReadBufferSize]; + var blockLength = 0; + var blockOffset = 0; + var valid = true; + while (true) + { + var n = ReadFgets(stream, window, block, ref blockLength, ref blockOffset); + if (n <= 0) + { + break; + } + + ProcessFgetsWindow(window.AsSpan(0, n), stats, ref valid); + } + } + + /// Applies INN procpaths to one fgets window (newline included when present). + internal static void ProcessFgetsWindow(ReadOnlySpan window, NinpathsStatistics stats, ref bool valid) + { + var body = window; + if (StartsWithPathPrefix(body)) + { + body = body[PathPrefix.Length..]; + } + + var end = 0; + for (; end < body.Length; end++) + { + if (IsSpace(body[end])) + { + break; + } + } + + if (end >= body.Length) + { + valid = false; + return; + } + + if (!valid) + { + valid = true; + return; + } + + var path = body[..end]; + var lastBang = path.LastIndexOf((byte)'!'); + var chopped = lastBang < 0 ? ReadOnlySpan.Empty : path[..lastBang]; + stats.AddChoppedPath(chopped); + stats.AddValidArticle(); + } + + private static int ReadFgets( + Stream stream, + byte[] window, + byte[] block, + ref int blockLength, + ref int blockOffset) + { + var written = 0; + while (written < window.Length) + { + if (blockOffset >= blockLength) + { + blockLength = stream.Read(block, 0, block.Length); + blockOffset = 0; + if (blockLength == 0) + { + return written; + } + } + + var b = block[blockOffset++]; + window[written++] = b; + if (b == (byte)'\n') + { + break; + } + } + + return written; + } + + private static bool StartsWithPathPrefix(ReadOnlySpan line) + { + var prefix = PathPrefix; + if (line.Length < prefix.Length) + { + return false; + } + + for (var i = 0; i < prefix.Length; i++) + { + var a = line[i]; + var b = prefix[i]; + if (a == b) + { + continue; + } + + if (a is >= (byte)'A' and <= (byte)'Z') + { + a += 32; + } + + if (b is >= (byte)'A' and <= (byte)'Z') + { + b += 32; + } + + if (a != b) + { + return false; + } + } + + return true; + } + + /// C isspace on unsigned char (C locale). + internal static bool IsSpace(byte value) => + value is 0x09 or 0x0A or 0x0B or 0x0C or 0x0D or 0x20; + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsProcessingService.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsProcessingService.cs new file mode 100644 index 00000000..e4c1c01b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsProcessingService.cs @@ -0,0 +1,243 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Background ninpaths worker. Consumes completed Path-survey files without + /// blocking rotation or NNTP request processing. + /// + /// + /// The pending queue holds at most + /// completed files (not Path records). Worker exceptions are logged and do not + /// fault . The source file is streamed; raw Path lines + /// are not retained. + /// + public sealed class NinpathsProcessingService : IApplicationService + { + private readonly Channel _queue; + private readonly ConcurrentDictionary _seen = new(StringComparer.OrdinalIgnoreCase); + private readonly IEmailService _email; + private readonly IOptions _nntpdOptions; + private readonly IOptions _emailOptions; + private readonly TimeProvider _time; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new ninpaths processing service. + public NinpathsProcessingService( + IEmailService email, + IOptions nntpdOptions, + IOptions emailOptions, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(email); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(emailOptions); + ArgumentNullException.ThrowIfNull(logger); + _email = email; + _nntpdOptions = nntpdOptions; + _emailOptions = emailOptions; + _logger = logger; + _time = timeProvider ?? TimeProvider.System; + _queue = Channel.CreateBounded( + new BoundedChannelOptions(NinpathsConstants.PendingFileCapacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait, + }); + } + + /// + public string Name => "NinpathsProcessing"; + + /// + public Task? Execution => _execution; + + /// Gets the number of completed files waiting for the worker. + internal int PendingCount => _queue.Reader.Count; + + /// + /// Enqueues an already-open completed file. Returns + /// on duplicate path or a full queue (the caller must dispose ). + /// + internal bool TryEnqueue(NinpathsCompletedFile file) + { + ArgumentNullException.ThrowIfNull(file); + var key = System.IO.Path.GetFullPath(file.Path); + if (!_seen.TryAdd(key, 0)) + { + NinpathsLogMessages.Duplicate(_logger, file.Path); + return false; + } + + if (_queue.Writer.TryWrite(file)) + { + return true; + } + + _seen.TryRemove(key, out _); + NinpathsLogMessages.QueueFull(_logger, file.Path); + return false; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + NinpathsLogMessages.Started(_logger); + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Writer.TryComplete(); + try + { + await _runCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.WorkerFailed(_logger, ex); + } + } + + DrainPending(); + NinpathsLogMessages.Stopped(_logger); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await foreach (var file in _queue.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + using (file) + { + try + { + await ProcessAsync(file, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.ProcessingFailed(_logger, ex, file.Path); + } + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NinpathsLogMessages.WorkerFailed(_logger, ex); + } + } + + private async Task ProcessAsync(NinpathsCompletedFile file, CancellationToken cancellationToken) + { + var nntpd = _nntpdOptions.Value; + var recipients = NinpathsTop1000.GetRecipients(nntpd); + if (recipients.Length == 0) + { + NinpathsLogMessages.Disabled(_logger, file.Path); + return; + } + + NinpathsLogMessages.Processing(_logger, file.Path); + var stats = NinpathsLogReader.Read(file.Stream); + if (stats.TotalArticles == 0) + { + NinpathsLogMessages.EmptyDump(_logger, file.Path); + return; + } + + var (start, end, average) = NinpathsReportPeriod.Resolve(file.Path, _time); + var report = NinpathsReportFormatter.Format(stats, start, end, average); + if (report.Length == 0) + { + NinpathsLogMessages.EmptyDump(_logger, file.Path); + return; + } + + if (!EmailOptionsValidator.TryValidateMailbox(_emailOptions.Value.DefaultFrom, out var from) + && !EmailOptionsValidator.TryValidateMailbox(_emailOptions.Value.EnvelopeSender, out from)) + { + NinpathsLogMessages.MissingFrom(_logger, file.Path); + return; + } + + var message = NinpathsReportMail.TryCreate(report, nntpd.Fqdn, from, recipients); + if (message is null) + { + return; + } + + var result = await _email.SendAsync(message, cancellationToken).ConfigureAwait(false); + switch (result.Status) + { + case EmailEnqueueStatus.Accepted: + NinpathsLogMessages.EmailAccepted(_logger, file.Path, recipients.Length, report.Length); + break; + case EmailEnqueueStatus.Disabled: + NinpathsLogMessages.EmailDisabled(_logger, file.Path); + break; + default: + NinpathsLogMessages.EmailFailed( + _logger, + file.Path, + result.Status.ToString(), + result.Detail); + break; + } + } + + private void DrainPending() + { + while (_queue.Reader.TryRead(out var leftover)) + { + leftover.Dispose(); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportFormatter.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportFormatter.cs new file mode 100644 index 00000000..6602ca09 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportFormatter.cs @@ -0,0 +1,148 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Formats compact INN 3.1.x !!NINP dump output from aggregated statistics. + /// + /// + /// Does not reread the Path-survey file. Site numbers are assigned in INN + /// hash-table walk order. Returns an empty buffer when + /// is 0 (INN writedump returns -1). + /// + public static class NinpathsReportFormatter + { + /// + /// Writes the compact dump for . Empty when there is no traffic. + /// + public static byte[] Format( + NinpathsStatistics stats, + long startUnix, + long endUnix, + long averageUnix) + { + ArgumentNullException.ThrowIfNull(stats); + if (stats.TotalArticles == 0) + { + return []; + } + + using var buffer = new MemoryStream(4096); + using (var writer = new StreamWriter(buffer, Encoding.Latin1, 1024, leaveOpen: true)) + { + writer.NewLine = "\n"; + writer.Write("!!NINP "); + writer.Write(NinpathsConstants.Version); + writer.Write(' '); + writer.Write(startUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(endUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(stats.SiteCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(stats.TotalArticles.ToString(CultureInfo.InvariantCulture)); + writer.Write(' '); + writer.Write(averageUnix.ToString(CultureInfo.InvariantCulture)); + writer.Write('\n'); + + long numbered = 0; + var lineLength = 0; + var buckets = stats.Buckets; + for (var i = 0; i < buckets.Length; i++) + { + for (var site = buckets[i]; site is not null; site = site.BucketNext) + { + site.Number = numbered++; + var written = WriteSiteRecord(writer, site); + lineLength += written; + if (lineLength > NinpathsConstants.RecordLineSoftLimit) + { + writer.Write('\n'); + lineLength = 0; + } + else + { + writer.Write(' '); + } + } + } + + writer.Write('\n'); + writer.Write("!!NLREC\n"); + + long relations = 0; + lineLength = 0; + for (var i = 0; i < buckets.Length; i++) + { + for (var site = buckets[i]; site is not null; site = site.BucketNext) + { + if (site.Relations is null) + { + continue; + } + + lineLength += WriteColonNumber(writer, site.Number); + for (var rel = site.Relations; rel is not null; rel = rel.Next) + { + lineLength += WriteBangNumber(writer, rel.Target.Number); + if (rel.Tally > 1) + { + lineLength += WriteCommaNumber(writer, rel.Tally); + } + + relations++; + } + + if (lineLength > NinpathsConstants.RecordLineSoftLimit) + { + writer.Write('\n'); + lineLength = 0; + } + } + } + + writer.Write('\n'); + writer.Write("!!NLEND "); + writer.Write(relations.ToString(CultureInfo.InvariantCulture)); + writer.Write('\n'); + } + + return buffer.ToArray(); + } + + private static int WriteSiteRecord(StreamWriter writer, NinpathsSite site) + { + var id = Encoding.Latin1.GetString(site.Id); + var count = site.SentTo.ToString(CultureInfo.InvariantCulture); + writer.Write(id); + writer.Write(' '); + writer.Write(count); + return id.Length + 1 + count.Length; + } + + private static int WriteColonNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write(':'); + writer.Write(text); + return 1 + text.Length; + } + + private static int WriteBangNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write('!'); + writer.Write(text); + return 1 + text.Length; + } + + private static int WriteCommaNumber(StreamWriter writer, long value) + { + var text = value.ToString(CultureInfo.InvariantCulture); + writer.Write(','); + writer.Write(text); + return 1 + text.Length; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportMail.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportMail.cs new file mode 100644 index 00000000..dbdde36f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportMail.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Builds one for a compact ninpaths dump (sendinpaths To/Subject/Auto-Submitted). + /// + internal static class NinpathsReportMail + { + /// + /// Creates a single message with every recipient in To. Returns + /// when there is no dump or no recipients. + /// + public static EmailMessage? TryCreate( + ReadOnlyMemory report, + string fqdn, + string fromMailbox, + IReadOnlyList recipients) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + ArgumentException.ThrowIfNullOrWhiteSpace(fromMailbox); + ArgumentNullException.ThrowIfNull(recipients); + if (report.IsEmpty || recipients.Count == 0) + { + return null; + } + + var to = new EmailAddress[recipients.Count]; + for (var i = 0; i < recipients.Count; i++) + { + to[i] = new EmailAddress(recipients[i]); + } + + return new EmailMessage + { + From = new EmailAddress(fromMailbox), + To = to, + Subject = NinpathsConstants.EmailSubjectPrefix + fqdn, + Body = report, + ContentType = "text/plain", + Charset = "us-ascii", + Headers = [new EmailHeader("Auto-Submitted", "auto-generated")], + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportPeriod.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportPeriod.cs new file mode 100644 index 00000000..8a9a09d6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsReportPeriod.cs @@ -0,0 +1,66 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Derives INN dump timestamps from a completed Path-survey file and . + /// + /// + /// INN starttime is process start and endtime is time(NULL) at dump. + /// For a completed daily inpaths-yyyyMMdd.log the reporting period start is local + /// midnight of that date. Articles are aggregated as a batch at dump time, so + /// avgtime = (atimes / total) + starttime equals dump time (INN would add + /// time(0) - starttime once per article during a same-moment batch). + /// + internal static class NinpathsReportPeriod + { + /// + /// Resolves starttime, endtime, and avgtime Unix seconds for a dump. + /// + public static (long StartUnix, long EndUnix, long AverageUnix) Resolve( + string completedFilePath, + TimeProvider time) + { + ArgumentException.ThrowIfNullOrWhiteSpace(completedFilePath); + ArgumentNullException.ThrowIfNull(time); + + var now = time.GetUtcNow(); + var end = now.ToUnixTimeSeconds(); + if (!TryParseSurveyDate(completedFilePath, out var date)) + { + return (end, end, end); + } + + var unspecified = new DateTime(date.Year, date.Month, date.Day, 0, 0, 0, DateTimeKind.Unspecified); + var offset = time.LocalTimeZone.GetUtcOffset(unspecified); + var start = new DateTimeOffset(unspecified, offset).ToUnixTimeSeconds(); + return (start, end, end); + } + + /// Parses inpaths-yyyyMMdd.log from a file name or path. + internal static bool TryParseSurveyDate(string completedFilePath, out DateOnly date) + { + date = default; + var name = Path.GetFileName(completedFilePath.AsSpan()); + const string prefix = "inpaths-"; + const string suffix = ".log"; + if (name.Length != prefix.Length + 8 + suffix.Length) + { + return false; + } + + if (!name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + || !name.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return DateOnly.TryParseExact( + name.Slice(prefix.Length, 8), + "yyyyMMdd", + CultureInfo.InvariantCulture, + DateTimeStyles.None, + out date); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsStatistics.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsStatistics.cs new file mode 100644 index 00000000..6372c414 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsStatistics.cs @@ -0,0 +1,217 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Compact INN ninpaths aggregation: unique sites and pairwise Path relations. + /// + /// + /// Memory is O(unique sites + unique directed relations), not O(articles). + /// Raw Path observations are not retained. Hash-table layout matches INN + /// hosthash (FNV-style hash, chain prepend, dump numbering). + /// + public sealed class NinpathsStatistics + { + private readonly NinpathsSite?[] _buckets = new NinpathsSite?[NinpathsConstants.HashTableSize]; + private long _sites; + private long _total; + + /// Gets the number of Path observations that INN would count in total. + public long TotalArticles => _total; + + /// Gets the number of unique interned sites. + public long SiteCount => _sites; + + /// + /// Applies one Path value after INN last-element chopping (may be empty). + /// Does not increment ; the reader does that + /// for valid lines even when this method stops early on an overlong token. + /// + internal void AddChoppedPath(ReadOnlySpan chopped) + { + NinpathsSite? previous = null; + var remaining = chopped; + while (!remaining.IsEmpty) + { + var bang = remaining.IndexOf((byte)'!'); + ReadOnlySpan token; + if (bang < 0) + { + token = remaining; + remaining = default; + } + else + { + token = remaining[..bang]; + remaining = remaining[(bang + 1)..]; + while (!remaining.IsEmpty && remaining[0] == (byte)'!') + { + remaining = remaining[1..]; + } + } + + if (token.Length > NinpathsConstants.MaxHostChars) + { + return; + } + + var site = GetOrAdd(token); + site.SentTo++; + if (previous is not null && !ReferenceEquals(previous, site)) + { + IncrementRelation(previous, site); + } + + previous = site; + } + } + + /// Counts one valid Path line (INN ++total). + internal void AddValidArticle() => _total++; + + internal NinpathsSite?[] Buckets => _buckets; + + /// Looks up INN sentto for a site id, using case-sensitive strcmp semantics. + public bool TryGetSentTo(ReadOnlySpan siteId, out long sentTo) + { + var site = Find(siteId); + if (site is null) + { + sentTo = 0; + return false; + } + + sentTo = site.SentTo; + return true; + } + + /// Looks up the directed relation tally from to . + public bool TryGetRelation(ReadOnlySpan from, ReadOnlySpan to, out long tally) + { + tally = 0; + var source = Find(from); + var target = Find(to); + if (source is null || target is null) + { + return false; + } + + for (var rel = source.Relations; rel is not null; rel = rel.Next) + { + if (ReferenceEquals(rel.Target, target)) + { + tally = rel.Tally; + return true; + } + } + + return false; + } + + /// INN hash() over a NUL-free token (FNV-style). + internal static int Hash(ReadOnlySpan str) + { + ulong val = 0; + for (var i = 0; i < str.Length; i++) + { + val *= NinpathsConstants.HashPrime; + val ^= str[i]; + } + + return (int)(val & (NinpathsConstants.HashTableSize - 1)); + } + + internal NinpathsSite? Find(ReadOnlySpan token) + { + var index = Hash(token); + for (var site = _buckets[index]; site is not null; site = site.BucketNext) + { + if (token.SequenceEqual(site.Id)) + { + return site; + } + } + + return null; + } + + /// Latin-1 helper for tests that start from managed strings. + public bool TryGetSentTo(string siteId, out long sentTo) + { + ArgumentNullException.ThrowIfNull(siteId); + return TryGetSentTo(Encoding.Latin1.GetBytes(siteId), out sentTo); + } + + /// Latin-1 helper for tests that start from managed strings. + public bool TryGetRelation(string from, string to, out long tally) + { + ArgumentNullException.ThrowIfNull(from); + ArgumentNullException.ThrowIfNull(to); + return TryGetRelation(Encoding.Latin1.GetBytes(from), Encoding.Latin1.GetBytes(to), out tally); + } + + private NinpathsSite GetOrAdd(ReadOnlySpan token) + { + var index = Hash(token); + for (var site = _buckets[index]; site is not null; site = site.BucketNext) + { + if (token.SequenceEqual(site.Id)) + { + return site; + } + } + + var created = new NinpathsSite + { + Id = token.ToArray(), + BucketNext = _buckets[index], + }; + _buckets[index] = created; + _sites++; + return created; + } + + private static void IncrementRelation(NinpathsSite from, NinpathsSite to) + { + for (var rel = from.Relations; rel is not null; rel = rel.Next) + { + if (ReferenceEquals(rel.Target, to)) + { + rel.Tally++; + return; + } + } + + from.Relations = new NinpathsRelation + { + Target = to, + Tally = 1, + Next = from.Relations, + }; + } + } + + /// One interned Path hop (INN nrec). + internal sealed class NinpathsSite + { + public NinpathsSite? BucketNext; + + public NinpathsRelation? Relations; + + public byte[] Id { get; set; } = []; + + public long Number; + + public long SentTo; + } + + /// One directed relation (INN trec). + internal sealed class NinpathsRelation + { + public NinpathsRelation? Next; + + public NinpathsSite Target { get; set; } = null!; + + public long Tally; + } +} diff --git a/src/VectorNNTPD.TransitService/Ninpaths/NinpathsTop1000.cs b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsTop1000.cs new file mode 100644 index 00000000..370ec370 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Ninpaths/NinpathsTop1000.cs @@ -0,0 +1,51 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Ninpaths +{ + /// + /// Resolves Nntpd:Top1000 recipients. Missing, null, empty, and + /// whitespace-only entries disable or skip; remaining mailboxes enable reporting. + /// + internal static class NinpathsTop1000 + { + /// Returns distinct trimmed mailboxes, ignoring whitespace-only entries. + public static string[] GetRecipients(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Top1000 is null || options.Top1000.Length == 0) + { + return []; + } + + List? list = null; + foreach (var raw in options.Top1000) + { + if (!EmailOptionsValidator.TryValidateMailbox(raw, out var mailbox)) + { + continue; + } + + list ??= new List(options.Top1000.Length); + var exists = false; + for (var i = 0; i < list.Count; i++) + { + if (string.Equals(list[i], mailbox, StringComparison.OrdinalIgnoreCase)) + { + exists = true; + break; + } + } + + if (!exists) + { + list.Add(mailbox); + } + } + + return list is null ? [] : [.. list]; + } + + /// True when at least one usable Top1000 mailbox is configured. + public static bool IsEnabled(NntpdOptions options) => GetRecipients(options).Length > 0; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/AccountByteConsumeResult.cs b/src/VectorNNTPD.TransitService/NntpDb/AccountByteConsumeResult.cs new file mode 100644 index 00000000..301f2094 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/AccountByteConsumeResult.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Outcome of a durable account_byte_limit consume or remaining-quota read. + public enum AccountByteConsumeStatus + { + /// The account row exists and the remaining value is valid. + Consumed, + + /// No nntpusers row matched the plaintext account name. + AccountNotFound, + } + + /// Durable byte-quota result. is never negative. + public readonly struct AccountByteConsumeResult + { + /// Initializes a clamped durable quota result. + public AccountByteConsumeResult(AccountByteConsumeStatus status, long remaining, long consumed) + { + ArgumentOutOfRangeException.ThrowIfNegative(remaining); + ArgumentOutOfRangeException.ThrowIfNegative(consumed); + Status = status; + Remaining = remaining; + Consumed = consumed; + } + + /// Gets whether the row was found or missing. + public AccountByteConsumeStatus Status { get; } + + /// Gets remaining bytes after the operation. Always >= 0. + public long Remaining { get; } + + /// Gets how many bytes were actually subtracted. Always >= 0. + public long Consumed { get; } + + /// Gets whether the row exists and remaining quota is known. + public bool Succeeded => Status == AccountByteConsumeStatus.Consumed; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/INntpArticlePolicySource.cs b/src/VectorNNTPD.TransitService/NntpDb/INntpArticlePolicySource.cs new file mode 100644 index 00000000..1a178e50 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/INntpArticlePolicySource.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// + /// Supplies the article-size and Path-tracker values captured for one NNTPD article operation. + /// + public interface INntpArticlePolicySource + { + /// Copies the published shared configuration. + /// Published maxartsize in bytes. + /// Published sitename. + /// when no snapshot has been published. + bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName); + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/INntpDbConnection.cs b/src/VectorNNTPD.TransitService/NntpDb/INntpDbConnection.cs new file mode 100644 index 00000000..34069759 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/INntpDbConnection.cs @@ -0,0 +1,91 @@ +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// + /// NNTPD query surface over one logical MySQL connection. + /// Dispose returns the physical connection to MySqlConnector's pool. + /// + public interface INntpDbConnection : INntpDbSession + { + /// + /// Executes and returns every row. + /// + /// Token used to cancel the query. + /// The complete result set. The caller must not retain this connection. + ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken); + + /// + /// Looks up one nntpusers row for reader authentication. + /// + /// Plaintext wire username. Bound as @account_name. + /// Token used to cancel the query. + /// The mapped account, or when no row exists. + ValueTask QueryUserAccountAsync( + string accountName, + CancellationToken cancellationToken); + + /// + /// Executes and returns every row. + /// + /// Token used to cancel the query. + /// Enabled moderator rows in moderator_id order. + ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken); + + /// + /// Loads one published PostFilter revision and the collection rows that + /// belong to that revision. + /// + /// Token used to cancel the queries. + /// + /// The mapped policy, or when + /// nntppostfiltercurrent.policy_id = 1 is missing. + /// + ValueTask QueryPostFilterPolicyAsync( + CancellationToken cancellationToken); + + /// + /// Loads the singleton published Transit catalogue. + /// + /// Token used to cancel the queries. + /// + /// The mapped catalogue, or when + /// nntptransitcurrent.policy_id = 1 is missing. + /// + ValueTask QueryTransitCatalogueAsync( + CancellationToken cancellationToken); + + /// + /// Inserts one PostFilter rejection evidence row. Not used on the accept path. + /// + ValueTask InsertPostFilterRejectionAsync( + PostFilter.PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken); + + /// + /// Atomically subtracts from + /// account_byte_limit, clamping at zero. Remaining is never negative. + /// + /// Plaintext wire username. Bound as @account_name. + /// Non-negative byte count to subtract. + /// Token used to cancel the transaction. + /// + /// Found status, bytes actually subtracted, and remaining after the update. + /// + ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken); + + /// + /// Reads remaining account_byte_limit without mutating. Does not use the + /// AUTHINFO user-record cache. + /// + /// Plaintext wire username. Bound as @account_name. + /// Token used to cancel the query. + ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/MySqlNntpDbConnection.cs b/src/VectorNNTPD.TransitService/NntpDb/MySqlNntpDbConnection.cs new file mode 100644 index 00000000..e7e2c740 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/MySqlNntpDbConnection.cs @@ -0,0 +1,803 @@ +using System.Data; +using System.Globalization; +using MySqlConnector; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// NNTPD queries over a shared logical session. Dispose returns it to the provider pool. + internal sealed class MySqlNntpDbConnection : INntpDbConnection + { + private readonly INntpDbSession _session; + + /// Initializes a new instance wrapping an already-open provider connection. + /// Open connection. This instance owns its disposal. + public MySqlNntpDbConnection(MySqlConnection connection) + : this(new MySqlNntpDbSession(connection)) + { + } + + /// Initializes a new instance wrapping a shared session. + /// Open session. This instance owns its disposal. + public MySqlNntpDbConnection(INntpDbSession session) + { + ArgumentNullException.ThrowIfNull(session); + _session = session; + } + + /// + public ValueTask SelectOneAsync(CancellationToken cancellationToken) => + _session.SelectOneAsync(cancellationToken); + + /// + public MySqlCommand CreateCommand() => _session.CreateCommand(); + + /// + public ValueTask BeginTransactionAsync(CancellationToken cancellationToken) => + _session.BeginTransactionAsync(cancellationToken); + + /// + public ValueTask BeginTransactionAsync( + IsolationLevel isolationLevel, + CancellationToken cancellationToken) => + _session.BeginTransactionAsync(isolationLevel, cancellationToken); + + /// + public async ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken) + { + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpGroupQueries.SelectNewsgroups; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var rows = new List(); + var index = 0; + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rows.Add(ReadGroupRow(reader, index)); + index++; + } + + return rows; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not Newsgroups.NewsgroupCatalogueException) + { + throw new NntpDbUnavailableException("MySQL newsgroup catalogue query failed.", ex); + } + } + + /// + public async ValueTask QueryUserAccountAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpUserQueries.SelectUserByName; + command.Parameters.AddWithValue("@account_name", accountName); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + return MapUserRecord(reader, accountName); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntpusers lookup failed.", ex); + } + } + + /// + public async ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken) + { + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpModeratorQueries.SelectEnabledModerators; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var rows = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rows.Add(MapModeratorRow(reader)); + } + + return rows; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntpmoderators catalogue query failed.", ex); + } + } + + /// + public async ValueTask QueryPostFilterPolicyAsync( + CancellationToken cancellationToken) + { + try + { + await using var transaction = await _session + .BeginTransactionAsync(IsolationLevel.RepeatableRead, cancellationToken) + .ConfigureAwait(false); + try + { + var revision = await ReadCurrentRevisionAsync(transaction, cancellationToken) + .ConfigureAwait(false); + if (revision is null) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return null; + } + + PostFilterPolicyRecord record; + await using (var policyCommand = CreateRevisionCommand( + NntpPostFilterQueries.SelectPolicy, + revision.Value, + transaction)) + { + await using var policyReader = await policyCommand + .ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false); + if (!await policyReader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException( + "nntppostfiltercurrent.revision " + + revision.Value.ToString(CultureInfo.InvariantCulture) + + " has no nntppostfilterpolicy row."); + } + + record = MapPolicyScalars(policyReader); + } + + var deniedAccounts = new List(); + var allowlistedAccounts = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectAccounts, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindDeny, + NntpPostFilterQueries.ListKindAllow, + deniedAccounts, + allowlistedAccounts, + cancellationToken) + .ConfigureAwait(false); + + var deniedCidrs = new List(); + var allowlistedCidrs = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectCidrs, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindDeny, + NntpPostFilterQueries.ListKindAllow, + deniedCidrs, + allowlistedCidrs, + cancellationToken) + .ConfigureAwait(false); + + var rejectArtTypes = new List(); + var excludeArtTypes = new List(); + await ReadKeyedStringsAsync( + NntpPostFilterQueries.SelectArtTypes, + revision.Value, + transaction, + NntpPostFilterQueries.ListKindReject, + NntpPostFilterQueries.ListKindSaExclude, + rejectArtTypes, + excludeArtTypes, + cancellationToken) + .ConfigureAwait(false); + + var hosts = await ReadHostsAsync(revision.Value, transaction, cancellationToken) + .ConfigureAwait(false); + record.Options.DeniedAccounts = [.. deniedAccounts]; + record.Options.AllowlistedAccounts = [.. allowlistedAccounts]; + record.Options.DeniedCidrs = [.. deniedCidrs]; + record.Options.AllowlistedCidrs = [.. allowlistedCidrs]; + record.Options.RejectArtTypes = [.. rejectArtTypes]; + record.Options.SpamAssassin.ExcludeArtTypes = [.. excludeArtTypes]; + record.Options.SpamAssassin.Hosts = [.. hosts]; + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return record; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not InvalidOperationException) + { + throw new NntpDbUnavailableException("MySQL nntppostfilterpolicy query failed.", ex); + } + } + + /// + public async ValueTask InsertPostFilterRejectionAsync( + PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(evidence); + try + { + await using var command = _session.CreateCommand(); + command.CommandText = NntpPostFilterQueries.InsertRejection; + command.Parameters.AddWithValue("@rejected_utc", evidence.RejectedUtc.UtcDateTime); + command.Parameters.AddWithValue("@revision", evidence.PolicyRevision); + command.Parameters.AddWithValue( + "@account_name", + evidence.AccountName is null ? DBNull.Value : evidence.AccountName); + var sourceIp = command.Parameters.Add("@source_ip", MySqlDbType.VarBinary, 16); + sourceIp.Value = SourceAddressIdentity.ToNetworkBytes(evidence.SourceAddress); + command.Parameters.AddWithValue("@art_type", (uint)evidence.ArtType); + command.Parameters.AddWithValue( + "@message_id", + evidence.MessageId is null ? DBNull.Value : evidence.MessageId); + command.Parameters.AddWithValue("@article_size", evidence.ArticleSize); + command.Parameters.AddWithValue("@stage", evidence.Stage.ToString()); + command.Parameters.AddWithValue("@reason", evidence.Reason); + command.Parameters.AddWithValue( + "@sa_status", + evidence.SpamAssassinStatus is { } saStatus ? saStatus.ToString() : DBNull.Value); + command.Parameters.AddWithValue( + "@sa_score", + evidence.SpamAssassinScore is { } score ? score : DBNull.Value); + command.Parameters.AddWithValue( + "@sa_threshold", + evidence.SpamAssassinThreshold is { } threshold ? threshold : DBNull.Value); + var payload = evidence.ArticlePayload; + command.Parameters.AddWithValue( + "@article_payload", + payload is { Length: > 0 } ? payload.Value.ToArray() : DBNull.Value); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL nntppostfilterrejections insert failed.", ex); + } + } + + /// + public async ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + try + { + await using var transaction = await _session + .BeginTransactionAsync(cancellationToken) + .ConfigureAwait(false); + try + { + var locked = await ReadByteQuotaAsync( + NntpUserQueries.SelectByteQuotaForUpdate, + accountName, + transaction, + cancellationToken) + .ConfigureAwait(false); + if (locked is null) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0); + } + + var current = ClampNonNegative(locked.Value.Remaining); + var consumed = bytes > current ? current : bytes; + await using (var update = _session.CreateCommand()) + { + update.Transaction = transaction; + update.CommandText = NntpUserQueries.ConsumeAccountBytes; + update.Parameters.AddWithValue("@account_name", accountName); + update.Parameters.AddWithValue("@bytes", bytes); + await update.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var after = await ReadByteQuotaAsync( + NntpUserQueries.SelectAccountByteRemaining, + accountName, + transaction, + cancellationToken) + .ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + var remaining = after is { } row + ? ClampNonNegative(row.Remaining) + : 0L; + return new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, consumed); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL account byte-quota consume failed.", ex); + } + } + + /// + public async ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + var row = await ReadByteQuotaAsync( + NntpUserQueries.SelectAccountByteRemaining, + accountName, + transaction: null, + cancellationToken) + .ConfigureAwait(false); + if (row is null) + { + return new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0); + } + + return new AccountByteConsumeResult( + AccountByteConsumeStatus.Consumed, + ClampNonNegative(row.Value.Remaining), + 0); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException and not NntpDbUnavailableException) + { + throw new NntpDbUnavailableException("MySQL account byte-quota query failed.", ex); + } + } + + /// + public ValueTask DisposeAsync() => _session.DisposeAsync(); + + /// Maps one enabled nntpmoderators row. CHAR columns are trimmed. + internal static NntpModeratorRow MapModeratorRow(MySqlDataReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + var id = Convert.ToInt64(reader.GetValue(0), System.Globalization.CultureInfo.InvariantCulture); + var pattern = reader.IsDBNull(1) ? string.Empty : reader.GetString(1).Trim(); + var address = reader.IsDBNull(2) ? string.Empty : reader.GetString(2).Trim(); + var account = reader.IsDBNull(3) ? string.Empty : reader.GetString(3).Trim(); + return new NntpModeratorRow(id, pattern, address, account); + } + + /// + /// Maps one nntpusers row from . + /// Flag columns are true only for Y. NULL rate/byte limits map to 0. + /// Ordinals: 0 pass, 1 salt, 2 iterations, 3 stored key, 4 server key, 5 plain, + /// 6 scram256, 7 rate bps, 8 byte remaining, 9 session limit, 10 srcip limit, + /// 11 enabled, 12 customer, 13 account_art_type. + /// + internal static NntpUserRecord MapUserRecord(MySqlDataReader reader, string accountName) + { + ArgumentNullException.ThrowIfNull(reader); + var password = reader.IsDBNull(0) ? string.Empty : reader.GetString(0); + var scramSalt = ReadBinaryColumn(reader, 1); + var scramIterations = reader.IsDBNull(2) ? 0 : Convert.ToInt32(reader.GetValue(2)); + var scramStoredKey = ReadBinaryColumn(reader, 3); + var scramServerKey = ReadBinaryColumn(reader, 4); + var allowAuthPlain = IsYesFlag(reader, 5); + var allowAuthScram256 = IsYesFlag(reader, 6); + var rateLimitBps = reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7)); + var byteLimit = reader.IsDBNull(8) ? 0L : ConvertByteLimit(reader.GetValue(8)); + var sessionLimit = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)); + var srcIpLimit = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)); + var isEnabled = IsYesFlag(reader, 11); + var customerId = ReadCustomerId(reader, 12); + var allowedArtTypes = ReadAccountArtType(reader, 13); + return new NntpUserRecord( + accountName, + password, + allowAuthPlain, + allowAuthScram256, + scramSalt, + scramIterations, + scramStoredKey, + scramServerKey, + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + isEnabled, + customerId, + allowedArtTypes); + } + + private async ValueTask ReadByteQuotaAsync( + string commandText, + string accountName, + MySqlTransaction? transaction, + CancellationToken cancellationToken) + { + await using var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = commandText; + command.Parameters.AddWithValue("@account_name", accountName); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + var remaining = reader.IsDBNull(0) ? 0L : ConvertByteLimit(reader.GetValue(0)); + return new ByteQuotaRow(remaining); + } + + /// + /// Converts a MySQL integer account_byte_limit to a non-negative . + /// Unsigned values above saturate rather than wrap. + /// Negative signed values become 0 (exhausted / never allowed). + /// + internal static long ConvertByteLimit(object? value) => + value switch + { + null or DBNull => 0, + long signed64 => ClampNonNegative(signed64), + ulong unsigned64 => unsigned64 > long.MaxValue ? long.MaxValue : (long)unsigned64, + int signed32 => ClampNonNegative(signed32), + uint unsigned32 => unsigned32, + short signed16 => ClampNonNegative(signed16), + ushort unsigned16 => unsigned16, + sbyte signed8 => ClampNonNegative(signed8), + byte unsigned8 => unsigned8, + decimal dec when dec < 0 => 0, + decimal dec when dec > long.MaxValue => long.MaxValue, + decimal dec => (long)dec, + _ => ClampNonNegative(Convert.ToInt64(value, System.Globalization.CultureInfo.InvariantCulture)), + }; + + private static long ClampNonNegative(long value) => value < 0 ? 0 : value; + + private readonly record struct ByteQuotaRow(long Remaining); + + private static bool IsYesFlag(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return false; + } + + return string.Equals(Convert.ToString(reader.GetValue(ordinal)), "Y", StringComparison.OrdinalIgnoreCase); + } + + private static string ReadCustomerId(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return string.Empty; + } + + var value = reader.GetValue(ordinal); + return value switch + { + Guid guid => guid.ToString("D"), + string text => text, + _ => Convert.ToString(value) ?? string.Empty, + }; + } + + private static ArticleType ReadAccountArtType(MySqlDataReader reader, int ordinal) + { + if (ordinal >= reader.FieldCount) + { + throw new InvalidOperationException( + "nntpusers.account_art_type was not selected. Apply docs/schema/nntpusers-account-art-type.sql."); + } + + if (reader.IsDBNull(ordinal)) + { + throw new InvalidOperationException("nntpusers.account_art_type is NULL."); + } + + return (ArticleType)Convert.ToUInt32(reader.GetValue(ordinal), CultureInfo.InvariantCulture); + } + + private static ReadOnlyMemory ReadBinaryColumn(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return ReadOnlyMemory.Empty; + } + + var value = reader.GetValue(ordinal); + return value switch + { + byte[] bytes when bytes.Length == 0 => ReadOnlyMemory.Empty, + byte[] bytes => bytes, + ReadOnlyMemory memory => memory, + _ => ReadOnlyMemory.Empty, + }; + } + + /// Reads one nntpgroups row, preserving unsigned integer width. + internal static NntpGroupRow ReadGroupRow(MySqlDataReader reader, int index) + { + ArgumentNullException.ThrowIfNull(reader); + if (reader.IsDBNull(0)) + { + throw new Newsgroups.NewsgroupCatalogueException($"nntpgroups row {index} has a NULL group_name."); + } + + var name = reader.GetString(0); + var description = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); + var high = ConvertUnsignedCount(reader.GetValue(2), "count_high", name); + var low = ConvertUnsignedCount(reader.GetValue(3), "count_low", name); + var status = ConvertPostingStatus(reader.GetValue(4), name); + return new NntpGroupRow(name, description, high, low, status); + } + + /// + /// Converts a MySQL unsigned/signed integer column to without + /// narrowing or floating-point conversion. + /// + internal static ulong ConvertUnsignedCount(object? value, string column, string groupName) => + value switch + { + ulong unsigned64 => unsigned64, + uint unsigned32 => unsigned32, + ushort unsigned16 => unsigned16, + byte unsigned8 => unsigned8, + long signed64 when signed64 >= 0 => (ulong)signed64, + int signed32 when signed32 >= 0 => (ulong)signed32, + short signed16 when signed16 >= 0 => (ulong)signed16, + sbyte signed8 when signed8 >= 0 => (ulong)signed8, + _ => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' column {column} is not a non-negative integer."), + }; + + /// + /// Reads posting_status as a single supported octet. + /// MySQL ENUM/CHAR typically arrives as a one-character string from MySqlConnector; + /// byte/sbyte/char are also accepted. Values other than y/n/m/x/j + /// are rejected, including RFC 6048 =<newsgroup>. + /// + internal static byte ConvertPostingStatus(object? value, string groupName) + { + var status = value switch + { + byte b => b, + sbyte sb => (byte)sb, + char ch => (byte)ch, + string text when text.Length == 1 => (byte)text[0], + string text when text.Length > 0 && text[0] == '=' => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' uses unsupported RFC 6048 = posting_status."), + _ => throw new Newsgroups.NewsgroupCatalogueException( + $"nntpgroups row '{groupName}' has an unreadable posting_status."), + }; + + if (!Newsgroups.NewsgroupPostingStatusOctets.IsSupported(status)) + { + var reason = status == (byte)'=' + ? $"nntpgroups row '{groupName}' uses unsupported RFC 6048 = posting_status." + : $"nntpgroups row '{groupName}' has invalid posting_status 0x{status:X2}."; + throw new Newsgroups.NewsgroupCatalogueException(reason); + } + + return status; + } + + /// + /// Converts a MySQL SELECT 1 scalar, including the connector's result type. + /// + internal static int ConvertSelectOneScalar(object? result) => + MySqlNntpDbSession.ConvertSelectOneScalar(result); + + private static PostFilterPolicyRecord MapPolicyScalars(MySqlDataReader reader) + { + var revision = Convert.ToInt64(reader.GetValue(0), System.Globalization.CultureInfo.InvariantCulture); + var updated = reader.GetDateTime(1); + var updatedUtc = updated.Kind == DateTimeKind.Unspecified + ? new DateTimeOffset(DateTime.SpecifyKind(updated, DateTimeKind.Utc)) + : new DateTimeOffset(updated.ToUniversalTime()); + var gateText = reader.GetString(2).Trim(); + if (!Enum.TryParse(gateText, ignoreCase: true, out var gate) + || !Enum.IsDefined(gate)) + { + throw new InvalidOperationException($"PostFilter gate is not a defined value: '{gateText}'."); + } + + var options = new PostFilterOptions + { + Gate = gate, + Quota = new PostFilterQuotaOptions + { + LongWindow = TimeSpan.FromMilliseconds(ReadInt64(reader, 3)), + ShortWindow = TimeSpan.FromMilliseconds(ReadInt64(reader, 4)), + MaxMessagesLong = ReadInt64(reader, 5), + MaxBytesLong = ReadInt64(reader, 6), + MaxIdenticalLong = ReadInt64(reader, 7), + MaxMessagesShort = ReadInt64(reader, 8), + MaxBytesShort = ReadInt64(reader, 9), + MaxIdenticalShort = ReadInt64(reader, 10), + }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = IsYes(reader, 11), + OnFailure = ReadOnFailure(reader, 12), + MaxArticleSize = checked((int)ReadInt64(reader, 13)), + Port = checked((int)ReadInt64(reader, 14)), + ProtocolVersion = reader.IsDBNull(15) ? string.Empty : reader.GetString(15).Trim(), + MaxConnections = checked((int)ReadInt64(reader, 16)), + HostSelection = ReadHostSelection(reader, 17), + ConnectTimeout = TimeSpan.FromMilliseconds(ReadInt64(reader, 18)), + OperationTimeout = TimeSpan.FromMilliseconds(ReadInt64(reader, 19)), + }, + }; + + return new PostFilterPolicyRecord(revision, updatedUtc, options); + } + + private async Task ReadCurrentRevisionAsync( + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpPostFilterQueries.SelectCurrentRevision; + var scalar = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return scalar is null or DBNull + ? null + : Convert.ToInt64(scalar, CultureInfo.InvariantCulture); + } + + private MySqlCommand CreateRevisionCommand( + string commandText, + long revision, + MySqlTransaction transaction) + { + var command = _session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = commandText; + command.Parameters.AddWithValue("@revision", revision); + return command; + } + + private async Task ReadKeyedStringsAsync( + string commandText, + long revision, + MySqlTransaction transaction, + string firstKind, + string secondKind, + List first, + List second, + CancellationToken cancellationToken) + { + await using var command = CreateRevisionCommand(commandText, revision, transaction); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var kind = reader.IsDBNull(0) ? string.Empty : reader.GetString(0).Trim(); + var value = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); + if (string.Equals(kind, firstKind, StringComparison.OrdinalIgnoreCase)) + { + first.Add(value); + } + else if (string.Equals(kind, secondKind, StringComparison.OrdinalIgnoreCase)) + { + second.Add(value); + } + else + { + throw new InvalidOperationException($"PostFilter collection list_kind is unknown: '{kind}'."); + } + } + } + + private async Task> ReadHostsAsync( + long revision, + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = CreateRevisionCommand( + NntpPostFilterQueries.SelectHosts, + revision, + transaction); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var hosts = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + hosts.Add(reader.IsDBNull(0) ? string.Empty : reader.GetString(0)); + } + + return hosts; + } + + private static long ReadInt64(MySqlDataReader reader, int ordinal) => + Convert.ToInt64(reader.GetValue(ordinal), System.Globalization.CultureInfo.InvariantCulture); + + private static bool IsYes(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return false; + } + + var value = reader.GetValue(ordinal); + return value switch + { + string text => text.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase), + char ch => ch is 'Y' or 'y', + byte b => b is (byte)'Y' or (byte)'y', + _ => false, + }; + } + + private static PostFilterSpamOnFailure? ReadOnFailure(MySqlDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return null; + } + + var text = reader.GetString(ordinal).Trim(); + if (text.Length == 0) + { + return null; + } + + if (!Enum.TryParse(text, ignoreCase: true, out var parsed) + || !Enum.IsDefined(parsed)) + { + throw new InvalidOperationException($"PostFilter SpamAssassin OnFailure is not a defined value: '{text}'."); + } + + return parsed; + } + + private static PostFilterSpamAssassinHostSelection ReadHostSelection(MySqlDataReader reader, int ordinal) + { + var text = reader.IsDBNull(ordinal) + ? PostFilterSpamAssassinHostSelection.RoundRobin.ToString() + : reader.GetString(ordinal).Trim(); + if (!Enum.TryParse(text, ignoreCase: true, out var parsed) + || !Enum.IsDefined(parsed)) + { + throw new InvalidOperationException($"PostFilter SpamAssassin HostSelection is not a defined value: '{text}'."); + } + + return parsed; + } + + /// + public async ValueTask QueryTransitCatalogueAsync(CancellationToken cancellationToken) + { + try + { + return await NntpTransitCatalogueLoader.LoadAsync(_session, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException + and not NntpDbUnavailableException + and not InvalidOperationException) + { + throw new NntpDbUnavailableException("MySQL Transit catalogue query failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpArticlePolicyCapture.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpArticlePolicyCapture.cs new file mode 100644 index 00000000..9ca7f09a --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpArticlePolicyCapture.cs @@ -0,0 +1,48 @@ +using System.Text; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// + /// Captures one article-size ceiling and Path tracker for a single article operation. + /// + internal static class NntpArticlePolicyCapture + { + /// + /// Copies the published maxartsize when the session has a shared-configuration snapshot. + /// Otherwise uses . + /// The returned ceiling is the smaller of that value and the connection's captured Receive + /// MaxArticleBytes. A peer cannot exceed the published global ceiling. + /// A non-positive peer size, or a peer size above , adds no further limit. + /// + /// Session that may hold the catalogue and the identified peer. + /// Limit used when no snapshot is published on the session. + /// Article-size boundary for this operation. + /// UTF-8 tracker component, or when the static tracker remains in use. + internal static void Capture( + NntpSession session, + int fallbackMaxArticleBytes, + out int maxArticleBytes, + out byte[]? siteNameUtf8) + { + ArgumentNullException.ThrowIfNull(session); + int globalOrFallback; + if (session.SharedConfiguration is { } catalogue + && catalogue.TryGetArticlePolicy(out var publishedMax, out var siteName)) + { + globalOrFallback = publishedMax; + siteNameUtf8 = Encoding.UTF8.GetBytes(siteName); + } + else + { + globalOrFallback = fallbackMaxArticleBytes; + siteNameUtf8 = null; + } + + maxArticleBytes = TransitReceivePolicy.EffectiveMaxArticleBytes( + globalOrFallback, + session.Authorization.TransitPeerPolicy); + } + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpDbConnections.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpDbConnections.cs new file mode 100644 index 00000000..060e22d0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpDbConnections.cs @@ -0,0 +1,29 @@ +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Opens an NNTPD query connection from the shared database service. + internal static class NntpDbConnections + { + /// + /// Opens a session and returns the NNTPD query surface. + /// A session that already implements is returned unchanged. + /// + /// Started database service. + /// Token used to cancel the open. + /// An open query connection. The caller disposes it. + public static async ValueTask OpenAsync( + NntpDbService service, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(service); + var session = await service.OpenAsync(cancellationToken).ConfigureAwait(false); + if (session is INntpDbConnection connection) + { + return connection; + } + + return new MySqlNntpDbConnection(session); + } + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpGroupQueries.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpGroupQueries.cs new file mode 100644 index 00000000..2ce176b4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpGroupQueries.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Authoritative NntpDB queries used by the newsgroup catalogue. + public static class NntpGroupQueries + { + /// + /// Complete newsgroup catalogue source. Column set is exactly the fields + /// required by LIST ACTIVE / LIST NEWSGROUPS / GROUP metadata. + /// + public const string SelectNewsgroups = + """ + SELECT + group_name, + group_description, + count_high, + count_low, + posting_status + FROM nntpgroups + """; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpGroupRow.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpGroupRow.cs new file mode 100644 index 00000000..8b6e3219 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpGroupRow.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// + /// One raw nntpgroups row as read from MySQL. Validation and wire encoding + /// happen when a newsgroup snapshot is constructed. + /// + /// Original group_name text. + /// group_description, empty when the column is null. + /// Unsigned count_high water mark. + /// Unsigned count_low water mark. + /// Single status octet as stored (y/n/m/x/j). The RFC 6048 =<newsgroup> form is not represented. + public readonly record struct NntpGroupRow( + string GroupName, + string GroupDescription, + ulong CountHigh, + ulong CountLow, + byte PostingStatus); +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpModeratorQueries.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpModeratorQueries.cs new file mode 100644 index 00000000..496caded --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpModeratorQueries.cs @@ -0,0 +1,16 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Authoritative NntpDB query for the moderator authorization catalogue. + public static class NntpModeratorQueries + { + /// + /// Enabled nntpmoderators rows in first-match order (moderator_id ASC). + /// + public const string SelectEnabledModerators = + "SELECT " + + "moderator_id, group_pattern, moderator_address, account_name " + + "FROM nntpmoderators " + + "WHERE is_enabled = 'Y' " + + "ORDER BY moderator_id"; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpPostFilterQueries.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpPostFilterQueries.cs new file mode 100644 index 00000000..bce5a49c --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpPostFilterQueries.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Authoritative NntpDB queries for the cluster PostFilter policy. + /// + /// NNTPD only SELECTs one published revision. Schema DDL is + /// docs/schema/postfilter.sql. There is no application migration runner. + /// + public static class NntpPostFilterQueries + { + /// Published revision pointer. policy_id is always 1. + public const string SelectCurrentRevision = + "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"; + + /// Scalar policy for one revision. + public const string SelectPolicy = + """ + SELECT + revision, + updated_utc, + gate, + long_window_ms, + short_window_ms, + max_messages_long, + max_bytes_long, + max_identical_long, + max_messages_short, + max_bytes_short, + max_identical_short, + sa_enabled, + sa_on_failure, + sa_max_article_size, + sa_port, + sa_protocol_version, + sa_max_connections, + sa_host_selection, + sa_connect_timeout_ms, + sa_operation_timeout_ms + FROM nntppostfilterpolicy + WHERE revision = @revision + """; + + /// Deny/allow MD5 account identifiers for @revision. + public const string SelectAccounts = + "SELECT list_kind, account_name FROM nntppostfilteraccounts " + + "WHERE revision = @revision ORDER BY list_kind, account_name"; + + /// + /// Inserts one PostFilter rejection evidence row. Payload is the unstuffed + /// article when available. Never used on the accept path. + /// + public const string InsertRejection = + """ + INSERT INTO nntppostfilterrejections ( + rejected_utc, + revision, + account_name, + source_ip, + art_type, + message_id, + article_size, + stage, + reason, + sa_status, + sa_score, + sa_threshold, + article_payload + ) VALUES ( + @rejected_utc, + @revision, + @account_name, + @source_ip, + @art_type, + @message_id, + @article_size, + @stage, + @reason, + @sa_status, + @sa_score, + @sa_threshold, + @article_payload + ) + """; + + /// Deny/allow client CIDRs for @revision. + public const string SelectCidrs = + "SELECT list_kind, cidr FROM nntppostfiltercidrs " + + "WHERE revision = @revision ORDER BY list_kind, cidr"; + + /// ArtType lists for @revision. + public const string SelectArtTypes = + "SELECT list_kind, art_type FROM nntppostfilterarttypes " + + "WHERE revision = @revision ORDER BY list_kind, art_type"; + + /// SPAMD hosts for @revision in compiled order. + public const string SelectHosts = + "SELECT host FROM nntppostfiltersahosts WHERE revision = @revision ORDER BY host_order"; + + /// Account list kind stored for deny. + public const string ListKindDeny = "deny"; + + /// Account/CIDR list kind stored for allowlist. + public const string ListKindAllow = "allow"; + + /// ArtType list kind stored for Stage 3 reject. + public const string ListKindReject = "reject"; + + /// ArtType list kind stored for SpamAssassin exclusion. + public const string ListKindSaExclude = "sa_exclude"; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationLogMessages.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationLogMessages.cs new file mode 100644 index 00000000..ab6939a2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Source-generated shared-configuration lifecycle log messages. + internal static partial class NntpSharedConfigurationLogMessages + { + [LoggerMessage( + EventId = 2260, + Level = LogLevel.Information, + Message = "Loading initial nntpsharedconfig")] + public static partial void InitialLoadStarted(ILogger logger); + + [LoggerMessage( + EventId = 2261, + Level = LogLevel.Information, + Message = "Published nntpsharedconfig maxArticleBytes={MaxArticleBytes} siteName={SiteName}")] + public static partial void SnapshotPublished(ILogger logger, int MaxArticleBytes, string SiteName); + + [LoggerMessage( + EventId = 2262, + Level = LogLevel.Error, + Message = "Initial nntpsharedconfig load failed; application will not start")] + public static partial void InitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2263, + Level = LogLevel.Error, + Message = "nntpsharedconfig refresh failed; retaining the last known-good snapshot")] + public static partial void RefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2264, + Level = LogLevel.Information, + Message = "nntpsharedconfig refresh loop started (interval {Interval})")] + public static partial void RefreshLoopStarted(ILogger logger, TimeSpan Interval); + + [LoggerMessage( + EventId = 2265, + Level = LogLevel.Information, + Message = "nntpsharedconfig refresh loop stopped")] + public static partial void RefreshLoopStopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationService.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationService.cs new file mode 100644 index 00000000..0e875833 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpSharedConfigurationService.cs @@ -0,0 +1,263 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// + /// Loads nntpsharedconfig during startup and refreshes it on this service's own loop. + /// + /// + /// The initial load fails startup. A later refresh failure keeps the last known-good snapshot. + /// One refresh runs at a time. This service does not query on an article path. + /// + internal sealed class NntpSharedConfigurationService : INntpSharedConfigurationCatalogue, INntpArticlePolicySource, IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. Owned by this service, not by a shared scheduler. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly NntpDbService _nntpDb; + private readonly IOptions _options; + private readonly IOptions _rabbitMq; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private PublishedSnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a new instance of the class. + /// Started database service. + /// Live NNTPD options. ACME, DNS, and Cloudflare credential fields are replaced from each published snapshot. + /// Live RabbitMQ options. Username and password are replaced from each published snapshot. + /// Lifecycle logger. + public NntpSharedConfigurationService( + NntpDbService nntpDb, + IOptions options, + IOptions rabbitMq, + ILogger logger) + : this(nntpDb, options, logger, TimeProvider.System, RefreshInterval, rabbitMq) + { + } + + /// Initializes a new instance with an explicit clock and interval (tests). + /// Started database service. + /// Live NNTPD options. ACME, DNS, and Cloudflare credential fields are replaced from each published snapshot. + /// Lifecycle logger. + /// Clock used by the refresh delay. + /// Delay between refreshes. + /// Live RabbitMQ options. Username and password are replaced from each published snapshot. + internal NntpSharedConfigurationService( + NntpDbService nntpDb, + IOptions options, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval, + IOptions? rabbitMq = null) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _options = options; + _rabbitMq = rabbitMq ?? Options.Create(new RabbitMqOptions()); + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "NntpSharedConfiguration"; + + /// + public Task? Execution => _execution; + + /// + public bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName) + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + maxArticleBytes = 0; + siteName = string.Empty; + return false; + } + + maxArticleBytes = snapshot.Value.MaxArticleBytes; + siteName = snapshot.Value.SiteName; + return true; + } + + /// + public NntpSharedConfiguration Current + { + get + { + var snapshot = Volatile.Read(ref _current); + return snapshot?.Value ?? throw new InvalidOperationException("nntpsharedconfig has not been published."); + } + } + + private sealed class PublishedSnapshot + { + internal PublishedSnapshot(NntpSharedConfiguration value) => Value = value; + + internal NntpSharedConfiguration Value { get; } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// Gets whether a refresh is currently running (tests). + internal bool IsRefreshing => Volatile.Read(ref _refreshing) == 1; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + NntpSharedConfigurationLogMessages.InitialLoadStarted(_logger); + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + NntpSharedConfigurationLogMessages.RefreshLoopStarted(_logger, _interval); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + NntpSharedConfigurationLogMessages.InitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + NntpSharedConfigurationLogMessages.RefreshLoopStopped(_logger); + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + NntpSharedConfigurationLogMessages.RefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadAsync(CancellationToken cancellationToken) + { + await using var session = await _nntpDb.OpenAsync(cancellationToken).ConfigureAwait(false); + return await NntpSharedConfigurationReader.ReadAsync(session, cancellationToken).ConfigureAwait(false); + } + + private void Publish(NntpSharedConfiguration snapshot) + { + var options = _options.Value; + if (options.ServerId is { } serverId && ServerIdRules.IsInRange(serverId)) + { + var prefix = string.IsNullOrWhiteSpace(options.HostApplicationPrefix) + ? NntpdOptions.ApplicationPrefix + : options.HostApplicationPrefix; + var fqdn = snapshot.RequireApplicationFqdn(prefix, serverId); + if (options.IsTlsListenerEnabled) + { + DnsZoneCoverage.RequireIdentitiesInDnsZone( + CertificateIdentities.ForFqdn(fqdn, options.IncludeNewsHostnameInCertificate), + snapshot.DnsSuffix); + } + } + + snapshot.CopyAcmeDnsTo(options); + snapshot.CopyRabbitMqCredentialsTo(_rabbitMq.Value); + Volatile.Write(ref _current, new PublishedSnapshot(snapshot)); + NntpSharedConfigurationLogMessages.SnapshotPublished(_logger, snapshot.MaxArticleBytes, snapshot.SiteName); + } + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpTransitCatalogueLoader.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpTransitCatalogueLoader.cs new file mode 100644 index 00000000..8ff4adf4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpTransitCatalogueLoader.cs @@ -0,0 +1,416 @@ +using System.Data; +using System.Globalization; +using MySqlConnector; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Reads one coherent Transit publication inside a repeatable-read transaction. + internal static class NntpTransitCatalogueLoader + { + /// + /// Loads nntptransitcurrent. Returns when the singleton row is missing. + /// + public static async ValueTask LoadAsync( + INntpDbSession session, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + await using var transaction = await session + .BeginTransactionAsync(IsolationLevel.RepeatableRead, cancellationToken) + .ConfigureAwait(false); + try + { + var header = await ReadHeaderAsync(session, transaction, cancellationToken).ConfigureAwait(false); + if (header is null) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return null; + } + + var receiveCount = await ReadCountAsync( + session, + transaction, + NntpTransitQueries.CountReceive, + "@receive_revision", + header.ReceiveRevision, + cancellationToken) + .ConfigureAwait(false); + var sendCount = await ReadCountAsync( + session, + transaction, + NntpTransitQueries.CountSend, + "@send_revision", + header.SendRevision, + cancellationToken) + .ConfigureAwait(false); + var peers = await ReadPeersAsync(session, transaction, header, cancellationToken).ConfigureAwait(false); + if (peers.Count != receiveCount || peers.Count != sendCount) + { + throw new InvalidOperationException( + "Transit receive revision " + + header.ReceiveRevision.ToString(CultureInfo.InvariantCulture) + + " and send revision " + + header.SendRevision.ToString(CultureInfo.InvariantCulture) + + " do not contain the same peer identifiers."); + } + + var allowFrom = await ReadStringsAsync( + session, + transaction, + NntpTransitQueries.SelectAllowFrom, + "@receive_revision", + header.ReceiveRevision, + "entry", + cancellationToken) + .ConfigureAwait(false); + var endpoints = await ReadEndpointsAsync(session, transaction, header.SendRevision, cancellationToken) + .ConfigureAwait(false); + var exclusions = await ReadStringsAsync( + session, + transaction, + NntpTransitQueries.SelectPathExclusions, + "@send_revision", + header.SendRevision, + "path_token", + cancellationToken) + .ConfigureAwait(false); + var records = new List(peers.Count); + foreach (var peer in peers) + { + if (!allowFrom.TryGetValue(peer.Identifier, out var allow)) + { + allow = []; + } + + if (!endpoints.TryGetValue(peer.Identifier, out var ends)) + { + ends = []; + } + + if (!exclusions.TryGetValue(peer.Identifier, out var excluded)) + { + excluded = []; + } + + records.Add(peer.WithChildren(allow, ends, excluded)); + } + + RejectUnknown(allowFrom, peers, "AllowFrom"); + RejectUnknown(endpoints, peers, "Send endpoint"); + RejectUnknown(exclusions, peers, "Path exclusion"); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return new TransitCatalogueRecord( + header.PublicationId, + header.ReceiveRevision, + header.SendRevision, + header.GlobalRevision, + header.WantTrash, + header.LogTrash, + records); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + private static async ValueTask ReadHeaderAsync( + INntpDbSession session, + MySqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectCurrent; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + var header = new Header( + reader.GetInt64(0), + reader.GetInt64(1), + reader.GetInt64(2), + reader.GetInt64(3), + ReadYn(reader.GetString(4)), + ReadYn(reader.GetString(5))); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("nntptransitcurrent returned more than one publication."); + } + + return header; + } + + private static async ValueTask ReadCountAsync( + INntpDbSession session, + MySqlTransaction transaction, + string sql, + string parameterName, + long revision, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = sql; + command.Parameters.AddWithValue(parameterName, revision); + var scalar = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return Convert.ToInt32(scalar, CultureInfo.InvariantCulture); + } + + private static async ValueTask> ReadPeersAsync( + INntpDbSession session, + MySqlTransaction transaction, + Header header, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectPeers; + command.Parameters.AddWithValue("@receive_revision", header.ReceiveRevision); + command.Parameters.AddWithValue("@send_revision", header.SendRevision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var peers = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + peers.Add(new PeerBuilder( + reader.GetString(0), + reader.GetString(1), + reader.GetInt32(2), + reader.GetString(3), + reader.GetString(4), + ReadYn(reader.GetString(5)), + reader.GetInt32(6), + reader.GetInt32(7), + reader.GetString(8), + reader.GetInt32(9), + reader.GetString(10), + reader.GetString(11), + reader.GetString(12), + ReadSendMaxArticleBytes(reader, 13), + reader.GetInt32(14), + reader.GetString(15), + reader.GetString(16))); + } + + return peers; + } + + private static async ValueTask>> ReadStringsAsync( + INntpDbSession session, + MySqlTransaction transaction, + string sql, + string parameterName, + long revision, + string valueColumn, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = sql; + command.Parameters.AddWithValue(parameterName, revision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var valueOrdinal = reader.GetOrdinal(valueColumn); + var map = new Dictionary>(StringComparer.Ordinal); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var identifier = reader.GetString(0); + if (!map.TryGetValue(identifier, out var list)) + { + list = []; + map.Add(identifier, list); + } + + list.Add(reader.GetString(valueOrdinal)); + } + + return map; + } + + private static async ValueTask>> ReadEndpointsAsync( + INntpDbSession session, + MySqlTransaction transaction, + long sendRevision, + CancellationToken cancellationToken) + { + await using var command = session.CreateCommand(); + command.Transaction = transaction; + command.CommandText = NntpTransitQueries.SelectEndpoints; + command.Parameters.AddWithValue("@send_revision", sendRevision); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var map = new Dictionary>(StringComparer.Ordinal); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var identifier = reader.GetString(0); + if (!map.TryGetValue(identifier, out var list)) + { + list = []; + map.Add(identifier, list); + } + + list.Add(new TransitCatalogueEndpointRecord(reader.GetString(1), reader.GetInt32(2))); + } + + return map; + } + + private static void RejectUnknown( + Dictionary children, + List peers, + string kind) + { + foreach (var identifier in children.Keys) + { + var found = false; + foreach (var peer in peers) + { + if (string.Equals(peer.Identifier, identifier, StringComparison.Ordinal)) + { + found = true; + break; + } + } + + if (!found) + { + throw new InvalidOperationException( + "Transit " + kind + " row '" + identifier + "' is not in the published peer set."); + } + } + } + + /// + /// Reads nntptransitsend.max_article_bytes. + /// SQL NULL is unlimited and stays . A stored integer stays that ceiling. + /// + /// The peer row. + /// The send max_article_bytes column. + /// when the column is SQL NULL; otherwise the stored byte ceiling. + internal static int? ReadSendMaxArticleBytes(IDataRecord record, int ordinal) + { + ArgumentNullException.ThrowIfNull(record); + return record.IsDBNull(ordinal) ? null : record.GetInt32(ordinal); + } + + private static bool ReadYn(string value) => + value switch + { + "Y" => true, + "N" => false, + _ => throw new InvalidOperationException("Transit catalogue flag is not Y or N."), + }; + + private sealed class Header( + long publicationId, + long receiveRevision, + long sendRevision, + long globalRevision, + bool wantTrash, + bool logTrash) + { + public long PublicationId { get; } = publicationId; + + public long ReceiveRevision { get; } = receiveRevision; + + public long SendRevision { get; } = sendRevision; + + public long GlobalRevision { get; } = globalRevision; + + public bool WantTrash { get; } = wantTrash; + + public bool LogTrash { get; } = logTrash; + } + + private sealed class PeerBuilder + { + private readonly string _identifier; + private readonly string _peerName; + private readonly int _maxInbound; + private readonly string _receiveUsername; + private readonly string _receivePassword; + private readonly bool _deferOnDuplicate; + private readonly int _receiveMaxArticleBytes; + private readonly int _receiveArticleTypes; + private readonly string _receivePatterns; + private readonly int _maxOutbound; + private readonly string _sslMode; + private readonly string _sendUsername; + private readonly string _sendPassword; + private readonly int? _sendMaxArticleBytes; + private readonly int _sendArticleTypes; + private readonly string _sendPatterns; + private readonly string _pathToken; + + public PeerBuilder( + string identifier, + string peerName, + int maxInbound, + string receiveUsername, + string receivePassword, + bool deferOnDuplicate, + int receiveMaxArticleBytes, + int receiveArticleTypes, + string receivePatterns, + int maxOutbound, + string sslMode, + string sendUsername, + string sendPassword, + int? sendMaxArticleBytes, + int sendArticleTypes, + string sendPatterns, + string pathToken) + { + _identifier = identifier; + _peerName = peerName; + _maxInbound = maxInbound; + _receiveUsername = receiveUsername; + _receivePassword = receivePassword; + _deferOnDuplicate = deferOnDuplicate; + _receiveMaxArticleBytes = receiveMaxArticleBytes; + _receiveArticleTypes = receiveArticleTypes; + _receivePatterns = receivePatterns; + _maxOutbound = maxOutbound; + _sslMode = sslMode; + _sendUsername = sendUsername; + _sendPassword = sendPassword; + _sendMaxArticleBytes = sendMaxArticleBytes; + _sendArticleTypes = sendArticleTypes; + _sendPatterns = sendPatterns; + _pathToken = pathToken; + } + + public string Identifier => _identifier; + + public TransitCataloguePeerRecord WithChildren( + IReadOnlyList allowFrom, + IReadOnlyList endpoints, + IReadOnlyList pathExclusions) => + new( + _identifier, + _peerName, + _maxInbound, + _receiveUsername, + _receivePassword, + _deferOnDuplicate, + _receiveMaxArticleBytes, + _receiveArticleTypes, + _receivePatterns, + allowFrom, + _maxOutbound, + _sslMode, + _sendUsername, + _sendPassword, + _sendMaxArticleBytes, + _sendArticleTypes, + _sendPatterns, + _pathToken, + endpoints, + pathExclusions); + } + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpTransitQueries.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpTransitQueries.cs new file mode 100644 index 00000000..2f9a70e7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpTransitQueries.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// SQL for the published Transit catalogue. NNTPD only reads these tables. + internal static class NntpTransitQueries + { + /// The singleton current-publication row and its global flags. + public const string SelectCurrent = """ + SELECT p.publication_id, p.receive_revision, p.send_revision, p.global_revision, + g.want_trash, g.log_trash + FROM nntptransitcurrent c + INNER JOIN nntptransitpublication p ON p.publication_id = c.publication_id + INNER JOIN nntptransitglobalrevision g ON g.revision = p.global_revision + WHERE c.policy_id = 1 + """; + + /// Receive and send rows for the publication's two revisions, joined on identifier. + public const string SelectPeers = """ + SELECT r.identifier, peer.peer_name, + r.max_inbound, r.username, r.password, r.defer_on_duplicate, + r.max_article_bytes, r.article_types, r.patterns, + s.max_outbound, s.ssl_mode, s.username, s.password, + s.max_article_bytes, s.article_types, s.patterns, s.path_token + FROM nntptransitreceive r + INNER JOIN nntptransitsend s + ON s.identifier = r.identifier AND s.revision = @send_revision + INNER JOIN nntptransitpeer peer ON peer.identifier = r.identifier + WHERE r.revision = @receive_revision + ORDER BY r.identifier + """; + + /// Counts receive rows so a dropped join cannot hide a missing send peer. + public const string CountReceive = """ + SELECT COUNT(*) FROM nntptransitreceive WHERE revision = @receive_revision + """; + + /// Counts send rows so a dropped join cannot hide a missing receive peer. + public const string CountSend = """ + SELECT COUNT(*) FROM nntptransitsend WHERE revision = @send_revision + """; + + /// AllowFrom entries for the receive revision, in configuration order. + public const string SelectAllowFrom = """ + SELECT identifier, entry + FROM nntptransitallowfrom + WHERE revision = @receive_revision + ORDER BY identifier, ordinal + """; + + /// Send endpoints for the send revision, in configuration order. + public const string SelectEndpoints = """ + SELECT identifier, host, port + FROM nntptransitendpoint + WHERE revision = @send_revision + ORDER BY identifier, ordinal + """; + + /// Send Path exclusions for the send revision, in configuration order. + public const string SelectPathExclusions = """ + SELECT identifier, path_token + FROM nntptransitsendpathexclude + WHERE revision = @send_revision + ORDER BY identifier, ordinal + """; + } +} diff --git a/src/VectorNNTPD.TransitService/NntpDb/NntpUserQueries.cs b/src/VectorNNTPD.TransitService/NntpDb/NntpUserQueries.cs new file mode 100644 index 00000000..5dd60e83 --- /dev/null +++ b/src/VectorNNTPD.TransitService/NntpDb/NntpUserQueries.cs @@ -0,0 +1,59 @@ +namespace VectorNNTPD.TransitService.NntpDb +{ + /// Authoritative NntpDB query for reader account authentication. + public static class NntpUserQueries + { + /// + /// Looks up one nntpusers row. account_name is stored as MD5 hex of the + /// plaintext wire username. account_pass is decrypted in SQL with + /// AES_DECRYPT(..., UNHEX(SHA2(account_name, 256))) using the stored hash column. + /// Does not select account_type; every account has both remaining-byte and rate + /// policies. + /// + public const string SelectUserByName = + "SELECT " + + "CAST(AES_DECRYPT(account_pass, UNHEX(SHA2(account_name, 256))) AS CHAR) AS account_pass, " + + "scram_salt, scram_iterations, scram_stored_key, scram_server_key, " + + "allow_auth_plain, allow_auth_scram256, " + + "account_rate_limit, account_byte_limit, account_session_limit, account_srcip_limit, " + + "is_enabled, customer_id, account_art_type " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name)"; + + /// + /// Additive nntpusers extension from + /// docs/schema/nntpusers-account-art-type.sql. Does not drop the table. + /// Default 65535 is ArticleTypeCapabilities.All. + /// + public const string AddAccountArtTypeColumn = + "ALTER TABLE nntpusers " + + "ADD COLUMN account_art_type INT UNSIGNED NOT NULL DEFAULT 65535 " + + "COMMENT 'ArticleType flags; 65535=ArticleTypeCapabilities.All (unrestricted)'"; + + /// Locks one nntpusers row so a consume can read remaining atomically. + public const string SelectByteQuotaForUpdate = + "SELECT account_byte_limit " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name) " + + "FOR UPDATE"; + + /// + /// Subtracts consumed bytes from remaining quota, clamping at zero. + /// Uses CASE rather than unchecked subtraction so an unsigned column + /// cannot wrap before the floor is applied. + /// + public const string ConsumeAccountBytes = + "UPDATE nntpusers " + + "SET account_byte_limit = CASE " + + "WHEN account_byte_limit IS NULL THEN 0 " + + "WHEN account_byte_limit > @bytes THEN account_byte_limit - @bytes " + + "ELSE 0 END " + + "WHERE account_name = MD5(@account_name)"; + + /// Reads remaining quota without locking. + public const string SelectAccountByteRemaining = + "SELECT account_byte_limit " + + "FROM nntpusers " + + "WHERE account_name = MD5(@account_name)"; + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilter.cs b/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilter.cs new file mode 100644 index 00000000..063cc2ab --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilter.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Accepts every article without quota or SpamAssassin. Used when no filter is wired. + internal sealed class DisabledPostFilter : IPostFilter + { + /// Shared instance. + public static DisabledPostFilter Instance { get; } = new(); + + /// + public ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + _ = request; + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(PostFilterResult.AcceptedDisabled()); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..d5d4b954 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/DisabledPostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,40 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// No-op evidence queue used when the writer is not registered. + public sealed class DisabledPostFilterRejectionEvidenceQueue : IPostFilterRejectionEvidenceQueue + { + /// Shared disabled instance. + public static DisabledPostFilterRejectionEvidenceQueue Instance { get; } = new(); + + /// + public int Capacity => 0; + + /// + public int Count => 0; + + /// + public long Dropped => 0; + + /// + public long Written => 0; + + /// + public long WriteFailures => 0; + + /// + public bool TryEnqueue(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + return true; + } + + /// + public void Complete() + { + } + + /// + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + ValueTask.FromResult(null); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/IPostFilter.cs b/src/VectorNNTPD.TransitService/PostFilter/IPostFilter.cs new file mode 100644 index 00000000..379ceea6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/IPostFilter.cs @@ -0,0 +1,67 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// NNTPD POST-only accept filter. Does not own CreateQueued / TryAdmit / History. + public interface IPostFilter + { + /// + /// Evaluates one CanonicalV1 article. On Accept, + /// is set when a reservation was taken. + /// + ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default); + } + + /// Inputs already available at the POST insertion point. + public readonly struct PostFilterRequest + { + /// Initializes a request. Does not copy . + public PostFilterRequest( + ArticleRecord article, + string? accountName, + ConnectionClientIdentity clientIdentity, + NntpAccountPolicy? accountPolicy, + DateTimeOffset injectionUtc, + string[] newsgroups, + DateTimeOffset now, + string? serverFqdn = null) + { + Article = article; + AccountName = accountName; + ClientIdentity = clientIdentity; + AccountPolicy = accountPolicy; + InjectionUtc = injectionUtc; + Newsgroups = newsgroups; + Now = now; + ServerFqdn = string.IsNullOrWhiteSpace(serverFqdn) ? "localhost" : serverFqdn.Trim(); + } + + /// Gets the CanonicalV1 record (read-only). + public ArticleRecord Article { get; } + + /// Gets the authenticated account, when present. + public string? AccountName { get; } + + /// Gets the PROXY-aware client identity. + public ConnectionClientIdentity ClientIdentity { get; } + + /// Gets the AUTH-time account policy, when present. + public NntpAccountPolicy? AccountPolicy { get; } + + /// Gets injection time from StreamingPost. + public DateTimeOffset InjectionUtc { get; } + + /// Gets newsgroup names already parsed by StreamingPost. + public string[] Newsgroups { get; } + + /// Gets the evaluation clock. + public DateTimeOffset Now { get; } + + /// Gets the server FQDN used for synthetic SPAMD scan headers. + public string ServerFqdn { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicyRepository.cs b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicyRepository.cs new file mode 100644 index 00000000..fd038824 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicyRepository.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Loads the cluster PostFilter policy from NntpDB. Must not be called from the POST hot path. + /// + public interface IPostFilterPolicyRepository + { + /// + /// Reads the singleton policy and its collection tables. + /// + /// Token used to cancel the load. + /// The complete policy document. + /// + /// MySQL is unavailable or the query failed. + /// + /// + /// The singleton row is missing or the row set cannot be mapped. + /// + ValueTask LoadAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicySource.cs b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicySource.cs new file mode 100644 index 00000000..6a1466af --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterPolicySource.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Publishes the current immutable PostFilter snapshot. + internal interface IPostFilterPolicySource + { + /// Gets the last-known-good snapshot. Throws when none has been published. + PostFilterPolicySnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/IPostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..df26ad25 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,37 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Bounded asynchronous queue for PostFilter rejection evidence. + /// Enqueue never changes the PostFilter or NNTP decision. + /// + public interface IPostFilterRejectionEvidenceQueue + { + /// Gets the configured capacity. + int Capacity { get; } + + /// Gets the approximate queued count. + int Count { get; } + + /// Gets dropped-enqueue count (queue full or completed). + long Dropped { get; } + + /// Gets successfully written rows. + long Written { get; } + + /// Gets failed database writes. + long WriteFailures { get; } + + /// + /// Attempts to enqueue evidence. Returns when the + /// queue is full or completed. The caller must still return the original + /// NNTP rejection. + /// + bool TryEnqueue(PostFilterRejectionEvidence evidence); + + /// Completes the writer so the consumer can drain and exit. + void Complete(); + + /// Dequeues the next item, or when completed and empty. + ValueTask DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/IPostFilterSpamAssassin.cs b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterSpamAssassin.cs new file mode 100644 index 00000000..17989107 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/IPostFilterSpamAssassin.cs @@ -0,0 +1,91 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// NNTPD-only SPAMD CHECK seam. Does not mutate . + internal interface IPostFilterSpamAssassin + { + /// Runs CHECK. Cancellation is rethrown and is not an on_failure event. + ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default); + } + + /// Peer and server identity used only to build the disposable SPAMD scan message. + internal readonly record struct SpamdScanContext( + IPAddress ClientAddress, + string ServerFqdn, + DateTimeOffset Now); + + /// SPAMD endpoints, pool, and timeouts captured with one PostFilter snapshot. + internal readonly record struct PostFilterSpamAssassinTarget( + IReadOnlyList Hosts, + int Port, + TimeSpan ConnectTimeout, + TimeSpan OperationTimeout, + string ProtocolVersion, + int MaxConnections, + PostFilterSpamAssassinHostSelection HostSelection); + + /// CHECK outcome classified for PostFilter policy. + public enum PostFilterSpamAssassinStatus + { + /// SPAMD reported ham. + Ham = 0, + + /// SPAMD reported spam. + Spam = 1, + + /// Connect, protocol, timeout, or malformed response. + Failed = 2, + } + + /// CHECK result. + internal readonly struct PostFilterSpamAssassinResult + { + /// Initializes a result. + public PostFilterSpamAssassinResult( + PostFilterSpamAssassinStatus status, + string detail, + decimal? score = null, + decimal? threshold = null) + { + Status = status; + Detail = detail; + Score = score; + Threshold = threshold; + } + + /// Gets the classified status. + public PostFilterSpamAssassinStatus Status { get; } + + /// Gets an internal detail. + public string Detail { get; } + + /// Gets the SPAMD score when the Spam header included one. + public decimal? Score { get; } + + /// Gets the SPAMD threshold when the Spam header included one. + public decimal? Threshold { get; } + + /// Ham. + public static PostFilterSpamAssassinResult Ham(decimal? score = null, decimal? threshold = null) => + new(PostFilterSpamAssassinStatus.Ham, "ham", score, threshold); + + /// Spam. + public static PostFilterSpamAssassinResult Spam( + string detail, + decimal? score = null, + decimal? threshold = null) => + new(PostFilterSpamAssassinStatus.Spam, detail, score, threshold); + + /// Scanner fault. + public static PostFilterSpamAssassinResult Failed(string detail) => + new(PostFilterSpamAssassinStatus.Failed, detail); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/MySqlPostFilterPolicyRepository.cs b/src/VectorNNTPD.TransitService/PostFilter/MySqlPostFilterPolicyRepository.cs new file mode 100644 index 00000000..20dd34ce --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/MySqlPostFilterPolicyRepository.cs @@ -0,0 +1,53 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Loads the published PostFilter revision through the existing NntpDB connection pool. + public sealed class MySqlPostFilterPolicyRepository : IPostFilterPolicyRepository + { + private readonly NntpDbService _nntpDb; + private readonly ILogger _logger; + + /// Initializes a new instance of the class. + public MySqlPostFilterPolicyRepository( + NntpDbService nntpDb, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _logger = logger; + } + + /// + public async ValueTask LoadAsync(CancellationToken cancellationToken = default) + { + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var record = await connection.QueryPostFilterPolicyAsync(cancellationToken).ConfigureAwait(false); + return record ?? throw new InvalidOperationException( + "nntppostfiltercurrent is missing policy_id = 1. NNTPD will not invent a local PostFilter policy."); + } + catch (OperationCanceledException) + { + throw; + } + catch (InvalidOperationException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PostFilterLogMessages.PolicyRepositoryFailed(_logger, ex); + throw new NntpDbUnavailableException("MySQL nntppostfilterpolicy lookup failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/NullPostFilterSpamAssassin.cs b/src/VectorNNTPD.TransitService/PostFilter/NullPostFilterSpamAssassin.cs new file mode 100644 index 00000000..aedb0bb1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/NullPostFilterSpamAssassin.cs @@ -0,0 +1,27 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// SpamAssassin seam used when CHECK is not configured. + internal sealed class NullPostFilterSpamAssassin : IPostFilterSpamAssassin + { + /// Shared instance. + public static NullPostFilterSpamAssassin Instance { get; } = new(); + + /// + public ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = article; + _ = accountName; + _ = target; + _ = scanContext; + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(PostFilterSpamAssassinResult.Failed("spamassassin not configured")); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterAccountIdentity.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterAccountIdentity.cs new file mode 100644 index 00000000..42927aa0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterAccountIdentity.cs @@ -0,0 +1,64 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// PostFilter account-list identity: lowercase hexadecimal MD5 of the AUTH + /// username using the same trim-on-policy / raw-on-request contract as the + /// evaluator. Does not replace ArticleId or ArtHash. + /// + public static class PostFilterAccountIdentity + { + /// Stored identifier length (CHAR(32) lowercase hex). + public const int HexLength = 32; + + /// + /// MD5 of as UTF-8 bytes, lowercase hex. + /// Does not trim; request matching uses the session username as stored. + /// + public static string FromUsername(string username) + { + ArgumentNullException.ThrowIfNull(username); + var digest = MD5.HashData(Encoding.UTF8.GetBytes(username)); + return Convert.ToHexString(digest).ToLowerInvariant(); + } + + /// + /// Policy-list normalization: trim then hash, unless the trimmed value is + /// already a stored 32-character lowercase hex identifier. + /// + public static string FromPolicyEntry(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + var trimmed = value.Trim(); + return IsStoredIdentifier(trimmed) ? trimmed : FromUsername(trimmed); + } + + /// Returns whether is 32 lowercase hex characters. + public static bool IsStoredIdentifier(ReadOnlySpan value) + { + if (value.Length != HexLength) + { + return false; + } + + foreach (var ch in value) + { + if (ch is not (>= '0' and <= '9' or >= 'a' and <= 'f')) + { + return false; + } + } + + return true; + } + + /// + /// Hashes a request username when it is non-empty; otherwise returns + /// . Does not trim. + /// + public static string? TryFromRequest(string? accountName) => + accountName is { Length: > 0 } ? FromUsername(accountName) : null; + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterBodyHash.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterBodyHash.cs new file mode 100644 index 00000000..3c2d3748 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterBodyHash.cs @@ -0,0 +1,44 @@ +using System.Globalization; +using System.IO.Hashing; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// XXH3-64 of canonical body octets for identical-body quota. + /// + /// Does not use . Encoded ArtType bits do not participate. + /// + internal static class PostFilterBodyHash + { + /// ArtType bits excluded from identical-body quota. + public const ArticleType EncodedTypes = + ArticleType.YEncoded + | ArticleType.Binary + | ArticleType.UuEncode + | ArticleType.Base64 + | ArticleType.BinHex; + + /// + /// Returns lowercase hex XXH3-64 of the body after the first CRLF CRLF, + /// or when the article must not participate. + /// + public static string? TryCompute(in ArticleRecord article) + { + if ((article.ArtType & EncodedTypes) != 0) + { + return null; + } + + var data = article.ArtData.Span; + var separator = data.IndexOf("\r\n\r\n"u8); + if (separator < 0) + { + return null; + } + + var body = data[(separator + 4)..]; + var hash = XxHash3.HashToUInt64(body); + return hash.ToString("x16", CultureInfo.InvariantCulture); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterEvaluator.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterEvaluator.cs new file mode 100644 index 00000000..59f5ceeb --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterEvaluator.cs @@ -0,0 +1,257 @@ +using System.Globalization; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// POST-only PostFilter pipeline. Does not call CreateQueued or History. + internal sealed class PostFilterEvaluator : IPostFilter + { + private readonly IPostFilterPolicySource _policy; + private readonly IPostFilterQuotaStore _quota; + private readonly PostFilterReservationIdentity _identity; + private readonly IPostFilterSpamAssassin _spamAssassin; + private readonly ILogger _logger; + + /// Initializes the evaluator. + public PostFilterEvaluator( + IPostFilterPolicySource policy, + IPostFilterQuotaStore quota, + PostFilterReservationIdentity identity, + IPostFilterSpamAssassin spamAssassin, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(policy); + ArgumentNullException.ThrowIfNull(quota); + ArgumentNullException.ThrowIfNull(identity); + ArgumentNullException.ThrowIfNull(spamAssassin); + ArgumentNullException.ThrowIfNull(logger); + _policy = policy; + _quota = quota; + _identity = identity; + _spamAssassin = spamAssassin; + _logger = logger; + } + + /// + public async ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var snapshot = _policy.Current; + var account = request.AccountName ?? string.Empty; + var artId = FormatArtId(request.Article); + + if (snapshot.Gate == PostFilterGateState.Disabled) + { + PostFilterLogMessages.Accepted(_logger, PostFilterStage.Gate, account, artId, reserved: false); + return PostFilterResult.AcceptedDisabled(snapshot.Revision); + } + + if (snapshot.Gate == PostFilterGateState.Closed) + { + return Reject(snapshot, PostFilterStage.Gate, "closed", account, artId); + } + + if (IsDenied(snapshot, request)) + { + return Reject(snapshot, PostFilterStage.Deny, "denied", account, artId); + } + + if (snapshot.RejectArtTypes != ArticleType.None + && (request.Article.ArtType & snapshot.RejectArtTypes) != 0) + { + return Reject(snapshot, PostFilterStage.ArtType, "arttype", account, artId); + } + + var allowlisted = IsAllowlisted(snapshot, request); + if (string.IsNullOrWhiteSpace(request.AccountName)) + { + return Reject(snapshot, PostFilterStage.Quota, "unauthenticated", account, artId); + } + + var reservation = _identity.Next(); + var bodyHex = PostFilterBodyHash.TryCompute(request.Article); + var mpUnits = bodyHex is null ? 0 : 1; + PostFilterQuotaReserveStatus reserved; + try + { + reserved = await _quota.ReserveAsync( + request.AccountName, + reservation, + request.Now, + snapshot.Windows, + snapshot.Ceilings, + messages: 1, + bytes: request.Article.ArtSize, + mpUnits, + bodyHex, + cancellationToken, + snapshot.ReservationTtlMs) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await _quota.ReleaseAsync( + request.AccountName, + reservation, + request.Now, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + if (reserved != PostFilterQuotaReserveStatus.Accepted) + { + return Reject(snapshot, PostFilterStage.Quota, reserved.ToString(), account, artId, reserved); + } + + var lease = new PostFilterLease( + _quota, + request.AccountName, + reservation, + snapshot.Windows, + snapshot.Ceilings); + + if (!allowlisted && ShouldScan(snapshot, request.Article)) + { + PostFilterSpamAssassinResult scan; + try + { + scan = await _spamAssassin + .CheckAsync( + request.Article, + request.AccountName, + snapshot.SpamAssassinTarget, + new SpamdScanContext( + request.ClientIdentity.ClientAddress, + request.ServerFqdn, + request.Now), + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + throw; + } + + if (scan.Status == PostFilterSpamAssassinStatus.Spam) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + PostFilterLogMessages.Rejected(_logger, PostFilterStage.SpamAssassin, "spam", account, artId); + return PostFilterResult.RejectedSpamAssassin("spam", snapshot.Revision, scan); + } + + if (scan.Status == PostFilterSpamAssassinStatus.Failed) + { + var action = snapshot.SpamAssassinOnFailure + ?? throw new InvalidOperationException( + "SpamAssassin is enabled but OnFailure is not configured."); + PostFilterLogMessages.SpamAssassinFailed( + _logger, + action.ToString(), + scan.Detail, + account, + artId); + if (action == PostFilterSpamOnFailure.Reject) + { + await ReleaseQuietAsync(lease, request.Now).ConfigureAwait(false); + PostFilterLogMessages.Rejected( + _logger, + PostFilterStage.SpamAssassin, + "scanner-failure", + account, + artId); + return PostFilterResult.RejectedSpamAssassin( + "scanner-failure", + snapshot.Revision, + scan); + } + } + } + + PostFilterLogMessages.Accepted(_logger, PostFilterStage.Complete, account, artId, reserved: true); + return PostFilterResult.Accepted(lease, snapshot.Revision); + } + + private PostFilterResult Reject( + PostFilterPolicySnapshot snapshot, + PostFilterStage stage, + string reason, + string account, + string artId, + PostFilterQuotaReserveStatus? quotaStatus = null) + { + PostFilterLogMessages.Rejected(_logger, stage, reason, account, artId); + return PostFilterResult.Rejected(stage, reason, quotaStatus, snapshot.Revision); + } + + private async ValueTask ReleaseQuietAsync(PostFilterLease lease, DateTimeOffset now) + { + var status = await lease.ReleaseAsync(now, CancellationToken.None).ConfigureAwait(false); + PostFilterLogMessages.Released(_logger, status, lease.AccountName, lease.Reservation.Token); + } + + private static bool ShouldScan(PostFilterPolicySnapshot snapshot, in ArticleRecord article) + { + if (!snapshot.SpamAssassinEnabled) + { + return false; + } + + if (snapshot.SpamAssassinMaxArticleSize > 0 + && article.ArtSize >= snapshot.SpamAssassinMaxArticleSize) + { + return false; + } + + return (article.ArtType & snapshot.SpamAssassinExcludeArtTypes) == 0; + } + + private static bool IsDenied(PostFilterPolicySnapshot snapshot, in PostFilterRequest request) + { + var accountId = PostFilterAccountIdentity.TryFromRequest(request.AccountName); + if (accountId is not null && snapshot.DeniedAccounts.Contains(accountId)) + { + return true; + } + + var address = request.ClientIdentity.ClientAddress; + foreach (var network in snapshot.DeniedCidrs) + { + if (network.Contains(address)) + { + return true; + } + } + + return false; + } + + private static bool IsAllowlisted(PostFilterPolicySnapshot snapshot, in PostFilterRequest request) + { + var accountId = PostFilterAccountIdentity.TryFromRequest(request.AccountName); + if (accountId is not null && snapshot.AllowlistedAccounts.Contains(accountId)) + { + return true; + } + + var address = request.ClientIdentity.ClientAddress; + foreach (var network in snapshot.AllowlistedCidrs) + { + if (network.Contains(address)) + { + return true; + } + } + + return false; + } + + private static string FormatArtId(in ArticleRecord article) => + article.ArtHash.ToString("x16", CultureInfo.InvariantCulture); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterLogMessages.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterLogMessages.cs new file mode 100644 index 00000000..14bdbb7d --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterLogMessages.cs @@ -0,0 +1,94 @@ +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Source-generated PostFilter logs. Reasons stay off the NNTP wire. + internal static partial class PostFilterLogMessages + { + [LoggerMessage(EventId = 2800, Level = LogLevel.Information, Message = "PostFilter rejected stage={Stage} reason={Reason} account={Account} artId={ArtId}")] + public static partial void Rejected(ILogger logger, PostFilterStage stage, string reason, string account, string artId); + + [LoggerMessage(EventId = 2801, Level = LogLevel.Debug, Message = "PostFilter accepted stage={Stage} account={Account} artId={ArtId} reserved={Reserved}")] + public static partial void Accepted(ILogger logger, PostFilterStage stage, string account, string artId, bool reserved); + + [LoggerMessage(EventId = 2802, Level = LogLevel.Warning, Message = "PostFilter COMMIT NOOP account={Account} token={Token} artId={ArtId}")] + public static partial void CommitNoop(ILogger logger, string account, string token, string artId); + + [LoggerMessage(EventId = 2803, Level = LogLevel.Warning, Message = "PostFilter COMMIT unavailable after admit account={Account} token={Token} artId={ArtId}")] + public static partial void CommitUnavailable(ILogger logger, string account, string token, string artId); + + [LoggerMessage(EventId = 2804, Level = LogLevel.Debug, Message = "PostFilter RELEASE status={Status} account={Account} token={Token}")] + public static partial void Released(ILogger logger, PostFilterQuotaReleaseStatus status, string account, string token); + + [LoggerMessage(EventId = 2805, Level = LogLevel.Warning, Message = "PostFilter policy refresh failed; retaining last-known-good snapshot")] + public static partial void PolicyRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2806, Level = LogLevel.Warning, Message = "PostFilter SpamAssassin failure action={Action} detail={Detail} account={Account} artId={ArtId}")] + public static partial void SpamAssassinFailed(ILogger logger, string action, string detail, string account, string artId); + + [LoggerMessage( + EventId = 2808, + Level = LogLevel.Information, + Message = "PostFilter policy published revision={Revision} gate={Gate} spamAssassin={SpamAssassin} maxArticleSize={MaxArticleSize} excludeArtTypes={ExcludeArtTypes} hosts={Hosts} maxConnections={MaxConnections}")] + public static partial void PolicyPublished( + ILogger logger, + long revision, + string gate, + bool spamAssassin, + int maxArticleSize, + string excludeArtTypes, + int hosts, + int maxConnections); + + [LoggerMessage(EventId = 2809, Level = LogLevel.Error, Message = "PostFilter initial policy load failed")] + public static partial void PolicyInitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2810, Level = LogLevel.Information, Message = "PostFilter policy revision changed from={FromRevision} to={ToRevision}")] + public static partial void PolicyRevisionChanged(ILogger logger, long fromRevision, long toRevision); + + [LoggerMessage(EventId = 2811, Level = LogLevel.Error, Message = "PostFilter policy repository failed")] + public static partial void PolicyRepositoryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2840, + Level = LogLevel.Information, + Message = "PostFilter evidence writer started capacity={Capacity}")] + public static partial void EvidenceWriterStarted(ILogger logger, int capacity); + + [LoggerMessage( + EventId = 2841, + Level = LogLevel.Information, + Message = "PostFilter evidence writer stopped written={Written} dropped={Dropped} writeFailures={WriteFailures}")] + public static partial void EvidenceWriterStopped(ILogger logger, long written, long dropped, long writeFailures); + + [LoggerMessage( + EventId = 2842, + Level = LogLevel.Warning, + Message = "PostFilter evidence writer stopped with queued items remaining={Remaining}")] + public static partial void EvidenceWriterStoppedWithQueued(ILogger logger, int remaining); + + [LoggerMessage( + EventId = 2843, + Level = LogLevel.Error, + Message = "PostFilter evidence write failed stage={Stage} reason={Reason}")] + public static partial void EvidenceWriteFailed(ILogger logger, Exception exception, PostFilterStage stage, string reason); + + [LoggerMessage( + EventId = 2844, + Level = LogLevel.Warning, + Message = "PostFilter evidence queue full; rejection evidence dropped stage={Stage} reason={Reason}")] + public static partial void EvidenceQueueFull(ILogger logger, PostFilterStage stage, string reason); + + [LoggerMessage( + EventId = 2807, + Level = LogLevel.Information, + Message = "PostFilter SPAMD transport stopped connects={Connects} checks={Checks} reuses={Reuses} reconnects={Reconnects} evictions={Evictions}")] + public static partial void SpamAssassinTransportStopped( + ILogger logger, + long connects, + long checks, + long reuses, + long reconnects, + long evictions); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterMetrics.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterMetrics.cs new file mode 100644 index 00000000..4f27693e --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterMetrics.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Process counters for PostFilter reservation outcomes. + public sealed class PostFilterMetrics + { + private long _commitNoop; + private long _commitUnavailable; + + /// Gets COMMIT NOOP after a successful TryAdmit. + public long CommitNoop => Volatile.Read(ref _commitNoop); + + /// Gets COMMIT Unavailable after a successful TryAdmit. + public long CommitUnavailable => Volatile.Read(ref _commitUnavailable); + + /// Records a COMMIT NOOP. The article remains queued. + public void RecordCommitNoop() => Interlocked.Increment(ref _commitNoop); + + /// Records COMMIT Unavailable after admit. Does not RELEASE. + public void RecordCommitUnavailable() => Interlocked.Increment(ref _commitUnavailable); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyCompiler.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyCompiler.cs new file mode 100644 index 00000000..cb52a103 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyCompiler.cs @@ -0,0 +1,222 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Compiles into an immutable snapshot. + internal static class PostFilterPolicyCompiler + { + /// Builds a snapshot or throws when the options are not usable. + public static PostFilterPolicySnapshot Compile(PostFilterOptions options, long revision = 0) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + ArgumentOutOfRangeException.ThrowIfNegative(revision); + if (!Enum.IsDefined(options.Gate)) + { + throw new InvalidOperationException($"PostFilter Gate is not a defined value: '{options.Gate}'."); + } + + if (options.Quota.LongWindow <= TimeSpan.Zero || options.Quota.ShortWindow <= TimeSpan.Zero) + { + throw new InvalidOperationException("PostFilter quota windows must be positive."); + } + + var windows = new PostFilterQuotaWindows( + (long)options.Quota.LongWindow.TotalMilliseconds, + (long)options.Quota.ShortWindow.TotalMilliseconds); + windows.Validate(); + + var sa = options.SpamAssassin; + if (sa.Enabled) + { + if (sa.OnFailure is null) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin OnFailure is required when SpamAssassin is enabled."); + } + + if (sa.Hosts.Length == 0) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin Hosts is required when SpamAssassin is enabled."); + } + + var uniqueHosts = new HashSet(StringComparer.Ordinal); + foreach (var host in sa.Hosts) + { + if (string.IsNullOrWhiteSpace(host)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin Hosts contains a malformed empty entry."); + } + + if (!uniqueHosts.Add(host.Trim())) + { + throw new InvalidOperationException( + $"PostFilter SpamAssassin Hosts contains a duplicate entry: '{host.Trim()}'."); + } + } + + if (sa.Port is < 1 or > 65535) + { + throw new InvalidOperationException("PostFilter SpamAssassin Port must be 1–65535."); + } + + if (sa.OperationTimeout < PostFilterSpamAssassinOptions.MinOperationTimeout + || sa.OperationTimeout > PostFilterSpamAssassinOptions.MaxOperationTimeout) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin OperationTimeout must be between 1 second and 2 minutes when SpamAssassin is enabled."); + } + + if (sa.ConnectTimeout <= TimeSpan.Zero || sa.ConnectTimeout > sa.OperationTimeout) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin ConnectTimeout must be positive and must not exceed OperationTimeout."); + } + + if (string.IsNullOrWhiteSpace(sa.ProtocolVersion)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin ProtocolVersion is required when SpamAssassin is enabled."); + } + + if (sa.MaxConnections < PostFilterSpamAssassinOptions.MinMaxConnections + || sa.MaxConnections > PostFilterSpamAssassinOptions.MaxMaxConnections) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin MaxConnections must be between 1 and 32 when SpamAssassin is enabled."); + } + + if (!Enum.IsDefined(sa.HostSelection)) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin HostSelection is not a defined strategy."); + } + } + + if (sa.MaxArticleSize < 0) + { + throw new InvalidOperationException( + "PostFilter SpamAssassin MaxArticleSize must be zero or positive."); + } + + var ceilings = new PostFilterQuotaCeilings( + options.Quota.MaxMessagesLong, + options.Quota.MaxBytesLong, + options.Quota.MaxIdenticalLong, + options.Quota.MaxMessagesShort, + options.Quota.MaxBytesShort, + options.Quota.MaxIdenticalShort); + ceilings.Validate(); + + return new PostFilterPolicySnapshot( + options.Gate, + ToSet(options.DeniedAccounts, "DeniedAccounts"), + ToNetworks(options.DeniedCidrs, "DeniedCidrs"), + ToSet(options.AllowlistedAccounts, "AllowlistedAccounts"), + ToNetworks(options.AllowlistedCidrs, "AllowlistedCidrs"), + ParseArtTypes(options.RejectArtTypes, "RejectArtTypes"), + windows, + ceilings, + sa.Enabled, + sa.OnFailure, + sa.MaxArticleSize, + ParseArtTypes(sa.ExcludeArtTypes, "SpamAssassin:ExcludeArtTypes"), + sa.Enabled ? sa.Hosts.Select(static host => host.Trim()).ToArray() : [], + sa.Port, + string.IsNullOrWhiteSpace(sa.ProtocolVersion) + ? PostFilterSpamAssassinOptions.DefaultProtocolVersion + : sa.ProtocolVersion.Trim(), + sa.MaxConnections, + sa.HostSelection, + sa.ConnectTimeout, + sa.OperationTimeout, + PostFilterQuotaDefaults.HoldMilliseconds(sa.Enabled, sa.OperationTimeout), + revision); + } + + private static HashSet ToSet(string[] values, string name) + { + var set = new HashSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + var identifier = PostFilterAccountIdentity.FromPolicyEntry(trimmed); + if (!set.Add(identifier)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + } + + return set; + } + + private static List ToNetworks(string[] values, string name) + { + var networks = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + if (!seen.Add(trimmed)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + + if (!IPNetwork.TryParse(trimmed, out var network)) + { + throw new InvalidOperationException($"PostFilter {name} contains an invalid CIDR: '{value}'."); + } + + networks.Add(network); + } + + return networks; + } + + private static ArticleType ParseArtTypes(string[] values, string name) + { + var mask = ArticleType.None; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"PostFilter {name} contains a malformed empty entry."); + } + + var trimmed = value.Trim(); + if (!seen.Add(trimmed)) + { + throw new InvalidOperationException($"PostFilter {name} contains a duplicate entry: '{trimmed}'."); + } + + if (!Enum.TryParse(trimmed, ignoreCase: true, out var flag) + || !Enum.IsDefined(flag) + || flag == ArticleType.None) + { + throw new InvalidOperationException($"PostFilter {name} contains an unknown ArtType: '{value}'."); + } + + mask |= flag; + } + + return mask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyRecord.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyRecord.cs new file mode 100644 index 00000000..c9f05ec4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyRecord.cs @@ -0,0 +1,29 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Authoritative cluster PostFilter policy loaded from NntpDB, before compile. + /// + public sealed class PostFilterPolicyRecord + { + /// Initializes a loaded policy document. + public PostFilterPolicyRecord(long revision, DateTimeOffset updatedUtc, PostFilterOptions options) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(revision); + ArgumentNullException.ThrowIfNull(options); + Revision = revision; + UpdatedUtc = updatedUtc; + Options = options; + } + + /// Gets the published NntpDB revision (nntppostfiltercurrent.revision). + public long Revision { get; } + + /// Gets nntppostfilterpolicy.updated_utc. + public DateTimeOffset UpdatedUtc { get; } + + /// Gets the compile input. Not bound from Nntpd:PostFilter. + public PostFilterOptions Options { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyService.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyService.cs new file mode 100644 index 00000000..13343deb --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicyService.cs @@ -0,0 +1,215 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Loads PostFilter policy from NntpDB, publishes an immutable snapshot, + /// and refreshes it every 60 seconds. + /// + /// + /// Initial load failure prevents RUNNING. Refresh failure retains last-known-good. + /// A successful refresh of the same published revision does not replace the snapshot. + /// POST observes via a volatile snapshot read and never queries MySQL. + /// There is no appsettings fallback. + /// + internal sealed class PostFilterPolicyService : IPostFilterPolicySource, IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly IPostFilterPolicyRepository _repository; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private PostFilterPolicySnapshot? _current; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a production policy service. + public PostFilterPolicyService( + IPostFilterPolicyRepository repository, + ILogger logger) + : this(repository, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a service with an explicit clock and interval (tests). + internal PostFilterPolicyService( + IPostFilterPolicyRepository repository, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(repository); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _repository = repository; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "PostFilterPolicy"; + + /// + public Task? Execution => _execution; + + /// + public PostFilterPolicySnapshot Current + { + get + { + var snapshot = Volatile.Read(ref _current); + if (snapshot is null) + { + throw new InvalidOperationException("PostFilter policy has not been published."); + } + + return snapshot; + } + } + + /// Gets whether a snapshot has been published (tests). + internal bool HasSnapshot => Volatile.Read(ref _current) is not null; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot, previousRevision: null); + _execution = RunAsync(_runCts.Token); + await Task.CompletedTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + PostFilterLogMessages.PolicyInitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is not null) + { + try + { + await _execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var previous = Volatile.Read(ref _current); + var snapshot = await LoadSnapshotAsync(cancellationToken).ConfigureAwait(false); + if (previous is not null && previous.Revision == snapshot.Revision) + { + return; + } + + Publish(snapshot, previous?.Revision); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + PostFilterLogMessages.PolicyRefreshFailed(_logger, ex); + } + finally + { + Volatile.Write(ref _refreshing, 0); + } + } + + private async Task LoadSnapshotAsync(CancellationToken cancellationToken) + { + var record = await _repository.LoadAsync(cancellationToken).ConfigureAwait(false); + return PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + } + + private void Publish(PostFilterPolicySnapshot snapshot, long? previousRevision) + { + Volatile.Write(ref _current, snapshot); + PostFilterLogMessages.PolicyPublished( + _logger, + snapshot.Revision, + snapshot.Gate.ToString(), + snapshot.SpamAssassinEnabled, + snapshot.SpamAssassinMaxArticleSize, + snapshot.SpamAssassinExcludeArtTypes.ToString(), + snapshot.SpamAssassinHosts.Count, + snapshot.SpamAssassinMaxConnections); + if (previousRevision is { } prior && prior != snapshot.Revision) + { + PostFilterLogMessages.PolicyRevisionChanged(_logger, prior, snapshot.Revision); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicySnapshot.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicySnapshot.cs new file mode 100644 index 00000000..c857bc5d --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterPolicySnapshot.cs @@ -0,0 +1,163 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Immutable PostFilter policy observed for one POST evaluation. + internal sealed class PostFilterPolicySnapshot + { + /// Initializes a compiled snapshot. + public PostFilterPolicySnapshot( + PostFilterGateState gate, + IReadOnlySet deniedAccounts, + IReadOnlyList deniedCidrs, + IReadOnlySet allowlistedAccounts, + IReadOnlyList allowlistedCidrs, + ArticleType rejectArtTypes, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + bool spamAssassinEnabled, + PostFilterSpamOnFailure? spamAssassinOnFailure, + int spamAssassinMaxArticleSize, + ArticleType spamAssassinExcludeArtTypes, + IReadOnlyList spamAssassinHosts, + int spamAssassinPort, + string spamAssassinProtocolVersion, + int spamAssassinMaxConnections, + PostFilterSpamAssassinHostSelection spamAssassinHostSelection, + TimeSpan spamAssassinConnectTimeout, + TimeSpan spamAssassinOperationTimeout, + long reservationTtlMs, + long revision = 0) + { + Gate = gate; + DeniedAccounts = deniedAccounts; + DeniedCidrs = deniedCidrs; + AllowlistedAccounts = allowlistedAccounts; + AllowlistedCidrs = allowlistedCidrs; + RejectArtTypes = rejectArtTypes; + Windows = windows; + Ceilings = ceilings; + SpamAssassinEnabled = spamAssassinEnabled; + SpamAssassinOnFailure = spamAssassinOnFailure; + SpamAssassinMaxArticleSize = spamAssassinMaxArticleSize; + SpamAssassinExcludeArtTypes = spamAssassinExcludeArtTypes; + SpamAssassinHosts = spamAssassinHosts; + SpamAssassinPort = spamAssassinPort; + SpamAssassinProtocolVersion = spamAssassinProtocolVersion; + SpamAssassinMaxConnections = spamAssassinMaxConnections; + SpamAssassinHostSelection = spamAssassinHostSelection; + SpamAssassinConnectTimeout = spamAssassinConnectTimeout; + SpamAssassinOperationTimeout = spamAssassinOperationTimeout; + ReservationTtlMs = reservationTtlMs; + Revision = revision; + } + + /// Gets the operational gate. + public PostFilterGateState Gate { get; } + + /// Gets denied account MD5 identifiers (ordinal lowercase hex). + public IReadOnlySet DeniedAccounts { get; } + + /// Gets denied client networks. + public IReadOnlyList DeniedCidrs { get; } + + /// Gets allowlisted account MD5 identifiers (ordinal lowercase hex). + public IReadOnlySet AllowlistedAccounts { get; } + + /// Gets allowlisted client networks. + public IReadOnlyList AllowlistedCidrs { get; } + + /// Gets the ArtType reject mask. disables the stage. + public ArticleType RejectArtTypes { get; } + + /// Gets quota windows. + public PostFilterQuotaWindows Windows { get; } + + /// Gets quota ceilings. + public PostFilterQuotaCeilings Ceilings { get; } + + /// Gets whether SpamAssassin CHECK is enabled. + public bool SpamAssassinEnabled { get; } + + /// Gets scanner-fault behaviour when SpamAssassin is enabled. + public PostFilterSpamOnFailure? SpamAssassinOnFailure { get; } + + /// + /// Gets the exclusive SA size gate. Articles with + /// ArtSize >= SpamAssassinMaxArticleSize skip CHECK. + /// 0 disables the size gate. + /// + public int SpamAssassinMaxArticleSize { get; } + + /// Gets ArtType bits excluded from CHECK. + public ArticleType SpamAssassinExcludeArtTypes { get; } + + /// Gets compiled SPAMD hosts for this evaluation. + public IReadOnlyList SpamAssassinHosts { get; } + + /// Gets the compiled SPAMD port. + public int SpamAssassinPort { get; } + + /// Gets the compiled SPAMC protocol version. + public string SpamAssassinProtocolVersion { get; } + + /// Gets the compiled persistent-connection pool size. + public int SpamAssassinMaxConnections { get; } + + /// Gets the compiled host-selection strategy. + public PostFilterSpamAssassinHostSelection SpamAssassinHostSelection { get; } + + /// Gets the compiled SPAMD connect timeout. + public TimeSpan SpamAssassinConnectTimeout { get; } + + /// Gets the compiled SPAMD CHECK wall-clock budget. + public TimeSpan SpamAssassinOperationTimeout { get; } + + /// + /// Gets the reservation hold compiled for this snapshot. + /// Floor is 10s; when SpamAssassin is enabled this covers + /// plus hold skew. + /// + public long ReservationTtlMs { get; } + + /// Gets the NntpDB policy revision compiled into this snapshot. + public long Revision { get; } + + /// Gets the CHECK target captured with this snapshot. + public PostFilterSpamAssassinTarget SpamAssassinTarget => + new( + SpamAssassinHosts, + SpamAssassinPort, + SpamAssassinConnectTimeout, + SpamAssassinOperationTimeout, + SpamAssassinProtocolVersion, + SpamAssassinMaxConnections, + SpamAssassinHostSelection); + + /// Disabled gate, no lists, no type policy, no SA, all quota dimensions off. + public static PostFilterPolicySnapshot Disabled { get; } = new( + PostFilterGateState.Disabled, + new HashSet(StringComparer.Ordinal), + [], + new HashSet(StringComparer.Ordinal), + [], + ArticleType.None, + new PostFilterQuotaWindows(86_400_000, 600_000), + new PostFilterQuotaCeilings(0, 0, 0, 0, 0, 0), + spamAssassinEnabled: false, + spamAssassinOnFailure: null, + spamAssassinMaxArticleSize: 0, + spamAssassinExcludeArtTypes: ArticleType.None, + spamAssassinHosts: [], + spamAssassinPort: 783, + spamAssassinProtocolVersion: PostFilterSpamAssassinOptions.DefaultProtocolVersion, + spamAssassinMaxConnections: PostFilterSpamAssassinOptions.DefaultMaxConnections, + spamAssassinHostSelection: PostFilterSpamAssassinHostSelection.RoundRobin, + spamAssassinConnectTimeout: TimeSpan.FromSeconds(5), + spamAssassinOperationTimeout: TimeSpan.FromSeconds(30), + reservationTtlMs: (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidence.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidence.cs new file mode 100644 index 00000000..dd3a1d24 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidence.cs @@ -0,0 +1,144 @@ +using System.Net; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// One PostFilter rejection queued for asynchronous NntpDB persistence. + /// The payload is the exact unstuffed article available at the decision, or + /// when those bytes were not available. + /// + public sealed class PostFilterRejectionEvidence + { + /// Creates an evidence snapshot. Does not copy . + public PostFilterRejectionEvidence( + DateTimeOffset rejectedUtc, + long policyRevision, + string? accountName, + IPAddress sourceAddress, + ArticleType artType, + string? messageId, + int articleSize, + PostFilterStage stage, + string reason, + PostFilterSpamAssassinStatus? spamAssassinStatus, + decimal? spamAssassinScore, + decimal? spamAssassinThreshold, + ReadOnlyMemory? articlePayload) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + ArgumentNullException.ThrowIfNull(sourceAddress); + ArgumentOutOfRangeException.ThrowIfNegative(articleSize); + RejectedUtc = rejectedUtc; + PolicyRevision = policyRevision; + AccountName = PostFilterAccountIdentity.TryFromRequest(accountName); + SourceAddress = sourceAddress; + ArtType = artType; + MessageId = string.IsNullOrWhiteSpace(messageId) ? null : messageId.Trim(); + ArticleSize = articleSize; + Stage = stage; + Reason = reason; + SpamAssassinStatus = spamAssassinStatus; + SpamAssassinScore = spamAssassinScore; + SpamAssassinThreshold = spamAssassinThreshold; + ArticlePayload = articlePayload; + } + + /// Gets the rejection timestamp. + public DateTimeOffset RejectedUtc { get; } + + /// Gets the published policy revision that produced the decision. + public long PolicyRevision { get; } + + /// Gets the MD5 account identifier, or when unauthenticated. + public string? AccountName { get; } + + /// Gets the client source address. + public IPAddress SourceAddress { get; } + + /// Gets the classifier flags from the article, not a policy ENUM. + public ArticleType ArtType { get; } + + /// Gets the Message-ID when available. + public string? MessageId { get; } + + /// Gets the article size in bytes (0 when unknown). + public int ArticleSize { get; } + + /// Gets the PostFilter stage. + public PostFilterStage Stage { get; } + + /// Gets the internal PostFilter reason. + public string Reason { get; } + + /// Gets the SA status for SA-stage rejections. + public PostFilterSpamAssassinStatus? SpamAssassinStatus { get; } + + /// Gets the SA score when present. + public decimal? SpamAssassinScore { get; } + + /// Gets the SA threshold when present. + public decimal? SpamAssassinThreshold { get; } + + /// + /// Gets the complete unstuffed article, or when the + /// complete article was not available at the rejection point. + /// + public ReadOnlyMemory? ArticlePayload { get; } + + /// Builds evidence from a POST evaluation that already has an . + public static PostFilterRejectionEvidence FromEvaluation( + in PostFilterRequest request, + in PostFilterResult result, + DateTimeOffset rejectedUtc) + { + var article = request.Article; + ReadOnlyMemory? payload = article.ArtSize > 0 ? article.ArtData : null; + return new( + rejectedUtc, + result.PolicyRevision, + request.AccountName, + request.ClientIdentity.ClientAddress, + article.ArtType, + EncodingMessageId(article), + article.ArtSize, + result.Stage, + result.Reason, + result.SpamAssassinStatus, + result.SpamAssassinScore, + result.SpamAssassinThreshold, + payload); + } + + /// Builds evidence for a post-accept admission failure while the article is still held. + public static PostFilterRejectionEvidence FromAdmissionFailure( + in PostFilterRequest request, + in PostFilterResult accepted, + string reason, + DateTimeOffset rejectedUtc) + { + var article = request.Article; + ReadOnlyMemory? payload = article.ArtSize > 0 ? article.ArtData : null; + return new( + rejectedUtc, + accepted.PolicyRevision, + request.AccountName, + request.ClientIdentity.ClientAddress, + article.ArtType, + EncodingMessageId(article), + article.ArtSize, + PostFilterStage.Admission, + reason, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + payload); + } + + private static string? EncodingMessageId(in ArticleRecord article) + { + var messageId = article.MessageId; + return messageId.IsEmpty ? null : System.Text.Encoding.ASCII.GetString(messageId); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceQueue.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceQueue.cs new file mode 100644 index 00000000..f62cabff --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceQueue.cs @@ -0,0 +1,87 @@ +using System.Threading.Channels; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Bounded rejection-evidence queue. Full enqueues are dropped and counted; + /// they never invert a 441/240 decision. + /// + public sealed class PostFilterRejectionEvidenceQueue : IPostFilterRejectionEvidenceQueue + { + /// Default bound. Evidence rows can include full article bytes. + public const int DefaultCapacity = 128; + + private readonly Channel _channel; + private int _count; + private long _dropped; + private long _written; + private long _writeFailures; + + /// Initializes a bounded queue. + public PostFilterRejectionEvidenceQueue(int capacity = DefaultCapacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + Capacity = capacity; + _channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false, + AllowSynchronousContinuations = false, + }); + } + + /// + public int Capacity { get; } + + /// + public int Count => Math.Max(0, Volatile.Read(ref _count)); + + /// + public long Dropped => Volatile.Read(ref _dropped); + + /// + public long Written => Volatile.Read(ref _written); + + /// + public long WriteFailures => Volatile.Read(ref _writeFailures); + + /// + public bool TryEnqueue(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + if (!_channel.Writer.TryWrite(evidence)) + { + Interlocked.Increment(ref _dropped); + return false; + } + + Interlocked.Increment(ref _count); + return true; + } + + /// + public void Complete() => _channel.Writer.TryComplete(); + + /// + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (_channel.Reader.TryRead(out var evidence)) + { + Interlocked.Decrement(ref _count); + return evidence; + } + } + + return null; + } + + /// Records a successful database insert. + internal void RecordWritten() => Interlocked.Increment(ref _written); + + /// Records a failed database insert. + internal void RecordWriteFailure() => Interlocked.Increment(ref _writeFailures); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceService.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceService.cs new file mode 100644 index 00000000..73d9a946 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterRejectionEvidenceService.cs @@ -0,0 +1,122 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Background writer for PostFilter rejection evidence. Completes independently + /// of the POST decision. Write failures are counted and logged; they never + /// change 441/240 semantics. + /// + public sealed class PostFilterRejectionEvidenceService : IApplicationService + { + private readonly NntpDbService _nntpDb; + private readonly PostFilterRejectionEvidenceQueue _queue; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes the writer. + public PostFilterRejectionEvidenceService( + NntpDbService nntpDb, + PostFilterRejectionEvidenceQueue queue, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(logger); + _nntpDb = nntpDb; + _queue = queue; + _logger = logger; + } + + /// + public string Name => "PostFilterRejectionEvidence"; + + /// + public Task? Execution => _execution; + + /// Gets the bounded queue consumed by this service. + public PostFilterRejectionEvidenceQueue Queue => _queue; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Complete(); + await _runCts.CancelAsync().ConfigureAwait(false); + + var execution = _execution; + if (execution is null) + { + return; + } + + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + PostFilterLogMessages.EvidenceWriterStoppedWithQueued(_logger, _queue.Count); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + PostFilterLogMessages.EvidenceWriterStarted(_logger, _queue.Capacity); + while (true) + { + PostFilterRejectionEvidence? evidence; + try + { + evidence = await _queue.DequeueAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (evidence is null) + { + break; + } + + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, CancellationToken.None) + .ConfigureAwait(false); + await connection.InsertPostFilterRejectionAsync(evidence, CancellationToken.None) + .ConfigureAwait(false); + _queue.RecordWritten(); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + _queue.RecordWriteFailure(); + PostFilterLogMessages.EvidenceWriteFailed(_logger, ex, evidence.Stage, evidence.Reason); + } + + if (cancellationToken.IsCancellationRequested && _queue.Count == 0) + { + break; + } + } + + PostFilterLogMessages.EvidenceWriterStopped(_logger, _queue.Written, _queue.Dropped, _queue.WriteFailures); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterReservationIdentity.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterReservationIdentity.cs new file mode 100644 index 00000000..6fce07e8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterReservationIdentity.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Process-lifetime reservation identity: node + incarnation + monotonic generation. + internal sealed class PostFilterReservationIdentity + { + private readonly string _nodeId; + private readonly string _incarnation; + private long _generation; + + /// Initializes identity for this process. + public PostFilterReservationIdentity(string nodeId, string? incarnation = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + _nodeId = nodeId; + _incarnation = string.IsNullOrWhiteSpace(incarnation) + ? Guid.NewGuid().ToString("N") + : incarnation; + } + + /// Gets the node identity. + public string NodeId => _nodeId; + + /// Gets the process incarnation. + public string Incarnation => _incarnation; + + /// Allocates the next per-POST generation. + public PostFilterReservationId Next() => + new(_nodeId, _incarnation, Interlocked.Increment(ref _generation)); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/PostFilterResults.cs b/src/VectorNNTPD.TransitService/PostFilter/PostFilterResults.cs new file mode 100644 index 00000000..a72ee231 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/PostFilterResults.cs @@ -0,0 +1,245 @@ +using VectorNNTPD.TransitService.PostFilter.Quota; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Client-visible PostFilter decision. Moderate is not implemented. + public enum PostFilterDecision + { + /// Continue to CreateQueued. + Accept = 0, + + /// Do not queue. Client receives 441. + Reject = 1, + } + + /// Stage that produced the decision. + public enum PostFilterStage + { + /// No remaining stage ran (disabled gate). + Gate = 0, + + /// Account or client-IP deny. + Deny = 1, + + /// ArtType policy mask. + ArtType = 2, + + /// Redis accept-quota reservation. + Quota = 3, + + /// SpamAssassin CHECK. + SpamAssassin = 4, + + /// Accepted after all enabled stages. + Complete = 5, + + /// Queue admission after an Accept decision (TryAdmit). + Admission = 6, + } + + /// Lease held after a successful RESERVE. POST owns commit/release. + public sealed class PostFilterLease + { + private readonly IPostFilterQuotaStore _store; + + /// Initializes a live reservation lease. + internal PostFilterLease( + IPostFilterQuotaStore store, + string accountName, + PostFilterReservationId reservation, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + _store = store; + AccountName = accountName; + Reservation = reservation; + Windows = windows; + Ceilings = ceilings; + } + + /// Gets the authenticated account key. + public string AccountName { get; } + + /// Gets the reservation token for logs. + public string Token => Reservation.Token; + + /// Gets the reservation identity. + internal PostFilterReservationId Reservation { get; } + + /// Gets the windows used at reserve time. + internal PostFilterQuotaWindows Windows { get; } + + /// Gets the ceilings used at reserve time. + internal PostFilterQuotaCeilings Ceilings { get; } + + /// COMMITs reserved units into the current buckets. + internal ValueTask CommitAsync( + DateTimeOffset now, + CancellationToken cancellationToken = default) => + _store.CommitAsync(AccountName, Reservation, now, Windows, Ceilings, cancellationToken); + + /// RELEASEs the reservation. Never decrements committed usage. + internal ValueTask ReleaseAsync( + DateTimeOffset now, + CancellationToken cancellationToken = default) => + _store.ReleaseAsync(AccountName, Reservation, now, cancellationToken); + } + + /// PostFilter evaluation outcome. + public readonly struct PostFilterResult + { + /// Initializes a result. + public PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease) + : this(decision, stage, reason, lease, quotaStatus: null) + { + } + + /// Initializes a result with a quota reserve status. + internal PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease, + PostFilterQuotaReserveStatus? quotaStatus) + : this( + decision, + stage, + reason, + lease, + quotaStatus, + policyRevision: 0, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null) + { + } + + /// Initializes a result with policy revision and optional SA metadata. + internal PostFilterResult( + PostFilterDecision decision, + PostFilterStage stage, + string reason, + PostFilterLease? lease, + PostFilterQuotaReserveStatus? quotaStatus, + long policyRevision, + PostFilterSpamAssassinStatus? spamAssassinStatus, + decimal? spamAssassinScore, + decimal? spamAssassinThreshold) + { + Decision = decision; + Stage = stage; + Reason = reason; + Lease = lease; + QuotaStatus = quotaStatus; + PolicyRevision = policyRevision; + SpamAssassinStatus = spamAssassinStatus; + SpamAssassinScore = spamAssassinScore; + SpamAssassinThreshold = spamAssassinThreshold; + } + + /// Gets Accept or Reject. + public PostFilterDecision Decision { get; } + + /// Gets the decisive stage. + public PostFilterStage Stage { get; } + + /// Gets an internal reason (never sent on the wire). + public string Reason { get; } + + /// Gets the reservation lease when Accept reserved quota. + public PostFilterLease? Lease { get; } + + /// Gets the RESERVE status when the quota stage ran. + internal PostFilterQuotaReserveStatus? QuotaStatus { get; } + + /// Gets the published policy revision observed for this decision. + public long PolicyRevision { get; } + + /// Gets the SA status when the SpamAssassin stage produced the decision. + internal PostFilterSpamAssassinStatus? SpamAssassinStatus { get; } + + /// Gets the SA score when the Spam header included one. + public decimal? SpamAssassinScore { get; } + + /// Gets the SA threshold when the Spam header included one. + public decimal? SpamAssassinThreshold { get; } + + /// Accept with no reservation (disabled gate). + public static PostFilterResult AcceptedDisabled(long policyRevision = 0) => + new( + PostFilterDecision.Accept, + PostFilterStage.Gate, + "disabled", + lease: null, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Accept after enabled stages. + public static PostFilterResult Accepted(PostFilterLease? lease, long policyRevision = 0) => + new( + PostFilterDecision.Accept, + PostFilterStage.Complete, + "accepted", + lease, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject. + public static PostFilterResult Rejected(PostFilterStage stage, string reason, long policyRevision = 0) => + new( + PostFilterDecision.Reject, + stage, + reason, + lease: null, + quotaStatus: null, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject with a quota reserve status. + internal static PostFilterResult Rejected( + PostFilterStage stage, + string reason, + PostFilterQuotaReserveStatus? quotaStatus, + long policyRevision = 0) => + new( + PostFilterDecision.Reject, + stage, + reason, + lease: null, + quotaStatus, + policyRevision, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null); + + /// Reject after SpamAssassin CHECK. + internal static PostFilterResult RejectedSpamAssassin( + string reason, + long policyRevision, + PostFilterSpamAssassinResult scan) => + new( + PostFilterDecision.Reject, + PostFilterStage.SpamAssassin, + reason, + lease: null, + quotaStatus: null, + policyRevision, + scan.Status, + scan.Score, + scan.Threshold); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/IPostFilterQuotaStore.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/IPostFilterQuotaStore.cs new file mode 100644 index 00000000..5fd0c042 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/IPostFilterQuotaStore.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Cluster-wide PostFilter accept-quota reservations. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Reserve, commit, and release each use one atomic operation. + /// + internal interface IPostFilterQuotaStore + { + /// + /// Atomically evaluates enabled ceilings and creates one reservation, or writes nothing. + /// reservationTtlMs of 0 uses . + /// When SpamAssassin CHECK can run, callers pass + /// . + /// + ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0); + + /// + /// Transfers units from the live reservation into the current buckets. + /// The caller does not supply units. + /// + ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default); + + /// Deletes a matching live reservation. Never decrements committed usage. + ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs new file mode 100644 index 00000000..114ab0f5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/InMemoryPostFilterQuotaStore.cs @@ -0,0 +1,124 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Process-local cluster fake for accept-quota. Two stores sharing one engine + /// behave as two NNTP nodes. Ordinary unit tests must not require live Redis. + /// + internal sealed class InMemoryPostFilterQuotaStore : IPostFilterQuotaStore + { + private readonly PostFilterQuotaEngine _engine; + + /// Initializes a store with a dedicated engine. + public InMemoryPostFilterQuotaStore() + : this(new PostFilterQuotaEngine()) + { + } + + /// Initializes a store that shares with other fakes. + public InMemoryPostFilterQuotaStore(PostFilterQuotaEngine engine) + { + ArgumentNullException.ThrowIfNull(engine); + _engine = engine; + } + + /// When set, operations fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Gets the shared algorithm (tests). + internal PostFilterQuotaEngine Engine => _engine; + + /// + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReserveStatus.Unavailable); + } + + var ttl = reservationTtlMs > 0 + ? reservationTtlMs + : (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds; + var code = _engine.Reserve( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds(), + ttl, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex ?? string.Empty); + return ValueTask.FromResult((PostFilterQuotaReserveStatus)code); + } + + /// + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Unavailable); + } + + var code = _engine.Commit( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds(), + windows, + ceilings); + return ValueTask.FromResult((PostFilterQuotaCommitStatus)code); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReleaseStatus.Unavailable); + } + + var code = _engine.Release( + QuotaKey(accountName), + MultipostKey(accountName), + reservation, + now.ToUnixTimeMilliseconds()); + return ValueTask.FromResult((PostFilterQuotaReleaseStatus)code); + } + + /// In-memory quota hash identity matching the Redis key bytes. + internal static string QuotaKey(string accountName) => + Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateQuota(accountName)); + + /// In-memory multipost hash identity matching the Redis key bytes. + internal static string MultipostKey(string accountName) => + Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateMultipost(accountName)); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaCeilings.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaCeilings.cs new file mode 100644 index 00000000..7cea97c6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaCeilings.cs @@ -0,0 +1,43 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Per-window ceilings. 0 disables that dimension. + /// + /// Message ceiling for L. 0 is disabled. + /// Byte ceiling for L. 0 is disabled. + /// Identical-body ceiling for L. 0 is disabled. + /// Message ceiling for S. 0 is disabled. + /// Byte ceiling for S. 0 is disabled. + /// Identical-body ceiling for S. 0 is disabled. + internal readonly record struct PostFilterQuotaCeilings( + long MaxMessagesLong, + long MaxBytesLong, + long MaxIdenticalLong, + long MaxMessagesShort, + long MaxBytesShort, + long MaxIdenticalShort) + { + /// Throws when any ceiling is negative. + public void Validate() + { + ArgumentOutOfRangeException.ThrowIfNegative(MaxMessagesLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxBytesLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxIdenticalLong); + ArgumentOutOfRangeException.ThrowIfNegative(MaxMessagesShort); + ArgumentOutOfRangeException.ThrowIfNegative(MaxBytesShort); + ArgumentOutOfRangeException.ThrowIfNegative(MaxIdenticalShort); + } + + /// Returns the message ceiling for . + public long MaxMessages(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxMessagesLong : MaxMessagesShort; + + /// Returns the byte ceiling for . + public long MaxBytes(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxBytesLong : MaxBytesShort; + + /// Returns the identical-body ceiling for . + public long MaxIdentical(char window) => + window == PostFilterQuotaKeys.LongWindow ? MaxIdenticalLong : MaxIdenticalShort; + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaDefaults.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaDefaults.cs new file mode 100644 index 00000000..72328b48 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaDefaults.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// Fixed crash-recovery timings for PostFilter accept-quota reservations. + /// + /// Reservation TTL is not a quota window and is not a v1 operator knob. + /// The floor () is the crash-recovery hold when + /// SpamAssassin CHECK is not in the POST path. When CHECK is enabled, the + /// effective hold is derived so a live reservation outlives the configured + /// SPAMD OperationTimeout plus . + /// Idle key TTL is garbage collection only; bucket fields define window expiry. + /// + internal static class PostFilterQuotaDefaults + { + /// Live-reservation crash-recovery floor written by RESERVE when CHECK is off. + public static readonly TimeSpan ReservationTtl = TimeSpan.FromSeconds(10); + + /// + /// Extra hold after the configured SpamAssassin operation budget so + /// CreateQueued / TryAdmit / COMMIT still see a live reservation. + /// + public static readonly TimeSpan ReservationHoldSkew = TimeSpan.FromSeconds(1); + + /// + /// Extra milliseconds added to max(window) + reservation hold when refreshing + /// idle Redis key expiry. Does not change bucket accounting. + /// + public const long IdleTtlSkewMs = 1_000; + + /// + /// Effective reservation hold in milliseconds for one POST. + /// Floor is . When SpamAssassin is enabled, + /// hold is at least plus + /// . + /// + public static long HoldMilliseconds(bool spamAssassinEnabled, TimeSpan operationTimeout) + { + var floor = (long)ReservationTtl.TotalMilliseconds; + if (!spamAssassinEnabled) + { + return floor; + } + + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(operationTimeout, TimeSpan.Zero); + var needed = (long)operationTimeout.TotalMilliseconds + + (long)ReservationHoldSkew.TotalMilliseconds; + return Math.Max(floor, needed); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaEngine.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaEngine.cs new file mode 100644 index 00000000..5c654e08 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaEngine.cs @@ -0,0 +1,547 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Atomic accept-quota algorithm. Redis Lua is a faithful port of these operations. + /// + internal sealed class PostFilterQuotaEngine + { + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// Atomically evaluates ceilings and creates one reservation, or writes nothing. + public long Reserve( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs, + long reservationTtlMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + ValidateReserve(quotaKey, multipostKey, nowMs, reservationTtlMs, windows, ceilings, messages, bytes, mpUnits, ref bodyHex); + + lock (_gate) + { + var quota = GetOrCreateHash(quotaKey); + var multipost = GetOrCreateHash(multipostKey); + PruneQuota(quota, windows, nowMs); + PruneMultipost(multipost, windows, nowMs); + + var field = reservation.Field; + if (quota.TryGetValue(field, out var existing) + && PostFilterQuotaKeys.TryParseReservation( + existing, out var exp, out var storedGen, out var sm, out var sb, out var smp, out var sbody) + && exp > nowMs) + { + if (storedGen == reservation.Generation + && sm == messages + && sb == bytes + && smp == mpUnits + && string.Equals(sbody, bodyHex, StringComparison.Ordinal)) + { + return PostFilterQuotaReserveCodes.Accept; + } + + return PostFilterQuotaReserveCodes.Conflict; + } + + var deny = FirstDenial(quota, multipost, windows, ceilings, nowMs, messages, bytes, mpUnits, bodyHex); + if (deny != PostFilterQuotaReserveCodes.Accept) + { + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return deny; + } + + quota[field] = PostFilterQuotaKeys.PackReservation( + nowMs + reservationTtlMs, + reservation.Generation, + messages, + bytes, + mpUnits, + bodyHex); + return PostFilterQuotaReserveCodes.Accept; + } + } + + /// Transfers a live reservation into the current buckets, or no-ops. + public long Commit( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + windows.Validate(); + ceilings.Validate(); + + lock (_gate) + { + var quota = GetOrCreateHash(quotaKey); + var multipost = GetOrCreateHash(multipostKey); + PruneQuota(quota, windows, nowMs); + PruneMultipost(multipost, windows, nowMs); + + var field = reservation.Field; + if (!quota.TryGetValue(field, out var raw) + || !PostFilterQuotaKeys.TryParseReservation( + raw, out var exp, out var storedGen, out var messages, out var bytes, out var mpUnits, out var bodyHex)) + { + if (quota.ContainsKey(field)) + { + quota.Remove(field); + } + + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Noop; + } + + if (storedGen != reservation.Generation) + { + return PostFilterQuotaCommitCodes.Noop; + } + + if (exp <= nowMs) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Noop; + } + + IncrementCurrent(quota, multipost, windows, ceilings, nowMs, messages, bytes, mpUnits, bodyHex); + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + RemoveIfEmpty(multipostKey, multipost); + return PostFilterQuotaCommitCodes.Committed; + } + } + + /// Deletes a matching live reservation. Never decrements committed usage. + public long Release( + string quotaKey, + string multipostKey, + PostFilterReservationId reservation, + long nowMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + + lock (_gate) + { + if (!_hashes.TryGetValue(quotaKey, out var quota)) + { + return PostFilterQuotaReleaseCodes.Noop; + } + + var field = reservation.Field; + if (!quota.TryGetValue(field, out var raw) + || !PostFilterQuotaKeys.TryParseReservation( + raw, out var exp, out var storedGen, out _, out _, out _, out _)) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Noop; + } + + if (storedGen != reservation.Generation) + { + return PostFilterQuotaReleaseCodes.Noop; + } + + if (exp <= nowMs) + { + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Noop; + } + + quota.Remove(field); + RemoveIfEmpty(quotaKey, quota); + return PostFilterQuotaReleaseCodes.Released; + } + } + + /// Returns committed messages in the current bucket, or zero. + internal long CommittedMessages(string quotaKey, char window, long bucketId) => + ReadLong(quotaKey, PostFilterQuotaKeys.CommittedMessagesField(window, bucketId)); + + /// Returns committed bytes in the current bucket, or zero. + internal long CommittedBytes(string quotaKey, char window, long bucketId) => + ReadLong(quotaKey, PostFilterQuotaKeys.CommittedBytesField(window, bucketId)); + + /// Returns committed identical count, or zero. + internal long CommittedIdentical(string multipostKey, string bodyHex, char window, long bucketId) => + ReadLong(multipostKey, PostFilterQuotaKeys.MultipostField(bodyHex, window, bucketId)); + + /// Returns whether a reservation field is present (including expired until pruned). + internal bool HasReservationField(string quotaKey, PostFilterReservationId reservation) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) && quota.ContainsKey(reservation.Field); + } + } + + /// Returns whether a live (unexpired) reservation exists. + internal bool HasLiveReservation(string quotaKey, PostFilterReservationId reservation, long nowMs) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) + && quota.TryGetValue(reservation.Field, out var raw) + && PostFilterQuotaKeys.TryParseReservation(raw, out var exp, out var gen, out _, out _, out _, out _) + && gen == reservation.Generation + && exp > nowMs; + } + } + + /// Returns the number of fields on the quota hash. + internal int QuotaFieldCount(string quotaKey) + { + lock (_gate) + { + return _hashes.TryGetValue(quotaKey, out var quota) ? quota.Count : 0; + } + } + + /// Returns the number of fields on the multipost hash. + internal int MultipostFieldCount(string multipostKey) + { + lock (_gate) + { + return _hashes.TryGetValue(multipostKey, out var multipost) ? multipost.Count : 0; + } + } + + /// Returns reservation field count (prefix r:). + internal int ReservationFieldCount(string quotaKey) + { + lock (_gate) + { + if (!_hashes.TryGetValue(quotaKey, out var quota)) + { + return 0; + } + + var count = 0; + // codeql[cs/linq/missed-where] Lock-held count without LINQ allocation. + foreach (var field in quota.Keys) + { + if (field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + count++; + } + } + + return count; + } + } + + /// Writes a reservation field (tests: expiry / generation mismatch). + internal void SeedReservation( + string quotaKey, + PostFilterReservationId reservation, + long expiryMs, + long storedGeneration, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + lock (_gate) + { + GetOrCreateHash(quotaKey)[reservation.Field] = PostFilterQuotaKeys.PackReservation( + expiryMs, storedGeneration, messages, bytes, mpUnits, bodyHex); + } + } + + /// Writes a committed counter field (tests). + internal void SeedCommitted(string key, string field, long value) + { + lock (_gate) + { + GetOrCreateHash(key)[field] = value.ToString(CultureInfo.InvariantCulture); + } + } + + private long ReadLong(string key, string field) + { + lock (_gate) + { + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var raw) + || !long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value)) + { + return 0; + } + + return value; + } + } + + private static long FirstDenial( + Dictionary quota, + Dictionary multipost, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long nowMs, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + SumLive(quota, nowMs, bodyHex, mpUnits, out var resMsg, out var resBytes, out var resIdentical); + var bucketL = windows.BucketId(PostFilterQuotaKeys.LongWindow, nowMs); + var bucketS = windows.BucketId(PostFilterQuotaKeys.ShortWindow, nowMs); + var cml = ReadField(quota, PostFilterQuotaKeys.CommittedMessagesField(PostFilterQuotaKeys.LongWindow, bucketL)); + var cbl = ReadField(quota, PostFilterQuotaKeys.CommittedBytesField(PostFilterQuotaKeys.LongWindow, bucketL)); + var cms = ReadField(quota, PostFilterQuotaKeys.CommittedMessagesField(PostFilterQuotaKeys.ShortWindow, bucketS)); + var cbs = ReadField(quota, PostFilterQuotaKeys.CommittedBytesField(PostFilterQuotaKeys.ShortWindow, bucketS)); + var cil = mpUnits == 1 + ? ReadField(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, PostFilterQuotaKeys.LongWindow, bucketL)) + : 0; + var cis = mpUnits == 1 + ? ReadField(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, PostFilterQuotaKeys.ShortWindow, bucketS)) + : 0; + + if (ceilings.MaxMessagesLong > 0 && cml + resMsg + messages > ceilings.MaxMessagesLong) + { + return PostFilterQuotaReserveCodes.DeniedMessagesLong; + } + + if (ceilings.MaxBytesLong > 0 && cbl + resBytes + bytes > ceilings.MaxBytesLong) + { + return PostFilterQuotaReserveCodes.DeniedBytesLong; + } + + if (mpUnits == 1 + && ceilings.MaxIdenticalLong > 0 + && cil + resIdentical + mpUnits > ceilings.MaxIdenticalLong) + { + return PostFilterQuotaReserveCodes.DeniedIdenticalLong; + } + + if (ceilings.MaxMessagesShort > 0 && cms + resMsg + messages > ceilings.MaxMessagesShort) + { + return PostFilterQuotaReserveCodes.DeniedMessagesShort; + } + + if (ceilings.MaxBytesShort > 0 && cbs + resBytes + bytes > ceilings.MaxBytesShort) + { + return PostFilterQuotaReserveCodes.DeniedBytesShort; + } + + if (mpUnits == 1 + && ceilings.MaxIdenticalShort > 0 + && cis + resIdentical + mpUnits > ceilings.MaxIdenticalShort) + { + return PostFilterQuotaReserveCodes.DeniedIdenticalShort; + } + + return PostFilterQuotaReserveCodes.Accept; + } + + private static void SumLive( + Dictionary quota, + long nowMs, + string bodyHex, + int requestMpUnits, + out long messages, + out long bytes, + out long identical) + { + messages = 0; + bytes = 0; + identical = 0; + foreach (var (field, value) in quota) + { + if (!field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + continue; + } + + if (!PostFilterQuotaKeys.TryParseReservation(value, out var exp, out _, out var m, out var b, out var mp, out var body) + || exp <= nowMs) + { + continue; + } + + messages += m; + bytes += b; + if (requestMpUnits == 1 && mp == 1 && string.Equals(body, bodyHex, StringComparison.Ordinal)) + { + identical += mp; + } + } + } + + private static void IncrementCurrent( + Dictionary quota, + Dictionary multipost, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long nowMs, + long messages, + long bytes, + int mpUnits, + string bodyHex) + { + foreach (var window in new[] { PostFilterQuotaKeys.LongWindow, PostFilterQuotaKeys.ShortWindow }) + { + var bucket = windows.BucketId(window, nowMs); + if (ceilings.MaxMessages(window) > 0) + { + Add(quota, PostFilterQuotaKeys.CommittedMessagesField(window, bucket), messages); + } + + if (ceilings.MaxBytes(window) > 0) + { + Add(quota, PostFilterQuotaKeys.CommittedBytesField(window, bucket), bytes); + } + + if (mpUnits == 1 && bodyHex.Length > 0 && ceilings.MaxIdentical(window) > 0) + { + Add(multipost, PostFilterQuotaKeys.MultipostField(bodyHex, window, bucket), mpUnits); + } + } + } + + private static void Add(Dictionary hash, string field, long delta) + { + var current = ReadField(hash, field); + hash[field] = (current + delta).ToString(CultureInfo.InvariantCulture); + } + + private static long ReadField(Dictionary hash, string field) => + hash.TryGetValue(field, out var raw) + && long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) + ? value + : 0; + + private static void PruneQuota( + Dictionary quota, + PostFilterQuotaWindows windows, + long nowMs) + { + List? dead = null; + foreach (var (field, value) in quota) + { + if (field.StartsWith(PostFilterQuotaKeys.ReservationFieldPrefix, StringComparison.Ordinal)) + { + if (!PostFilterQuotaKeys.TryParseReservation(value, out var exp, out _, out _, out _, out _, out _) + || exp <= nowMs) + { + (dead ??= []).Add(field); + } + + continue; + } + + if (PostFilterQuotaKeys.TryParseCommittedField(field, out _, out var window, out var bucket) + && windows.BucketEnded(window, bucket, nowMs)) + { + (dead ??= []).Add(field); + } + } + + DeleteDead(quota, dead); + } + + private static void PruneMultipost( + Dictionary multipost, + PostFilterQuotaWindows windows, + long nowMs) + { + List? dead = null; + // codeql[cs/linq/missed-where] TryParse out-params; collect-then-remove without LINQ allocation. + foreach (var field in multipost.Keys) + { + if (PostFilterQuotaKeys.TryParseMultipostField(field, out _, out var window, out var bucket) + && windows.BucketEnded(window, bucket, nowMs)) + { + (dead ??= []).Add(field); + } + } + + DeleteDead(multipost, dead); + } + + private static void DeleteDead(Dictionary hash, List? dead) + { + if (dead is null) + { + return; + } + + foreach (var field in dead) + { + hash.Remove(field); + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private static void ValidateReserve( + string quotaKey, + string multipostKey, + long nowMs, + long reservationTtlMs, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + ref string bodyHex) + { + ArgumentException.ThrowIfNullOrWhiteSpace(quotaKey); + ArgumentException.ThrowIfNullOrWhiteSpace(multipostKey); + ArgumentOutOfRangeException.ThrowIfNegative(nowMs); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(reservationTtlMs, 0); + windows.Validate(); + ceilings.Validate(); + ArgumentOutOfRangeException.ThrowIfNegative(messages); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + if (mpUnits is not (0 or 1)) + { + throw new ArgumentOutOfRangeException(nameof(mpUnits), mpUnits, "mpUnits must be 0 or 1."); + } + + bodyHex ??= string.Empty; + if (mpUnits == 0) + { + bodyHex = string.Empty; + return; + } + + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaKeys.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaKeys.cs new file mode 100644 index 00000000..5b6e6b31 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaKeys.cs @@ -0,0 +1,211 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Redis key and field material for PostFilter accept-quota state. + /// + /// + /// Keys: nntpd:pf:q:{sha256hex(account)} and nntpd:pf:m:{sha256hex(account)}. + /// Account hashing matches nntpd:sess: / nntpd:account: + /// (UTF-8 SHA-256, lowercase hex). Reservation value: + /// expiryMs|generation|messages|bytes|mpUnits|bodyHex. + /// + internal static class PostFilterQuotaKeys + { + /// ASCII namespace for committed + reservation hashes. + public static ReadOnlySpan QuotaNamespacePrefix => "nntpd:pf:q:"u8; + + /// ASCII namespace for committed identical-body hashes. + public static ReadOnlySpan MultipostNamespacePrefix => "nntpd:pf:m:"u8; + + /// Prefix of live reservation fields. + public const string ReservationFieldPrefix = "r:"; + + /// Separates packed reservation fields. + public const char ValueSeparator = '|'; + + /// Long / sustained window identifier. + public const char LongWindow = 'L'; + + /// Short / burst window identifier. + public const char ShortWindow = 'S'; + + /// Builds the quota HASH key for . + public static byte[] CreateQuota(string accountName) => Create(QuotaNamespacePrefix, accountName); + + /// Builds the multipost HASH key for . + public static byte[] CreateMultipost(string accountName) => Create(MultipostNamespacePrefix, accountName); + + /// Builds c:m:{W}:{bucketId}. + public static string CommittedMessagesField(char window, long bucketId) => + string.Concat("c:m:", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + + /// Builds c:b:{W}:{bucketId}. + public static string CommittedBytesField(char window, long bucketId) => + string.Concat("c:b:", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + + /// Builds {bodyHex}:{W}:{bucketId}. + public static string MultipostField(string bodyHex, char window, long bucketId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + return string.Concat(bodyHex, ":", window, ":", bucketId.ToString(CultureInfo.InvariantCulture)); + } + + /// Packs a reservation value. + public static string PackReservation( + long expiryMs, + long generation, + long messages, + long bytes, + int mpUnits, + string bodyHex) => + string.Concat( + expiryMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + messages.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + bytes.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + mpUnits.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + bodyHex); + + /// Parses a reservation value. Returns when malformed. + public static bool TryParseReservation( + string value, + out long expiryMs, + out long generation, + out long messages, + out long bytes, + out int mpUnits, + out string bodyHex) + { + expiryMs = 0; + generation = 0; + messages = 0; + bytes = 0; + mpUnits = 0; + bodyHex = string.Empty; + if (string.IsNullOrEmpty(value)) + { + return false; + } + + // Lua split_res takes the first five '|' fields; the remainder is bodyHex + // and may itself contain '|'. Require exactly those five separators. + var fields = new string[5]; + var start = 0; + for (var i = 0; i < 5; i++) + { + var sep = value.IndexOf(ValueSeparator, start); + if (sep < 0) + { + return false; + } + + fields[i] = value[start..sep]; + start = sep + 1; + } + + if (!long.TryParse(fields[0], NumberStyles.None, CultureInfo.InvariantCulture, out expiryMs) + || !long.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out generation) + || !long.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out messages) + || !long.TryParse(fields[3], NumberStyles.None, CultureInfo.InvariantCulture, out bytes) + || !int.TryParse(fields[4], NumberStyles.None, CultureInfo.InvariantCulture, out mpUnits)) + { + return false; + } + + bodyHex = value[start..]; + return true; + } + + /// Parses c:m|b:{W}:{bucket}. Returns when not a committed field. + public static bool TryParseCommittedField(string field, out char kind, out char window, out long bucketId) + { + kind = default; + window = default; + bucketId = 0; + if (field.Length < 7 || field[0] != 'c' || field[1] != ':') + { + return false; + } + + if (field[2] is not ('m' or 'b') || field[3] != ':') + { + return false; + } + + kind = field[2]; + if (field[4] is not (LongWindow or ShortWindow) || field[5] != ':') + { + return false; + } + + window = field[4]; + return long.TryParse(field.AsSpan(6), NumberStyles.None, CultureInfo.InvariantCulture, out bucketId); + } + + /// Parses {bodyHex}:{W}:{bucket} from the right. + public static bool TryParseMultipostField(string field, out string bodyHex, out char window, out long bucketId) + { + bodyHex = string.Empty; + window = default; + bucketId = 0; + var last = field.LastIndexOf(':'); + if (last <= 2 || last >= field.Length - 1) + { + return false; + } + + var mid = field.LastIndexOf(':', last - 1); + if (mid <= 0 || last != mid + 2) + { + return false; + } + + var w = field[mid + 1]; + if (w is not (LongWindow or ShortWindow)) + { + return false; + } + + if (!long.TryParse(field.AsSpan(last + 1), NumberStyles.None, CultureInfo.InvariantCulture, out bucketId)) + { + return false; + } + + window = w; + bodyHex = field[..mid]; + return bodyHex.Length > 0; + } + + private static byte[] Create(ReadOnlySpan prefix, string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaResults.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaResults.cs new file mode 100644 index 00000000..83f58cf2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaResults.cs @@ -0,0 +1,107 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// Integer codes returned by the RESERVE EVAL (and the matching engine). + internal static class PostFilterQuotaReserveCodes + { + /// Accepted, including an idempotent live-token hit. + public const long Accept = 0; + + /// First failing enabled dimension: messages L. + public const long DeniedMessagesLong = 1; + + /// Bytes L. + public const long DeniedBytesLong = 2; + + /// Identical L. + public const long DeniedIdenticalLong = 3; + + /// Messages S. + public const long DeniedMessagesShort = 4; + + /// Bytes S. + public const long DeniedBytesShort = 5; + + /// Identical S. + public const long DeniedIdenticalShort = 6; + + /// Live token exists with different units or body hash. No write. + public const long Conflict = 7; + } + + /// Mapped RESERVE outcome including store-level Redis unavailability. + internal enum PostFilterQuotaReserveStatus + { + /// Reservation is live and counts toward the ceiling. + Accepted = 0, + + /// Messages L would exceed. + DeniedMessagesLong = 1, + + /// Bytes L would exceed. + DeniedBytesLong = 2, + + /// Identical L would exceed. + DeniedIdenticalLong = 3, + + /// Messages S would exceed. + DeniedMessagesShort = 4, + + /// Bytes S would exceed. + DeniedBytesShort = 5, + + /// Identical S would exceed. + DeniedIdenticalShort = 6, + + /// Same token, different payload. State unchanged. + Conflict = 7, + + /// Redis unavailable. No reservation. Caller fail-closes. + Unavailable = 8, + } + + /// Mapped COMMIT outcome. + internal enum PostFilterQuotaCommitStatus + { + /// No live matching reservation. Committed counters unchanged. + Noop = 0, + + /// Units transferred into the current buckets and the reservation deleted. + Committed = 1, + + /// Redis unavailable. Reservation, if any, is unchanged. + Unavailable = 2, + } + + /// Mapped RELEASE outcome. + internal enum PostFilterQuotaReleaseStatus + { + /// Nothing deleted. + Noop = 0, + + /// Matching live reservation deleted. + Released = 1, + + /// Redis unavailable. Reservation, if any, is unchanged. + Unavailable = 2, + } + + /// Integer codes returned by the COMMIT EVAL. + internal static class PostFilterQuotaCommitCodes + { + /// No transfer. + public const long Noop = 0; + + /// Transferred. + public const long Committed = 1; + } + + /// Integer codes returned by the RELEASE EVAL. + internal static class PostFilterQuotaReleaseCodes + { + /// No delete. + public const long Noop = 0; + + /// Deleted. + public const long Released = 1; + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaScripts.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaScripts.cs new file mode 100644 index 00000000..35182235 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaScripts.cs @@ -0,0 +1,454 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Redis Lua scripts for atomic PostFilter accept-quota reserve / commit / release. + /// Semantics match . + /// + internal static class PostFilterQuotaScripts + { + /// + /// Evaluates all enabled ceilings and creates one reservation, or writes nothing. + /// + /// + /// KEYS[1] quota hash, KEYS[2] multipost hash. + /// ARGV: nowMs, reservationTtlMs, reservationToken, generation, messages, bytes, + /// mpUnits, bodyHex, longMs, shortMs, maxML, maxBL, maxIL, maxMS, maxBS, maxIS, idleMs. + /// Returns 0 accept, 1–6 first deny, 7 conflict. + /// + public const string Reserve = + """ + -- nntpd-pf-reserve + local qKey = KEYS[1] + local mKey = KEYS[2] + local now = tonumber(ARGV[1]) + local ttl = tonumber(ARGV[2]) + local resId = ARGV[3] + local gen = ARGV[4] + local messages = tonumber(ARGV[5]) + local bytes = tonumber(ARGV[6]) + local mpUnits = tonumber(ARGV[7]) + local bodyHex = ARGV[8] + local longMs = tonumber(ARGV[9]) + local shortMs = tonumber(ARGV[10]) + local maxML = tonumber(ARGV[11]) + local maxBL = tonumber(ARGV[12]) + local maxIL = tonumber(ARGV[13]) + local maxMS = tonumber(ARGV[14]) + local maxBS = tonumber(ARGV[15]) + local maxIS = tonumber(ARGV[16]) + local idleMs = tonumber(ARGV[17]) + local rField = 'r:' .. resId + + local function window_ms(w) + if w == 'L' then return longMs end + return shortMs + end + + local function bucket_id(w) + return math.floor(now / window_ms(w)) + end + + local function bucket_ended(w, b) + return ((b + 1) * window_ms(w)) <= now + end + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local function parse_c_field(f) + if string.sub(f, 1, 2) ~= 'c:' then return nil end + local kind = string.sub(f, 3, 3) + if (kind ~= 'm' and kind ~= 'b') or string.sub(f, 4, 4) ~= ':' then return nil end + local w = string.sub(f, 5, 5) + if (w ~= 'L' and w ~= 'S') or string.sub(f, 6, 6) ~= ':' then return nil end + local b = tonumber(string.sub(f, 7)) + if b == nil then return nil end + return kind, w, b + end + + local function parse_m_field(f) + local last = string.len(f) + local colon2 = nil + for i = last, 1, -1 do + if string.sub(f, i, i) == ':' then + if colon2 == nil then + colon2 = i + else + local w = string.sub(f, i + 1, colon2 - 1) + local b = tonumber(string.sub(f, colon2 + 1)) + local body = string.sub(f, 1, i - 1) + if (w == 'L' or w == 'S') and b ~= nil and string.len(body) > 0 then + return body, w, b + end + return nil + end + end + end + return nil + end + + local function hdel_list(key, dead) + if #dead == 0 then return end + redis.call('HDEL', key, unpack(dead)) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + local function prune_q() + local entries = redis.call('HGETALL', qKey) + local dead = {} + for i = 1, #entries, 2 do + local f = entries[i] + if string.sub(f, 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec == nil or rec.exp <= now then + dead[#dead + 1] = f + end + else + local _, w, b = parse_c_field(f) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = f + end + end + end + hdel_list(qKey, dead) + end + + local function prune_m() + local entries = redis.call('HGETALL', mKey) + local dead = {} + for i = 1, #entries, 2 do + local body, w, b = parse_m_field(entries[i]) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = entries[i] + end + end + hdel_list(mKey, dead) + end + + local function touch() + if idleMs ~= nil and idleMs > 0 then + if redis.call('EXISTS', qKey) == 1 then + redis.call('PEXPIRE', qKey, idleMs) + end + if redis.call('EXISTS', mKey) == 1 then + redis.call('PEXPIRE', mKey, idleMs) + end + end + end + + if mpUnits == 0 then + bodyHex = '' + end + + prune_q() + prune_m() + + local existing = redis.call('HGET', qKey, rField) + if existing then + local rec = split_res(existing) + if rec and rec.exp > now and rec.gen == gen and rec.messages == messages and rec.bytes == bytes and rec.mp == mpUnits and rec.body == bodyHex then + return 0 + end + return 7 + end + + local bL = bucket_id('L') + local bS = bucket_id('S') + local function hnum(key, field) + local v = redis.call('HGET', key, field) + if not v then return 0 end + return tonumber(v) or 0 + end + + local cml = hnum(qKey, 'c:m:L:' .. bL) + local cbl = hnum(qKey, 'c:b:L:' .. bL) + local cms = hnum(qKey, 'c:m:S:' .. bS) + local cbs = hnum(qKey, 'c:b:S:' .. bS) + local cil = 0 + local cis = 0 + if mpUnits == 1 and bodyHex ~= '' then + cil = hnum(mKey, bodyHex .. ':L:' .. bL) + cis = hnum(mKey, bodyHex .. ':S:' .. bS) + end + + local resMsg, resBytes, resIdent = 0, 0, 0 + local entries = redis.call('HGETALL', qKey) + for i = 1, #entries, 2 do + if string.sub(entries[i], 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec and rec.exp > now then + resMsg = resMsg + rec.messages + resBytes = resBytes + rec.bytes + if mpUnits == 1 and rec.mp == 1 and rec.body == bodyHex then + resIdent = resIdent + rec.mp + end + end + end + end + + if maxML > 0 and (cml + resMsg + messages) > maxML then return 1 end + if maxBL > 0 and (cbl + resBytes + bytes) > maxBL then return 2 end + if mpUnits == 1 and maxIL > 0 and (cil + resIdent + mpUnits) > maxIL then return 3 end + if maxMS > 0 and (cms + resMsg + messages) > maxMS then return 4 end + if maxBS > 0 and (cbs + resBytes + bytes) > maxBS then return 5 end + if mpUnits == 1 and maxIS > 0 and (cis + resIdent + mpUnits) > maxIS then return 6 end + + redis.call('HSET', qKey, rField, tostring(now + ttl) .. '|' .. gen .. '|' .. tostring(messages) .. '|' .. tostring(bytes) .. '|' .. tostring(mpUnits) .. '|' .. bodyHex) + touch() + return 0 + """; + + /// + /// Transfers reservation units into the current buckets, or no-ops. + /// + /// + /// KEYS[1] quota, KEYS[2] multipost. + /// ARGV: nowMs, reservationToken, generation, longMs, shortMs, + /// maxML, maxBL, maxIL, maxMS, maxBS, maxIS, idleMs. + /// Units are read only from the reservation. Returns 0 no-op, 1 committed. + /// + public const string Commit = + """ + -- nntpd-pf-commit + local qKey = KEYS[1] + local mKey = KEYS[2] + local now = tonumber(ARGV[1]) + local resId = ARGV[2] + local gen = ARGV[3] + local longMs = tonumber(ARGV[4]) + local shortMs = tonumber(ARGV[5]) + local maxML = tonumber(ARGV[6]) + local maxBL = tonumber(ARGV[7]) + local maxIL = tonumber(ARGV[8]) + local maxMS = tonumber(ARGV[9]) + local maxBS = tonumber(ARGV[10]) + local maxIS = tonumber(ARGV[11]) + local idleMs = tonumber(ARGV[12]) + local rField = 'r:' .. resId + + local function window_ms(w) + if w == 'L' then return longMs end + return shortMs + end + + local function bucket_id(w) + return math.floor(now / window_ms(w)) + end + + local function bucket_ended(w, b) + return ((b + 1) * window_ms(w)) <= now + end + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local function parse_c_field(f) + if string.sub(f, 1, 2) ~= 'c:' then return nil end + local kind = string.sub(f, 3, 3) + if (kind ~= 'm' and kind ~= 'b') or string.sub(f, 4, 4) ~= ':' then return nil end + local w = string.sub(f, 5, 5) + if (w ~= 'L' and w ~= 'S') or string.sub(f, 6, 6) ~= ':' then return nil end + local b = tonumber(string.sub(f, 7)) + if b == nil then return nil end + return kind, w, b + end + + local function parse_m_field(f) + local last = string.len(f) + local colon2 = nil + for i = last, 1, -1 do + if string.sub(f, i, i) == ':' then + if colon2 == nil then + colon2 = i + else + local w = string.sub(f, i + 1, colon2 - 1) + local b = tonumber(string.sub(f, colon2 + 1)) + local body = string.sub(f, 1, i - 1) + if (w == 'L' or w == 'S') and b ~= nil and string.len(body) > 0 then + return body, w, b + end + return nil + end + end + end + return nil + end + + local function hdel_list(key, dead) + if #dead == 0 then return end + redis.call('HDEL', key, unpack(dead)) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + local function prune_q() + local entries = redis.call('HGETALL', qKey) + local dead = {} + for i = 1, #entries, 2 do + local f = entries[i] + if string.sub(f, 1, 2) == 'r:' then + local rec = split_res(entries[i + 1]) + if rec == nil or rec.exp <= now then + dead[#dead + 1] = f + end + else + local _, w, b = parse_c_field(f) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = f + end + end + end + hdel_list(qKey, dead) + end + + local function prune_m() + local entries = redis.call('HGETALL', mKey) + local dead = {} + for i = 1, #entries, 2 do + local body, w, b = parse_m_field(entries[i]) + if w ~= nil and bucket_ended(w, b) then + dead[#dead + 1] = entries[i] + end + end + hdel_list(mKey, dead) + end + + prune_q() + prune_m() + + local existing = redis.call('HGET', qKey, rField) + local rec = split_res(existing) + if rec == nil then + if existing then + redis.call('HDEL', qKey, rField) + end + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + if rec.gen ~= gen then + return 0 + end + if rec.exp <= now then + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + + local bL = bucket_id('L') + local bS = bucket_id('S') + if maxML > 0 then redis.call('HINCRBY', qKey, 'c:m:L:' .. bL, rec.messages) end + if maxBL > 0 then redis.call('HINCRBY', qKey, 'c:b:L:' .. bL, rec.bytes) end + if maxMS > 0 then redis.call('HINCRBY', qKey, 'c:m:S:' .. bS, rec.messages) end + if maxBS > 0 then redis.call('HINCRBY', qKey, 'c:b:S:' .. bS, rec.bytes) end + if rec.mp == 1 and rec.body ~= '' then + if maxIL > 0 then redis.call('HINCRBY', mKey, rec.body .. ':L:' .. bL, rec.mp) end + if maxIS > 0 then redis.call('HINCRBY', mKey, rec.body .. ':S:' .. bS, rec.mp) end + end + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + if idleMs ~= nil and idleMs > 0 then + if redis.call('EXISTS', qKey) == 1 then redis.call('PEXPIRE', qKey, idleMs) end + if redis.call('EXISTS', mKey) == 1 then redis.call('PEXPIRE', mKey, idleMs) end + end + return 1 + """; + + /// + /// Deletes a matching live reservation. Never decrements committed usage. + /// + /// + /// KEYS[1] quota, KEYS[2] multipost (unused except for KEYS contract). + /// ARGV: nowMs, reservationToken, generation. + /// Returns 0 no-op, 1 released. + /// + public const string Release = + """ + -- nntpd-pf-release + local qKey = KEYS[1] + local now = tonumber(ARGV[1]) + local resId = ARGV[2] + local gen = ARGV[3] + local rField = 'r:' .. resId + + local function split_res(v) + if not v then return nil end + local p = {} + local start = 1 + for i = 1, 5 do + local sep = string.find(v, '|', start, true) + if not sep then return nil end + p[i] = string.sub(v, start, sep - 1) + start = sep + 1 + end + p[6] = string.sub(v, start) + local exp = tonumber(p[1]) + local g = p[2] + local m = tonumber(p[3]) + local b = tonumber(p[4]) + local mp = tonumber(p[5]) + if exp == nil or m == nil or b == nil or mp == nil then return nil end + return { exp = exp, gen = g, messages = m, bytes = b, mp = mp, body = p[6] } + end + + local existing = redis.call('HGET', qKey, rField) + local rec = split_res(existing) + if rec == nil then + if existing then + redis.call('HDEL', qKey, rField) + end + if redis.call('EXISTS', qKey) == 1 and redis.call('HLEN', qKey) == 0 then + redis.call('DEL', qKey) + end + return 0 + end + if rec.gen ~= gen then + return 0 + end + if rec.exp <= now then + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 0 + end + redis.call('HDEL', qKey, rField) + if redis.call('HLEN', qKey) == 0 then redis.call('DEL', qKey) end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaWindows.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaWindows.cs new file mode 100644 index 00000000..61a4d4b4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterQuotaWindows.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// Fixed-epoch window lengths in milliseconds. + /// Sustained (L) window. Must be greater than zero. + /// Burst (S) window. Must be greater than zero. + internal readonly record struct PostFilterQuotaWindows(long LongWindowMs, long ShortWindowMs) + { + /// Returns the current bucket id for at . + public long BucketId(char window, long nowMs) + { + var ms = WindowMs(window); + return nowMs / ms; + } + + /// Returns whether bucket of has ended. + public bool BucketEnded(char window, long bucketId, long nowMs) => + (bucketId + 1) * WindowMs(window) <= nowMs; + + /// Returns the millisecond length of . + public long WindowMs(char window) => window == PostFilterQuotaKeys.LongWindow + ? LongWindowMs + : ShortWindowMs; + + /// Throws when either window length is not positive. + public void Validate() + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(LongWindowMs, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(ShortWindowMs, 0); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterReservationId.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterReservationId.cs new file mode 100644 index 00000000..35a5284b --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/PostFilterReservationId.cs @@ -0,0 +1,41 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// One in-flight POST reservation: {nodeId}:{incarnation}:{generation}. + /// + /// + /// Not session occupancy. nodeId and incarnation follow + /// DistributedSessionStateTracker ownership. generation is unique + /// per reservation so one process can hold many concurrent POSTs. + /// + internal readonly record struct PostFilterReservationId + { + /// Initializes a reservation identity. + public PostFilterReservationId(string nodeId, string incarnation, long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentException.ThrowIfNullOrWhiteSpace(incarnation); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(generation, 0); + NodeId = nodeId; + Incarnation = incarnation; + Generation = generation; + } + + /// Gets the NNTPD node identity. + public string NodeId { get; } + + /// Gets the process-lifetime incarnation. + public string Incarnation { get; } + + /// Gets the per-reservation generation. + public long Generation { get; } + + /// Gets {nodeId}:{incarnation}:{generation}. + public string Token => string.Concat(NodeId, ":", Incarnation, ":", Generation.ToString(CultureInfo.InvariantCulture)); + + /// Gets the quota-hash field r:{token}. + public string Field => string.Concat(PostFilterQuotaKeys.ReservationFieldPrefix, Token); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/Quota/RedisPostFilterQuotaStore.cs b/src/VectorNNTPD.TransitService/PostFilter/Quota/RedisPostFilterQuotaStore.cs new file mode 100644 index 00000000..5e34079b --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/Quota/RedisPostFilterQuotaStore.cs @@ -0,0 +1,212 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.PostFilter.Quota +{ + /// + /// Redis-backed accept-quota using one atomic Lua EVAL per reserve / commit / release + /// on the shared connection. + /// + internal sealed class RedisPostFilterQuotaStore : IPostFilterQuotaStore + { + private readonly IRedisService _redis; + + /// Initializes a Redis-backed quota store. + public RedisPostFilterQuotaStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + windows.Validate(); + ceilings.Validate(); + ArgumentOutOfRangeException.ThrowIfNegative(messages); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + if (mpUnits is not (0 or 1)) + { + throw new ArgumentOutOfRangeException(nameof(mpUnits), mpUnits, "mpUnits must be 0 or 1."); + } + + var ttl = ResolveReservationTtlMs(reservationTtlMs); + var hash = mpUnits == 1 ? bodyHex : string.Empty; + if (mpUnits == 1) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bodyHex); + hash = bodyHex; + } + + return ExecuteAsync( + PostFilterQuotaScripts.Reserve, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(ttl), + Utf8(reservation.Token), + Utf8(reservation.Generation), + Utf8(messages), + Utf8(bytes), + Utf8(mpUnits), + Utf8(hash ?? string.Empty), + Utf8(windows.LongWindowMs), + Utf8(windows.ShortWindowMs), + Utf8(ceilings.MaxMessagesLong), + Utf8(ceilings.MaxBytesLong), + Utf8(ceilings.MaxIdenticalLong), + Utf8(ceilings.MaxMessagesShort), + Utf8(ceilings.MaxBytesShort), + Utf8(ceilings.MaxIdenticalShort), + Utf8(IdleTtlMs(windows, ttl)), + ], + static result => result is >= 0 and <= 7 + ? (PostFilterQuotaReserveStatus)result + : PostFilterQuotaReserveStatus.Unavailable, + PostFilterQuotaReserveStatus.Unavailable, + cancellationToken); + } + + /// + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + windows.Validate(); + ceilings.Validate(); + return ExecuteAsync( + PostFilterQuotaScripts.Commit, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(reservation.Token), + Utf8(reservation.Generation), + Utf8(windows.LongWindowMs), + Utf8(windows.ShortWindowMs), + Utf8(ceilings.MaxMessagesLong), + Utf8(ceilings.MaxBytesLong), + Utf8(ceilings.MaxIdenticalLong), + Utf8(ceilings.MaxMessagesShort), + Utf8(ceilings.MaxBytesShort), + Utf8(ceilings.MaxIdenticalShort), + Utf8(IdleTtlMs(windows, (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds)), + ], + static result => result == PostFilterQuotaCommitCodes.Committed + ? PostFilterQuotaCommitStatus.Committed + : PostFilterQuotaCommitStatus.Noop, + PostFilterQuotaCommitStatus.Unavailable, + cancellationToken); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) => + ExecuteAsync( + PostFilterQuotaScripts.Release, + accountName, + [ + Utf8(now.ToUnixTimeMilliseconds()), + Utf8(reservation.Token), + Utf8(reservation.Generation), + ], + static result => result == PostFilterQuotaReleaseCodes.Released + ? PostFilterQuotaReleaseStatus.Released + : PostFilterQuotaReleaseStatus.Noop, + PostFilterQuotaReleaseStatus.Unavailable, + cancellationToken); + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + long result; + try + { + // Observe EVAL even if the caller cancelled. WaitAsync on the Redis + // task would otherwise abandon a completed Lua write and leak a + // reservation until TTL. After the result is known, propagate + // cancel so EvaluateAsync RELEASE can delete the row. + result = await database.ScriptEvaluateAsync( + script, + [PostFilterQuotaKeys.CreateQuota(accountName), PostFilterQuotaKeys.CreateMultipost(accountName)], + values, + CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + cancellationToken.ThrowIfCancellationRequested(); + return map(result); + } + + private static long ResolveReservationTtlMs(long reservationTtlMs) => + reservationTtlMs > 0 + ? reservationTtlMs + : (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds; + + private static long IdleTtlMs(PostFilterQuotaWindows windows, long reservationTtlMs) => + Math.Max(windows.LongWindowMs, windows.ShortWindowMs) + + reservationTtlMs + + PostFilterQuotaDefaults.IdleTtlSkewMs; + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + + private static byte[] Utf8(long value) => + Encoding.UTF8.GetBytes(value.ToString(CultureInfo.InvariantCulture)); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/SpamdCheckClient.cs b/src/VectorNNTPD.TransitService/PostFilter/SpamdCheckClient.cs new file mode 100644 index 00000000..67f1f3ee --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/SpamdCheckClient.cs @@ -0,0 +1,639 @@ +using System.Globalization; +using System.Net.Sockets; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Persistent SPAMD CHECK client. Builds a disposable scan message and never + /// mutates . + /// + /// + /// + /// SPAMD PROTOCOL has no request identifiers and says each side shuts down after + /// writing. This client therefore does not pipeline or multiplex CHECKs on one + /// socket. Each pooled connection serializes requests. Concurrent CHECKs use + /// distinct connections up to the snapshot pool size. + /// + /// + /// After a complete CHECK response the socket is returned to the pool when it is + /// still connected and has no leftover bytes. A broken connection is evicted + /// before another request can use it. Hosts, port, pool size, and timeouts come + /// from the PostFilter snapshot captured for this POST. + /// + /// + internal sealed class SpamdCheckClient : IPostFilterSpamAssassin, IAsyncDisposable + { + private readonly object _gate = new(); + private readonly Dictionary<(string Host, int Port), Queue> _idle = new(); + private readonly List _waiters = []; + private readonly ILogger? _logger; + private int _live; + private int _roundRobin; + private bool _disposed; + + /// Initializes the client with process-wide transport counters. + public SpamdCheckClient() + : this(new SpamdTransportMetrics(), logger: null) + { + } + + /// Initializes the client with injected counters and optional shutdown logging. + public SpamdCheckClient(SpamdTransportMetrics metrics, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(metrics); + Metrics = metrics; + _logger = logger; + } + + /// Gets process-wide transport counters. + public SpamdTransportMetrics Metrics { get; } + + /// + public async ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_disposed) + { + return PostFilterSpamAssassinResult.Failed("disposed"); + } + + if (target.Hosts.Count == 0) + { + return PostFilterSpamAssassinResult.Failed("no-hosts"); + } + + byte[] scan; + try + { + scan = SpamdScanArticleBuilder.Build(article, scanContext); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + } + + var start = SelectStartIndex(target); + Exception? lastConnectFailure = null; + for (var attempt = 0; attempt < target.Hosts.Count; attempt++) + { + var host = target.Hosts[(start + attempt) % target.Hosts.Count]; + if (string.IsNullOrWhiteSpace(host)) + { + lastConnectFailure = new InvalidOperationException("empty-host"); + continue; + } + + host = host.Trim(); + SpamdConnection? leased = null; + var reused = false; + try + { + (leased, reused) = await RentAsync(host, target, cancellationToken).ConfigureAwait(false); + var started = TimeProvider.System.GetTimestamp(); + try + { + var result = await CheckOnConnectionAsync( + leased, + scan, + accountName, + target, + cancellationToken) + .ConfigureAwait(false); + if (reused && IsStaleIdleFailure(result)) + { + throw new IOException(result.Detail); + } + + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + Return(leased, reusable: result.Status != PostFilterSpamAssassinStatus.Failed); + return result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Evict(leased); + throw; + } + catch (OperationCanceledException) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (reused && IsConnectionFailure(ex)) + { + Evict(leased); + (leased, reused) = await RentAsync(host, target, cancellationToken).ConfigureAwait(false); + started = TimeProvider.System.GetTimestamp(); + try + { + var retry = await CheckOnConnectionAsync( + leased, + scan, + accountName, + target, + cancellationToken) + .ConfigureAwait(false); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + Return(leased, reusable: retry.Status != PostFilterSpamAssassinStatus.Failed); + return retry; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Evict(leased); + throw; + } + catch (OperationCanceledException) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception retryEx) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return ClassifyAfterWrite(retryEx); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + Evict(leased); + Metrics.RecordCheck(TimeProvider.System.GetElapsedTime(started)); + return ClassifyAfterWrite(ex); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (leased is null) + { + lastConnectFailure = new TimeoutException("connect-timeout"); + if (attempt + 1 < target.Hosts.Count) + { + continue; + } + + return PostFilterSpamAssassinResult.Failed("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (leased is null && IsConnectionFailure(ex) && attempt + 1 < target.Hosts.Count) + { + lastConnectFailure = ex; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (leased is null) + { + return ClassifyConnect(ex); + } + finally + { + _ = reused; + } + } + + return PostFilterSpamAssassinResult.Failed(lastConnectFailure?.GetType().Name ?? "no-hosts"); + } + + /// + public async ValueTask DisposeAsync() + { + List idle; + List waiters; + lock (_gate) + { + if (_disposed) + { + return; + } + + _disposed = true; + idle = []; + foreach (var queue in _idle.Values) + { + while (queue.Count > 0) + { + idle.Add(queue.Dequeue()); + } + } + + _idle.Clear(); + _live = 0; + waiters = [.. _waiters]; + _waiters.Clear(); + } + + foreach (var waiter in waiters) + { + waiter.TrySetCanceled(); + } + + foreach (var connection in idle) + { + connection.Dispose(); + } + + if (_logger is not null) + { + PostFilterLogMessages.SpamAssassinTransportStopped( + _logger, + Metrics.ConnectionsEstablished, + Metrics.CheckRequests, + Metrics.ConnectionReuses, + Metrics.Reconnects, + Metrics.Evictions); + } + + await ValueTask.CompletedTask.ConfigureAwait(false); + } + + private int SelectStartIndex(in PostFilterSpamAssassinTarget target) + { + if (target.Hosts.Count == 0) + { + return 0; + } + + return SpamdHostSelector.NextStart(target.HostSelection, target.Hosts.Count, ref _roundRobin); + } + + private async Task<(SpamdConnection Connection, bool Reused)> RentAsync( + string host, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + Task wait; + lock (_gate) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (TryDequeueIdle(host, target.Port, out var idle)) + { + Metrics.RecordReuse(); + return (idle, true); + } + + if (_live < target.MaxConnections) + { + _live++; + wait = Task.CompletedTask; + } + else + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add(tcs); + wait = tcs.Task; + } + } + + if (wait.IsCompleted) + { + try + { + var connection = await ConnectAsync(host, target, cancellationToken).ConfigureAwait(false); + return (connection, false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + lock (_gate) + { + _live--; + PulseWaiter(); + } + + throw; + } + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private bool TryDequeueIdle(string host, int port, out SpamdConnection connection) + { + var key = (host, port); + if (_idle.TryGetValue(key, out var queue)) + { + while (queue.Count > 0) + { + var candidate = queue.Dequeue(); + if (candidate.IsReusable) + { + connection = candidate; + return true; + } + + // codeql[cs/dispose-not-called-on-throw] Candidate is pool-owned from Dequeue; Dispose here is best-effort eviction. Dequeue cannot create a new undisposed resource, and ConnectAsync already disposes TcpClient on connect failure. + candidate.Dispose(); + _live--; + Metrics.RecordEviction(); + Metrics.RecordReconnect(); + } + } + + connection = null!; + return false; + } + + private async Task ConnectAsync( + string host, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + var client = new TcpClient(); + try + { + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(target.ConnectTimeout); + await client.ConnectAsync(host, target.Port, connectCts.Token).ConfigureAwait(false); + Metrics.RecordConnect(); + return new SpamdConnection(client, host, target.Port); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + client.Dispose(); + throw; + } + } + + private void Return(SpamdConnection connection, bool reusable) + { + lock (_gate) + { + if (_disposed || !reusable || !connection.IsReusable) + { + connection.Dispose(); + if (!_disposed) + { + _live--; + Metrics.RecordEviction(); + PulseWaiter(); + } + + return; + } + + var key = (connection.Host, connection.Port); + if (!_idle.TryGetValue(key, out var queue)) + { + queue = new Queue(); + _idle[key] = queue; + } + + queue.Enqueue(connection); + PulseWaiter(); + } + } + + private void Evict(SpamdConnection connection) + { + connection.Dispose(); + lock (_gate) + { + if (!_disposed) + { + _live--; + Metrics.RecordEviction(); + Metrics.RecordReconnect(); + PulseWaiter(); + } + } + } + + private void PulseWaiter() + { + if (_waiters.Count == 0) + { + return; + } + + var waiter = _waiters[0]; + _waiters.RemoveAt(0); + waiter.TrySetResult(); + } + + private static async Task CheckOnConnectionAsync( + SpamdConnection connection, + ReadOnlyMemory scan, + string? accountName, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(target.OperationTimeout); + await WriteCheckAsync(connection.Stream, scan, accountName, target.ProtocolVersion, timeoutCts.Token) + .ConfigureAwait(false); + return await ReadCheckAsync(connection.Stream, timeoutCts.Token).ConfigureAwait(false); + } + + private static async Task WriteCheckAsync( + NetworkStream stream, + ReadOnlyMemory scan, + string? accountName, + string protocolVersion, + CancellationToken cancellationToken) + { + var version = string.IsNullOrWhiteSpace(protocolVersion) + ? PostFilterSpamAssassinOptions.DefaultProtocolVersion + : protocolVersion.Trim(); + var header = new StringBuilder(64); + header.Append("CHECK SPAMC/"); + header.Append(version); + header.Append("\r\n"); + header.Append("Content-length: "); + header.Append(scan.Length.ToString(CultureInfo.InvariantCulture)); + header.Append("\r\n"); + if (!string.IsNullOrWhiteSpace(accountName)) + { + header.Append("User: "); + header.Append(accountName); + header.Append("\r\n"); + } + + header.Append("\r\n"); + var headerBytes = Encoding.ASCII.GetBytes(header.ToString()); + await stream.WriteAsync(headerBytes, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(scan, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadCheckAsync( + NetworkStream stream, + CancellationToken cancellationToken) + { + var buffer = new byte[2048]; + var total = 0; + while (total < buffer.Length) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + break; + } + + total += read; + if (buffer.AsSpan(0, total).IndexOf("\r\n\r\n"u8) >= 0) + { + break; + } + } + + if (total == 0) + { + return PostFilterSpamAssassinResult.Failed("closed"); + } + + var terminator = buffer.AsSpan(0, total).IndexOf("\r\n\r\n"u8); + if (terminator < 0) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + if (terminator + 4 != total) + { + return PostFilterSpamAssassinResult.Failed("leftover"); + } + + var text = Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + if (!text.StartsWith("SPAMD/", StringComparison.Ordinal)) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + var firstLineEnd = text.IndexOf("\r\n", StringComparison.Ordinal); + if (firstLineEnd < 0) + { + return PostFilterSpamAssassinResult.Failed("malformed"); + } + + var statusLine = text[..firstLineEnd]; + var parts = statusLine.Split(' ', 3, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2 || parts[1] != "0") + { + return PostFilterSpamAssassinResult.Failed(statusLine); + } + + var spamIndex = text.IndexOf("Spam:", StringComparison.OrdinalIgnoreCase); + if (spamIndex < 0) + { + return PostFilterSpamAssassinResult.Failed("missing-spam-header"); + } + + var lineEnd = text.IndexOf("\r\n", spamIndex, StringComparison.Ordinal); + var spamLine = lineEnd < 0 ? text[spamIndex..] : text[spamIndex..lineEnd]; + TryParseSpamScores(spamLine, out var score, out var threshold); + if (spamLine.Contains("True", StringComparison.OrdinalIgnoreCase)) + { + return PostFilterSpamAssassinResult.Spam(spamLine, score, threshold); + } + + if (spamLine.Contains("False", StringComparison.OrdinalIgnoreCase)) + { + return PostFilterSpamAssassinResult.Ham(score, threshold); + } + + return PostFilterSpamAssassinResult.Failed(spamLine); + } + + private static void TryParseSpamScores(string spamLine, out decimal? score, out decimal? threshold) + { + score = null; + threshold = null; + var separator = spamLine.IndexOf(';'); + if (separator < 0) + { + return; + } + + var scores = spamLine[(separator + 1)..].Split('/', 2, StringSplitOptions.TrimEntries); + if (scores.Length == 2 + && decimal.TryParse(scores[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsedScore) + && decimal.TryParse(scores[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var parsedThreshold)) + { + score = parsedScore; + threshold = parsedThreshold; + } + } + + private static PostFilterSpamAssassinResult ClassifyConnect(Exception ex) => + ex is OperationCanceledException + ? PostFilterSpamAssassinResult.Failed("timeout") + : PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + + private static PostFilterSpamAssassinResult ClassifyAfterWrite(Exception ex) => + ex is IOException or SocketException + ? PostFilterSpamAssassinResult.Failed("connection") + : PostFilterSpamAssassinResult.Failed(ex.GetType().Name); + + private static bool IsConnectionFailure(Exception ex) => + ex is SocketException or IOException or TimeoutException or ObjectDisposedException; + + private static bool IsStaleIdleFailure(in PostFilterSpamAssassinResult result) => + result.Status == PostFilterSpamAssassinStatus.Failed + && result.Detail is "closed" or "connection"; + + private sealed class SpamdConnection : IDisposable + { + private readonly TcpClient _client; + + public SpamdConnection(TcpClient client, string host, int port) + { + _client = client; + Host = host; + Port = port; + Stream = client.GetStream(); + } + + public string Host { get; } + + public int Port { get; } + + public NetworkStream Stream { get; } + + public bool IsReusable + { + get + { + try + { + var socket = _client.Client; + if (!_client.Connected || socket is null || !socket.Connected) + { + return false; + } + + if (socket.Poll(0, SelectMode.SelectRead) && socket.Available == 0) + { + return false; + } + + return !Stream.DataAvailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return false; + } + } + } + + public void Dispose() => _client.Dispose(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/SpamdHostSelector.cs b/src/VectorNNTPD.TransitService/PostFilter/SpamdHostSelector.cs new file mode 100644 index 00000000..d5c329ab --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/SpamdHostSelector.cs @@ -0,0 +1,33 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Deterministic SPAMD host index selection. Does not allocate. + internal static class SpamdHostSelector + { + /// Returns the first host index to try for this CHECK. + public static int NextStart( + PostFilterSpamAssassinHostSelection selection, + int hostCount, + ref int roundRobin) + { + if (hostCount <= 0) + { + return 0; + } + + if (selection == PostFilterSpamAssassinHostSelection.Failover) + { + return 0; + } + + var next = Interlocked.Increment(ref roundRobin) - 1; + if (next < 0) + { + next = 0; + } + + return next % hostCount; + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/SpamdScanArticleBuilder.cs b/src/VectorNNTPD.TransitService/PostFilter/SpamdScanArticleBuilder.cs new file mode 100644 index 00000000..c0f8cd80 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/SpamdScanArticleBuilder.cs @@ -0,0 +1,267 @@ +using System.Buffers; +using System.Globalization; +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.PostFilter +{ + /// + /// Builds a disposable email-like SPAMD CHECK payload from a canonical article. + /// + /// + /// Does not mutate or any other article field. + /// The returned buffer is independent of the article and is only the bytes sent to SPAMD. + /// + internal static class SpamdScanArticleBuilder + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + private static readonly byte[] HeaderSeparator = "\r\n\r\n"u8.ToArray(); + + /// Constructs the SPAMD scan representation. + /// Canonical article. Read-only. + /// Peer address, server FQDN, and evaluation clock. + /// Independent scan bytes. Never aliases . + public static byte[] Build(in ArticleRecord article, in SpamdScanContext context) + { + ArgumentNullException.ThrowIfNull(context.ClientAddress); + var fqdn = SanitizeToken(context.ServerFqdn); + var artData = article.ArtData.Span; + SplitArticle(artData, out var headerBlock, out var body); + ScanOriginalHeaders( + headerBlock, + out var hasDate, + out var messageId, + out var newsgroups, + out var preservedHeaders); + + if (messageId.Length == 0 && article.Fields.MessageId.IsPresent) + { + messageId = article.MessageId.ToArray(); + } + + if (newsgroups.Length == 0 && article.Fields.Newsgroups.IsPresent) + { + newsgroups = article.Newsgroups.ToArray(); + } + + var writer = new ArrayBufferWriter(256 + artData.Length); + WriteReceived(writer, context.ClientAddress, fqdn, messageId, context.Now); + WriteAscii(writer, "To: usenet@"); + WriteAscii(writer, fqdn); + writer.Write(Crlf); + if (newsgroups.Length > 0) + { + WriteAscii(writer, "X-Usenet-Newsgroups: "); + writer.Write(newsgroups); + writer.Write(Crlf); + } + + if (!hasDate) + { + WriteAscii(writer, "Date: "); + WriteAscii(writer, FormatDate(context.Now)); + writer.Write(Crlf); + } + + writer.Write(preservedHeaders); + writer.Write(Crlf); + writer.Write(body); + return writer.WrittenSpan.ToArray(); + } + + private static void WriteReceived( + ArrayBufferWriter writer, + IPAddress client, + string fqdn, + ReadOnlySpan messageId, + DateTimeOffset now) + { + WriteAscii(writer, "Received: from ["); + WriteAscii(writer, client.ToString()); + WriteAscii(writer, "] by "); + WriteAscii(writer, fqdn); + WriteAscii(writer, " with NNTP"); + if (!messageId.IsEmpty) + { + WriteAscii(writer, " id "); + writer.Write(messageId); + } + + WriteAscii(writer, "; "); + WriteAscii(writer, FormatDate(now)); + writer.Write(Crlf); + } + + private static void SplitArticle( + ReadOnlySpan artData, + out ReadOnlySpan headers, + out ReadOnlySpan body) + { + var separator = artData.IndexOf(HeaderSeparator); + if (separator < 0) + { + headers = artData; + body = ReadOnlySpan.Empty; + return; + } + + headers = artData[..separator]; + body = artData[(separator + HeaderSeparator.Length)..]; + } + + private static void ScanOriginalHeaders( + ReadOnlySpan headerBlock, + out bool hasDate, + out byte[] messageId, + out byte[] newsgroups, + out byte[] preservedHeaders) + { + hasDate = false; + messageId = []; + newsgroups = []; + if (headerBlock.IsEmpty) + { + preservedHeaders = []; + return; + } + + var preserved = new ArrayBufferWriter(headerBlock.Length); + var offset = 0; + while (offset < headerBlock.Length) + { + var blockStart = offset; + if (!TryReadLine(headerBlock, ref offset, out var line)) + { + break; + } + + while (offset < headerBlock.Length + && headerBlock[offset] is (byte)' ' or (byte)'\t') + { + if (!TryReadLine(headerBlock, ref offset, out _)) + { + break; + } + } + + var block = headerBlock[blockStart..offset]; + var colon = line.IndexOf((byte)':'); + var name = colon < 0 ? line : line[..colon]; + if (IsStripped(name)) + { + continue; + } + + if (AsciiEqualsIgnoreCase(name, "Date"u8)) + { + hasDate = true; + } + else if (AsciiEqualsIgnoreCase(name, "Message-ID"u8) && messageId.Length == 0) + { + messageId = ExtractValue(line, colon); + } + else if (AsciiEqualsIgnoreCase(name, "Newsgroups"u8) && newsgroups.Length == 0) + { + newsgroups = ExtractValue(line, colon); + } + + preserved.Write(block); + if (block.Length < 2 + || block[^2] != (byte)'\r' + || block[^1] != (byte)'\n') + { + preserved.Write(Crlf); + } + } + + preservedHeaders = preserved.WrittenSpan.ToArray(); + } + + private static bool TryReadLine(ReadOnlySpan headers, ref int offset, out ReadOnlySpan line) + { + var remaining = headers[offset..]; + var eol = remaining.IndexOf("\r\n"u8); + if (eol < 0) + { + line = remaining; + offset = headers.Length; + return !line.IsEmpty; + } + + line = remaining[..eol]; + offset += eol + 2; + return true; + } + + private static byte[] ExtractValue(ReadOnlySpan line, int colon) + { + if (colon < 0 || colon + 1 >= line.Length) + { + return []; + } + + var value = line[(colon + 1)..]; + while (!value.IsEmpty && value[0] is (byte)' ' or (byte)'\t') + { + value = value[1..]; + } + + return value.ToArray(); + } + + private static bool IsStripped(ReadOnlySpan name) => + AsciiEqualsIgnoreCase(name, "Path"u8) + || AsciiEqualsIgnoreCase(name, "Xref"u8) + || AsciiEqualsIgnoreCase(name, "Injection-Info"u8) + || AsciiEqualsIgnoreCase(name, "X-Trace"u8) + || AsciiEqualsIgnoreCase(name, "X-Complaints-To"u8) + || AsciiEqualsIgnoreCase(name, "NNTP-Posting-Host"u8); + + private static bool AsciiEqualsIgnoreCase(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (ToLower(left[i]) != ToLower(right[i])) + { + return false; + } + } + + return true; + } + + private static byte ToLower(byte value) => + value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; + + private static string SanitizeToken(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "localhost"; + } + + var trimmed = value.Trim(); + foreach (var ch in trimmed) + { + if (ch is '\r' or '\n' or '\0') + { + return "localhost"; + } + } + + return trimmed; + } + + private static string FormatDate(DateTimeOffset now) => + now.UtcDateTime.ToString("ddd, dd MMM yyyy HH:mm:ss +0000", CultureInfo.InvariantCulture); + + private static void WriteAscii(ArrayBufferWriter writer, string value) => + writer.Write(Encoding.ASCII.GetBytes(value)); + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/SpamdTransportMetrics.cs b/src/VectorNNTPD.TransitService/PostFilter/SpamdTransportMetrics.cs new file mode 100644 index 00000000..75a0d17d --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/SpamdTransportMetrics.cs @@ -0,0 +1,56 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Process-wide SPAMD transport counters. Not per-POST logging. + internal sealed class SpamdTransportMetrics + { + private long _connectionsEstablished; + private long _checkRequests; + private long _connectionReuses; + private long _reconnects; + private long _evictions; + private long _totalCheckTicks; + + /// Gets TCP connections opened to SPAMD. + public long ConnectionsEstablished => Volatile.Read(ref _connectionsEstablished); + + /// Gets CHECK operations attempted after a connection was obtained. + public long CheckRequests => Volatile.Read(ref _checkRequests); + + /// Gets CHECKs that reused an idle persistent connection. + public long ConnectionReuses => Volatile.Read(ref _connectionReuses); + + /// Gets replacements opened after a broken or half-closed connection. + public long Reconnects => Volatile.Read(ref _reconnects); + + /// Gets connections removed from the pool before another request could use them. + public long Evictions => Volatile.Read(ref _evictions); + + /// Gets total CHECK duration in . + public long TotalCheckTicks => Volatile.Read(ref _totalCheckTicks); + + /// Gets average CHECK latency, or when none completed. + public TimeSpan AverageCheckLatency => + CheckRequests == 0 + ? TimeSpan.Zero + : TimeSpan.FromTicks(TotalCheckTicks / CheckRequests); + + /// Records a newly established TCP connection. + public void RecordConnect() => Interlocked.Increment(ref _connectionsEstablished); + + /// Records a CHECK that reused an idle connection. + public void RecordReuse() => Interlocked.Increment(ref _connectionReuses); + + /// Records a reconnect after eviction or unexpected close. + public void RecordReconnect() => Interlocked.Increment(ref _reconnects); + + /// Records removal of a broken connection. + public void RecordEviction() => Interlocked.Increment(ref _evictions); + + /// Records one CHECK duration. + public void RecordCheck(TimeSpan elapsed) + { + Interlocked.Increment(ref _checkRequests); + Interlocked.Add(ref _totalCheckTicks, elapsed.Ticks); + } + } +} diff --git a/src/VectorNNTPD.TransitService/PostFilter/StaticPostFilterPolicySource.cs b/src/VectorNNTPD.TransitService/PostFilter/StaticPostFilterPolicySource.cs new file mode 100644 index 00000000..30ab9085 --- /dev/null +++ b/src/VectorNNTPD.TransitService/PostFilter/StaticPostFilterPolicySource.cs @@ -0,0 +1,16 @@ +namespace VectorNNTPD.TransitService.PostFilter +{ + /// Fixed snapshot for tests and the disabled-gate default. + internal sealed class StaticPostFilterPolicySource : IPostFilterPolicySource + { + /// Initializes a source that always returns . + public StaticPostFilterPolicySource(PostFilterPolicySnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Current = snapshot; + } + + /// + public PostFilterPolicySnapshot Current { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Program.cs b/src/VectorNNTPD.TransitService/Program.cs new file mode 100644 index 00000000..2f0e1856 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Program.cs @@ -0,0 +1,80 @@ +using Serilog; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService +{ + /// Process entry point for the NNTP transit-server host. + internal static class Program + { + /// + /// Builds and runs the transit Generic Host, or runs startup smoke when that environment variable is set. + /// + /// Command-line arguments passed to the host builder. + /// Zero on clean shutdown or successful smoke; one on unexpected termination. + private static async Task Main(string[] args) + { + NntpdLoggingExtensions.UseAutoFlushConsoleOutput(); + Log.Logger = NntpdLoggingExtensions.CreateBootstrapLogger(); + + try + { + Log.Information("{Application} host starting", ApplicationJsonConfiguration.EntryAssemblyName); + + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + Args = args, + ContentRootPath = AppContext.BaseDirectory, + }); + builder.Environment.ContentRootPath = AppContext.BaseDirectory; + ApplicationJsonConfiguration.UseEntryAssemblyJsonFiles( + builder.Configuration, + builder.Environment.EnvironmentName); + ApplicationJsonConfiguration.AddSharedRabbitMqJsonFile( + builder.Configuration, + builder.Environment.EnvironmentName, + NntpServerJsonFiles.SharedRabbitMqFileName); + + var startupSmoke = TransitStartupSmoke.IsRequested( + Environment.GetEnvironmentVariable(TransitStartupSmoke.EnvironmentVariable)); + if (startupSmoke) + { + TransitStartupSmoke.UseIsolatedConfiguration(builder.Configuration); + } + + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpTransitHosting(); + + var host = builder.Build(); + NntpCommandLoggers.Configure(host.Services.GetRequiredService()); + NntpdLoggingExtensions.WriteLoggingInitialized( + host.Services, + builder.Environment.EnvironmentName, + builder.Environment.ContentRootPath); + if (startupSmoke) + { + return TransitStartupSmoke.Complete(host, Console.Out); + } + + await host.RunAsync().ConfigureAwait(false); + + Log.Information("{Application} host stopped cleanly", ApplicationJsonConfiguration.EntryAssemblyName); + return 0; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + Log.Fatal(ex, "{Application} host terminated unexpectedly", ApplicationJsonConfiguration.EntryAssemblyName); + return 1; + } + finally + { + await Log.CloseAndFlushAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs new file mode 100644 index 00000000..a44d80be --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkAggregatePolicy.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// NNTPD policy for sequential Backfill Scheduler attempts. + /// + /// + /// + /// BackFiller v1 outcomes classify a single work item. The scheduler publishes at most + /// one ArticleWork request at a time and never fans out concurrently. + /// + /// + /// Attempt policy: + /// + /// + /// + /// completes the logical lookup immediately. + /// + /// + /// + /// + /// is a definitive, lookup-terminal + /// miss (provider 430). BackFiller ACKs the delivery. NNTPD must not publish + /// another request to a different backbone, requeue, or invent a replacement work item. + /// + /// + /// + /// + /// is definitive for the selected + /// backbone. BackFiller ACKs the delivery. NNTPD may publish a NEW request to another + /// eligible backbone (unusable article on one provider is not proof of absence elsewhere). + /// + /// + /// + /// + /// is also attempt-terminal for the + /// selected backbone (protocol rejection from that destination); another eligible + /// backbone may still be attempted. + /// + /// + /// + /// + /// + /// In-flight attempts cannot be cancelled on the broker. A per-attempt wait that + /// expires without a definitive wire outcome ends the logical lookup without starting + /// another backbone (avoids duplicate upstream retrieval). + /// + /// + internal static class ArticleWorkAggregatePolicy + { + /// Returns whether this outcome completes the whole logical lookup successfully. + internal static bool IsLookupSuccess(ArticleWorkOutcome outcome) => + outcome == ArticleWorkOutcome.Success; + + /// + /// Returns whether this wire outcome ends the current backbone attempt + /// (success or definitive absence/rejection). + /// + internal static bool IsAttemptTerminal(ArticleWorkOutcome outcome) => + outcome is ArticleWorkOutcome.Success + or ArticleWorkOutcome.ArticleNotFound + or ArticleWorkOutcome.InvalidArticle + or ArticleWorkOutcome.InvalidRequest; + + /// + /// Returns whether the scheduler may try another eligible backbone after this outcome. + /// + /// + /// is intentionally omitted: a definitive + /// provider 430 ends the entire logical lookup. + /// + internal static bool ShouldTryNextBackbone(ArticleWorkOutcome outcome) => + outcome is ArticleWorkOutcome.InvalidArticle + or ArticleWorkOutcome.InvalidRequest; + + /// Legacy name: only Success completed the old parallel aggregate. + internal static bool IsAggregateTerminal(ArticleWorkOutcome outcome) => + IsLookupSuccess(outcome); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs new file mode 100644 index 00000000..6111162d --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkLookupOperation.cs @@ -0,0 +1,93 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// In-flight article-work lookup shared by every publication of one requestId. + internal sealed class ArticleWorkLookupOperation + { + private readonly TaskCompletionSource _completion = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Initializes a new lookup operation. + internal ArticleWorkLookupOperation(Guid requestId, string messageId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + RequestId = requestId; + MessageId = messageId; + } + + /// Gets the application request identity shared by every publication. + internal Guid RequestId { get; } + + /// Gets the Message-ID being retrieved. + internal string MessageId { get; } + + /// Gets the task that completes when the lookup reaches a terminal classification. + internal Task Completion => _completion.Task; + + /// Returns whether the operation has already completed. + internal bool IsCompleted => _completion.Task.IsCompleted; + + /// Returns whether a response belongs to this operation's application identity. + internal bool Matches(ArticleWorkResponse response) + { + ArgumentNullException.ThrowIfNull(response); + if (response.RequestId.HasValue && response.RequestId.Value != RequestId) + { + return false; + } + + if (response.MessageId is not null + && !string.Equals(response.MessageId, MessageId, StringComparison.Ordinal)) + { + return false; + } + + return true; + } + + /// + /// Applies a validated response immediately. Attempt-terminal outcomes + /// (, NotFound, InvalidArticle, InvalidRequest) + /// complete this attempt's wait so the scheduler can decide whether to continue. + /// + internal void OnResponse(string exchange, ArticleWorkResponse response) + { + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + ArgumentNullException.ThrowIfNull(response); + + if (!ArticleWorkAggregatePolicy.IsAttemptTerminal(response.Outcome)) + { + return; + } + + TryComplete(new ArticleWorkRpcResult( + response.Outcome, + RequestId, + MessageId, + response.Backbone, + response.Fqdn, + response.VatpPort, + response.ArticleId, + response.Error, + exchange)); + } + + /// Completes the operation if it is still pending. + internal bool TryComplete(ArticleWorkRpcResult result) => _completion.TrySetResult(result); + + /// Cancels the operation during shutdown. + internal bool TryCancel() => _completion.TrySetCanceled(); + + /// Returns the completed result when available. + internal bool TryGetResult(out ArticleWorkRpcResult result) + { + if (_completion.Task is { IsCompletedSuccessfully: true } task) + { + result = task.Result; + return true; + } + + result = default!; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs new file mode 100644 index 00000000..a1859c95 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkOutcome.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Canonical BackFiller v1 article-work response outcomes that NNTPD classifies. + public enum ArticleWorkOutcome + { + /// The source located the article and supplied a cache URI. + Success = 0, + + /// The source did not have the article. + ArticleNotFound = 1, + + /// The source rejected the article as invalid. + InvalidArticle = 2, + + /// The source rejected the request as protocol-invalid. + InvalidRequest = 3, + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRequest.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRequest.cs new file mode 100644 index 00000000..34da52e9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRequest.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Application JSON fields for a BackFiller v1 article-work request. + /// + /// AMQP CorrelationId and ReplyTo are transport metadata and are never + /// serialized into this payload. + /// + /// Application protocol version. Current value is 1. + /// Application work identity retained across storage and provider publications. + /// Canonical NNTP Message-ID as supplied on the command line. + /// Destination backbone identity for the publication target. + internal sealed record ArticleWorkRequest( + int Version, + Guid RequestId, + string MessageId, + string Backbone); +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkResponse.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkResponse.cs new file mode 100644 index 00000000..63d0a2f6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkResponse.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Parsed BackFiller v1 article-work response JSON. + /// Application protocol version. + /// Application request identity when the responder could establish it. + /// Request Message-ID when the responder could establish it. + /// Request backbone when the responder could establish it. + /// Terminal response outcome. + /// Success-only BackFiller FQDN. Absent for non-success outcomes. + /// Success-only TLS VATP listen port. Absent for non-success outcomes. + /// Success-only CanonicalV1 ArtId. Absent for non-success outcomes. + /// Terminal-failure detail. Absent for success. + internal sealed record ArticleWorkResponse( + int Version, + Guid? RequestId, + string? MessageId, + string? Backbone, + ArticleWorkOutcome Outcome, + string? Fqdn, + int? VatpPort, + VectorNNTPD.Common.Articles.ArticleId? ArticleId, + string? Error); +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs new file mode 100644 index 00000000..b81be186 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcAmqp.cs @@ -0,0 +1,56 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// AMQP transport constants for article-work RPC publications. + internal static class ArticleWorkRpcAmqp + { + /// AMQP header/property name carrying the logical lookup UUID. + internal const string RequestIdPropertyName = "RequestId"; + + /// + /// Documented BackFiller response AMQP Expiration (milliseconds). + /// NNTPD work requests omit Expiration so RabbitMQ may buffer ready work while + /// consumers are busy. A one-second request TTL silently discarded accepted lookups and + /// conflicted with the five-second . + /// Leftover replies on exclusive ReplyTo still use this value on the BackFiller publish path. + /// NNTPD does not delete or purge queues. + /// + internal const string ResponseExpirationMilliseconds = "1000"; + + /// + /// Obsolete alias retained only for transitional call sites that assert response TTL. + /// Prefer . Not applied to request publishes. + /// + internal const string ExpirationMilliseconds = ResponseExpirationMilliseconds; + + /// Creates the AMQP header table that carries . + internal static Dictionary CreateRequestIdHeaders(string requestId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(requestId); + return new Dictionary(StringComparer.Ordinal) + { + [RequestIdPropertyName] = requestId, + }; + } + + /// Reads the logical lookup UUID from AMQP headers, if present. + internal static string? ReadRequestId(IDictionary? headers) + { + if (headers is null + || !headers.TryGetValue(RequestIdPropertyName, out var value) + || value is null) + { + return null; + } + + return value switch + { + string text => text, + byte[] bytes => Encoding.UTF8.GetString(bytes), + ReadOnlyMemory memory => Encoding.UTF8.GetString(memory.Span), + _ => null, + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs new file mode 100644 index 00000000..97d2745e --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcClient.cs @@ -0,0 +1,304 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// Sequential Backfill Scheduler: one ArticleWork publish per attempt, weighted by + /// active consumer count, excluding already-attempted backbones. + /// + /// + /// + /// Never fans out concurrently. Definitive ArticleNotFound (provider 430) + /// from BackFiller is ACKed there and ends the logical lookup — NNTPD does not publish + /// to another backbone. InvalidArticle / InvalidRequest remain + /// attempt-terminal for the selected backbone and may advance to another eligible + /// backbone. In-flight attempts cannot be cancelled on the broker — an attempt wait + /// that expires without a definitive wire outcome ends the logical lookup without + /// starting another backbone. + /// + /// + internal sealed class ArticleWorkRpcClient : IArticleWorkRpcClient + { + private readonly IArticleWorkRpcPublisher _publisher; + private readonly ArticleWorkRpcResponseRouter _router; + private readonly IBackfillConsumerAvailability _availability; + private readonly IBackboneSelector _selector; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly Func _currentGeneration; + + /// Initializes a new orchestrator. + internal ArticleWorkRpcClient( + IArticleWorkRpcPublisher publisher, + ArticleWorkRpcResponseRouter router, + IBackfillConsumerAvailability availability, + IBackboneSelector selector, + TimeProvider timeProvider, + ILogger logger, + Func? currentGeneration = null) + { + ArgumentNullException.ThrowIfNull(publisher); + ArgumentNullException.ThrowIfNull(router); + ArgumentNullException.ThrowIfNull(availability); + ArgumentNullException.ThrowIfNull(selector); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(logger); + _publisher = publisher; + _router = router; + _availability = availability; + _selector = selector; + _timeProvider = timeProvider; + _logger = logger; + _currentGeneration = currentGeneration ?? (() => 0); + } + + /// + public async Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var started = _timeProvider.GetUtcNow(); + var decoded = DecodeMessageId(messageId); + var requestId = Guid.NewGuid(); + var attempted = new HashSet(StringComparer.OrdinalIgnoreCase); + var deadline = started + ArticleWorkRpcTiming.LookupDeadline; + + using var deadlineCts = new CancellationTokenSource(ArticleWorkRpcTiming.LookupDeadline, _timeProvider); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, deadlineCts.Token); + var lookupToken = linked.Token; + + try + { + var attempt = 0; + while (true) + { + lookupToken.ThrowIfCancellationRequested(); + var now = _timeProvider.GetUtcNow(); + if (now >= deadline) + { + var timedOut = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article lookup deadline elapsed"); + LogCompleted(timedOut, started); + return timedOut; + } + + var selected = SelectNext(attempted); + if (selected is null) + { + var exhausted = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + attempted.Count == 0 + ? "No BackFiller ArticleWork consumers are active" + : "All eligible BackFiller backbones returned not-found"); + LogCompleted(exhausted, started); + return exhausted; + } + + attempted.Add(selected.Value.Backbone); + attempt++; + ArticleWorkRpcLogMessages.SchedulerAttempt( + _logger, + requestId, + decoded, + selected.Value.Backbone, + selected.Value.ConsumerCount, + attempt); + + var operation = new ArticleWorkLookupOperation(requestId, decoded); + try + { + await PublishAsync(operation, selected.Value, lookupToken).ConfigureAwait(false); + lookupToken.ThrowIfCancellationRequested(); + + await WaitUntilAsync(operation, deadline, lookupToken).ConfigureAwait(false); + + if (!operation.TryGetResult(out var attemptResult)) + { + // No definitive wire outcome before the remaining lookup deadline. + // Do not start another backbone: in-flight work cannot be cancelled. + var uncertain = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article-work attempt timed out without a definitive response"); + LogCompleted(uncertain, started); + return uncertain; + } + + if (ArticleWorkAggregatePolicy.IsLookupSuccess(attemptResult.Outcome)) + { + LogCompleted(attemptResult, started); + return attemptResult; + } + + if (!ArticleWorkAggregatePolicy.ShouldTryNextBackbone(attemptResult.Outcome)) + { + // Includes definitive ArticleNotFound (430): lookup ends here. + LogCompleted(attemptResult, started); + return attemptResult; + } + + // InvalidArticle / InvalidRequest for this backbone — try another eligible. + } + finally + { + _router.UnregisterAll(operation); + } + } + } + catch (OperationCanceledException) when (deadlineCts.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + var expired = ArticleWorkRpcResult.NotFound( + requestId, + decoded, + "Article lookup deadline elapsed"); + LogCompleted(expired, started); + return expired; + } + } + + private BackfillEligibleBackbone? SelectNext(HashSet attempted) + { + var eligible = _availability.GetEligibleBackbones(); + if (eligible.Count == 0) + { + return null; + } + + List? filtered = null; + for (var i = 0; i < eligible.Count; i++) + { + var candidate = eligible[i]; + if (candidate.ConsumerCount < 1 || attempted.Contains(candidate.Backbone)) + { + continue; + } + + filtered ??= new List(eligible.Count); + filtered.Add(candidate); + } + + if (filtered is null || filtered.Count == 0) + { + return null; + } + + return _selector.Select(filtered); + } + + private async Task PublishAsync( + ArticleWorkLookupOperation operation, + BackfillEligibleBackbone destination, + CancellationToken cancellationToken) + { + var correlationId = Guid.NewGuid().ToString("D"); + var request = new ArticleWorkRequest( + ArticleWorkWireProtocol.CurrentVersion, + operation.RequestId, + operation.MessageId, + destination.Backbone); + var body = ArticleWorkWireProtocol.SerializeRequestV1(request); + var generation = _currentGeneration(); + if (!_router.TryRegister(correlationId, operation, destination.Exchange, generation)) + { + return; + } + + try + { + await _publisher + .PublishAsync( + destination.Exchange, + destination.RoutingKey, + operation.RequestId, + correlationId, + body, + cancellationToken) + .ConfigureAwait(false); + + ArticleWorkRpcLogMessages.Published( + _logger, + operation.RequestId, + correlationId, + operation.MessageId, + destination.Exchange, + generation); + } + catch (OperationCanceledException) + { + _router.Unregister(correlationId); + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _router.Unregister(correlationId); + ArticleWorkRpcLogMessages.PublishFailed( + _logger, + ex, + operation.RequestId, + correlationId, + operation.MessageId, + destination.Exchange, + generation); + throw; + } + } + + private async Task WaitUntilAsync( + ArticleWorkLookupOperation operation, + DateTimeOffset deadline, + CancellationToken cancellationToken) + { + var remaining = deadline - _timeProvider.GetUtcNow(); + if (remaining <= TimeSpan.Zero || operation.IsCompleted) + { + return; + } + + using var delayCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var delay = Task.Delay(remaining, _timeProvider, delayCts.Token); + var completed = await Task.WhenAny(operation.Completion, delay).ConfigureAwait(false); + if (completed == operation.Completion) + { + await delayCts.CancelAsync().ConfigureAwait(false); + _ = await operation.Completion.ConfigureAwait(false); + } + else + { + await delay.ConfigureAwait(false); + } + } + + private void LogCompleted(ArticleWorkRpcResult result, DateTimeOffset started) + { + var elapsedMs = (long)Math.Max(0, (_timeProvider.GetUtcNow() - started).TotalMilliseconds); + ArticleWorkRpcLogMessages.Completed( + _logger, + result.RequestId, + result.MessageId, + result.Outcome.ToString(), + result.SourceExchange, + elapsedMs); + } + + /// + /// Converts command-line Message-ID bytes to the JSON string at the RPC application boundary. + /// NNTP Message-IDs are ASCII; this is not a protocol-data-plane default representation. + /// + internal static string DecodeMessageId(ReadOnlyMemory messageId) + { + if (messageId.IsEmpty || !NntpMessageIdValidation.IsValidMessageId(messageId.Span)) + { + throw new ArgumentException("Article-work RPC requires a well-formed Message-ID.", nameof(messageId)); + } + + return Encoding.ASCII.GetString(messageId.Span); + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs new file mode 100644 index 00000000..957ac222 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcLogMessages.cs @@ -0,0 +1,111 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Source-generated article-work RPC log messages. Never includes payloads or credentials. + internal static partial class ArticleWorkRpcLogMessages + { + [LoggerMessage( + EventId = 2830, + Level = LogLevel.Information, + Message = "Article-work RPC ready (replyTo={ReplyTo}, generation={Generation})")] + public static partial void Ready(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2831, + Level = LogLevel.Information, + Message = "Article-work RPC session replaced (replyTo={ReplyTo}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2832, + Level = LogLevel.Error, + Message = "Article-work RPC is not ready")] + public static partial void NotReady(ILogger logger); + + [LoggerMessage( + EventId = 2833, + Level = LogLevel.Debug, + Message = "Article-work request published (requestId={RequestId}, correlationId={CorrelationId}, messageId={MessageId}, exchange={Exchange}, generation={Generation})")] + public static partial void Published( + ILogger logger, + Guid RequestId, + string CorrelationId, + string MessageId, + string Exchange, + long Generation); + + [LoggerMessage( + EventId = 2834, + Level = LogLevel.Warning, + Message = "Article-work publication failed (requestId={RequestId}, correlationId={CorrelationId}, messageId={MessageId}, exchange={Exchange}, generation={Generation})")] + public static partial void PublishFailed( + ILogger logger, + Exception exception, + Guid RequestId, + string CorrelationId, + string MessageId, + string Exchange, + long Generation); + + [LoggerMessage( + EventId = 2835, + Level = LogLevel.Information, + Message = "Article-work RPC completed (requestId={RequestId}, messageId={MessageId}, outcome={Outcome}, exchange={Exchange}, elapsedMs={ElapsedMs})")] + public static partial void Completed( + ILogger logger, + Guid RequestId, + string MessageId, + string Outcome, + string? Exchange, + long ElapsedMs); + + [LoggerMessage( + EventId = 2836, + Level = LogLevel.Debug, + Message = "Article-work RPC response ignored (correlationId={CorrelationId}, reason={Reason})")] + public static partial void ResponseIgnored(ILogger logger, string CorrelationId, string Reason); + + [LoggerMessage( + EventId = 2837, + Level = LogLevel.Warning, + Message = "Article-work RPC consumer callback failed")] + public static partial void ConsumerCallbackFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2838, + Level = LogLevel.Error, + Message = "Article-work RPC session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2839, + Level = LogLevel.Warning, + Message = "ARTICLE message-id RPC lookup failed (messageId={MessageId})")] + public static partial void ArticleLookupFailed(ILogger logger, Exception exception, string MessageId); + + [LoggerMessage( + EventId = 2840, + Level = LogLevel.Warning, + Message = "Article-work Management API availability refresh failed; retaining last successful snapshot")] + public static partial void ManagementAvailabilityRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2841, + Level = LogLevel.Debug, + Message = "Article-work scheduler attempt (requestId={RequestId}, messageId={MessageId}, backbone={Backbone}, consumers={Consumers}, attempt={Attempt})")] + public static partial void SchedulerAttempt( + ILogger logger, + Guid RequestId, + string MessageId, + string Backbone, + int Consumers, + int Attempt); + + [LoggerMessage( + EventId = 2842, + Level = LogLevel.Debug, + Message = "Article-work availability snapshot refreshed (eligibleCount={EligibleCount})")] + public static partial void AvailabilitySnapshotRefreshed(ILogger logger, int EligibleCount); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs new file mode 100644 index 00000000..bbc84db7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouter.cs @@ -0,0 +1,176 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// Routes consumed RPC responses to outstanding publications by AMQP + /// CorrelationId and logical RequestId. + /// + /// + /// This type owns NNTPD in-process correlation state only. It does not delete, + /// purge, or cancel RabbitMQ messages. Unknown and stale correlations are a + /// normal cheap ignore path after the logical lookup has completed. + /// + internal sealed class ArticleWorkRpcResponseRouter + { + private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); + private readonly ILogger _logger; + + /// Initializes a new router. + internal ArticleWorkRpcResponseRouter(ILogger logger) + { + ArgumentNullException.ThrowIfNull(logger); + _logger = logger; + } + + /// Gets the number of outstanding correlation registrations. + internal int OutstandingCount => _pending.Count; + + /// + /// Registers a publication so a later response can be correlated. + /// Completed lookups are not registered. + /// + internal bool TryRegister( + string correlationId, + ArticleWorkLookupOperation operation, + string exchange, + long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + ArgumentNullException.ThrowIfNull(operation); + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + + if (operation.IsCompleted) + { + return false; + } + + if (!_pending.TryAdd(correlationId, new PendingPublication(operation, exchange, generation))) + { + throw new InvalidOperationException($"Duplicate article-work correlation '{correlationId}'."); + } + + if (operation.IsCompleted) + { + _pending.TryRemove(correlationId, out _); + return false; + } + + return true; + } + + /// Registers a publication so a later response can be correlated. + internal void Register( + string correlationId, + ArticleWorkLookupOperation operation, + string exchange, + long generation) + { + _ = TryRegister(correlationId, operation, exchange, generation); + } + + /// Removes one publication registration. Safe when already removed. + internal void Unregister(string correlationId) + { + if (string.IsNullOrWhiteSpace(correlationId)) + { + return; + } + + _pending.TryRemove(correlationId, out _); + } + + /// Removes every registration owned by . + internal void UnregisterAll(ArticleWorkLookupOperation operation) + { + ArgumentNullException.ThrowIfNull(operation); + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _pending) + { + if (ReferenceEquals(pair.Value.Operation, operation)) + { + _pending.TryRemove(pair.Key, out _); + } + } + } + + /// Cancels every outstanding operation during shutdown. + internal void CancelAll() + { + foreach (var pair in _pending) + { + pair.Value.Operation.TryCancel(); + _pending.TryRemove(pair.Key, out _); + } + } + + /// + /// Dispatches one consumed body. Unknown, stale, malformed, generation-mismatched, + /// already-completed, or identity-mismatched responses are ignored. + /// + internal void Dispatch( + string? correlationId, + ReadOnlyMemory body, + string? amqpRequestId, + long deliveryGeneration) + { + if (string.IsNullOrWhiteSpace(correlationId)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, "(missing)", "missing-correlation"); + return; + } + + if (!_pending.TryGetValue(correlationId, out var pending)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "unknown-correlation"); + return; + } + + if (pending.Operation.IsCompleted) + { + UnregisterAll(pending.Operation); + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "lookup-already-complete"); + return; + } + + if (pending.Generation != deliveryGeneration) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "generation-mismatch"); + return; + } + + if (string.IsNullOrWhiteSpace(amqpRequestId) + || !Guid.TryParse(amqpRequestId, out var headerRequestId) + || headerRequestId != pending.Operation.RequestId) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "request-id-mismatch"); + return; + } + + if (!ArticleWorkWireProtocol.TryParseResponseV1(body.Span, out var response, out var reason) + || response is null) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, reason); + return; + } + + if (!pending.Operation.Matches(response)) + { + ArticleWorkRpcLogMessages.ResponseIgnored(_logger, correlationId, "identity-mismatch"); + return; + } + + pending.Operation.OnResponse(pending.Exchange, response); + if (pending.Operation.IsCompleted) + { + UnregisterAll(pending.Operation); + } + } + + private readonly record struct PendingPublication( + ArticleWorkLookupOperation Operation, + string Exchange, + long Generation); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs new file mode 100644 index 00000000..787fd71d --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcResult.cs @@ -0,0 +1,41 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Structured article-work RPC result returned to NNTP command callers. + /// + /// This type does not contain article bytes. Callers that need the article dial + /// and , then OPEN with . + /// + /// Classified RPC outcome. + /// Application request identity used for the lookup. + /// Message-ID that was queried. + /// Winning source backbone when a wire response completed the operation. + /// Success-only BackFiller FQDN from the winning response. + /// Success-only TLS VATP listen port from the winning response. + /// Success-only CanonicalV1 ArtId from the winning response. + /// Failure detail when the classified outcome is not success. + /// Exchange that produced the winning response, when applicable. + public sealed record ArticleWorkRpcResult( + ArticleWorkOutcome Outcome, + Guid RequestId, + string MessageId, + string? Backbone, + string? Fqdn, + int? VatpPort, + VectorNNTPD.Common.Articles.ArticleId? ArticleId, + string? Error, + string? SourceExchange) + { + /// Builds an overall not-found result after the lookup deadline elapses. + internal static ArticleWorkRpcResult NotFound(Guid requestId, string messageId, string error) => + new( + ArticleWorkOutcome.ArticleNotFound, + requestId, + messageId, + Backbone: null, + Fqdn: null, + VatpPort: null, + ArticleId: null, + error, + SourceExchange: null); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs new file mode 100644 index 00000000..104d984f --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcService.cs @@ -0,0 +1,395 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// NNTPD-owned article-work RPC session: shared reply consumer, generation-scoped channels, + /// and sequential Backfill Scheduler orchestration. + /// + /// + /// remains the sole TCP connection owner. This service + /// opens caller-owned channels, declares an exclusive auto-delete reply queue, and + /// rebuilds that session after . + /// Outstanding application operations survive generation replacement in process memory; + /// a stale channel cannot publish into or mutate the replacement session. + /// BackFiller availability is discovered via the RabbitMQ Management HTTP API + /// (eventually consistent ~5s snapshot), not via AMQP passive queue declares. + /// + internal sealed class ArticleWorkRpcService : IApplicationService, IArticleWorkRpcClient, IArticleWorkRpcPublisher + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly ArticleWorkRpcResponseRouter _router; + private readonly BackfillConsumerAvailabilityService _availability; + private readonly ArticleWorkRpcClient _client; + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly object _sessionGate = new(); + private readonly Guid _instanceId = Guid.NewGuid(); + private CancellationTokenSource? _shutdownCts; + private RpcSession? _session; + private int _started; + + /// Initializes a new article-work RPC service. + public ArticleWorkRpcService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IRabbitMqManagementQueueInventory managementInventory, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(managementInventory); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _router = new ArticleWorkRpcResponseRouter(logger); + _availability = new BackfillConsumerAvailabilityService(managementInventory, logger, _timeProvider); + _client = new ArticleWorkRpcClient( + this, + _router, + _availability, + new WeightedBackboneSelector(), + _timeProvider, + logger, + () => _rabbitMq.ConnectionGeneration); + } + + /// + public string Name => "RabbitMQArticleWorkRpc"; + + /// + public Task? Execution => null; + + /// + public string ReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo + ?? throw new InvalidOperationException("Article-work RPC reply queue is not ready."); + } + } + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + await _availability.RefreshNowAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + /// + public async Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref _started) == 0 || _shutdownCts is null) + { + throw new InvalidOperationException("Article-work RPC is not started."); + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token); + return await _client.LookupByMessageIdAsync(messageId, linked.Token).ConfigureAwait(false); + } + + /// + public async Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(exchange); + ArgumentNullException.ThrowIfNull(routingKey); + if (requestId == Guid.Empty) + { + throw new ArgumentException("Article-work RPC requires a non-empty RequestId.", nameof(requestId)); + } + + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + + RpcSession session; + lock (_sessionGate) + { + session = _session ?? throw new InvalidOperationException("Article-work RPC session is not ready."); + } + + if (!_rabbitMq.TryGetCurrent(out var handle) + || !handle.IsCurrent + || handle.Generation != session.Generation) + { + throw new InvalidOperationException( + $"Article-work RPC publication failed: connection generation {session.Generation} is not current."); + } + + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + lock (_sessionGate) + { + if (!ReferenceEquals(_session, session)) + { + throw new InvalidOperationException( + "Article-work RPC publication failed: session was replaced."); + } + } + + await session.PublishChannel + .PublishAsync( + exchange, + routingKey, + correlationId, + requestId.ToString("D"), + session.ReplyTo, + ArticleWorkWireProtocol.JsonContentType, + expiration: string.Empty, + body, + cancellationToken) + .ConfigureAwait(false); + } + finally + { + _publishGate.Release(); + } + } + + internal int OutstandingCorrelations => _router.OutstandingCount; + + internal string? CurrentReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo; + } + } + } + + internal long? CurrentSessionGeneration + { + get + { + lock (_sessionGate) + { + return _session?.Generation; + } + } + } + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + ArticleWorkRpcLogMessages.NotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for article-work RPC."); + } + + var generation = handle.Generation; + IRabbitMqRpcChannel? publishChannel = null; + IRabbitMqRpcChannel? consumeChannel = null; + try + { + publishChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + consumeChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + var replyTo = CreateReplyQueueName(); + await consumeChannel.QueueDeclareAsync( + replyTo, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + _ = await consumeChannel + .ConsumeAsync(replyTo, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new RpcSession(generation, replyTo, publishChannel, consumeChannel); + RpcSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + ArticleWorkRpcLogMessages.SessionReplaced(_logger, replyTo, generation); + } + else + { + ArticleWorkRpcLogMessages.Ready(_logger, replyTo, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + ArticleWorkRpcLogMessages.SessionAttachFailed(_logger, ex, generation); + if (publishChannel is not null) + { + await publishChannel.DisposeAsync().ConfigureAwait(false); + } + + if (consumeChannel is not null) + { + await consumeChannel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqRpcDelivery delivery) + { + try + { + _router.Dispatch( + delivery.CorrelationId, + delivery.Body, + delivery.RequestId, + delivery.Generation); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.ConsumerCallbackFailed(_logger, ex); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + _router.CancelAll(); + + RpcSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + await _availability.DisposeAsync().ConfigureAwait(false); + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + } + + private string CreateReplyQueueName() + { + var serverId = _nntpdOptions.Value.ServerId ?? 0; + return RabbitMqTopologyNames.Normalize($"nntpd.{serverId:00}.{_instanceId:N}.rpc"); + } + + private sealed class RpcSession : IAsyncDisposable + { + internal RpcSession( + long generation, + string replyTo, + IRabbitMqRpcChannel publishChannel, + IRabbitMqRpcChannel consumeChannel) + { + Generation = generation; + ReplyTo = replyTo; + PublishChannel = publishChannel; + ConsumeChannel = consumeChannel; + } + + internal long Generation { get; } + + internal string ReplyTo { get; } + + internal IRabbitMqRpcChannel PublishChannel { get; } + + internal IRabbitMqRpcChannel ConsumeChannel { get; } + + public async ValueTask DisposeAsync() + { + await PublishChannel.DisposeAsync().ConfigureAwait(false); + await ConsumeChannel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs new file mode 100644 index 00000000..fae0f154 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkRpcTiming.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Fixed article-work RPC deadlines. These are not configuration keys. + internal static class ArticleWorkRpcTiming + { + /// + /// Maximum total Backfill Scheduler lookup duration measured from request start. + /// Also the per-attempt wait budget (remaining time until this deadline). + /// + internal static readonly TimeSpan LookupDeadline = TimeSpan.FromSeconds(5); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs new file mode 100644 index 00000000..946e38b0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/ArticleWorkWireProtocol.cs @@ -0,0 +1,628 @@ +using System.Buffers; +using System.Text.Encodings.Web; +using System.Text.Json; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// Canonical compact System.Text.Json contract for BackFiller article-work v1 request and response bodies. + /// + /// + /// Property names are version, requestId, messageId, backbone, + /// and on responses outcome, fqdn, vatpPort, articleId, and error. + /// A response that includes uri is rejected. AMQP CorrelationId, + /// ReplyTo, and Expiration are never JSON fields. JSON requestId must + /// match the AMQP RequestId property. Serialization writes compact UTF-8 with no indentation. + /// + internal static partial class ArticleWorkWireProtocol + { + /// Current application protocol version. + internal const int CurrentVersion = 1; + + /// AMQP ContentType required by the BackFiller RPC contract. + internal const string JsonContentType = "application/json"; + + /// Serializes a version-1 request into compact UTF-8 JSON. + /// Application fields to write. Must be a concrete valid v1 request. + /// UTF-8 bytes containing only the four canonical request properties. + internal static byte[] SerializeRequestV1(ArticleWorkRequest request) + { + ArgumentNullException.ThrowIfNull(request); + if (request.Version != CurrentVersion) + { + throw new InvalidOperationException($"Unsupported article-work request version '{request.Version}'."); + } + + if (request.RequestId == Guid.Empty) + { + throw new InvalidOperationException("Canonical request serialization requires a concrete non-empty requestId."); + } + + if (string.IsNullOrWhiteSpace(request.MessageId)) + { + throw new InvalidOperationException("Canonical request serialization requires a canonical non-empty messageId."); + } + + if (string.IsNullOrWhiteSpace(request.Backbone)) + { + throw new InvalidOperationException("Canonical request serialization requires a non-empty backbone."); + } + + var writer = new ArrayBufferWriter(); + using var jsonWriter = new Utf8JsonWriter(writer, new JsonWriterOptions + { + Indented = false, + SkipValidation = false, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }); + + jsonWriter.WriteStartObject(); + jsonWriter.WriteNumber("version", request.Version); + jsonWriter.WriteString("requestId", request.RequestId); + jsonWriter.WriteString("messageId", request.MessageId); + jsonWriter.WriteString("backbone", request.Backbone); + jsonWriter.WriteEndObject(); + jsonWriter.Flush(); + return writer.WrittenSpan.ToArray(); + } + + /// Attempts to parse and validate a version-1 response payload. + /// UTF-8 JSON body. + /// Parsed response when validation succeeds. + /// Rejection reason when parsing fails. + /// when the payload is a valid v1 response. + internal static bool TryParseResponseV1( + ReadOnlySpan payload, + out ArticleWorkResponse? response, + out string reason) + { + response = null; + reason = "Response payload is empty."; + if (payload.IsEmpty) + { + return false; + } + + JsonDocument document; + try + { + document = JsonDocument.Parse(payload.ToArray()); + } + catch (JsonException) + { + reason = "Response payload was not valid JSON."; + return false; + } + + using (document) + { + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + reason = "Response payload must be a JSON object."; + return false; + } + + if (!root.TryGetProperty("version", out var versionElement) + || versionElement.ValueKind != JsonValueKind.Number + || !versionElement.TryGetInt32(out var version)) + { + reason = "Response payload is missing required integer property 'version'."; + return false; + } + + if (version != CurrentVersion) + { + reason = $"Unsupported response version '{version}'."; + return false; + } + + if (!TryReadRequiredString(root, "outcome", out var outcomeText) || outcomeText is null) + { + reason = "Response payload is missing required 'outcome'."; + return false; + } + + if (!TryParseOutcome(outcomeText, out var outcome)) + { + reason = $"Unsupported response outcome '{outcomeText}'."; + return false; + } + + if (!TryReadOptionalGuid(root, "requestId", out var requestId, out reason)) + { + return false; + } + + if (!TryReadMessageId(payload, out var messageId, out reason)) + { + return false; + } + + if (!TryReadOptionalString(root, "backbone", out var backbone, out reason)) + { + return false; + } + + if (root.TryGetProperty("uri", out _)) + { + reason = "Response payload must not include 'uri'."; + return false; + } + + var hasFqdn = root.TryGetProperty("fqdn", out var fqdnElement); + var hasVatpPort = root.TryGetProperty("vatpPort", out var vatpPortElement); + var hasError = root.TryGetProperty("error", out var errorElement); + var hasArticleId = root.TryGetProperty("articleId", out var articleIdElement); + string? fqdn = null; + int? vatpPort = null; + string? error = null; + ArticleId? articleId = null; + if (hasFqdn && !TryReadStringValue(fqdnElement, "fqdn", out fqdn, out reason)) + { + return false; + } + + if (hasVatpPort) + { + if (vatpPortElement.ValueKind != JsonValueKind.Number || !vatpPortElement.TryGetInt32(out var parsedPort)) + { + reason = "Response payload property 'vatpPort' must be an integer."; + return false; + } + + vatpPort = parsedPort; + } + + if (hasError && !TryReadStringValue(errorElement, "error", out error, out reason)) + { + return false; + } + + if (hasArticleId) + { + if (!TryReadStringValue(articleIdElement, "articleId", out var articleIdText, out reason)) + { + return false; + } + + if (articleIdText is not null) + { + if (!ArticleId.TryParseLowerHex(articleIdText, out var parsedArticleId)) + { + reason = "Response payload property 'articleId' is not a 64-character lowercase hexadecimal ArticleId."; + return false; + } + + articleId = parsedArticleId; + } + } + + if (!TryValidateOutcomeContract( + outcome, + requestId, + messageId, + backbone, + hasFqdn, + fqdn, + hasVatpPort, + vatpPort, + hasArticleId, + articleId, + hasError, + error, + out reason)) + { + return false; + } + + response = new ArticleWorkResponse( + version, + requestId, + messageId, + backbone, + outcome, + fqdn, + vatpPort, + articleId, + error); + reason = string.Empty; + return true; + } + } + + private static bool TryValidateOutcomeContract( + ArticleWorkOutcome outcome, + Guid? requestId, + string? messageId, + string? backbone, + bool hasFqdn, + string? fqdn, + bool hasVatpPort, + int? vatpPort, + bool hasArticleId, + ArticleId? articleId, + bool hasError, + string? error, + out string reason) + { + reason = string.Empty; + if (outcome == ArticleWorkOutcome.Success) + { + if (!requestId.HasValue || requestId.Value == Guid.Empty) + { + reason = "Success response payload requires a concrete non-empty 'requestId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(messageId)) + { + reason = "Success response payload requires a canonical non-empty 'messageId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(backbone)) + { + reason = "Success response payload requires a non-empty 'backbone'."; + return false; + } + + if (!hasFqdn || !VatpEndpointFields.IsCanonicalFqdn(fqdn)) + { + reason = "Success response payload requires a lowercase dotted DNS 'fqdn'."; + return false; + } + + if (!hasVatpPort || vatpPort is not int port || !VatpEndpointFields.IsCanonicalPort(port)) + { + reason = "Success response payload requires a 'vatpPort' in the range 1–65535."; + return false; + } + + if (!hasArticleId || articleId is null) + { + reason = "Success response payload requires a concrete 'articleId'."; + return false; + } + + if (hasError) + { + reason = "Success response payload must not include 'error'."; + return false; + } + + return true; + } + + if (outcome is ArticleWorkOutcome.ArticleNotFound or ArticleWorkOutcome.InvalidArticle) + { + if (!requestId.HasValue || requestId.Value == Guid.Empty) + { + reason = "Terminal non-invalid-request response payload requires a concrete non-empty 'requestId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(messageId)) + { + reason = "Terminal non-invalid-request response payload requires a canonical non-empty 'messageId'."; + return false; + } + + if (string.IsNullOrWhiteSpace(backbone)) + { + reason = "Terminal non-invalid-request response payload requires a non-empty 'backbone'."; + return false; + } + + if (hasFqdn || hasVatpPort) + { + reason = "Terminal failure response payload must not include 'fqdn' or 'vatpPort'."; + return false; + } + + if (hasArticleId) + { + reason = "Terminal failure response payload must not include 'articleId'."; + return false; + } + + if (!hasError || string.IsNullOrWhiteSpace(error)) + { + reason = "Terminal failure response payload requires a non-empty 'error'."; + return false; + } + + return true; + } + + if (requestId.HasValue && requestId.Value == Guid.Empty) + { + reason = "InvalidRequest payload must not use Guid.Empty sentinel request identity."; + return false; + } + + if (messageId is not null && string.IsNullOrWhiteSpace(messageId)) + { + reason = "InvalidRequest payload messageId, when provided, must be canonical and non-empty."; + return false; + } + + if (backbone is not null && string.IsNullOrWhiteSpace(backbone)) + { + reason = "InvalidRequest payload backbone, when provided, must be non-empty."; + return false; + } + + if (hasFqdn || hasVatpPort) + { + reason = "InvalidRequest payload must not include 'fqdn' or 'vatpPort'."; + return false; + } + + if (hasArticleId) + { + reason = "InvalidRequest payload must not include 'articleId'."; + return false; + } + + if (!hasError || string.IsNullOrWhiteSpace(error)) + { + reason = "InvalidRequest payload requires a non-empty 'error'."; + return false; + } + + return true; + } + + private static bool TryParseOutcome(string outcome, out ArticleWorkOutcome parsed) + { + if (string.Equals(outcome, nameof(ArticleWorkOutcome.Success), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.Success; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.ArticleNotFound), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.ArticleNotFound; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.InvalidArticle), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.InvalidArticle; + return true; + } + + if (string.Equals(outcome, nameof(ArticleWorkOutcome.InvalidRequest), StringComparison.Ordinal)) + { + parsed = ArticleWorkOutcome.InvalidRequest; + return true; + } + + parsed = default; + return false; + } + + private static bool TryReadRequiredString(JsonElement root, string propertyName, out string? value) + { + value = null; + return root.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.String + && (value = property.GetString()) is not null; + } + + private static bool TryReadOptionalGuid( + JsonElement root, + string propertyName, + out Guid? value, + out string reason) + { + value = null; + reason = string.Empty; + if (!root.TryGetProperty(propertyName, out var property)) + { + reason = $"Response payload is missing required '{propertyName}'."; + return false; + } + + if (property.ValueKind == JsonValueKind.Null) + { + return true; + } + + if (property.ValueKind != JsonValueKind.String) + { + reason = $"Response payload property '{propertyName}' must be a JSON string or null."; + return false; + } + + var text = property.GetString(); + if (string.IsNullOrWhiteSpace(text) || !Guid.TryParse(text, out var parsed) || parsed == Guid.Empty) + { + reason = $"Response payload property '{propertyName}' is not a concrete GUID."; + return false; + } + + value = parsed; + return true; + } + + /// + /// Reads the top-level messageId property from the original UTF-8 payload and validates those bytes. + /// + /// Response payload. Already known to be a JSON object. + /// ASCII text of a valid Message-ID, or null when the property is JSON null. + /// Rejection text when the result is . + /// when the property is missing, not a string, or not a valid Message-ID. + private static bool TryReadMessageId(ReadOnlySpan json, out string? messageId, out string reason) + { + messageId = null; + reason = string.Empty; + Span buffer = stackalloc byte[NntpMessageIdValidation.MaxMessageIdLength]; + switch (TryCopyTopLevelJsonString(json, "messageId"u8, buffer, out var length)) + { + case TopLevelJsonString.Missing: + reason = "Response payload is missing required 'messageId'."; + return false; + case TopLevelJsonString.Null: + return true; + case TopLevelJsonString.NotString: + reason = "Response payload property 'messageId' must be a JSON string or null."; + return false; + case TopLevelJsonString.TooLong: + reason = "Response payload property 'messageId' must be a canonical Message-ID."; + return false; + case TopLevelJsonString.Copied: + var token = buffer[..length]; + if (length == 0 || !NntpMessageIdValidation.IsValidMessageId(token)) + { + reason = "Response payload property 'messageId' must be a canonical Message-ID."; + return false; + } + + messageId = System.Text.Encoding.ASCII.GetString(token); + return true; + default: + reason = "Response payload is missing required 'messageId'."; + return false; + } + } + + private enum TopLevelJsonString + { + Missing, + Null, + NotString, + TooLong, + Copied, + } + + /// Copies one top-level JSON string property into without allocating. + private static TopLevelJsonString TryCopyTopLevelJsonString( + ReadOnlySpan json, + ReadOnlySpan propertyName, + Span destination, + out int written) + { + written = 0; + var reader = new Utf8JsonReader(json); + if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject) + { + return TopLevelJsonString.Missing; + } + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject && reader.CurrentDepth == 0) + { + return TopLevelJsonString.Missing; + } + + if (reader.TokenType != JsonTokenType.PropertyName || reader.CurrentDepth != 1) + { + continue; + } + + var match = reader.ValueTextEquals(propertyName); + if (!reader.Read()) + { + return TopLevelJsonString.Missing; + } + + if (!match) + { + if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + reader.Skip(); + } + + continue; + } + + if (reader.TokenType == JsonTokenType.Null) + { + return TopLevelJsonString.Null; + } + + if (reader.TokenType != JsonTokenType.String) + { + return TopLevelJsonString.NotString; + } + + if (!reader.ValueIsEscaped) + { + var value = reader.ValueSpan; + if (value.Length > destination.Length) + { + return TopLevelJsonString.TooLong; + } + + value.CopyTo(destination); + written = value.Length; + return TopLevelJsonString.Copied; + } + + try + { + written = reader.CopyString(destination); + return TopLevelJsonString.Copied; + } + catch (ArgumentException) + { + written = 0; + return TopLevelJsonString.TooLong; + } + } + + return TopLevelJsonString.Missing; + } + + private static bool TryReadOptionalString( + JsonElement root, + string propertyName, + out string? value, + out string reason) + { + value = null; + reason = string.Empty; + if (!root.TryGetProperty(propertyName, out var property)) + { + reason = $"Response payload is missing required '{propertyName}'."; + return false; + } + + return TryReadStringValue(property, propertyName, out value, out reason); + } + + private static bool TryReadStringValue( + JsonElement property, + string propertyName, + out string? value, + out string reason) + { + value = null; + reason = string.Empty; + switch (property.ValueKind) + { + case JsonValueKind.Null: + return true; + case JsonValueKind.String: + value = property.GetString(); + if (value is null) + { + reason = $"Response payload property '{propertyName}' string value was null."; + return false; + } + + return true; + default: + reason = $"Response payload property '{propertyName}' must be a JSON string or null."; + return false; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs new file mode 100644 index 00000000..93bc1aac --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillConsumerAvailability.cs @@ -0,0 +1,207 @@ +using System.Collections.Immutable; +using VectorNNTPD.TransitService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// Cached BackFiller ArticleWork consumer counts from the RabbitMQ Management HTTP API. + /// + /// + /// + /// Discovery plane: Management API queue inventory refreshed on a fixed ~5s interval. + /// AMQP remains the ArticleWork data plane. Availability is intentionally eventually + /// consistent and may be approximately 5 seconds stale. + /// + /// + /// A provider queue is eligible only when the Management API reports + /// consumers > 0 for that queue name. Missing queues and zero-consumer queues + /// are unavailable and are omitted from the snapshot. Absent/zero-consumer states are + /// normal and do not produce exceptions. + /// + /// + /// A failed Management API refresh retains the last successfully retrieved snapshot + /// (last-known-good). There is no additional long-term stale expiry beyond the normal + /// ~5s refresh cadence. + /// + /// + internal sealed class BackfillConsumerAvailabilityService : IBackfillConsumerAvailability, IAsyncDisposable + { + /// Minimum interval between Management API refresh attempts. + internal static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5); + + private readonly IRabbitMqManagementQueueInventory _inventory; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly object _gate = new(); + private readonly SemaphoreSlim _refreshGate = new(1, 1); + private ImmutableArray _snapshot = []; + private DateTimeOffset _nextRefresh = DateTimeOffset.MinValue; + private int _disposed; + + /// Initializes a new availability cache. + internal BackfillConsumerAvailabilityService( + IRabbitMqManagementQueueInventory inventory, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(inventory); + ArgumentNullException.ThrowIfNull(logger); + _inventory = inventory; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public IReadOnlyList GetEligibleBackbones() + { + RefreshIfDueAsync(force: false, CancellationToken.None).GetAwaiter().GetResult(); + return _snapshot; + } + + /// Forces a refresh regardless of interval (startup / tests). + internal Task RefreshNowAsync(CancellationToken cancellationToken = default) => + RefreshIfDueAsync(force: true, cancellationToken); + + private async Task RefreshIfDueAsync(bool force, CancellationToken cancellationToken) + { + if (Volatile.Read(ref _disposed) != 0) + { + return; + } + + var now = _timeProvider.GetUtcNow(); + if (!force) + { + lock (_gate) + { + if (now < _nextRefresh) + { + return; + } + } + } + + await _refreshGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (Volatile.Read(ref _disposed) != 0) + { + return; + } + + now = _timeProvider.GetUtcNow(); + if (!force) + { + lock (_gate) + { + if (now < _nextRefresh) + { + return; + } + } + } + + await RefreshCoreAsync(cancellationToken).ConfigureAwait(false); + lock (_gate) + { + _nextRefresh = _timeProvider.GetUtcNow() + RefreshInterval; + } + } + finally + { + _refreshGate.Release(); + } + } + + private async Task RefreshCoreAsync(CancellationToken cancellationToken) + { + try + { + var queues = await _inventory.ListQueuesAsync(cancellationToken).ConfigureAwait(false); + var consumersByName = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < queues.Count; i++) + { + var queue = queues[i]; + if (string.IsNullOrWhiteSpace(queue.Name)) + { + continue; + } + + // Last duplicate name wins; Management API should not duplicate names in one vhost. + consumersByName[queue.Name] = queue.Consumers; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + if (!consumersByName.TryGetValue(definition.QueueName, out var consumers) + || consumers <= 0) + { + continue; + } + + builder.Add(new BackfillEligibleBackbone( + definition.Provider, + definition.ExchangeName, + definition.RoutingKey, + definition.QueueName, + consumers)); + } + + var next = builder.ToImmutable(); + lock (_gate) + { + _snapshot = next; + } + + ArticleWorkRpcLogMessages.AvailabilitySnapshotRefreshed(_logger, next.Length); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // Retain last-known-good snapshot. Do not treat Management outage as zero consumers. + ArticleWorkRpcLogMessages.ManagementAvailabilityRefreshFailed(_logger, ex); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _refreshGate.Dispose(); + await Task.CompletedTask.ConfigureAwait(false); + } + } + + /// Test double / static snapshot availability. + internal sealed class StaticBackfillConsumerAvailability : IBackfillConsumerAvailability + { + private volatile IReadOnlyList _eligible; + + /// Creates availability from an initial snapshot. + internal StaticBackfillConsumerAvailability(IReadOnlyList eligible) + { + ArgumentNullException.ThrowIfNull(eligible); + _eligible = eligible; + } + + /// Replaces the snapshot (tests). + internal void SetEligible(IReadOnlyList eligible) + { + ArgumentNullException.ThrowIfNull(eligible); + _eligible = eligible; + } + + /// + public IReadOnlyList GetEligibleBackbones() => _eligible; + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs new file mode 100644 index 00000000..5debf9bd --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/BackfillSchedulerSelection.cs @@ -0,0 +1,119 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// One eligible BackFiller backbone with a positive active consumer count. + /// Canonical provider identifier (JSON backbone casing). + /// Normalized exchange name. + /// Normalized routing key. + /// Normalized queue name used for passive consumer inspection. + /// Broker-reported consumer count (must be > 0). + internal readonly record struct BackfillEligibleBackbone( + string Backbone, + string Exchange, + string RoutingKey, + string QueueName, + int ConsumerCount); + + /// + /// Snapshot of BackFiller ArticleWork consumer availability for scheduler selection. + /// + internal interface IBackfillConsumerAvailability + { + /// + /// Returns provider backbones with a positive active consumer count. + /// Zero-consumer and unknown queues are omitted. + /// + IReadOnlyList GetEligibleBackbones(); + } + + /// + /// Selects one eligible backbone weighted by active consumer count. + /// + internal interface IBackboneSelector + { + /// + /// Chooses one candidate. Returns when + /// is empty. + /// + BackfillEligibleBackbone? Select(IReadOnlyList candidates); + } + + /// + /// Weighted random backbone selection. Isolates randomness for deterministic tests. + /// + internal sealed class WeightedBackboneSelector : IBackboneSelector + { + private readonly Func _nextInt; + + /// Creates a selector using . + public WeightedBackboneSelector() + : this(static exclusiveMax => Random.Shared.Next(exclusiveMax)) + { + } + + /// + /// Creates a selector with an injectable RNG. receives + /// an exclusive upper bound and must return a value in [0, exclusiveMax). + /// + internal WeightedBackboneSelector(Func nextInt) + { + ArgumentNullException.ThrowIfNull(nextInt); + _nextInt = nextInt; + } + + /// + public BackfillEligibleBackbone? Select(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + if (candidates.Count == 0) + { + return null; + } + + long total = 0; + for (var i = 0; i < candidates.Count; i++) + { + var weight = candidates[i].ConsumerCount; + if (weight < 1) + { + continue; + } + + total += weight; + } + + if (total < 1) + { + return null; + } + + if (total > int.MaxValue) + { + total = int.MaxValue; + } + + var roll = _nextInt((int)total); + if (roll < 0 || roll >= total) + { + throw new InvalidOperationException("Backbone selector RNG returned a value outside [0, total)."); + } + + long cursor = 0; + for (var i = 0; i < candidates.Count; i++) + { + var candidate = candidates[i]; + if (candidate.ConsumerCount < 1) + { + continue; + } + + cursor += candidate.ConsumerCount; + if (roll < cursor) + { + return candidate; + } + } + + return candidates[^1]; + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs new file mode 100644 index 00000000..019e8935 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcClient.cs @@ -0,0 +1,25 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// + /// Reusable article-work RPC entry point for ARTICLE, HEAD, BODY, and STAT message-id lookups. + /// + /// + /// This boundary converts command-line Message-ID bytes into the JSON application payload. + /// It does not retrieve article bytes or write NNTP success responses. + /// + public interface IArticleWorkRpcClient + { + /// + /// Runs the sequential Backfill Scheduler: selects one eligible backbone at a time + /// (weighted by active consumer count), publishes one ArticleWork request per attempt, + /// and returns on Success, exhausted not-found, cancellation, or the 5-second lookup + /// deadline. The scheduler never exceeds that 5-second budget from its start time. + /// + /// Command-line Message-ID bytes, including angle brackets. + /// Caller or session cancellation. + /// The structured RPC result. Never includes article content. + Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs new file mode 100644 index 00000000..8c0bd03d --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/ArticleWork/IArticleWorkRpcPublisher.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.RabbitMq.ArticleWork +{ + /// Publishes one article-work RPC request onto the current RabbitMQ generation. + internal interface IArticleWorkRpcPublisher + { + /// Gets the NNTPD-owned reply queue name used as AMQP ReplyTo. + string ReplyTo { get; } + + /// + /// Publishes a compact v1 JSON body with AMQP RPC metadata: + /// RequestId, CorrelationId, ReplyTo, and + /// ContentType=application/json. Work requests omit AMQP Expiration + /// so the broker may buffer ready messages until a BackFiller consumer delivers them. + /// + /// Destination fanout exchange. + /// Routing key matching the declared binding. + /// Logical lookup UUID. Also written to JSON requestId. + /// Fresh AMQP correlation identity for this publication only. + /// Canonical compact request JSON. + /// Token used to cancel the publish. + Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/BackfillArticleRetrievalTopology.cs b/src/VectorNNTPD.TransitService/RabbitMq/BackfillArticleRetrievalTopology.cs new file mode 100644 index 00000000..90935653 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/BackfillArticleRetrievalTopology.cs @@ -0,0 +1,149 @@ +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// Immutable BackFiller-compatible topology for one article-retrieval provider. + /// + /// Canonical provider identifier. Casing is preserved. + /// Exchange that receives article-retrieval work for the provider. + /// RabbitMQ exchange type. BackFiller uses fanout. + /// Whether the exchange survives broker restart. + /// Whether the exchange is auto-deleted when unused. + /// Queue bound to the provider exchange. + /// Whether the queue survives broker restart. + /// Whether the queue is exclusive to a single connection. + /// Whether the queue is auto-deleted when unused. + /// Routing key used when binding the queue to the exchange. + /// + /// Queue arguments applied during declaration. Durable provider work queues require + /// x-queue-type=quorum. + /// + internal sealed record BackfillArticleRetrievalTopologyDefinition( + string Provider, + string ExchangeName, + string ExchangeType, + bool ExchangeDurable, + bool ExchangeAutoDelete, + string QueueName, + bool QueueDurable, + bool QueueExclusive, + bool QueueAutoDelete, + string RoutingKey, + IReadOnlyDictionary QueueArguments) + { + /// Projects this BackFiller definition onto the shared declare shape. + internal RabbitMqArticleRetrievalEndpoint ToEndpoint() => + new( + ExchangeName, + ExchangeType, + ExchangeDurable, + ExchangeAutoDelete, + QueueName, + QueueDurable, + QueueExclusive, + QueueAutoDelete, + RoutingKey, + QueueArguments); + } + + /// + /// Fixed article-retrieval topology shared with VectorNNTP.BackFiller. + /// + /// + /// + /// Provider identifiers are application constants, not configuration. The twelve names + /// match BackFiller's backbone set. Entity names are backfiller.{provider} + /// after normalization, for the exchange, queue, + /// and routing key. + /// + /// + /// Every queue name matches the durable quorum fanout shape BackFiller declares when a + /// backbone becomes usable (x-queue-type=quorum). NNTPD does not declare or consume + /// these queues; it publishes sequential ArticleWork RPC requests onto selected provider + /// queues that already exist and have active consumers. + /// + /// + internal static class BackfillArticleRetrievalTopology + { + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName; + + /// Required queue type for durable BackFiller ArticleWork provider queues. + internal const string QuorumQueueType = RabbitMqArticleRetrievalEndpoints.QuorumQueueType; + + /// + /// Canonical BackFiller provider identifiers. Casing is significant and is not normalized. + /// + internal static readonly string[] Providers = + [ + "Abavia", + "Altopia", + "BaseIP", + "Eweka", + "Elbracht", + "Giganews", + "GTT", + "Highwinds", + "ItsHosted", + "Novia", + "UExpress", + "UsenetNode1", + ]; + + /// Complete article-retrieval topology in provider-list order. + internal static IReadOnlyList Definitions { get; } = + BuildDefinitions(Providers); + + /// + /// Builds topology definitions for the supplied provider identifiers. + /// + /// Provider names whose exchanges and queues must exist. + /// One definition per non-blank unique provider, preserving first-seen order. + internal static IReadOnlyList BuildDefinitions( + IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + + var uniqueProviders = new HashSet(StringComparer.OrdinalIgnoreCase); + var definitions = new List(); + + foreach (var provider in providers) + { + if (string.IsNullOrWhiteSpace(provider)) + { + continue; + } + + var canonicalProvider = provider.Trim(); + if (!uniqueProviders.Add(canonicalProvider)) + { + continue; + } + + var endpoint = RabbitMqArticleRetrievalEndpoints.CreateFanoutQuorumBinding( + BuildProviderEntityName(canonicalProvider)); + definitions.Add(new BackfillArticleRetrievalTopologyDefinition( + Provider: canonicalProvider, + ExchangeName: endpoint.ExchangeName, + ExchangeType: endpoint.ExchangeType, + ExchangeDurable: endpoint.ExchangeDurable, + ExchangeAutoDelete: endpoint.ExchangeAutoDelete, + QueueName: endpoint.QueueName, + QueueDurable: endpoint.QueueDurable, + QueueExclusive: endpoint.QueueExclusive, + QueueAutoDelete: endpoint.QueueAutoDelete, + RoutingKey: endpoint.RoutingKey, + QueueArguments: endpoint.QueueArguments)); + } + + return definitions; + } + + /// + /// Builds the NNTPD BackFiller exchange, queue, and routing-key name for a provider. + /// + /// Provider identifier. Casing and surrounding whitespace are normalized. + /// The normalized entity name in the form backfiller.{provider}. + internal static string BuildProviderEntityName(string provider) => + RabbitMqTopologyNames.Compose("backfiller", provider); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/CacheBroadcastTopology.cs b/src/VectorNNTPD.TransitService/RabbitMq/CacheBroadcastTopology.cs new file mode 100644 index 00000000..e145721b --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/CacheBroadcastTopology.cs @@ -0,0 +1,61 @@ +using VectorNNTPD.Common.Messaging.Cache; +using RabbitMQ.Client; + +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// NNTPD-owned cache.requests fleet article-lookup fanout exchange. + /// + /// + /// There is no shared cache.requests work queue. Each StorageServer binds an + /// ephemeral cache.<storage-fqdn> queue. Per-NNTPD reply queues for lookup + /// responses are owned by StorageArticleLookupService. + /// + internal static class CacheRequestsTopology + { + /// Fanout exchange that carries "who has article X?" lookups. + internal const string ExchangeName = CacheFleetTopology.RequestsExchangeName; + + /// Exchange type. + internal const string ExchangeTypeName = ExchangeType.Fanout; + + /// Whether the exchange survives broker restart. + internal const bool ExchangeDurable = true; + + /// Whether the exchange auto-deletes when unused. + internal const bool ExchangeAutoDelete = false; + + /// Per-message AMQP expiration for lookup requests/responses. + internal const string ExpirationMilliseconds = CacheFleetTopology.LookupExpirationMilliseconds; + } + + /// + /// NNTPD-owned cache.broadcast fanout exchange declaration constants. + /// + /// + /// Per-instance ephemeral consume queues are declared by + /// , not by + /// . + /// + internal static class CacheBroadcastTopology + { + /// Fanout exchange that carries StorageServer advertisements. + internal const string ExchangeName = CacheFleetTopology.BroadcastExchangeName; + + /// Exchange type. + internal const string ExchangeTypeName = ExchangeType.Fanout; + + /// Whether the exchange survives broker restart. + internal const bool ExchangeDurable = true; + + /// Whether the exchange auto-deletes when unused. + internal const bool ExchangeAutoDelete = false; + + /// Per-message AMQP expiration for advertisements. + internal const string ExpirationMilliseconds = CacheFleetTopology.AdvertisementExpirationMilliseconds; + + /// Builds cache.{nntpdFqdn} after normalization. + internal static string BuildInstanceQueueName(string nntpdFqdn) => + CacheFleetTopology.BuildNntpdBroadcastQueueName(nntpdFqdn); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs b/src/VectorNNTPD.TransitService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs new file mode 100644 index 00000000..a4f77e32 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/Management/IRabbitMqManagementQueueInventory.cs @@ -0,0 +1,20 @@ +using VectorNNTPD.Common.Messaging.RabbitMq; +namespace VectorNNTPD.TransitService.RabbitMq.Management +{ + /// + /// Reads the RabbitMQ Management HTTP API queue inventory for one virtual host. + /// + /// + /// Discovery plane only. Does not publish or consume ArticleWork AMQP messages. + /// + internal interface IRabbitMqManagementQueueInventory + { + /// + /// Lists all queues in the configured virtual host via + /// GET /api/queues/{vhost} (vhost URL-encoded). + /// + /// Token used to cancel the HTTP request. + /// Queue name and consumer count rows. Never . + Task> ListQueuesAsync(CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementHttpClient.cs b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementHttpClient.cs new file mode 100644 index 00000000..411c8c5a --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementHttpClient.cs @@ -0,0 +1,148 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.RabbitMq.Management +{ + /// + /// HTTP client for RabbitMQ Management API queue inventory + /// (GET /api/queues/{vhost}). + /// + /// + /// Authenticates with HTTP Basic using / + /// . Never logs credentials or the + /// Authorization header. Uses source-generated JSON deserialization. + /// + internal sealed class RabbitMqManagementHttpClient : IRabbitMqManagementQueueInventory + { + /// Named client for Management API calls. + internal const string HttpClientName = "RabbitMqManagement"; + + private readonly HttpClient _httpClient; + private readonly IOptions _options; + + /// Initializes a new Management API client. + public RabbitMqManagementHttpClient( + HttpClient httpClient, + IOptions options) + { + ArgumentNullException.ThrowIfNull(httpClient); + ArgumentNullException.ThrowIfNull(options); + _httpClient = httpClient; + _options = options; + } + + /// + public async Task> ListQueuesAsync( + CancellationToken cancellationToken) + { + var rabbitMq = _options.Value; + var management = rabbitMq.Management + ?? throw new InvalidOperationException("RabbitMQ:Management is required."); + var baseUrl = management.BaseUrl?.Trim(); + if (string.IsNullOrWhiteSpace(baseUrl)) + { + throw new InvalidOperationException("RabbitMQ:Management:BaseUrl is required."); + } + + var username = rabbitMq.Username?.Trim(); + var password = rabbitMq.Password; + if (string.IsNullOrWhiteSpace(username) || password is null) + { + throw new InvalidOperationException( + "RabbitMQ:Username and RabbitMQ:Password are required for Management API access."); + } + + var virtualHost = string.IsNullOrWhiteSpace(rabbitMq.VirtualHost) + ? "/" + : rabbitMq.VirtualHost.Trim(); + var path = $"api/queues/{Uri.EscapeDataString(virtualHost)}"; + + var timeoutSeconds = management.RequestTimeoutSeconds is > 0 + ? management.RequestTimeoutSeconds.Value + : 5; + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds)); + + using var request = new HttpRequestMessage(HttpMethod.Get, path); + request.Headers.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}"))); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + HttpResponseMessage response; + try + { + response = await _httpClient + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"RabbitMQ Management API request timed out after {timeoutSeconds} seconds."); + } + + using (response) + { + if (!response.IsSuccessStatusCode) + { + // Do not include response bodies that may echo credentials or tokens. + throw new HttpRequestException( + $"RabbitMQ Management API returned {(int)response.StatusCode} ({response.StatusCode}) for queue inventory."); + } + + RabbitMqManagementQueueDto[]? payload; + try + { + payload = await response.Content + .ReadFromJsonAsync( + RabbitMqManagementJsonContext.Default.RabbitMqManagementQueueDtoArray, + timeoutCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"RabbitMQ Management API response read timed out after {timeoutSeconds} seconds."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + "RabbitMQ Management API returned a malformed queue inventory payload.", + ex); + } + + if (payload is null) + { + throw new InvalidOperationException( + "RabbitMQ Management API returned an empty queue inventory payload."); + } + + var results = new List(payload.Length); + for (var i = 0; i < payload.Length; i++) + { + var row = payload[i]; + if (row is null || string.IsNullOrWhiteSpace(row.Name)) + { + continue; + } + + var consumers = row.Consumers; + if (consumers < 0) + { + consumers = 0; + } + + results.Add(new RabbitMqManagementQueueInfo(row.Name.Trim(), consumers)); + } + + return results; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementJsonContext.cs b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementJsonContext.cs new file mode 100644 index 00000000..9ee88ebf --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementJsonContext.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace VectorNNTPD.TransitService.RabbitMq.Management +{ + /// + /// Source-generated JSON metadata for RabbitMQ Management API queue inventory responses. + /// + [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] + [JsonSerializable(typeof(RabbitMqManagementQueueDto[]))] + [JsonSerializable(typeof(List))] + internal sealed partial class RabbitMqManagementJsonContext : JsonSerializerContext; +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementQueueDto.cs b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementQueueDto.cs new file mode 100644 index 00000000..69fc7662 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/Management/RabbitMqManagementQueueDto.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace VectorNNTPD.TransitService.RabbitMq.Management +{ + /// + /// Minimal RabbitMQ Management HTTP API queue inventory entry + /// (GET /api/queues/{vhost}). + /// + internal sealed class RabbitMqManagementQueueDto + { + /// Queue name as reported by the Management API (name). + [JsonPropertyName("name")] + public string? Name { get; set; } + + /// Broker-reported AMQP consumer count (consumers). + [JsonPropertyName("consumers")] + public int Consumers { get; set; } + } + + /// Normalized queue inventory row used by availability discovery. + /// Queue name. + /// AMQP consumer count. + internal readonly record struct RabbitMqManagementQueueInfo(string Name, int Consumers); +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/OverviewDbTopology.cs b/src/VectorNNTPD.TransitService/RabbitMq/OverviewDbTopology.cs new file mode 100644 index 00000000..a80bf388 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/OverviewDbTopology.cs @@ -0,0 +1,57 @@ +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// Broker topology for the one-way OverviewDB ingest handoff. + /// + /// + /// Publishes through the default exchange with routing key + /// . No dedicated exchange, bind, reply queue, or RPC + /// topology is declared. + /// + internal static class OverviewDbTopology + { + /// + /// Durable quorum queue consumed independently by OverviewDB. + /// + /// + /// Declared with x-queue-type=quorum. Per-message AMQP expiration is + /// supported on the current broker; there is no queue-wide x-message-ttl. + /// + internal const string QueueName = "overviewdb.queue"; + + /// Default-exchange routing key; equal to . + internal const string RoutingKey = QueueName; + + /// AMQP default exchange (empty name). + internal const string DefaultExchange = ""; + + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName; + + /// Required queue type for the OverviewDB ingest queue. + internal const string QuorumQueueType = RabbitMqArticleRetrievalEndpoints.QuorumQueueType; + + /// Queue arguments applied during declaration. + internal static IReadOnlyDictionary QueueArguments { get; } = + new Dictionary(StringComparer.Ordinal) + { + [QueueTypeArgumentName] = QuorumQueueType, + }; + + /// Per-message AMQP expiration in milliseconds (expiration shortstr). + /// + /// The broker interprets this as milliseconds. RabbitMQ discards expired + /// messages when they reach the head of the queue; Ready counts can still + /// include expired messages queued behind a non-expired head. This is not a + /// queue-wide x-message-ttl. Value restored from the OverviewDB handoff + /// design: 2000. + /// + internal const string ExpirationMilliseconds = "2000"; + + /// + /// AMQP mandatory flag. Unroutable publications are returned to the publisher + /// instead of being confirmed and dropped. + /// + internal const bool Mandatory = true; + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs new file mode 100644 index 00000000..48a983b7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqArticleRetrievalEndpoint.cs @@ -0,0 +1,86 @@ +using RabbitMQ.Client; + +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// One declare-only article-retrieval exchange, durable quorum queue, and binding. + /// + /// Exchange that receives article-retrieval requests. + /// RabbitMQ exchange type. Article-retrieval uses fanout. + /// Whether the exchange survives broker restart. + /// Whether the exchange is auto-deleted when unused. + /// Queue bound to the exchange. + /// Whether the queue survives broker restart. + /// Whether the queue is exclusive to a single connection. + /// Whether the queue is auto-deleted when unused. + /// Routing key used when binding the queue to the exchange. + /// + /// Queue arguments applied during declaration. Durable article-retrieval work queues + /// require x-queue-type=quorum. + /// + internal sealed record RabbitMqArticleRetrievalEndpoint( + string ExchangeName, + string ExchangeType, + bool ExchangeDurable, + bool ExchangeAutoDelete, + string QueueName, + bool QueueDurable, + bool QueueExclusive, + bool QueueAutoDelete, + string RoutingKey, + IReadOnlyDictionary QueueArguments); + + /// + /// Shared factory for NNTPD/BackFiller article-retrieval broker entities. + /// + /// + /// Used by BackFiller provider endpoints. Storage fleet lookup uses + /// (fanout exchange only) and is not declared through + /// this factory. + /// + internal static class RabbitMqArticleRetrievalEndpoints + { + /// Broker argument that selects the RabbitMQ queue type. + internal const string QueueTypeArgumentName = "x-queue-type"; + + /// Required queue type for every durable article-retrieval work queue. + internal const string QuorumQueueType = "quorum"; + + /// + /// Builds a durable fanout exchange, durable quorum queue, and binding that share + /// the normalized as the exchange, queue, and routing key. + /// + internal static RabbitMqArticleRetrievalEndpoint CreateFanoutQuorumBinding(string entityName) + { + var name = RabbitMqTopologyNames.Normalize(entityName); + return new RabbitMqArticleRetrievalEndpoint( + ExchangeName: name, + ExchangeType: ExchangeType.Fanout, + ExchangeDurable: true, + ExchangeAutoDelete: false, + QueueName: name, + QueueDurable: true, + QueueExclusive: false, + QueueAutoDelete: false, + RoutingKey: name, + QueueArguments: new Dictionary(StringComparer.Ordinal) + { + [QueueTypeArgumentName] = QuorumQueueType, + }); + } + } + + /// + /// NNTPD-owned BackFiller-compatible article-retrieval endpoints declared at topology start. + /// + /// + /// Empty: per-backbone backfiller.* topology is declared by BackFiller. + /// Storage fleet lookup declares only the cache.requests fanout exchange (no shared queue). + /// + internal static class ArticleRetrievalTopology + { + /// NNTPD-owned BackFiller-style article-retrieval endpoints declared at startup. + internal static IReadOnlyList Required { get; } = + Array.Empty(); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyLogMessages.cs b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyLogMessages.cs new file mode 100644 index 00000000..6af6eefc --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyLogMessages.cs @@ -0,0 +1,53 @@ +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// Source-generated RabbitMQ topology log messages. Never includes credentials. + internal static partial class RabbitMqTopologyLogMessages + { + [LoggerMessage( + EventId = 2820, + Level = LogLevel.Information, + Message = "Establishing NNTPD RabbitMQ topology (fanoutExchanges={ExchangeCount})")] + public static partial void Establishing(ILogger logger, int ExchangeCount); + + [LoggerMessage( + EventId = 2821, + Level = LogLevel.Information, + Message = "NNTPD RabbitMQ topology established (fanoutExchanges={ExchangeCount}, generation={Generation})")] + public static partial void Established(ILogger logger, int ExchangeCount, long Generation); + + [LoggerMessage( + EventId = 2822, + Level = LogLevel.Error, + Message = "NNTPD RabbitMQ topology declaration failed (endpoint={Endpoint}, exchange={Exchange}, queue={Queue})")] + public static partial void DeclarationFailed( + ILogger logger, + Exception exception, + string Endpoint, + string Exchange, + string Queue); + + [LoggerMessage( + EventId = 2823, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for topology declaration")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2824, + Level = LogLevel.Information, + Message = "Declared OverviewDB handoff queue {Queue} (generation={Generation})")] + public static partial void OverviewQueueDeclared(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2825, + Level = LogLevel.Information, + Message = "Declared cache broadcast exchange {Exchange} (generation={Generation})")] + public static partial void CacheBroadcastDeclared(ILogger logger, string Exchange, long Generation); + + [LoggerMessage( + EventId = 2826, + Level = LogLevel.Information, + Message = "Declared cache requests fanout exchange {Exchange} (generation={Generation})")] + public static partial void CacheRequestsDeclared(ILogger logger, string Exchange, long Generation); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyNames.cs b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyNames.cs new file mode 100644 index 00000000..e693765c --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyNames.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// Deterministic, culture-independent names for RabbitMQ topology entities. + /// + /// + /// Every exchange name, queue name, and routing key is trimmed and converted with + /// . Callers must not assume their input is already + /// normalized. + /// + internal static class RabbitMqTopologyNames + { + /// + /// Normalizes a topology entity name: trim surrounding whitespace, then invariant lower-case. + /// + /// Exchange, queue, routing-key, or other topology entity name. + /// The trimmed invariant-lowercase name. + /// Thrown when is null or whitespace. + internal static string Normalize(string entityName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityName); + return entityName.Trim().ToLowerInvariant(); + } + + /// + /// Builds a two-part entity name {prefix}.{name} after normalizing both parts. + /// + /// Namespace such as backfiller. + /// Unqualified identifier such as a provider name. + /// The composed normalized entity name. + internal static string Compose(string prefix, string name) => + Normalize($"{Normalize(prefix)}.{Normalize(name)}"); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyService.cs b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyService.cs new file mode 100644 index 00000000..a7ea5771 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/RabbitMqTopologyService.cs @@ -0,0 +1,171 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// Declares NNTPD-owned RabbitMQ topology: cache fleet exchanges and OverviewDB. + /// + /// + /// + /// Declares the durable cache.requests fanout exchange (no shared work queue), + /// the durable cache.broadcast fanout exchange, and the OverviewDB ingest queue + /// overviewdb.queue. Per-backbone backfiller.* provider topology is declared + /// by VectorNNTP.BackFiller. Per-NNTPD broadcast queues and StorageServer request queues + /// are ephemeral and owned by their respective consumers. Article-work and storage-lookup + /// reply queues are exclusive auto-delete non-durable classic queues owned by their RPC + /// services. + /// + /// + /// Declaration uses RabbitMQ's normal idempotent declare/bind operations. Existing + /// entities are never deleted, purged, or mutated. An incompatible existing entity + /// fails startup. + /// + /// + public sealed class RabbitMqTopologyService : IApplicationService + { + private readonly IRabbitMqService _rabbitMq; + private readonly ILogger _logger; + private int _started; + + /// Initializes a new instance of the class. + public RabbitMqTopologyService( + IRabbitMqService rabbitMq, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _logger = logger; + } + + /// + public string Name => "RabbitMQTopology"; + + /// + public Task? Execution => null; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + await DeclareRequiredTopologyAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + Interlocked.Exchange(ref _started, 0); + throw; + } + catch (OperationCanceledException) + { + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + /// Declares cache.requests, cache.broadcast, and overviewdb.queue. + /// + internal async Task DeclareRequiredTopologyAsync(CancellationToken cancellationToken) + { + RabbitMqTopologyLogMessages.Establishing(_logger, ExchangeCount: 2); + + if (!_rabbitMq.TryGetCurrent(out var handle)) + { + RabbitMqTopologyLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for topology declaration."); + } + + IRabbitMqTopologyChannel? channel = null; + try + { + channel = await handle.Connection + .CreateTopologyChannelAsync(cancellationToken) + .ConfigureAwait(false); + + await DeclareFanoutExchangeAsync( + channel, + CacheRequestsTopology.ExchangeName, + CacheRequestsTopology.ExchangeDurable, + CacheRequestsTopology.ExchangeAutoDelete, + cancellationToken) + .ConfigureAwait(false); + + await DeclareFanoutExchangeAsync( + channel, + CacheBroadcastTopology.ExchangeName, + CacheBroadcastTopology.ExchangeDurable, + CacheBroadcastTopology.ExchangeAutoDelete, + cancellationToken) + .ConfigureAwait(false); + + await DeclareOverviewDbQueueAsync(channel, cancellationToken).ConfigureAwait(false); + + RabbitMqTopologyLogMessages.Established(_logger, ExchangeCount: 2, handle.Generation); + RabbitMqTopologyLogMessages.CacheRequestsDeclared( + _logger, + CacheRequestsTopology.ExchangeName, + handle.Generation); + RabbitMqTopologyLogMessages.CacheBroadcastDeclared( + _logger, + CacheBroadcastTopology.ExchangeName, + handle.Generation); + RabbitMqTopologyLogMessages.OverviewQueueDeclared(_logger, OverviewDbTopology.QueueName, handle.Generation); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + RabbitMqTopologyLogMessages.DeclarationFailed( + _logger, + ex, + CacheFleetTopology.RequestsExchangeName, + CacheFleetTopology.RequestsExchangeName, + OverviewDbTopology.QueueName); + throw; + } + finally + { + if (channel is not null) + { + await channel.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static Task DeclareFanoutExchangeAsync( + IRabbitMqTopologyChannel channel, + string exchange, + bool durable, + bool autoDelete, + CancellationToken cancellationToken) => + channel.ExchangeDeclareAsync( + exchange, + CacheFleetTopology.FanoutExchangeType, + durable, + autoDelete, + arguments: null, + cancellationToken); + + private static Task DeclareOverviewDbQueueAsync( + IRabbitMqTopologyChannel channel, + CancellationToken cancellationToken) => + channel.QueueDeclareAsync( + OverviewDbTopology.QueueName, + durable: true, + exclusive: false, + autoDelete: false, + OverviewDbTopology.QueueArguments, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/RabbitMq/StorageArticleRetrievalTopology.cs b/src/VectorNNTPD.TransitService/RabbitMq/StorageArticleRetrievalTopology.cs new file mode 100644 index 00000000..0522f102 --- /dev/null +++ b/src/VectorNNTPD.TransitService/RabbitMq/StorageArticleRetrievalTopology.cs @@ -0,0 +1,26 @@ +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.TransitService.RabbitMq +{ + /// + /// Constants for the StorageServer fleet article-presence lookup path. + /// + /// + /// cache.requests is a fanout exchange only. There is no shared quorum work queue + /// and no BackFiller provider semantics. See . + /// + internal static class StorageArticleRetrievalTopology + { + /// Fanout exchange name for fleet article-presence lookups. + internal const string ExchangeName = CacheFleetTopology.RequestsExchangeName; + + /// Alias for used by older ArticleWork test fixtures. + internal const string EntityName = ExchangeName; + + /// + /// Historical JSON backbone label used only in ArticleWork tests that assert + /// Storage is not a BackFiller provider. Not a BackFiller backbone identifier. + /// + internal const string Backbone = "Storage"; + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/IRedisConnection.cs b/src/VectorNNTPD.TransitService/Redis/IRedisConnection.cs new file mode 100644 index 00000000..c1bc9920 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/IRedisConnection.cs @@ -0,0 +1,9 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// Long-lived Redis connection owned by . + public interface IRedisConnection : IAsyncDisposable + { + /// Gets the shared logical database for this connection. + IRedisDatabase GetDatabase(); + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/IRedisConnectionFactory.cs b/src/VectorNNTPD.TransitService/Redis/IRedisConnectionFactory.cs new file mode 100644 index 00000000..f27b4c3e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/IRedisConnectionFactory.cs @@ -0,0 +1,11 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Redis +{ + /// Creates the process-wide Redis connection from configuration. + public interface IRedisConnectionFactory + { + /// Establishes one long-lived Redis connection for the configured topology. + Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/IRedisDatabase.cs b/src/VectorNNTPD.TransitService/Redis/IRedisDatabase.cs new file mode 100644 index 00000000..7ac81725 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/IRedisDatabase.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// Generic Redis key/value operations used by application consumers. + public interface IRedisDatabase + { + /// Sends PING and returns the observed round-trip time. + Task PingAsync(CancellationToken cancellationToken = default); + + /// Returns whether exists. + ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Returns the value of , or when missing. + ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Sets to with a Redis-native TTL. + ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default); + + /// + /// Evaluates a Lua script atomically and returns the integer result. + /// + ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/IRedisService.cs b/src/VectorNNTPD.TransitService/Redis/IRedisService.cs new file mode 100644 index 00000000..c05dde2a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/IRedisService.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// + /// Application Redis infrastructure: one shared connection and generic key operations. + /// + /// + /// Does not contain HistoryDB or CHECK policy. Consumers compose keys and TTLs themselves. + /// + public interface IRedisService + { + /// Gets the shared Redis database after a successful start. + IRedisDatabase Database { get; } + + /// + /// Gets whether Redis is in the cooldown window after a failure. CHECK uses + /// rather than this property so a recovery probe can proceed. + /// + bool IsUnavailable { get; } + + /// + /// Attempts to start a Redis operation. Returns during cooldown + /// or when a recovery probe is already in flight. + /// + /// + /// Set when this caller is the single recovery probe after cooldown. + /// Only that probe may mark the circuit healthy. + /// + bool TryBeginOperation(out bool isRecoveryProbe); + + /// Completes an operation started by . + void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null); + + /// + /// Releases a grant without treating the attempt as a Redis failure. + /// Used for caller/session cancellation. + /// + void AbandonOperation(bool isRecoveryProbe); + + /// Returns whether exists. + ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Returns the value of , or when missing. + ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default); + + /// Sets to with a Redis-native TTL. + ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/RedisAvailability.cs b/src/VectorNNTPD.TransitService/Redis/RedisAvailability.cs new file mode 100644 index 00000000..7e6220dc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/RedisAvailability.cs @@ -0,0 +1,111 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// + /// Short-lived Redis unavailable / recovery-probe state shared by EXISTS, SET, and EVAL. + /// + /// + /// Healthy operations proceed without a lock. After a failure, callers see + /// return until the cooldown expires. + /// Exactly one caller may become the recovery probe. Only that probe may return + /// the circuit to healthy. Caller cancellation abandons the grant and does not + /// open or close the cooldown. CHECK never takes a global lock. + /// + internal sealed class RedisAvailability + { + internal static readonly TimeSpan DefaultCooldown = TimeSpan.FromSeconds(1); + + private readonly TimeProvider _timeProvider; + private readonly long _cooldownTicks; + private long _unavailableUntilTicks; + private int _probeInFlight; + private int _openLogged; + + /// Initializes a new instance of the class. + public RedisAvailability(TimeProvider timeProvider, TimeSpan cooldown) + { + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(cooldown, TimeSpan.Zero); + _timeProvider = timeProvider; + _cooldownTicks = cooldown.Ticks; + } + + /// Gets whether a cooldown is active (a recovery probe is not counted as available). + public bool IsUnavailable + { + get + { + var until = Volatile.Read(ref _unavailableUntilTicks); + return until != 0 && _timeProvider.GetUtcNow().UtcTicks < until; + } + } + + /// + /// Attempts to start a Redis call. is set only for the + /// single caller allowed to probe after cooldown. + /// + public bool TryBegin(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + var until = Volatile.Read(ref _unavailableUntilTicks); + if (until == 0) + { + return true; + } + + if (_timeProvider.GetUtcNow().UtcTicks < until) + { + return false; + } + + if (Interlocked.CompareExchange(ref _probeInFlight, 1, 0) != 0) + { + return false; + } + + isRecoveryProbe = true; + return true; + } + + /// + /// Records a finished Redis call. Only a recovery probe may clear cooldown on success. + /// + public void Complete( + bool isRecoveryProbe, + bool succeeded, + out bool becameUnavailable, + out bool recovered) + { + becameUnavailable = false; + recovered = false; + if (succeeded) + { + if (isRecoveryProbe) + { + var previous = Interlocked.Exchange(ref _unavailableUntilTicks, 0); + Interlocked.Exchange(ref _openLogged, 0); + Interlocked.Exchange(ref _probeInFlight, 0); + recovered = previous != 0; + } + + return; + } + + var until = _timeProvider.GetUtcNow().UtcTicks + _cooldownTicks; + Interlocked.Exchange(ref _unavailableUntilTicks, until); + becameUnavailable = Interlocked.Exchange(ref _openLogged, 1) == 0; + if (isRecoveryProbe) + { + Interlocked.Exchange(ref _probeInFlight, 0); + } + } + + /// Releases a recovery-probe grant without changing cooldown state. + public void Abandon(bool isRecoveryProbe) + { + if (isRecoveryProbe) + { + Interlocked.Exchange(ref _probeInFlight, 0); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/RedisLogMessages.cs b/src/VectorNNTPD.TransitService/Redis/RedisLogMessages.cs new file mode 100644 index 00000000..533e5c02 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/RedisLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// Source-generated Redis infrastructure log messages. + internal static partial class RedisLogMessages + { + [LoggerMessage( + EventId = 2100, + Level = LogLevel.Information, + Message = "Connecting to Redis ({HostCount} endpoint(s), port {Port})")] + public static partial void Connecting(ILogger logger, int HostCount, int Port); + + [LoggerMessage( + EventId = 2101, + Level = LogLevel.Information, + Message = "Redis connection established (PING {ElapsedMs} ms)")] + public static partial void Connected(ILogger logger, long ElapsedMs); + + [LoggerMessage( + EventId = 2102, + Level = LogLevel.Error, + Message = "Redis connection failed during startup")] + public static partial void StartupFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2103, + Level = LogLevel.Information, + Message = "Redis connection disposed")] + public static partial void Disposed(ILogger logger); + + [LoggerMessage( + EventId = 2104, + Level = LogLevel.Error, + Message = "Redis operation failed ({Operation})")] + public static partial void OperationFailed(ILogger logger, Exception exception, string Operation); + + [LoggerMessage( + EventId = 2105, + Level = LogLevel.Warning, + Message = "Redis unavailable; CHECK and writes will return failure until a recovery probe")] + public static partial void BecameUnavailable(ILogger logger, Exception? exception); + + [LoggerMessage( + EventId = 2106, + Level = LogLevel.Information, + Message = "Redis recovered")] + public static partial void Recovered(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/RedisService.cs b/src/VectorNNTPD.TransitService/Redis/RedisService.cs new file mode 100644 index 00000000..7f9e6b0d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/RedisService.cs @@ -0,0 +1,159 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Redis +{ + /// + /// Process-wide Redis infrastructure: one long-lived connection and generic key operations. + /// + public sealed class RedisService : IRedisService, IApplicationService, IAsyncDisposable + { + private readonly IRedisConnectionFactory _connectionFactory; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly RedisAvailability _availability; + private IRedisConnection? _connection; + private IRedisDatabase? _database; + private int _started; + private int _disposed; + + /// Initializes a new instance of the class. + public RedisService( + IRedisConnectionFactory connectionFactory, + IOptions options, + ILogger logger) + : this(connectionFactory, options, logger, TimeProvider.System, RedisAvailability.DefaultCooldown) + { + } + + /// Initializes a new instance with an explicit circuit cooldown (tests). + internal RedisService( + IRedisConnectionFactory connectionFactory, + IOptions options, + ILogger logger, + TimeProvider timeProvider, + TimeSpan circuitCooldown) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + _connectionFactory = connectionFactory; + _options = options; + _logger = logger; + _availability = new RedisAvailability(timeProvider, circuitCooldown); + } + + /// + public string Name => "Redis"; + + /// + public Task? Execution => null; + + /// Gets the number of times a connection was established (tests). + internal int ConnectionCount { get; private set; } + + /// + public IRedisDatabase Database => + _database ?? throw new InvalidOperationException("Redis has not been started."); + + /// + public bool IsUnavailable => _availability.IsUnavailable; + + /// + public bool TryBeginOperation(out bool isRecoveryProbe) => _availability.TryBegin(out isRecoveryProbe); + + /// + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + _availability.Complete(isRecoveryProbe, succeeded, out var becameUnavailable, out var recovered); + if (becameUnavailable) + { + RedisLogMessages.BecameUnavailable(_logger, exception); + } + + if (recovered) + { + RedisLogMessages.Recovered(_logger); + } + } + + /// + public void AbandonOperation(bool isRecoveryProbe) => _availability.Abandon(isRecoveryProbe); + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + var options = _options.Value; + RedisLogMessages.Connecting(_logger, options.Host.Length, options.Port); + try + { + _connection = await _connectionFactory.ConnectAsync(options, cancellationToken).ConfigureAwait(false); + _database = _connection.GetDatabase(); + var ping = await _database.PingAsync(cancellationToken).ConfigureAwait(false); + ConnectionCount++; + RedisLogMessages.Connected(_logger, (long)ping.TotalMilliseconds); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + RedisLogMessages.StartupFailed(_logger, ex); + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + _connection = null; + _database = null; + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await DisposeAsync().ConfigureAwait(false); + } + + /// + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.KeyExistsAsync(key, cancellationToken); + + /// + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.GetAsync(key, cancellationToken); + + /// + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + Database.SetAsync(key, value, expiry, cancellationToken); + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + RedisLogMessages.Disposed(_logger); + } + + _connection = null; + _database = null; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/RedisUnavailableException.cs b/src/VectorNNTPD.TransitService/Redis/RedisUnavailableException.cs new file mode 100644 index 00000000..c66a4bf6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/RedisUnavailableException.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.TransitService.Redis +{ + /// + /// Redis infrastructure failure (unavailable, timeout, or protocol error). + /// Distinct from a successful reply that the key does not exist. + /// + public sealed class RedisUnavailableException : Exception + { + /// Initializes a new instance of the class. + public RedisUnavailableException(string message, Exception? innerException = null) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnection.cs b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnection.cs new file mode 100644 index 00000000..e4869a57 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnection.cs @@ -0,0 +1,37 @@ +using StackExchange.Redis; + +namespace VectorNNTPD.TransitService.Redis +{ + /// Owns one for the process. + internal sealed class StackExchangeRedisConnection : IRedisConnection + { + private readonly ConnectionMultiplexer _multiplexer; + private readonly StackExchangeRedisDatabase _database; + private int _disposed; + + /// Initializes a new instance of the class. + public StackExchangeRedisConnection(ConnectionMultiplexer multiplexer) + { + ArgumentNullException.ThrowIfNull(multiplexer); + _multiplexer = multiplexer; + _database = new StackExchangeRedisDatabase(multiplexer.GetDatabase()); + } + + /// Gets the shared multiplexer (tests). + internal ConnectionMultiplexer Multiplexer => _multiplexer; + + /// + public IRedisDatabase GetDatabase() => _database; + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await _multiplexer.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnectionFactory.cs b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnectionFactory.cs new file mode 100644 index 00000000..f5d8791f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisConnectionFactory.cs @@ -0,0 +1,47 @@ +using StackExchange.Redis; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Redis +{ + /// Creates one from . + public sealed class StackExchangeRedisConnectionFactory : IRedisConnectionFactory + { + /// + public async Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + + var configuration = new ConfigurationOptions + { + AbortOnConnectFail = true, + ConnectRetry = 2, + ConnectTimeout = 5000, + }; + + foreach (var host in options.Host) + { + configuration.EndPoints.Add(host.Trim(), options.Port); + } + + ConnectionMultiplexer multiplexer; + try + { + multiplexer = await ConnectionMultiplexer.ConnectAsync(configuration).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is RedisConnectionException or RedisException or TimeoutException) + { + throw new RedisUnavailableException("Redis ConnectAsync failed.", ex); + } + + if (cancellationToken.IsCancellationRequested) + { + await multiplexer.DisposeAsync().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } + + return new StackExchangeRedisConnection(multiplexer); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisDatabase.cs b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisDatabase.cs new file mode 100644 index 00000000..798514aa --- /dev/null +++ b/src/VectorNNTPD.TransitService/Redis/StackExchangeRedisDatabase.cs @@ -0,0 +1,242 @@ +using System.Runtime.InteropServices; +using StackExchange.Redis; + +namespace VectorNNTPD.TransitService.Redis +{ + /// Adapts to . + internal sealed class StackExchangeRedisDatabase : IRedisDatabase + { + private readonly IDatabase _database; + + /// Initializes a new instance of the class. + public StackExchangeRedisDatabase(IDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + _database = database; + } + + /// + public async Task PingAsync(CancellationToken cancellationToken = default) => + await Await(_database.PingAsync(), cancellationToken).ConfigureAwait(false); + + /// + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + try + { + var pending = _database.KeyExistsAsync(ToRedisKey(key)); + return AwaitExists(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis EXISTS failed.", ex)); + } + } + + /// + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + try + { + var pending = _database.StringGetAsync(ToRedisKey(key)); + return AwaitGet(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis GET failed.", ex)); + } + } + + /// + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) + { + try + { + var pending = _database.StringSetAsync(ToRedisKey(key), ToRedisValue(value), expiry); + return AwaitSet(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis SET failed.", ex)); + } + } + + /// + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(script); + ArgumentNullException.ThrowIfNull(keys); + ArgumentNullException.ThrowIfNull(values); + try + { + var redisKeys = new RedisKey[keys.Length]; + for (var i = 0; i < keys.Length; i++) + { + redisKeys[i] = ToRedisKey(keys[i]); + } + + var redisValues = new RedisValue[values.Length]; + for (var i = 0; i < values.Length; i++) + { + redisValues[i] = ToRedisValue(values[i]); + } + + var pending = _database.ScriptEvaluateAsync(script, redisKeys, redisValues); + return AwaitEval(pending, cancellationToken); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + return ValueTask.FromException(new RedisUnavailableException("Redis EVAL failed.", ex)); + } + } + + private static RedisKey ToRedisKey(ReadOnlyMemory key) + { + // RedisKey is byte[]-backed and has no ReadOnlyMemory implicit. + // HistoryRedisKeys.Create returns a dedicated array; reuse it. + if (MemoryMarshal.TryGetArray(key, out var segment) + && segment.Array is { } array + && segment.Offset == 0 + && segment.Count == array.Length) + { + return (RedisKey)array; + } + + return (RedisKey)key.ToArray(); + } + + private static RedisValue ToRedisValue(ReadOnlyMemory value) => + value.IsEmpty ? RedisValue.EmptyString : (RedisValue)value; + + private static ValueTask AwaitExists(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(pending.Result); + } + + return AwaitExistsSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitExistsSlow(Task pending, CancellationToken cancellationToken) + { + try + { + return await Await(pending, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis EXISTS failed.", ex); + } + } + + private static ValueTask AwaitGet(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(ToBytes(pending.Result)); + } + + return AwaitGetSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitGetSlow(Task pending, CancellationToken cancellationToken) + { + try + { + var value = await Await(pending, cancellationToken).ConfigureAwait(false); + return ToBytes(value); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis GET failed.", ex); + } + } + + private static byte[]? ToBytes(RedisValue value) => value.IsNull ? null : (byte[]?)value; + + private static ValueTask AwaitEval(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return new ValueTask(ToInt64(pending.Result)); + } + + return AwaitEvalSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitEvalSlow(Task pending, CancellationToken cancellationToken) + { + try + { + var result = await Await(pending, cancellationToken).ConfigureAwait(false); + return ToInt64(result); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis EVAL failed.", ex); + } + } + + private static long ToInt64(RedisResult result) + { + if (result.IsNull) + { + return 0; + } + + try + { + return (long)result; + } + catch (InvalidCastException) + { + return 0; + } + } + + private static ValueTask AwaitSet(Task pending, CancellationToken cancellationToken) + { + if (pending.IsCompletedSuccessfully) + { + return ValueTask.CompletedTask; + } + + return AwaitSetSlow(pending, cancellationToken); + } + + private static async ValueTask AwaitSetSlow(Task pending, CancellationToken cancellationToken) + { + try + { + _ = await Await(pending, cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (IsInfrastructureFailure(ex)) + { + throw new RedisUnavailableException("Redis SET failed.", ex); + } + } + + private static Task Await(Task pending, CancellationToken cancellationToken) => + cancellationToken.CanBeCanceled ? pending.WaitAsync(cancellationToken) : pending; + + private static bool IsInfrastructureFailure(Exception exception) => + exception is RedisException or RedisTimeoutException or TimeoutException or IOException; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs b/src/VectorNNTPD.TransitService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs new file mode 100644 index 00000000..1880d8f4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/DenyAllNntpAuthenticationProvider.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// + /// Default authentication provider that rejects all credentials. + /// + /// + /// Register a real in DI when an account backend exists. + /// Cleartext AUTHINFO policy is controlled by Nntpd:AllowCleartextAuth, not this type. + /// + public sealed class DenyAllNntpAuthenticationProvider : INntpAuthenticationProvider + { + /// Shared singleton instance. + public static DenyAllNntpAuthenticationProvider Instance { get; } = new(); + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/INntpAuthenticationProvider.cs b/src/VectorNNTPD.TransitService/Session/Authentication/INntpAuthenticationProvider.cs new file mode 100644 index 00000000..0258a045 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/INntpAuthenticationProvider.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// + /// Verifies NNTP AUTHINFO USER/PASS credentials and returns identity plus authorization. + /// + /// + /// Implementations must not log passwords. The session layer never stores the password after + /// this call returns. Whether AUTHINFO USER/PASS is allowed without TLS is a server policy + /// (Nntpd:AllowCleartextAuth), not a provider concern. + /// + public interface INntpAuthenticationProvider + { + /// Verifies username/password and returns authentication + authorization outcome. + ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default); + + /// + /// Verifies username/password with the effective client address for diagnostics and admission. + /// Default implementation ignores . + /// + ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) => + AuthenticateAsync(username, password, cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/ITransitPeerAuthenticator.cs b/src/VectorNNTPD.TransitService/Session/Authentication/ITransitPeerAuthenticator.cs new file mode 100644 index 00000000..9c6b67ad --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/ITransitPeerAuthenticator.cs @@ -0,0 +1,22 @@ +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// Validates AUTHINFO USER/PASS against an identified Transit peer policy. + public interface ITransitPeerAuthenticator + { + /// + /// Authenticates / against + /// . On success, gains + /// and keeps its peer identity. + /// Reader and posting privileges are not granted. A blank username or password on the + /// peer disables this gate (); it is not a mismatch. + /// + NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs b/src/VectorNNTPD.TransitService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs new file mode 100644 index 00000000..02b50663 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/NewsmasterNntpAuthenticationProvider.cs @@ -0,0 +1,80 @@ +using System.Security.Cryptography; +using System.Text; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// + /// AUTHINFO USER/PASS provider that recognizes only the configured newsmaster account. + /// + /// + /// Success grants posting plus so the + /// newsmaster can submit a well-formed Control: cancel article. It does not grant + /// transit or streaming privileges and does not accept any other Control verb. Passwords are + /// compared in constant time and are never logged. + /// When newsmaster credentials are not configured, returns + /// . + /// + public sealed class NewsmasterNntpAuthenticationProvider : INntpAuthenticationProvider + { + private readonly string _username; + private readonly byte[] _passwordUtf8; + + /// + /// Privileges granted to the configured newsmaster: authenticated reader + poster with + /// cancel-control permission only. + /// + public static NntpAuthorization Privileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false, + controlCancelPermitted: true); + + private NewsmasterNntpAuthenticationProvider(string username, string password) + { + _username = username; + _passwordUtf8 = Encoding.UTF8.GetBytes(password); + } + + /// + /// Returns a newsmaster provider when both credentials are configured; otherwise deny-all. + /// + public static INntpAuthenticationProvider Create(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.NewsmasterUser) + || string.IsNullOrEmpty(options.NewsmasterPassword)) + { + return DenyAllNntpAuthenticationProvider.Instance; + } + + return new NewsmasterNntpAuthenticationProvider( + options.NewsmasterUser.Trim(), + options.NewsmasterPassword); + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + + var provided = Encoding.UTF8.GetBytes(password); + var passwordMatch = provided.Length == _passwordUtf8.Length + && CryptographicOperations.FixedTimeEquals(provided, _passwordUtf8); + var userMatch = string.Equals(username, _username, StringComparison.Ordinal); + if (userMatch && passwordMatch) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, Privileges)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/NntpAuthenticationResult.cs b/src/VectorNNTPD.TransitService/Session/Authentication/NntpAuthenticationResult.cs new file mode 100644 index 00000000..d20121e8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/NntpAuthenticationResult.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.TransitService.Authentication; + +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// Classifies a non-success authentication outcome for wire mapping. + public enum NntpAuthenticationFailureKind + { + /// Success, or unused. + None, + + /// Invalid credentials, disabled account, or mechanism denied. Wire 481. + InvalidCredentials, + + /// Backend/store failure. Wire 503. Must not be presented as a bad password. + TransientFailure, + + /// Session admission rejected. Wire 481 Too many sessions. + TooManySessions, + + /// Source-IP admission rejected. Wire 481 Too many source addresses. + TooManySourceAddresses, + } + + /// Result of a credential verification attempt. + public sealed class NntpAuthenticationResult + { + /// Failed authentication (no identity, no authorization). + public static NntpAuthenticationResult Failed { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.InvalidCredentials, + policy: null); + + /// Backend failure. Distinct from invalid credentials. + public static NntpAuthenticationResult TransientFailure { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TransientFailure, + policy: null); + + /// Admission rejected: too many concurrent sessions. + public static NntpAuthenticationResult TooManySessions { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TooManySessions, + policy: null); + + /// Admission rejected: too many distinct source addresses. + public static NntpAuthenticationResult TooManySourceAddresses { get; } = new( + succeeded: false, + username: null, + authorization: null, + NntpAuthenticationFailureKind.TooManySourceAddresses, + policy: null); + + private NntpAuthenticationResult( + bool succeeded, + string? username, + NntpAuthorization? authorization, + NntpAuthenticationFailureKind failure, + NntpAccountPolicy? policy) + { + Succeeded = succeeded; + Username = username; + Authorization = authorization; + Failure = failure; + Policy = policy; + } + + /// Gets a value indicating whether credentials were accepted. + public bool Succeeded { get; } + + /// Gets the authenticated identity username when is . + public string? Username { get; } + + /// + /// Gets authorization granted by the provider on success. + /// Authentication success does not imply any particular privilege flags. + /// + public NntpAuthorization? Authorization { get; } + + /// Gets the failure classification when is . + public NntpAuthenticationFailureKind Failure { get; } + + /// Gets account policy after a successful MySQL authentication. + public NntpAccountPolicy? Policy { get; } + + /// Creates a successful result with explicit authorization (flags are provider-defined). + public static NntpAuthenticationResult Success( + string username, + NntpAuthorization authorization, + NntpAccountPolicy? policy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(authorization); + return new(succeeded: true, username: username, authorization: authorization, NntpAuthenticationFailureKind.None, policy); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Authentication/TransitPeerAuthenticator.cs b/src/VectorNNTPD.TransitService/Session/Authentication/TransitPeerAuthenticator.cs new file mode 100644 index 00000000..4b846f20 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Authentication/TransitPeerAuthenticator.cs @@ -0,0 +1,31 @@ +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.Authentication +{ + /// Ordinal Transit peer USER/PASS check. No MySQL fall-through. + public sealed class TransitPeerAuthenticator : ITransitPeerAuthenticator + { + /// Shared production instance. + public static TransitPeerAuthenticator Instance { get; } = new(); + + /// + public NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(currentAuthorization); + if (policy is not null && policy.CredentialsMatch(username, password)) + { + return NntpAuthenticationResult.Success( + username, + currentAuthorization.With(authorizedTransit: true, streamingPermitted: true)); + } + + return NntpAuthenticationResult.Failed; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CheckPipeline.cs b/src/VectorNNTPD.TransitService/Session/CheckPipeline.cs new file mode 100644 index 00000000..94b0c1b0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CheckPipeline.cs @@ -0,0 +1,538 @@ +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Per-session bounded CHECK execution window: overlap Redis lookups, emit responses in send order. + /// + /// + /// Only CHECK uses this window. Non-CHECK commands drain it first and then run on the serial + /// dispatcher. A slot is occupied from admission until + /// + /// accepts the response (or the slot is cancelled on shutdown). Slots own a copy of the + /// Message-ID so STREAM/READER parser scratch can be reused. Every CHECK response passes through + /// the emit gate; there is no immediate path that bypasses it. + /// + internal sealed class CheckPipeline + { + /// + /// Outstanding CHECK executions per session. Architectural constant; not configurable. + /// + public const int Depth = 16; + + private readonly NntpSession _session; + private readonly NntpResponseWriter _response; + private readonly ILogger _logger; + private readonly Slot?[] _slots; + private readonly object _gate = new(); + // codeql[cs/missed-using-statement] Owned by CheckPipeline until ShutdownAsync; a using at this declaration would release it when the constructor returns. + private readonly SemaphoreSlim _emitGate = new(1, 1); + // codeql[cs/missed-using-statement] Owned by CheckPipeline until ShutdownAsync; a using at this declaration would cancel in-flight work when the constructor returns. + private readonly CancellationTokenSource _lifetimeCts = new(); + private int _head; + private int _count; + private int _peakOccupied; + private int _shutDown; + private int _inFlightCompletions; + private TaskCompletionSource _progress = NewProgress(); + + /// Initializes a new instance of the class. + public CheckPipeline(NntpSession session, NntpResponseWriter response) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + _session = session; + _response = response; + _logger = NntpCommandLoggers.For(typeof(Check)); + _slots = new Slot?[Depth]; + } + + /// Gets the number of occupied slots (lookup, waiting for prefix, or awaiting TX accept). + public int Occupied + { + get + { + lock (_gate) + { + return _count; + } + } + } + + /// Gets whether the window cannot accept another CHECK without waiting. + public bool IsFull => Occupied >= Depth; + + /// Gets the peak occupied slot count observed on this session. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Gets the number of CompleteWhen tasks still running (at most ). + internal int InFlightCompletions => Volatile.Read(ref _inFlightCompletions); + + /// + /// Test-only: awaited after response composition and before + /// . + /// The slot remains occupied. Used to model blocked TX without filling the writer Channel. + /// + internal Func? BeforeEnqueueProbe { get; set; } + + /// + /// Waits until a slot is free. Does not parse or drop commands; the caller must not read RX + /// while this is pending so the input Pipe can apply backpressure. + /// + public async ValueTask WaitForCapacityAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfShutDown(); + if (_count < Depth) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Starts one authorized, syntactically valid CHECK. The Message-ID is copied into the slot + /// so parser scratch can be reused immediately. + /// + public async ValueTask SubmitAsync( + NntpCommand command, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ThrowIfShutDown(); + var probe = _session.FeedProbe; + probe?.RecordCheck(); + _session.RecordPeerCheck(); + _session.SetActivityState(FeedSessionState.WaitingHistory); + var owned = command.ArgumentMemory(line).ToArray(); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var lookup = Check.LookupAsync(_session, owned, cancellationToken); + var emit = false; + lock (_gate) + { + ThrowIfShutDown(); + if (_count >= Depth) + { + throw new InvalidOperationException("CHECK pipeline accepted a command without a free slot."); + } + + var slot = new Slot + { + MessageId = owned, + StartedTimestamp = started, + Index = (_head + _count) % Depth, + }; + _slots[slot.Index] = slot; + _count++; + _session.BeginCommandWork(); + if (_count > _peakOccupied) + { + _peakOccupied = _count; + } + + if (lookup.IsCompletedSuccessfully) + { + slot.Completed = true; + slot.Result = lookup.Result; + probe?.RecordHistory(lookup.Result, System.Diagnostics.Stopwatch.GetTimestamp() - started); + emit = HeadIsReadyNoLock(); + } + else + { + _ = CompleteWhenAsync(slot, lookup, cancellationToken); + } + } + + if (emit) + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Waits until every slot has been accepted by the TX writer (or cancelled). Required before + /// any non-CHECK command. + /// + public async ValueTask DrainAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Stops emission, cancels in-flight enqueue waits, and waits until every slot is released. + /// Must run before the session disposes the writer. + /// + public async ValueTask ShutdownAsync() + { + if (Interlocked.Exchange(ref _shutDown, 1) == 1) + { + return; + } + + try + { + await _lifetimeCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + lock (_gate) + { + for (var i = 0; i < Depth; i++) + { + if (_slots[i] is { } slot) + { + slot.Cancelled = true; + } + } + + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + try + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.ConfigureAwait(false); + } + } + finally + { + _lifetimeCts.Dispose(); + } + } + + private async Task CompleteWhenAsync( + Slot slot, + ValueTask lookup, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _inFlightCompletions); + try + { + HistoryLookupResult result; + var historyStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + result = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) + { + result = HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + result = HistoryLookupResult.Unavailable; + } + + _session.FeedProbe?.RecordHistory( + result, + System.Diagnostics.Stopwatch.GetTimestamp() - historyStart); + + var emit = false; + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + if (Volatile.Read(ref _shutDown) == 1) + { + slot.Completed = true; + slot.Cancelled = true; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot.Completed = true; + slot.Result = result; + emit = HeadIsReadyNoLock(); + SignalProgressNoLock(); + } + + if (emit) + { + try + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + finally + { + Interlocked.Decrement(ref _inFlightCompletions); + } + } + + private async ValueTask EmitReadySerializedAsync(CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetimeCts.Token); + var token = linked.Token; + await _emitGate.WaitAsync(token).ConfigureAwait(false); + try + { + var emitted = 0; + while (true) + { + Slot? slot; + lock (_gate) + { + if (Volatile.Read(ref _shutDown) == 1) + { + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot = TryBeginEmitHeadNoLock(); + } + + if (slot is null) + { + if (emitted > 0 + && _response.HasCoalescedUnflushed + && Volatile.Read(ref _shutDown) == 0 + && !token.IsCancellationRequested) + { + await _response.FlushCoalescedAsync(token).ConfigureAwait(false); + } + + return; + } + + try + { + await EmitOccupiedAsync(slot, token).ConfigureAwait(false); + lock (_gate) + { + ReleaseHeadIfNoLock(slot); + SignalProgressNoLock(); + } + + emitted++; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + lock (_gate) + { + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + throw; + } + } + } + finally + { + _emitGate.Release(); + } + } + + private async ValueTask EmitOccupiedAsync(Slot slot, CancellationToken cancellationToken) + { + var deferOnDuplicate = Check.DeferOnDuplicate(_session); + var owned = Check.Compose(slot.Result, slot.MessageId, deferOnDuplicate); + if (BeforeEnqueueProbe is { } probe) + { + await probe().ConfigureAwait(false); + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + } + + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + await _response.EnqueueLineAsync(owned, cancellationToken).ConfigureAwait(false); + if (_logger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + _logger, + _session, + "CHECK", + System.Diagnostics.Stopwatch.GetElapsedTime(slot.StartedTimestamp), + statusLine: NntpCommandStatusText.FormatCheck(slot.Result, slot.MessageId, deferOnDuplicate)); + } + } + + private bool HeadIsReadyNoLock() + { + if (_count == 0) + { + return false; + } + + var slot = _slots[_head]; + return slot is { Completed: true, Cancelled: false, Emitting: false }; + } + + private Slot? TryBeginEmitHeadNoLock() + { + if (!HeadIsReadyNoLock()) + { + return null; + } + + var slot = _slots[_head]!; + slot.Emitting = true; + return slot; + } + + private void ReleaseHeadIfNoLock(Slot slot) + { + if (_count == 0 || !ReferenceEquals(_slots[_head], slot)) + { + return; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + slot.Emitting = false; + } + + private void CancelAndRelease(Slot slot) + { + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.Completed = true; + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + } + + private void ReleaseCancelledPrefixNoLock() + { + while (_count > 0) + { + var slot = _slots[_head]; + if (slot is null || slot.Emitting || !slot.Cancelled) + { + break; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + } + } + + private bool IsPresentNoLock(Slot slot) => + (uint)slot.Index < Depth && ReferenceEquals(_slots[slot.Index], slot); + + private void ThrowIfShutDown() + { + if (Volatile.Read(ref _shutDown) == 1) + { + throw new OperationCanceledException("CHECK pipeline has shut down."); + } + } + + private void SignalProgressNoLock() + { + var previous = _progress; + _progress = NewProgress(); + previous.TrySetResult(); + } + + private static TaskCompletionSource NewProgress() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class Slot + { + public required byte[] MessageId { get; init; } + + public required int Index { get; init; } + + public long StartedTimestamp { get; init; } + + public HistoryLookupResult Result { get; set; } + + public bool Completed { get; set; } + + public bool Cancelled { get; set; } + + public bool Emitting { get; set; } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/CommandLogMessages.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/CommandLogMessages.cs new file mode 100644 index 00000000..44617be1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/CommandLogMessages.cs @@ -0,0 +1,153 @@ +using System.Net.Sockets; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Source-generated command RX/TX and handler log messages. + internal static partial class CommandLogMessages + { + [LoggerMessage( + EventId = 1600, + Level = LogLevel.Debug, + Message = "[{Client}] RX: {Command}")] + public static partial void CommandRx(ILogger logger, string Client, string Command); + + [LoggerMessage( + EventId = 1601, + Level = LogLevel.Debug, + Message = "[{Client}] RX rejected: verb={Verb} qualifier={Qualifier} status={Status} [{Detail}]")] + public static partial void CommandRejected( + ILogger logger, + string Client, + NntpVerb Verb, + NntpVerb Qualifier, + NntpParseStatus Status, + string Detail); + + [LoggerMessage( + EventId = 1602, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} executed in {ElapsedSeconds:F3}s")] + public static partial void CommandTx(ILogger logger, string Client, string Command, double ElapsedSeconds); + + [LoggerMessage( + EventId = 1603, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} executed in {ElapsedSeconds:F3}s [{Detail}]")] + public static partial void CommandTxWithDetail( + ILogger logger, + string Client, + string Command, + double ElapsedSeconds, + string Detail); + + [LoggerMessage( + EventId = 1604, + Level = LogLevel.Error, + Message = "[{Client}] {Command} failed")] + public static partial void CommandFailed(ILogger logger, Exception exception, string Client, string Command); + + [LoggerMessage( + EventId = 1605, + Level = LogLevel.Warning, + Message = "[{Client}] TAKETHIS temporary failure; closing connection")] + public static partial void TakeThisTemporaryFailure(ILogger logger, string Client); + + [LoggerMessage( + EventId = 1606, + Level = LogLevel.Error, + Message = "[{Client}] STARTTLS handshake failed")] + public static partial void StartTlsHandshakeFailed(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1607, + Level = LogLevel.Warning, + Message = "[{Client}] COMPRESS DEFLATE refused before activation")] + public static partial void CompressRefusedBeforeActivation(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1608, + Level = LogLevel.Error, + Message = "[{Client}] COMPRESS DEFLATE activation failed after 206")] + public static partial void CompressActivationFailed(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1609, + Level = LogLevel.Debug, + Message = "[{Client}] Peer disconnected during QUIT termination")] + public static partial void QuitPeerDisconnected(ILogger logger, Exception exception, string Client); + + [LoggerMessage( + EventId = 1610, + Level = LogLevel.Error, + Message = "[{Client}] QUIT uncaught during termination: {Type} socket={Socket}")] + public static partial void QuitUncaught( + ILogger logger, + Exception exception, + string Client, + string? Type, + SocketError? Socket); + + [LoggerMessage( + EventId = 1611, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST started peer={PeerId} peerName={PeerName}")] + public static partial void SpeedTestStarted(ILogger logger, string Client, string PeerId, string PeerName); + + [LoggerMessage( + EventId = 1612, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST completed peer={PeerId} peerName={PeerName} bytes={Bytes} durationMs={DurationMs:F3}")] + public static partial void SpeedTestCompleted( + ILogger logger, + string Client, + string PeerId, + string PeerName, + long Bytes, + double DurationMs); + + [LoggerMessage( + EventId = 1613, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST rejected peer={PeerId} peerName={PeerName} reason={Reason}")] + public static partial void SpeedTestRejected( + ILogger logger, + string Client, + string PeerId, + string PeerName, + string Reason); + + [LoggerMessage( + EventId = 1614, + Level = LogLevel.Information, + Message = "[{Client}] SPEEDTEST cancelled peer={PeerId} peerName={PeerName} reason={Reason}")] + public static partial void SpeedTestCancelled( + ILogger logger, + string Client, + string PeerId, + string PeerName, + string Reason); + + [LoggerMessage( + EventId = 1615, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} [{StatusLine}] executed in {ElapsedSeconds:F3}s")] + public static partial void CommandTxWithStatus( + ILogger logger, + string Client, + string Command, + string StatusLine, + double ElapsedSeconds); + + [LoggerMessage( + EventId = 1616, + Level = LogLevel.Debug, + Message = "[{Client}] TX: {Command} [{StatusLine}] executed in {ElapsedSeconds:F3}s [{Detail}]")] + public static partial void CommandTxWithStatusAndDetail( + ILogger logger, + string Client, + string Command, + string StatusLine, + double ElapsedSeconds, + string Detail); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs new file mode 100644 index 00000000..49a929a5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/DefaultNntpCommandCatalog.cs @@ -0,0 +1,127 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Built-in NNTP command inventory, access flags, and display names. + /// + /// See docs/commands.md for the implementation checklist. Dispatch is enum/switch, not a dictionary. + /// + public static class DefaultNntpCommandCatalog + { + /// + /// Expected registry keys for the complete command inventory (stable order for tests). + /// + public static IReadOnlyList InventoryKeys { get; } = + [ + "ARTICLE", + "AUTHINFO PASS", + "AUTHINFO SASL", + "AUTHINFO USER", + "BODY", + "CAPABILITIES", + "CHECK", + "COMPRESS", + "DATE", + "GROUP", + "HDR", + "HEAD", + "HELP", + "IHAVE", + "LAST", + "LIST", + "LIST ACTIVE", + "LIST COUNTS", + "LIST HEADERS", + "LIST MOTD", + "LIST NEWSGROUPS", + "LIST OVERVIEW.FMT", + "LISTGROUP", + "MODE READER", + "MODE STREAM", + "NEXT", + "OVER", + "POST", + "QUIT", + "SPEEDTEST", + "STARTTLS", + "STAT", + "TAKETHIS", + ]; + + /// Registered keys including the internal BENCHIT facility. + public static IReadOnlyList GetRegisteredKeys() + { + var keys = new List(InventoryKeys.Count + 1); + keys.AddRange(InventoryKeys); + keys.Add("BENCHIT"); + keys.Sort(StringComparer.Ordinal); + return keys; + } + + /// Returns access flags for a syntactically valid built-in command. + public static NntpCommandAccess GetAccess(NntpVerb verb, NntpVerb qualifier) + { + var readerAccess = NntpCommandAccess.RequiresAuthentication | NntpCommandAccess.RequiresReader; + return verb switch + { + NntpVerb.Capabilities or NntpVerb.Help or NntpVerb.Date or NntpVerb.Quit + or NntpVerb.StartTls or NntpVerb.Compress or NntpVerb.BenchIt + or NntpVerb.AuthInfo => NntpCommandAccess.Public, + NntpVerb.Mode when qualifier == NntpVerb.Reader => NntpCommandAccess.Public, + NntpVerb.Mode when qualifier == NntpVerb.Stream => NntpCommandAccess.RequiresStreaming, + NntpVerb.Check or NntpVerb.TakeThis or NntpVerb.Ihave or NntpVerb.SpeedTest + => NntpCommandAccess.RequiresTransit, + NntpVerb.Post => NntpCommandAccess.RequiresAuthentication + | NntpCommandAccess.RequiresReader, + NntpVerb.List or NntpVerb.ListGroup or NntpVerb.Group + or NntpVerb.Article or NntpVerb.Head or NntpVerb.Body + or NntpVerb.Stat or NntpVerb.Last or NntpVerb.Next or NntpVerb.Over + or NntpVerb.Hdr => readerAccess, + _ => NntpCommandAccess.Public, + }; + } + + /// Returns the static uppercase display name used by TX completion logs. + public static string DisplayName(NntpVerb verb, NntpVerb qualifier) + { + return (verb, qualifier) switch + { + (NntpVerb.Mode, NntpVerb.Reader) => "MODE READER", + (NntpVerb.Mode, NntpVerb.Stream) => "MODE STREAM", + (NntpVerb.AuthInfo, NntpVerb.User) => "AUTHINFO USER", + (NntpVerb.AuthInfo, NntpVerb.Pass) => "AUTHINFO PASS", + (NntpVerb.AuthInfo, NntpVerb.Sasl) => "AUTHINFO SASL", + (NntpVerb.List, NntpVerb.Active) => "LIST ACTIVE", + (NntpVerb.List, NntpVerb.Counts) => "LIST COUNTS", + (NntpVerb.List, NntpVerb.Headers) => "LIST HEADERS", + (NntpVerb.List, NntpVerb.Motd) => "LIST MOTD", + (NntpVerb.List, NntpVerb.Newsgroups) => "LIST NEWSGROUPS", + (NntpVerb.List, NntpVerb.OverviewFmt) => "LIST OVERVIEW.FMT", + (NntpVerb.Article, _) => "ARTICLE", + (NntpVerb.AuthInfo, _) => "AUTHINFO", + (NntpVerb.Body, _) => "BODY", + (NntpVerb.Capabilities, _) => "CAPABILITIES", + (NntpVerb.Check, _) => "CHECK", + (NntpVerb.Compress, _) => "COMPRESS", + (NntpVerb.Date, _) => "DATE", + (NntpVerb.Group, _) => "GROUP", + (NntpVerb.Hdr, _) => "HDR", + (NntpVerb.Head, _) => "HEAD", + (NntpVerb.Help, _) => "HELP", + (NntpVerb.Ihave, _) => "IHAVE", + (NntpVerb.Last, _) => "LAST", + (NntpVerb.List, _) => "LIST", + (NntpVerb.ListGroup, _) => "LISTGROUP", + (NntpVerb.Mode, _) => "MODE", + (NntpVerb.Next, _) => "NEXT", + (NntpVerb.Over, _) => "OVER", + (NntpVerb.Post, _) => "POST", + (NntpVerb.Quit, _) => "QUIT", + (NntpVerb.SpeedTest, _) => "SPEEDTEST", + (NntpVerb.StartTls, _) => "STARTTLS", + (NntpVerb.Stat, _) => "STAT", + (NntpVerb.TakeThis, _) => "TAKETHIS", + (NntpVerb.BenchIt, _) => "BENCHIT", + _ => "INVALID", + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/DirectTxPipeLease.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/DirectTxPipeLease.cs new file mode 100644 index 00000000..8731e4da --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/DirectTxPipeLease.cs @@ -0,0 +1,45 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// DIAGNOSTIC-ONLY exclusive lease of the existing TX . + /// + /// + /// SPEEDTEST holds this while writing the synthetic payload so the ResponseWriter pump + /// cannot produce to the same pipe. Dispose returns ownership to the Channel pump. + /// Does not construct a pipe, socket, or transport. + /// + internal sealed class DirectTxPipeLease : IAsyncDisposable + { + private readonly NntpResponseWriter _writer; + private int _disposed; + + internal DirectTxPipeLease(NntpResponseWriter writer) + { + ArgumentNullException.ThrowIfNull(writer); + _writer = writer; + } + + /// Gets the existing session TX (not a new pipe). + public PipeWriter PipeWriter => _writer.UnderlyingOutput; + + /// Gets direct-path FlushAsync calls recorded on this writer. + public long FlushCount => _writer.DirectPipeFlushCount; + + /// Copies to the existing TX pipe and flushes. + public ValueTask WriteAndFlushAsync(ReadOnlyMemory payload, CancellationToken cancellationToken = default) => + _writer.WriteDirectToOutputAsync(payload, cancellationToken); + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writer.ReleaseDirectTxPipe(); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/IHaveLogMessages.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/IHaveLogMessages.cs new file mode 100644 index 00000000..f7fa25e0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/IHaveLogMessages.cs @@ -0,0 +1,72 @@ +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Source-generated IHAVE receive/queue diagnostics. Does not log bodies or headers. + internal static partial class IHaveLogMessages + { + [LoggerMessage( + EventId = 1700, + Level = LogLevel.Information, + Message = "[{Client}] IHAVE {MessageId} size={Size} pipeReads={PipeReads} receiveMs={ReceiveMs:F1} queued={Queued}")] + public static partial void Received( + ILogger logger, + string Client, + string MessageId, + int Size, + int PipeReads, + double ReceiveMs, + bool Queued); + + [LoggerMessage( + EventId = 1710, + Level = LogLevel.Warning, + Message = "IHAVE deferred: TransitQueueMemoryLimit exhausted (queued {QueuedBytes}/{MemoryLimit} bytes, {Count} articles)")] + public static partial void QueueBudgetExhausted( + ILogger logger, + long QueuedBytes, + long MemoryLimit, + int Count); + } + + /// Rate-limits IHAVE queue-budget exhaustion warnings. + internal static class IHaveAdmissionLog + { + internal static TimeSpan WarningInterval { get; set; } = TimeSpan.FromSeconds(5); + + private static long _nextWarningTicks; + + /// Emits at most one budget-exhausted warning per . + internal static void BudgetExhausted(ILogger logger, IArticleIngestionQueue queue) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(queue); + + var now = DateTime.UtcNow.Ticks; + var next = Volatile.Read(ref _nextWarningTicks); + if (now < next) + { + return; + } + + var until = now + WarningInterval.Ticks; + if (Interlocked.CompareExchange(ref _nextWarningTicks, until, next) != next) + { + return; + } + + IHaveLogMessages.QueueBudgetExhausted( + logger, + queue.QueuedBytes, + queue.MemoryLimitBytes, + queue.Count); + } + + /// Resets rate-limit state for tests. + internal static void ResetForTests() + { + Volatile.Write(ref _nextWarningTicks, 0); + WarningInterval = TimeSpan.FromSeconds(5); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs new file mode 100644 index 00000000..ccb44c74 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxChunkBudget.cs @@ -0,0 +1,38 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Chunk budget for the shared article TX data plane (Phase 1). + /// + /// + /// Architecture: default 64 KiB; allowed production range 64–256 KiB + /// (docs/NNTP-DATA-PLANE-IMPLEMENTATION.md). Kept as an internal seam for Phase 6 + /// measurement rather than a full configuration surface. + /// + public static class NntpArticleTxChunkBudget + { + /// Default owned TX chunk size (64 KiB). + public const int DefaultBytes = 64 * 1024; + + /// Minimum production chunk size (64 KiB). + public const int MinBytes = 64 * 1024; + + /// Maximum production chunk size (256 KiB). + public const int MaxBytes = 256 * 1024; + + /// Returns when is in the production range. + public static bool IsProductionRange(int chunkBytes) => + chunkBytes is >= MinBytes and <= MaxBytes; + + /// Throws if is outside the production range. + public static void ThrowIfNotProductionRange(int chunkBytes) + { + if (!IsProductionRange(chunkBytes)) + { + throw new ArgumentOutOfRangeException( + nameof(chunkBytes), + chunkBytes, + $"Article TX chunk budget must be between {MinBytes} and {MaxBytes} bytes."); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxFraming.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxFraming.cs new file mode 100644 index 00000000..de55f8e3 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpArticleTxFraming.cs @@ -0,0 +1,91 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Wire framing kind for shared article TX. Mode-independent: callers choose framing; the TX + /// primitive does not branch on MODE READER vs MODE STREAM and does not perform article lookup. + /// + /// + /// Customer* names mean RFC 3977 ARTICLE/BODY response shapes for externally supplied + /// bytes — not an NNTPD customer article store. + /// + public enum NntpArticleTxFrameKind + { + /// ARTICLE wire shape: 220 n|0 <message-id>\r\n + full article + terminator. + CustomerArticle = 0, + + /// Peer TAKETHIS-style feed: TAKETHIS <message-id>\r\n + article + terminator. + PeerTakeThis = 1, + + /// BODY wire shape: 222 n|0 <message-id>\r\n + body only + terminator. + CustomerBody = 2, + + /// HEAD wire shape: 221 n|0 <message-id>\r\n + headers only + terminator. + CustomerHead = 3, + } + + /// Framing parameters for shared article TX via . + public readonly struct NntpArticleTxFraming + { + /// Initializes framing. + /// Framing kind. + /// Message-id (including angle brackets when applicable). + /// + /// Article number for the status line, or 0 when the article was selected by message-id + /// (RFC 3977 §6.2.1 / §6.2.3). + /// + public NntpArticleTxFraming(NntpArticleTxFrameKind kind, string messageId, long articleNumber = 0) + { + ArgumentException.ThrowIfNullOrEmpty(messageId); + ArgumentOutOfRangeException.ThrowIfNegative(articleNumber); + Kind = kind; + MessageId = messageId; + ArticleNumber = articleNumber; + } + + /// Gets the framing kind. + public NntpArticleTxFrameKind Kind { get; } + + /// Gets the message-id (including angle brackets when applicable). + public string MessageId { get; } + + /// Gets the article number used in the status line (0 for message-id selection). + public long ArticleNumber { get; } + + /// Creates ARTICLE-style framing (220 n mid). + public static NntpArticleTxFraming CustomerArticle(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerArticle, messageId, articleNumber); + + /// Creates BODY-style framing (222 n mid). + public static NntpArticleTxFraming CustomerBody(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerBody, messageId, articleNumber); + + /// Creates HEAD-style framing (221 n mid). + public static NntpArticleTxFraming CustomerHead(string messageId, long articleNumber = 0) => + new(NntpArticleTxFrameKind.CustomerHead, messageId, articleNumber); + + /// Creates TAKETHIS-style framing (TAKETHIS mid). + public static NntpArticleTxFraming PeerTakeThis(string messageId) => + new(NntpArticleTxFrameKind.PeerTakeThis, messageId); + + /// Builds the ASCII status/command header line including trailing CRLF. + public byte[] BuildHeaderBytes() + { + var number = ArticleNumber.ToString(CultureInfo.InvariantCulture); + return Kind switch + { + NntpArticleTxFrameKind.CustomerArticle => + Encoding.ASCII.GetBytes("220 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.CustomerHead => + Encoding.ASCII.GetBytes("221 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.CustomerBody => + Encoding.ASCII.GetBytes("222 " + number + " " + MessageId + "\r\n"), + NntpArticleTxFrameKind.PeerTakeThis => + Encoding.ASCII.GetBytes("TAKETHIS " + MessageId + "\r\n"), + _ => throw new ArgumentOutOfRangeException(nameof(Kind), Kind, "Unknown article TX framing kind."), + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpAscii.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpAscii.cs new file mode 100644 index 00000000..7bf96ec4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpAscii.cs @@ -0,0 +1,89 @@ +using System.Runtime.CompilerServices; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// ASCII helpers for NNTP command parsing. RFC 3977 §3.1 SP/TAB. No culture. + internal static class NntpAscii + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsWhitespace(byte b) => b == (byte)' ' || b == (byte)'\t'; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte FoldUpper(byte b) + { + if (b >= (byte)'a' && b <= (byte)'z') + { + return (byte)(b - 32); + } + + return b; + } + + public static bool EqualsFolded(ReadOnlySpan value, ReadOnlySpan upperAscii) + { + if (value.Length != upperAscii.Length) + { + return false; + } + + for (int i = 0; i < value.Length; i++) + { + if (FoldUpper(value[i]) != upperAscii[i]) + { + return false; + } + } + + return true; + } + + public static int SkipWhitespace(ReadOnlySpan line, int index) + { + while (index < line.Length && IsWhitespace(line[index])) + { + index++; + } + + return index; + } + + public static int SkipToken(ReadOnlySpan line, int index) + { + while (index < line.Length && !IsWhitespace(line[index])) + { + index++; + } + + return index; + } + + public static int TrimEndWhitespace(ReadOnlySpan line, int start, int end) + { + while (end > start && IsWhitespace(line[end - 1])) + { + end--; + } + + return end; + } + + public static int CountTokens(ReadOnlySpan line, int start, int end) + { + int count = 0; + int i = start; + while (i < end) + { + if (IsWhitespace(line[i])) + { + i++; + continue; + } + + count++; + i = SkipToken(line, i); + } + + return count; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommand.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommand.cs new file mode 100644 index 00000000..8ba280cc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommand.cs @@ -0,0 +1,85 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Immutable parse result: enums and indexes only. Safe to retain across await. + /// Argument bytes live in the current command buffer, not in this struct. + /// + public readonly struct NntpCommand + { + /// Initializes a new instance of the struct. + public NntpCommand( + NntpVerb verb, + NntpVerb qualifier, + int argumentStart, + int argumentLength, + int tokenCount, + NntpParseStatus status) + { + Verb = verb; + Qualifier = qualifier; + ArgumentStart = argumentStart; + ArgumentLength = argumentLength; + TokenCount = tokenCount; + Status = status; + } + + /// Gets the classified verb. + public NntpVerb Verb { get; } + + /// Gets the qualifier (MODE/AUTHINFO/LIST), or . + public NntpVerb Qualifier { get; } + + /// Gets the argument start index in the current command buffer. + public int ArgumentStart { get; } + + /// Gets the argument length in the current command buffer. + public int ArgumentLength { get; } + + /// Gets the number of tokens in the argument region. + public int TokenCount { get; } + + /// Gets the syntactic status. Only may reach a handler. + public NntpParseStatus Status { get; } + + /// Gets whether the command may be dispatched to a normal handler. + public bool IsValid => Status == NntpParseStatus.Ok; + + /// + /// A command line that exceeded . + /// Not a parsed prefix. Handlers must not run. + /// + public static NntpCommand OverlongLine { get; } = new( + NntpVerb.None, + NntpVerb.None, + argumentStart: 0, + argumentLength: 0, + tokenCount: 0, + NntpParseStatus.CommandLineTooLong); + + /// Returns the argument bytes from a still-valid command buffer. + public ReadOnlySpan ArgumentSpan(ReadOnlySpan line) + { + if (ArgumentLength <= 0 + || ArgumentStart < 0 + || ArgumentStart + ArgumentLength > line.Length) + { + return ReadOnlySpan.Empty; + } + + return line.Slice(ArgumentStart, ArgumentLength); + } + + /// Returns the argument bytes from a still-valid command buffer. + public ReadOnlyMemory ArgumentMemory(ReadOnlyMemory line) + { + if (ArgumentLength <= 0 + || ArgumentStart < 0 + || ArgumentStart + ArgumentLength > line.Length) + { + return ReadOnlyMemory.Empty; + } + + return line.Slice(ArgumentStart, ArgumentLength); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandAccess.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandAccess.cs new file mode 100644 index 00000000..637f50f8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandAccess.cs @@ -0,0 +1,31 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Authorization requirements evaluated by the command dispatcher before invoking a handler. + [Flags] + public enum NntpCommandAccess + { + /// Command is allowed without authentication (public/pre-auth). + Public = 0, + + /// Peer must be authenticated (480 if not). + RequiresAuthentication = 1, + + /// Peer must have reader authorization (502 if authenticated but not authorized). + RequiresReader = 2, + + /// Peer must have transit authorization (502 if not). + RequiresTransit = 4, + + /// Peer must have posting permission (in addition to reader where applicable). + RequiresPosting = 8, + + /// Session must be in reader mode. + RequiresReaderMode = 16, + + /// Session must be in stream mode. + RequiresStreamMode = 32, + + /// Peer must have streaming permission to enter stream/transit mode. + RequiresStreaming = 64, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandContext.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandContext.cs new file mode 100644 index 00000000..39c7570f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandContext.cs @@ -0,0 +1,77 @@ +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Per-command execution context supplied to handlers after syntactic validation. + public sealed class NntpCommandContext + { + /// Initializes a new instance of the class. + public NntpCommandContext( + NntpSession session, + NntpCommand command, + ReadOnlyMemory line, + NntpResponseWriter response, + NntpMultilineReadResult? preReadArticle = null) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + if (!command.IsValid) + { + throw new ArgumentException("Handlers receive only syntactically valid commands.", nameof(command)); + } + + Session = session; + Command = command; + Line = line; + Response = response; + PreReadArticle = preReadArticle; + } + + /// Gets the owning session. + public NntpSession Session { get; } + + /// Gets the validated command (indexes into ). + public NntpCommand Command { get; } + + /// + /// Gets the current command-line buffer. Valid until the next command is parsed. + /// Pipelined CHECK copies the Message-ID before the next parse; do not retain this + /// memory across a later command. + /// + public ReadOnlyMemory Line { get; } + + /// Gets the response writer for this command. + public NntpResponseWriter Response { get; } + + /// + /// Semantic first/status line for TX Debug logging. Set only when Debug is enabled; + /// first assignment wins. Never derived from wire bytes. + /// + internal string? StatusLine { get; set; } + + /// + /// Gets an article already consumed by the continuous RX scanner, when the session + /// pre-read a TAKETHIS body so the handler must not read the Pipe again. + /// + public NntpMultilineReadResult? PreReadArticle { get; } + + /// + /// Optional TX completion detail set by the handler after successful work + /// (for example negotiated TLS parameters after STARTTLS). + /// + public string? CompletionDetail { get; set; } + + /// Gets the underlying transport connection. + public INntpConnection Connection => Session.Connection; + + /// Gets the argument bytes from the current command buffer. + public ReadOnlySpan ArgumentSpan => Command.ArgumentSpan(Line.Span); + + /// + /// Gets the argument bytes as memory. Safe across await only while this command's + /// parser scratch remains the current line; CHECK pipeline slots copy the Message-ID first. + /// + public ReadOnlyMemory ArgumentMemory => Command.ArgumentMemory(Line); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandDispatcher.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandDispatcher.cs new file mode 100644 index 00000000..f5af1071 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandDispatcher.cs @@ -0,0 +1,426 @@ +using System.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Rejects invalid syntax, applies session/authorization gates, then switch-dispatches the handler. + /// + /// + /// Validation order: parse (caller) → syntax reject → authentication → authorization → mode → handler. + /// Command RX/TX completion records are owned by the session (RX) and command modules (TX), not this type. + /// + public sealed class NntpCommandDispatcher + { + private readonly ILogger _unknownCommandLogger; + + /// Initializes a new instance of the class. + public NntpCommandDispatcher(ILoggerFactory? loggerFactory = null) + { + NntpCommandLoggers.Configure(loggerFactory); + _unknownCommandLogger = NntpCommandLoggers.For(typeof(NntpCommandExecution)); + } + + /// Dispatches one already-parsed command. Invalid syntax never reaches a handler. + public async ValueTask DispatchAsync( + NntpSession session, + NntpCommand command, + ReadOnlyMemory line, + NntpResponseWriter response, + CancellationToken cancellationToken, + NntpMultilineReadResult? preReadArticle = null) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + + if (!command.IsValid) + { + await RejectInvalidAsync(session, command, response, preReadArticle, cancellationToken) + .ConfigureAwait(false); + return; + } + + var access = DefaultNntpCommandCatalog.GetAccess(command.Verb, command.Qualifier); + var display = DefaultNntpCommandCatalog.DisplayName(command.Verb, command.Qualifier); + var logger = LoggerFor(command); + var authz = session.Authorization; + var gateStarted = Stopwatch.GetTimestamp(); + + // Role isolation: CAPABILITIES is authoritative; opposite-role commands are unavailable. + if (session.ServerRole == NntpServerRole.Reader + && (access.HasFlag(NntpCommandAccess.RequiresTransit) + || access.HasFlag(NntpCommandAccess.RequiresStreaming))) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + "command not available on this server", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (session.ServerRole == NntpServerRole.Transit + && access.HasFlag(NntpCommandAccess.RequiresReader)) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + "command not available on this server", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresAuthentication) && !authz.IsAuthenticated) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.AuthenticationRequired, + NntpResponseStatus.AuthenticationRequired, + "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresReader) && !authz.AuthorizedReader) + { + var wire = authz.IsAuthenticated + ? NntpResponses.PermissionDenied + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.PermissionDenied + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "permission denied" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresTransit) && !authz.AuthorizedTransit) + { + var wire = authz.IsAuthenticated + ? NntpResponses.PermissionDenied + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.PermissionDenied + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "permission denied" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresPosting) && !authz.PostingPermitted) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.PostingNotPermitted, + NntpResponseStatus.PostingNotPermitted, + "posting not permitted", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresStreaming) && !authz.StreamingPermitted) + { + var wire = authz.IsAuthenticated + ? NntpResponses.StreamingNotPermitted + : NntpResponses.AuthenticationRequired; + var status = authz.IsAuthenticated + ? NntpResponseStatus.StreamingNotPermitted + : NntpResponseStatus.AuthenticationRequired; + await WriteGatedResponseAsync( + response, + logger, + session, + display, + wire, + status, + authz.IsAuthenticated ? "streaming not permitted" : "authentication required", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresReaderMode) && session.Mode != NntpSessionMode.Reader) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.NotInReaderMode, + NntpResponseStatus.NotInReaderMode, + "not in reader mode", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (access.HasFlag(NntpCommandAccess.RequiresStreamMode) && session.Mode != NntpSessionMode.Stream) + { + await WriteGatedResponseAsync( + response, + logger, + session, + display, + NntpResponses.NotInStreamMode, + NntpResponseStatus.NotInStreamMode, + "not in stream mode", + gateStarted, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var context = new NntpCommandContext(session, command, line, response, preReadArticle); + try + { + await InvokeHandlerAsync(context, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + if (session.Connection.IsCompleted || session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + try + { + await response + .WriteLineAsync(NntpResponses.CommandFailed, cancellationToken) + .ConfigureAwait(false); + } + catch (InvalidOperationException) + { + // Output writer completed between the check and the write. + } + } + } + + private async ValueTask RejectInvalidAsync( + NntpSession session, + NntpCommand command, + NntpResponseWriter response, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + if (command.Verb == NntpVerb.TakeThis && preReadArticle is null) + { + try + { + NntpArticlePolicyCapture.Capture( + session, + session.ArticleIngestion.MaxArticleBytes, + out var maxArticleBytes, + out _); + _ = await NntpMultilineDataReader + .ReadArticleAsync(session.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + } + + var started = Stopwatch.GetTimestamp(); + var display = command.Status is NntpParseStatus.Empty + or NntpParseStatus.UnknownVerb + or NntpParseStatus.CommandLineTooLong + ? "INVALID" + : DefaultNntpCommandCatalog.DisplayName(command.Verb, command.Qualifier); + var (wire, detail, statusLine) = Rejection(command); + await response.WriteLineAsync(wire, cancellationToken).ConfigureAwait(false); + session.LogCommandRejected(command, detail); + if (_unknownCommandLogger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + _unknownCommandLogger, + session, + display, + Stopwatch.GetElapsedTime(started), + detail, + statusLine); + } + } + + private static async ValueTask WriteGatedResponseAsync( + NntpResponseWriter response, + ILogger logger, + NntpSession session, + string display, + ReadOnlyMemory wire, + string statusLine, + string detail, + long startedTimestamp, + CancellationToken cancellationToken) + { + await response.WriteLineAsync(wire, cancellationToken).ConfigureAwait(false); + if (logger.IsEnabled(LogLevel.Debug)) + { + NntpCommandExecution.WriteCompletion( + logger, + session, + display, + Stopwatch.GetElapsedTime(startedTimestamp), + detail, + statusLine); + } + } + + private static (ReadOnlyMemory Wire, string Detail, string StatusLine) Rejection(NntpCommand command) + { + return command.Status switch + { + NntpParseStatus.CommandLineTooLong => + (NntpResponses.SyntaxError, "syntax error", NntpResponseStatus.SyntaxError), + NntpParseStatus.UnknownVerb => + (NntpResponses.UnknownCommand, "unknown command", NntpResponseStatus.UnknownCommand), + NntpParseStatus.UnknownQualifier => + (NntpResponses.UnknownCommandVariant, "unknown variant", NntpResponseStatus.UnknownCommandVariant), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.User => + (NntpResponses.AuthinfoUserRequiresUsername, "syntax error", NntpResponseStatus.AuthinfoUserRequiresUsername), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.Pass => + (NntpResponses.AuthinfoPassRequiresPassword, "syntax error", NntpResponseStatus.AuthinfoPassRequiresPassword), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.AuthInfo + && command.Qualifier == NntpVerb.Sasl => + (NntpResponses.SaslMechanismRequired, "syntax error", NntpResponseStatus.SaslMechanismRequired), + NntpParseStatus.MissingArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressRequiresAlgorithm, "syntax error", NntpResponseStatus.CompressRequiresAlgorithm), + NntpParseStatus.InvalidArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressAlgorithmSyntaxInvalid, "syntax error", NntpResponseStatus.CompressAlgorithmSyntaxInvalid), + NntpParseStatus.ExtraArgument when command.Verb == NntpVerb.Compress => + (NntpResponses.CompressRequiresAlgorithm, "syntax error", NntpResponseStatus.CompressRequiresAlgorithm), + _ => (NntpResponses.SyntaxError, "syntax error", NntpResponseStatus.SyntaxError), + }; + } + + private static ILogger LoggerFor(NntpCommand command) + { + var type = command.Verb switch + { + NntpVerb.Check => typeof(Check), + NntpVerb.TakeThis => typeof(TakeThis), + NntpVerb.Mode => typeof(Mode), + NntpVerb.AuthInfo => typeof(AuthInfo), + NntpVerb.Quit => typeof(Quit), + NntpVerb.Help => typeof(Help), + NntpVerb.Date => typeof(Date), + NntpVerb.Capabilities => typeof(Capabilities), + NntpVerb.StartTls => typeof(StartTls), + NntpVerb.Compress => typeof(Compress), + NntpVerb.List => typeof(List), + NntpVerb.ListGroup => typeof(ListGroup), + NntpVerb.Group => typeof(Group), + NntpVerb.Article => typeof(Article), + NntpVerb.Head => typeof(Article), + NntpVerb.Body => typeof(Article), + NntpVerb.Stat => typeof(Article), + NntpVerb.Last => typeof(Last), + NntpVerb.Next => typeof(Next), + NntpVerb.Over => typeof(Over), + NntpVerb.Hdr => typeof(Hdr), + NntpVerb.Post => typeof(Post), + NntpVerb.Ihave => typeof(IHave), + NntpVerb.BenchIt => typeof(BenchIt), + NntpVerb.SpeedTest => typeof(Commands.SpeedTest), + _ => typeof(NntpCommandExecution), + }; + + return NntpCommandLoggers.For(type); + } + + private static ValueTask InvokeHandlerAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + return (context.Command.Verb, context.Command.Qualifier) switch + { + (NntpVerb.Capabilities, _) => Capabilities.HandleAsync(context, cancellationToken), + (NntpVerb.Mode, NntpVerb.Reader) => Mode.HandleReaderAsync(context, cancellationToken), + (NntpVerb.Mode, NntpVerb.Stream) => Mode.HandleStreamAsync(context, cancellationToken), + (NntpVerb.Help, _) => Help.HandleAsync(context, cancellationToken), + (NntpVerb.Date, _) => Date.HandleAsync(context, cancellationToken), + (NntpVerb.Quit, _) => Quit.HandleAsync(context, cancellationToken), + (NntpVerb.StartTls, _) => StartTls.HandleAsync(context, context.Session.CertificateProvider, cancellationToken), + (NntpVerb.Compress, _) => Compress.HandleAsync(context, cancellationToken), + (NntpVerb.BenchIt, _) => BenchIt.HandleAsync(context, cancellationToken), + (NntpVerb.SpeedTest, _) => Commands.SpeedTest.HandleAsync(context, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.User) => AuthInfo.HandleUserAsync( + context, context.Session.AuthenticationProvider, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.Pass) => AuthInfo.HandlePassAsync( + context, context.Session.AuthenticationProvider, cancellationToken), + (NntpVerb.AuthInfo, NntpVerb.Sasl) => AuthInfo.HandleSaslAsync(context, cancellationToken), + (NntpVerb.List, _) => List.HandleAsync(context, cancellationToken), + (NntpVerb.Group, _) => Group.HandleAsync(context, cancellationToken), + (NntpVerb.ListGroup, _) => ListGroup.HandleAsync(context, cancellationToken), + (NntpVerb.Article, _) => Article.HandleArticleAsync(context, cancellationToken), + (NntpVerb.Head, _) => Article.HandleHeadAsync(context, cancellationToken), + (NntpVerb.Body, _) => Article.HandleBodyAsync(context, cancellationToken), + (NntpVerb.Stat, _) => Article.HandleStatAsync(context, cancellationToken), + (NntpVerb.Last, _) => Last.HandleAsync(context, cancellationToken), + (NntpVerb.Next, _) => Next.HandleAsync(context, cancellationToken), + (NntpVerb.Over, _) => Over.HandleAsync(context, cancellationToken), + (NntpVerb.Hdr, _) => Hdr.HandleAsync(context, cancellationToken), + (NntpVerb.Post, _) => Post.HandleAsync(context, cancellationToken), + (NntpVerb.Ihave, _) => IHave.HandleAsync(context, cancellationToken), + (NntpVerb.Check, _) => Check.HandleAsync(context, cancellationToken), + (NntpVerb.TakeThis, _) => TakeThis.HandleAsync(context, cancellationToken), + _ => NntpCommandNotImplemented.HandleAsync( + context, + NntpCommandLoggers.For(typeof(NntpCommandExecution)), + cancellationToken), + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandExecution.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandExecution.cs new file mode 100644 index 00000000..4a5ded7b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandExecution.cs @@ -0,0 +1,133 @@ +using System.Diagnostics; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Times command-handler execution and emits exactly one TX completion record owned by the command module. + /// + /// + /// Timing covers handler work only (not socket receive wait). Completion is always written from + /// finally so cancellation and failures still produce one TX line. + /// + internal static class NntpCommandExecution + { + /// + /// Runs under a monotonic stopwatch and logs + /// [client] TX: COMMAND [status] executed in 0.000s exactly once when + /// the handler produced an NNTP status line; otherwise the historical no-status + /// completion format is retained. + /// + public static async ValueTask RunAsync( + ILogger logger, + NntpCommandContext context, + string command, + Func execute, + CancellationToken cancellationToken, + string? successDetail = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(context); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + ArgumentNullException.ThrowIfNull(execute); + + var started = Stopwatch.GetTimestamp(); + string? detail = successDetail; + var completedBefore = context.Connection.IsCompleted; + try + { + await execute(context, cancellationToken).ConfigureAwait(false); + if (context.CompletionDetail is not null) + { + // Handler-owned detail (e.g. QUIT peer disconnect, STARTTLS cipher, COMPRESS). + detail = context.CompletionDetail; + } + else if (!completedBefore && context.Connection.IsCompleted) + { + detail = "failed"; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + detail = "cancelled"; + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + detail = "failed"; + CommandLogMessages.CommandFailed( + logger, + ex, + NntpCommandLogFormat.Client(context.Session), + command); + throw; + } + finally + { + WriteCompletion( + logger, + context.Session, + command, + Stopwatch.GetElapsedTime(started), + detail, + context.StatusLine); + } + } + + /// + /// Emits a TX completion record for gate/reject paths that never enter a command handler body. + /// + /// + /// Uses the command module's logger category (not ). + /// + public static void WriteCompletion( + ILogger logger, + NntpSession session, + string command, + TimeSpan elapsed, + string? detail = null, + string? statusLine = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(session); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + + if (!logger.IsEnabled(LogLevel.Debug) + || NntpCommandLogFormat.SuppressHotPathCommandLog(command)) + { + return; + } + + var client = NntpCommandLogFormat.Client(session); + var seconds = elapsed.TotalSeconds; + if (statusLine is not null) + { + if (detail is null) + { + CommandLogMessages.CommandTxWithStatus(logger, client, command, statusLine, seconds); + } + else + { + CommandLogMessages.CommandTxWithStatusAndDetail( + logger, + client, + command, + statusLine, + seconds, + detail); + } + + return; + } + + if (detail is null) + { + CommandLogMessages.CommandTx(logger, client, command, seconds); + } + else + { + CommandLogMessages.CommandTxWithDetail(logger, client, command, seconds, detail); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLineReader.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLineReader.cs new file mode 100644 index 00000000..5d984810 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLineReader.cs @@ -0,0 +1,113 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Reads CRLF-delimited NNTP command lines from a . + public static class NntpCommandLineReader + { + /// Maximum command-line content before CRLF. Longer lines are rejected, not parsed. + public const int MaxCommandLineBytes = 2048; + + /// + /// result when the line before CRLF exceeds the destination + /// scratch. The line has been consumed. -1 remains EOF. + /// + public const int OverlongLine = -2; + + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + + /// + /// Reads the next command line (without trailing CRLF). Returns on EOF. + /// + public static async ValueTask ReadLineAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + if (TryReadLine(ref buffer, out var line)) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return line; + } + + reader.AdvanceTo(buffer.Start, buffer.End); + if (result.IsCompleted) + { + return null; + } + } + } + + private static bool TryReadLine(ref ReadOnlySequence buffer, out string? line) + { + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence lineBytes, Crlf)) + { + line = null; + return false; + } + + line = Encoding.ASCII.GetString(lineBytes); + buffer = buffer.Slice(reader.Position); + return true; + } + + /// + /// Reads the next command line into (without CRLF). + /// Returns the number of bytes written, or -1 on EOF. + /// + public static async ValueTask ReadLineBytesAsync( + PipeReader reader, + byte[] scratch, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(scratch); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + if (TryCopyLine(ref buffer, scratch, out var length)) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return length; + } + + reader.AdvanceTo(buffer.Start, buffer.End); + if (result.IsCompleted) + { + return -1; + } + } + } + + private static bool TryCopyLine(ref ReadOnlySequence buffer, byte[] scratch, out int length) + { + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence lineBytes, Crlf)) + { + length = 0; + return false; + } + + buffer = buffer.Slice(reader.Position); + if (lineBytes.Length > scratch.Length) + { + length = OverlongLine; + return true; + } + + length = checked((int)lineBytes.Length); + lineBytes.CopyTo(scratch); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLogFormat.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLogFormat.cs new file mode 100644 index 00000000..60fcf571 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLogFormat.cs @@ -0,0 +1,155 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Formats effective client identity and redacts secrets for NNTP command RX/TX logs. + internal static class NntpCommandLogFormat + { + /// Formats src-ip:src-port from the session's effective client identity. + public static string Client(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + return string.Create( + CultureInfo.InvariantCulture, + $"{session.ClientAddress}:{session.ClientPort}"); + } + + /// + /// Returns whether per-command RX logging should be skipped for . + /// + /// + /// Previously suppressed because console Information + /// could not keep up. File logging is now Debug/Verbose; TAKETHIS diagnostics are restored. + /// + public static bool SuppressHotPathCommand(NntpVerb verb) + { + _ = verb; + return false; + } + + /// + /// Logging-boundary representation of a command. AUTHINFO secrets are redacted. + /// + /// + /// / + /// cannot accept + /// of bytes, and byte[] is not a useful operational + /// log representation. This ASCII conversion is the explicit logging string boundary. + /// Callers must check before + /// invoking this method. + /// + public static string RedactRxCommand(NntpCommand command, ReadOnlySpan line) + { + if (command.Verb == NntpVerb.AuthInfo && command.Qualifier == NntpVerb.Pass) + { + return "AUTHINFO PASS "; + } + + if (command.Verb == NntpVerb.AuthInfo && command.Qualifier == NntpVerb.Sasl) + { + return "AUTHINFO SASL "; + } + + return System.Text.Encoding.ASCII.GetString(line); + } + + /// + /// Returns a log-safe RX command line. Passwords and SASL credential material are redacted. + /// + public static string RedactRxLine(string rawLine) + { + ArgumentNullException.ThrowIfNull(rawLine); + if (rawLine.Length == 0) + { + return rawLine; + } + + // AUTHINFO PASS → AUTHINFO PASS + if (StartsWithCommand(rawLine, "AUTHINFO PASS")) + { + return "AUTHINFO PASS "; + } + + // AUTHINFO SASL may carry an initial response with credentials. + if (StartsWithCommand(rawLine, "AUTHINFO SASL")) + { + return "AUTHINFO SASL "; + } + + return rawLine; + } + + /// Extracts a display command name from a raw line when resolution is unavailable. + public static string DisplayNameFromRawLine(string rawLine) + { + ArgumentNullException.ThrowIfNull(rawLine); + if (rawLine.Length == 0) + { + return "INVALID"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO PASS")) + { + return "AUTHINFO PASS"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO SASL")) + { + return "AUTHINFO SASL"; + } + + if (StartsWithCommand(rawLine, "AUTHINFO USER")) + { + return "AUTHINFO USER"; + } + + if (StartsWithCommand(rawLine, "MODE READER")) + { + return "MODE READER"; + } + + if (StartsWithCommand(rawLine, "MODE STREAM")) + { + return "MODE STREAM"; + } + + if (StartsWithCommand(rawLine, "COMPRESS")) + { + return "COMPRESS"; + } + + var span = rawLine.AsSpan().Trim(); + var space = span.IndexOf(' '); + return space <= 0 ? span.ToString().ToUpperInvariant() : span[..space].ToString().ToUpperInvariant(); + } + + /// + /// Returns whether per-command RX/TX INFO traffic logging should be suppressed for this + /// command name or raw command line. + /// + /// + /// Previously suppressed TAKETHIS so feed volume did not flood the console. + /// Console is now Information-only; TAKETHIS Debug/Trace is restored for the file sink. + /// + public static bool SuppressHotPathCommandLog(string commandOrRawLine) + { + ArgumentNullException.ThrowIfNull(commandOrRawLine); + return false; + } + + private static bool StartsWithCommand(string line, string command) + { + if (line.Length < command.Length) + { + return false; + } + + if (!line.AsSpan(0, command.Length).Equals(command, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return line.Length == command.Length || char.IsWhiteSpace(line[command.Length]); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLoggers.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLoggers.cs new file mode 100644 index 00000000..f71a10f7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandLoggers.cs @@ -0,0 +1,33 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Resolves per-command instances from the host . + /// + /// + /// Command modules own completion logging; categories use the command type's full name + /// (for example VectorNNTPD.TransitService.Session.Commands.StartTls). + /// + internal static class NntpCommandLoggers + { + private static ILoggerFactory _factory = NullLoggerFactory.Instance; + private static readonly ConcurrentDictionary Cache = new(StringComparer.Ordinal); + + /// Configures the factory used for subsequent calls. + public static void Configure(ILoggerFactory? loggerFactory) + { + _factory = loggerFactory ?? NullLoggerFactory.Instance; + Cache.Clear(); + } + + /// Returns an whose category is 's full name. + public static ILogger For(Type commandType) + { + ArgumentNullException.ThrowIfNull(commandType); + var name = commandType.FullName ?? commandType.Name; + return Cache.GetOrAdd(name, static (category, factory) => factory.CreateLogger(category), _factory); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandNotImplemented.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandNotImplemented.cs new file mode 100644 index 00000000..6e43a15f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandNotImplemented.cs @@ -0,0 +1,18 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Shared placeholder response for commands not yet implemented. + internal static class NntpCommandNotImplemented + { + /// Returns 500 Command not implemented (RFC 3977 for unsupported optional commands). + public static ValueTask HandleAsync( + NntpCommandContext context, + ILogger logger, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + logger, + NntpResponses.CommandNotImplemented, + NntpResponseStatus.CommandNotImplemented, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandParser.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandParser.cs new file mode 100644 index 00000000..be67468b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandParser.cs @@ -0,0 +1,341 @@ +using VectorNNTPD.Common.Articles.Validation; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Authoritative byte-oriented NNTP command parser and syntactic validation boundary. + /// + /// + /// Only may reach a command handler. Invalid commands are + /// rejected at dispatch. Argument bytes remain indexes into the caller's buffer. + /// Keywords are case-insensitive; SP and TAB are separators (RFC 3977 §3.1). + /// + public static class NntpCommandParser + { + /// Parses one command line (CRLF already removed). Never throws for ordinary malformed input. + public static NntpCommand Parse(ReadOnlySpan line) + { + var verb = NntpVerbClassifier.Classify(line, out _, out int verbEnd); + if (verb == NntpVerb.None) + { + return new NntpCommand(NntpVerb.None, NntpVerb.None, 0, 0, 0, NntpParseStatus.Empty); + } + + if (verb == NntpVerb.Unknown) + { + return Finish(NntpVerb.Unknown, NntpVerb.None, line, verbEnd, NntpParseStatus.UnknownVerb); + } + + int index = NntpAscii.SkipWhitespace(line, verbEnd); + var qualifier = NntpVerb.None; + + if (NeedsQualifier(verb)) + { + if (index >= line.Length) + { + return new NntpCommand(verb, NntpVerb.None, 0, 0, 0, NntpParseStatus.UnknownQualifier); + } + + int qEnd = NntpAscii.SkipToken(line, index); + qualifier = ClassifyQualifier(verb, line.Slice(index, qEnd - index)); + if (qualifier is NntpVerb.Unknown or NntpVerb.None) + { + return Finish(verb, NntpVerb.Unknown, line, qEnd, NntpParseStatus.UnknownQualifier); + } + + index = NntpAscii.SkipWhitespace(line, qEnd); + } + else if (verb == NntpVerb.List && index < line.Length) + { + int qEnd = NntpAscii.SkipToken(line, index); + var listQualifier = ClassifyListQualifier(line.Slice(index, qEnd - index)); + if (listQualifier == NntpVerb.Unknown) + { + return Finish(verb, NntpVerb.Unknown, line, qEnd, NntpParseStatus.UnknownQualifier); + } + + qualifier = listQualifier; + index = NntpAscii.SkipWhitespace(line, qEnd); + } + + int argEnd = NntpAscii.TrimEndWhitespace(line, index, line.Length); + int tokenCount = NntpAscii.CountTokens(line, index, argEnd); + int argLength = argEnd - index; + var status = Validate(verb, qualifier, line, index, argLength, tokenCount); + return new NntpCommand(verb, qualifier, index, argLength, tokenCount, status); + } + + private static NntpCommand Finish( + NntpVerb verb, + NntpVerb qualifier, + ReadOnlySpan line, + int after, + NntpParseStatus status) + { + int index = NntpAscii.SkipWhitespace(line, after); + int argEnd = NntpAscii.TrimEndWhitespace(line, index, line.Length); + return new NntpCommand( + verb, + qualifier, + index, + argEnd - index, + NntpAscii.CountTokens(line, index, argEnd), + status); + } + + private static bool NeedsQualifier(NntpVerb verb) => verb is NntpVerb.Mode or NntpVerb.AuthInfo; + + private static NntpParseStatus Validate( + NntpVerb verb, + NntpVerb qualifier, + ReadOnlySpan line, + int argumentStart, + int argumentLength, + int tokenCount) + { + var argument = argumentLength <= 0 + ? ReadOnlySpan.Empty + : line.Slice(argumentStart, argumentLength); + + switch (verb) + { + case NntpVerb.Check: + case NntpVerb.TakeThis: + case NntpVerb.Ihave: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpMessageIdValidation.IsValidMessageId(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.SpeedTest: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return IsSpeedTestPeerSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.Quit: + case NntpVerb.Help: + case NntpVerb.Post: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.Compress: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return IsCompressAlgorithmSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.AuthInfo when qualifier is NntpVerb.User or NntpVerb.Pass: + return argument.IsEmpty ? NntpParseStatus.MissingArgument : NntpParseStatus.Ok; + + case NntpVerb.AuthInfo when qualifier is NntpVerb.Sasl: + return argument.IsEmpty ? NntpParseStatus.MissingArgument : NntpParseStatus.Ok; + + case NntpVerb.Article: + case NntpVerb.Head: + case NntpVerb.Body: + case NntpVerb.Stat: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpMessageIdValidation.IsValidMessageId(argument) || NntpArticleNumber.IsSyntax(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.Group: + if (tokenCount == 0) + { + return NntpParseStatus.MissingArgument; + } + + return tokenCount == 1 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.ListGroup: + return tokenCount <= 2 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.None: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.Active or NntpVerb.Counts or NntpVerb.Newsgroups: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpWildmat.TryValidate(argument) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + case NntpVerb.List when qualifier is NntpVerb.Motd or NntpVerb.OverviewFmt: + return tokenCount == 0 ? NntpParseStatus.Ok : NntpParseStatus.ExtraArgument; + + case NntpVerb.List when qualifier is NntpVerb.Headers: + if (tokenCount == 0) + { + return NntpParseStatus.Ok; + } + + if (tokenCount != 1) + { + return NntpParseStatus.ExtraArgument; + } + + return NntpAscii.EqualsFolded(argument, "MSGID"u8) + || NntpAscii.EqualsFolded(argument, "RANGE"u8) + ? NntpParseStatus.Ok + : NntpParseStatus.InvalidArgument; + + default: + return NntpParseStatus.Ok; + } + } + + /// + /// SPEEDTEST identifier: one NNTP token matching the Transit identifier grammar + /// (1–256 visible ASCII octets, no whitespace or controls). Exact ordinal match + /// is applied later against the snapshot. Not a quoted string or multi-token name. + /// + internal static bool IsSpeedTestPeerSyntax(ReadOnlySpan peer) => + TransitPeersOptionsValidator.IsPeerIdentifier(peer); + + /// RFC 8054 §5.3 algorithm syntax (case-sensitive). + internal static bool IsCompressAlgorithmSyntax(ReadOnlySpan algorithm) + { + if (algorithm.Length is < 1 or > 20) + { + return false; + } + + for (int i = 0; i < algorithm.Length; i++) + { + byte b = algorithm[i]; + if ((b >= (byte)'A' && b <= (byte)'Z') + || (b >= (byte)'0' && b <= (byte)'9') + || b == (byte)'-' + || b == (byte)'_') + { + continue; + } + + return false; + } + + return true; + } + + private static NntpVerb ClassifyQualifier(NntpVerb verb, ReadOnlySpan token) + { + if (verb == NntpVerb.Mode) + { + if (NntpAscii.EqualsFolded(token, "STREAM"u8)) + { + return NntpVerb.Stream; + } + + if (NntpAscii.EqualsFolded(token, "READER"u8)) + { + return NntpVerb.Reader; + } + + return NntpVerb.Unknown; + } + + if (verb == NntpVerb.AuthInfo) + { + if (NntpAscii.EqualsFolded(token, "USER"u8)) + { + return NntpVerb.User; + } + + if (NntpAscii.EqualsFolded(token, "PASS"u8)) + { + return NntpVerb.Pass; + } + + if (NntpAscii.EqualsFolded(token, "SASL"u8)) + { + return NntpVerb.Sasl; + } + + return NntpVerb.Unknown; + } + + return NntpVerb.None; + } + + private static NntpVerb ClassifyListQualifier(ReadOnlySpan token) + { + if (NntpAscii.EqualsFolded(token, "ACTIVE"u8)) + { + return NntpVerb.Active; + } + + if (NntpAscii.EqualsFolded(token, "COUNTS"u8)) + { + return NntpVerb.Counts; + } + + if (NntpAscii.EqualsFolded(token, "HEADERS"u8)) + { + return NntpVerb.Headers; + } + + if (NntpAscii.EqualsFolded(token, "MOTD"u8)) + { + return NntpVerb.Motd; + } + + if (NntpAscii.EqualsFolded(token, "NEWSGROUPS"u8)) + { + return NntpVerb.Newsgroups; + } + + if (NntpAscii.EqualsFolded(token, "OVERVIEW.FMT"u8)) + { + return NntpVerb.OverviewFmt; + } + + return NntpVerb.Unknown; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandReply.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandReply.cs new file mode 100644 index 00000000..ae7d102d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandReply.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Writes existing wire bytes and, only when Debug command logging is enabled, records + /// the independently authored semantic status line on the command context. + /// + internal static class NntpCommandReply + { + /// + /// Records when Debug is enabled and no status is set yet. + /// Immortal strings are stored by reference; nothing is allocated when Debug is off. + /// + internal static void TryNote(NntpCommandContext context, ILogger logger, string statusLine) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(statusLine); + if (context.StatusLine is not null || !logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + context.StatusLine = statusLine; + } + + /// Writes and notes if Debug is on. + internal static ValueTask WriteAsync( + NntpCommandContext context, + ILogger logger, + ReadOnlyMemory wire, + string statusLine, + CancellationToken cancellationToken = default) + { + TryNote(context, logger, statusLine); + return context.Response.WriteLineAsync(wire, cancellationToken); + } + + /// + /// Enqueues immediately and notes if Debug is on. + /// + internal static ValueTask EnqueueImmediateAsync( + NntpCommandContext context, + ILogger logger, + ReadOnlyMemory wire, + string statusLine, + CancellationToken cancellationToken = default) + { + TryNote(context, logger, statusLine); + return context.Response.EnqueueLineImmediateAsync(wire, cancellationToken); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandStatusText.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandStatusText.cs new file mode 100644 index 00000000..b5eaaa81 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpCommandStatusText.cs @@ -0,0 +1,53 @@ +using System.Text; +using VectorNNTPD.TransitService.History; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Debug-only semantic TX status lines for dynamic CHECK/TAKETHIS/DATE replies. + /// + internal static class NntpCommandStatusText + { + /// + /// Formats the CHECK status line from the lookup result and Message-ID octets. + /// Call only after . + /// + /// History lookup that selected the CHECK reply. It is not rewritten. + /// Message-ID octets already accepted for the command. + /// + /// Connection DeferOnDuplicate. When , a seen identifier is logged as 431. + /// + internal static string FormatCheck( + HistoryLookupResult result, + ReadOnlySpan messageId, + bool deferOnDuplicate = false) + { + var id = Encoding.ASCII.GetString(messageId); + return result switch + { + HistoryLookupResult.Seen when deferOnDuplicate => "431 " + id, + HistoryLookupResult.Seen => "438 " + id, + HistoryLookupResult.Unavailable => "431 " + id, + _ => "238 " + id + " send article to be transferred", + }; + } + + /// + /// Formats TAKETHIS 239 message-id. Call only after . + /// + internal static string FormatTakeThisAccepted(ReadOnlySpan messageId) => + "239 " + Encoding.ASCII.GetString(messageId); + + /// + /// Formats TAKETHIS 439 message-id. Call only after . + /// + internal static string FormatTakeThisRejected(ReadOnlySpan messageId) => + "439 " + Encoding.ASCII.GetString(messageId); + + /// + /// Formats 111 YYYYMMDDhhmmss from the same stamp characters used for the wire line. + /// Call only after . + /// + internal static string FormatDate(ReadOnlySpan stamp) => string.Concat("111 ".AsSpan(), stamp); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpParseStatus.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpParseStatus.cs new file mode 100644 index 00000000..69457f59 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpParseStatus.cs @@ -0,0 +1,33 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// Syntactic outcome of . + public enum NntpParseStatus : byte + { + /// Command is syntactically valid and may be dispatched to a handler. + Ok = 0, + + /// Empty or whitespace-only line. + Empty, + + /// Verb is not a supported command. + UnknownVerb, + + /// Verb is known but the qualifier/variant is missing or unsupported. + UnknownQualifier, + + /// A required argument is missing. + MissingArgument, + + /// An argument is present but syntactically invalid. + InvalidArgument, + + /// The command has more tokens than its grammar allows. + ExtraArgument, + + /// + /// The CRLF-terminated line is longer than . + /// The prefix is not parsed. + /// + CommandLineTooLong, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpPeerDisconnect.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpPeerDisconnect.cs new file mode 100644 index 00000000..c5182d9e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpPeerDisconnect.cs @@ -0,0 +1,59 @@ +using System.Net.Sockets; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Classifies transport failures that indicate the peer has already gone away. + /// + /// + /// Used by QUIT and by the session command-loop catch. Classification stays + /// reset/EOF-shaped so unrelated I/O failures are not treated as normal disconnects. + /// Authenticated-session release is guaranteed by session finalization, not by this classifier. + /// + internal static class NntpPeerDisconnect + { + /// + /// Returns whether indicates the peer disconnected or the + /// connection became terminal in a way expected during QUIT response/close. + /// + public static bool IsPeerDisconnect(Exception exception, INntpConnection connection) + { + ArgumentNullException.ThrowIfNull(exception); + ArgumentNullException.ThrowIfNull(connection); + + for (Exception? current = exception; current is not null; current = current.InnerException) + { + switch (current) + { + case SocketException socket when IsResetLike(socket.SocketErrorCode): + return true; + case IOException { InnerException: SocketException nested } when IsResetLike(nested.SocketErrorCode): + return true; + case IOException { Message: not null } io when + io.Message.Contains("zero bytes", StringComparison.OrdinalIgnoreCase): + return true; + case ObjectDisposedException when connection.IsCompleted + || connection.ConnectionClosed.IsCancellationRequested: + return true; + case InvalidOperationException { Message: not null } ioe when + ioe.Message.Contains("writer was completed", StringComparison.OrdinalIgnoreCase) + || ioe.Message.Contains("pipe completed", StringComparison.OrdinalIgnoreCase): + // Output PipeWriter completed (peer gone / CompleteAsync raced with QUIT flush). + return true; + } + } + + return false; + } + + private static bool IsResetLike(SocketError error) => + error is SocketError.ConnectionReset + or SocketError.ConnectionAborted + or SocketError.Shutdown + or SocketError.NotConnected + or SocketError.NetworkReset + // EPIPE (32) — not always present as SocketError.BrokenPipe on Windows TFMs. + or (SocketError)32; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseCompose.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseCompose.cs new file mode 100644 index 00000000..5db472c4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseCompose.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Builds one owned wire buffer from static prefixes and dynamic protocol bytes. + /// + /// + /// The returned array is the caller's until it is passed to + /// + /// or . + /// The TX pump only reads it. Session scratch and Pipe memory must not be enqueued; + /// those spans are copied into this buffer first. + /// + internal static class NntpResponseCompose + { + /// Allocates prefix + dynamic + suffix as one owned array. + internal static byte[] Concat( + ReadOnlySpan prefix, + ReadOnlySpan dynamic, + ReadOnlySpan suffix) + { + var result = new byte[checked(prefix.Length + dynamic.Length + suffix.Length)]; + prefix.CopyTo(result); + dynamic.CopyTo(result.AsSpan(prefix.Length)); + suffix.CopyTo(result.AsSpan(prefix.Length + dynamic.Length)); + return result; + } + + /// Allocates the concatenation of as one owned array. + internal static byte[] Concatenate(ReadOnlySpan> parts) + { + var total = 0; + for (var i = 0; i < parts.Length; i++) + { + total = checked(total + parts[i].Length); + } + + var result = new byte[total]; + var offset = 0; + for (var i = 0; i < parts.Length; i++) + { + var part = parts[i].Span; + part.CopyTo(result.AsSpan(offset)); + offset += part.Length; + } + + return result; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseStatus.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseStatus.cs new file mode 100644 index 00000000..ecdd3f79 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseStatus.cs @@ -0,0 +1,91 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Immortal semantic first/status lines matching wire text + /// (without CRLF). Authored independently of the byte buffers; not decoded from them. + /// + internal static class NntpResponseStatus + { + internal const string HelpTextFollows = "100 Help text follows"; + internal const string CapabilityListFollows = "101 Capability list:"; + internal const string ListOfNewsgroupsFollows = "215 list of newsgroups follows"; + internal const string ListOverviewFmtFollows = "215 Order of fields in overview database."; + internal const string ListHeadersFollows = "215 headers and metadata items supported:"; + internal const string NoSuchNewsgroup = "411 No such newsgroup"; + internal const string NoNewsgroupSelected = "412 No newsgroup selected"; + internal const string CurrentArticleNumberInvalid = "420 Current article number is invalid"; + internal const string NoArticleWithNumber = "423 No article with that number"; + internal const string NoArticleWithMessageId = "430 No article with that message-id"; + internal const string GroupSelected = "211"; + internal const string ReaderModePostingPermitted = "200 Reader mode, posting permitted"; + internal const string ReaderModePostingProhibited = "201 Reader mode, posting prohibited"; + internal const string StreamingPermitted = "203 Streaming permitted"; + internal const string ConnectionClosing = "205 Connection closing"; + internal const string CompressionActive = "206 Compression active"; + internal const string IhaveTransferredOk = "235 Article transferred OK"; + internal const string ArticleReceivedOk = "240 Article received OK"; + internal const string IhaveSendArticle = "335 Send article to be transferred"; + internal const string PostSendArticle = "340 Input article; end with ."; + internal const string PostingProhibited = "440 Posting not permitted"; + internal const string PostingFailed = "441 Posting failed"; + internal const string AuthenticationAccepted = "281 Authentication accepted"; + internal const string PasswordRequired = "381 Password required"; + internal const string ContinueWithTls = "382 Continue with TLS negotiation"; + internal const string ServiceTemporarilyUnavailable = "400 Service temporarily unavailable"; + internal const string SpeedTestBusy = "400 SPEEDTEST BUSY"; + internal const string CommandFailed = "403 Command failed"; + internal const string UnableToActivateCompression = "403 Unable to activate compression"; + internal const string IhaveNotWanted = "435 Article not wanted"; + internal const string IhaveTryLater = "436 Transfer not possible; try again later"; + internal const string IhaveTransferFailed = "436 Transfer failed; try again later"; + internal const string IhaveRejected = "437 Transfer rejected; do not retry"; + internal const string AuthenticationRequired = "480 Authentication required"; + internal const string AuthenticationFailed = "481 Authentication failed"; + internal const string AuthenticationCancelled = "481 Authentication cancelled"; + internal const string TooManySessions = "481 Too many sessions"; + internal const string TooManySourceAddresses = "481 Too many source addresses"; + internal const string AuthenticationOutOfSequence = "482 Authentication commands issued out of sequence"; + internal const string PrivacyRequired = "483 Encryption or stronger authentication required"; + internal const string UnknownCommand = "500 Unknown command"; + internal const string CommandNotImplemented = "500 Command not implemented"; + internal const string SyntaxError = "501 Syntax error"; + internal const string UnknownCommandVariant = "501 Unknown command variant"; + internal const string AuthinfoUserRequiresUsername = "501 AUTHINFO USER requires a username"; + internal const string AuthinfoPassRequiresPassword = "501 AUTHINFO PASS requires a password"; + internal const string CompressRequiresAlgorithm = "501 COMPRESS requires a single algorithm argument"; + internal const string CompressAlgorithmSyntaxInvalid = "501 Syntactically incorrect compression algorithm"; + internal const string AuthinfoSaslNotImplemented = "501 AUTHINFO SASL not implemented"; + internal const string SaslMechanismRequired = "501 SASL mechanism required"; + internal const string SaslInvalidBase64 = "504 Base64 encoding error"; + internal const string SaslLoginUsernameChallenge = "383 VXNlcm5hbWU6"; + internal const string SaslLoginPasswordChallenge = "383 UGFzc3dvcmQ6"; + internal const string SaslEmptyChallenge = "383 ="; + internal const string SaslChallenge = "383 SASL challenge"; + internal const string SaslSuccessWithData = "283 Authentication accepted"; + internal const string SaslProtocolError = "482 SASL protocol error"; + internal const string SaslMechanismNotSupported = "503 Mechanism not supported"; + internal const string SaslContinuationNotSupported = "503 SASL continuation not supported"; + internal const string NoSaslExchangeInProgress = "503 No SASL exchange in progress"; + internal const string TemporaryAuthenticationFailure = "503 Temporary authentication failure"; + internal const string SaslScramUnavailable = "503 SCRAM not available"; + internal const string SaslCramUnavailable = "503 CRAM-MD5 not available"; + internal const string AlreadyAuthenticated = "502 Already authenticated"; + internal const string CommandUnavailableAfterAuthentication = "502 Command unavailable after authentication"; + internal const string CommandUnavailableAfterCompression = "502 Command unavailable after compression"; + internal const string DeflateAlreadyActive = "502 DEFLATE compression already active"; + internal const string CompressionAlreadyActive = "502 Compression already active"; + internal const string TlsAlreadyActive = "502 TLS already active"; + internal const string TlsProviderUnavailable = "502 TLS provider unavailable"; + internal const string PostingNotPermitted = "502 Posting not permitted"; + internal const string PermissionDenied = "502 Permission denied"; + internal const string CommandNotAvailable = "502 Command not available on this server"; + internal const string StreamingNotPermitted = "502 Streaming not permitted"; + internal const string NotInReaderMode = "502 Not in reader mode"; + internal const string NotInStreamMode = "502 Not in stream mode"; + internal const string SpeedTestUnknownPeer = "502 UNKNOWN SPEEDTEST PEER"; + internal const string SpeedTestPeerMismatch = "502 SPEEDTEST PEER MISMATCH"; + internal const string CompressionAlgorithmNotSupported = "503 Compression algorithm not supported"; + internal const string SpeedTestNotSupported = "503 SPEEDTEST NOT SUPPORTED"; + internal const string ListDataItemNotStored = "503 Data item not stored"; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseWriter.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseWriter.cs new file mode 100644 index 00000000..3feae325 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponseWriter.cs @@ -0,0 +1,853 @@ +using System.IO.Pipelines; +using System.Text; +using System.Threading.Channels; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Writes NNTP single-line and multi-line responses to a through a + /// single ordered outbound pump. + /// + /// + /// + /// All writes are serialized onto one pump so response bytes remain ordered. Ordinary + /// awaits pump flush completion + /// (existing command semantics). + /// only waits until the line is + /// accepted by the ordered queue, + /// so TAKETHIS can continue reading the next pipelined article without waiting for network + /// delivery of the prior 239/439. + /// + /// + /// The pump coalesces fire-and-forget lines into batches of at most + /// before one . + /// Logical response bytes, CRLF framing, and channel order are unchanged; only write + /// granularity changes. Awaiting writes (status lines, article chunks, STARTTLS/COMPRESS/QUIT) + /// flush any held batch first. + /// + /// + /// Large article bodies use the WriteArticleAsync APIs: externally supplied destuffed bytes → + /// restuff → bounded owned chunks → the same ordered Channel (no second TX pipeline; + /// mode-independent). The writer does not own article storage or lookup. + /// + /// + public sealed class NntpResponseWriter : IAsyncDisposable + { + /// + /// Maximum fire-and-forget responses held into one . + /// + /// + /// Chosen from the measured N=8 reverse-path cell in + /// .artifacts/STREAM-TCP-WIRE-MECHANISM-AUDIT.md (~200 × 264-byte TCP segments/s + /// versus N=1 ~1450 × 33-byte segments/s). Not a configuration setting. + /// + internal const int CoalesceResponseBatchSize = 8; + + private static readonly byte[] DotCrlf = ".\r\n"u8.ToArray(); + + private readonly PipeWriter _output; + private readonly Channel _channel; + private readonly Task _pump; + // codeql[cs/missed-using-statement] Owned by NntpResponseWriter until Dispose; a using at this declaration would cancel the pump when the constructor returns. + private readonly CancellationTokenSource _pumpCts = new(); + private int _disposed; + private long _channelEnqueueCount; + private long _pipeFlushCount; + private long _ownedPayloadBytesEnqueued; + private int _coalescedUnflushed; + private int _directExclusive; + private long _directPipeFlushCount; + private IAccountByteSink _byteSink = NullAccountByteSink.Instance; + + private readonly struct WriteRequest + { + public WriteRequest( + ReadOnlyMemory payload, + TaskCompletionSource? completed, + bool flushAfter = false) + { + Payload = payload; + Completed = completed; + FlushAfter = flushAfter; + } + + /// + /// Payload bytes valid until the pump finishes writing this item. Immortal static + /// responses may share the same backing array across requests; the pump only reads. + /// + public ReadOnlyMemory Payload { get; } + + public TaskCompletionSource? Completed { get; } + + /// + /// When , the pump flushes after this fire-and-forget line + /// instead of holding it in a TAKETHIS coalesce batch. + /// + public bool FlushAfter { get; } + } + + /// Initializes a new instance of the class. + public NntpResponseWriter(PipeWriter output) + : this(output, channelCapacity: 4096) + { + } + + /// Test/harness constructor with an explicit bounded Channel capacity. + internal NntpResponseWriter(PipeWriter output, int channelCapacity) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentOutOfRangeException.ThrowIfLessThan(channelCapacity, 1); + _output = output; + var channelOptions = new BoundedChannelOptions(channelCapacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait, + AllowSynchronousContinuations = true, + }; + ChannelAllowsSynchronousContinuations = channelOptions.AllowSynchronousContinuations; + _channel = Channel.CreateBounded(channelOptions); + _pump = Task.Run(() => PumpAsync(_pumpCts.Token), CancellationToken.None); + } + + /// + /// Gets the TX response Channel AllowSynchronousContinuations value applied at construction. + /// + internal bool ChannelAllowsSynchronousContinuations { get; } + + /// + /// Installs the account byte sink. Counting starts after the next successful + /// PipeWriter.Advance, never at Channel enqueue. + /// + public void SetByteSink(IAccountByteSink sink) + { + ArgumentNullException.ThrowIfNull(sink); + Volatile.Write(ref _byteSink, sink); + } + + /// Gets the number of Channel items accepted (diagnostics / tests). + internal long ChannelEnqueueCount => Volatile.Read(ref _channelEnqueueCount); + + /// Gets the existing TX this writer pumps into. Not a second pipe. + internal PipeWriter UnderlyingOutput => _output; + + /// Gets whether SPEEDTEST currently holds exclusive TX PipeWriter ownership. + internal bool DirectPipeExclusive => Volatile.Read(ref _directExclusive) == 1; + + /// Gets calls made on the SPEEDTEST direct-pipe lease. + internal long DirectPipeFlushCount => Volatile.Read(ref _directPipeFlushCount); + + /// Gets the number of calls from the TX pump. + internal long PipeFlushCount => Volatile.Read(ref _pipeFlushCount); + + /// Gets total owned payload bytes accepted into the Channel. + internal long OwnedPayloadBytesEnqueued => Volatile.Read(ref _ownedPayloadBytesEnqueued); + + /// + /// Gets whether any fire-and-forget lines have been accepted and not yet flushed. + /// + internal bool HasCoalescedUnflushed => Volatile.Read(ref _coalescedUnflushed) > 0; + + /// + /// Writes a single-line response (code text), waiting until the ordered pump has + /// flushed it into the outbound pipe (subject to pipe backpressure). + /// + public ValueTask WriteLineAsync(int code, string text, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(code); + ArgumentNullException.ThrowIfNull(text); + return WriteAndAwaitFlushAsync(EncodeLine(code, text), cancellationToken); + } + + /// + /// Writes a complete pre-encoded wire response (including CRLF) and awaits pump flush. + /// + /// + /// Does not copy . The memory must remain unchanged until this + /// write has been flushed into the outbound pipe. Immortal static responses satisfy that + /// for the process lifetime. Dynamically composed responses must be an owned buffer + /// (not Pipe memory, not session scratch). Channel order, coalesce-flush-first, and TCS + /// barrier semantics match . + /// + public ValueTask WriteLineAsync(ReadOnlyMemory wireLine, CancellationToken cancellationToken = default) + { + return WriteAndAwaitFlushAsync(wireLine, cancellationToken); + } + + /// + /// Enqueues a single-line response without waiting for outbound pipe/network flush. + /// + /// + /// Ordering is preserved. Returns when the ordered queue accepts the line (may wait if the + /// response queue is saturated). Used by TAKETHIS so receive can proceed immediately. + /// + public ValueTask EnqueueLineAsync(int code, string text, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(code); + ArgumentNullException.ThrowIfNull(text); + return EnqueueAsync(new WriteRequest(EncodeLine(code, text), completed: null), cancellationToken); + } + + /// + /// Enqueues a complete pre-encoded wire response without waiting for outbound flush. + /// + /// + /// Does not copy . The memory must remain unchanged until the + /// coalesced batch that contains this item is flushed. Dynamically composed responses + /// must be an owned buffer (not Pipe memory, not session scratch). Coalesce/order + /// semantics match . + /// + public ValueTask EnqueueLineAsync(ReadOnlyMemory wireLine, CancellationToken cancellationToken = default) => + EnqueueAsync(new WriteRequest(wireLine, completed: null), cancellationToken); + + /// + /// Enqueues a pre-encoded wire line and flushes it without waiting for a coalesce batch. + /// Returns when the Channel accepts the line (may wait if the writer is saturated). + /// + /// + /// Used by TAKETHIS so 239/439 are not held for + /// siblings. Channel order and pump serialization are unchanged. CHECK still uses + /// . + /// + public ValueTask EnqueueLineImmediateAsync( + ReadOnlyMemory wireLine, + CancellationToken cancellationToken = default) => + EnqueueAsync(new WriteRequest(wireLine, completed: null, flushAfter: true), cancellationToken); + + /// + /// Flushes any held fire-and-forget lines without writing additional response bytes. + /// + /// + /// No-op when the coalesce buffer is empty so MODE READER / WriteLineAsync paths are not + /// delayed. Used when the session is about to wait for more inbound octets, and by tests. + /// + internal ValueTask FlushCoalescedAsync(CancellationToken cancellationToken = default) + { + if (!HasCoalescedUnflushed) + { + return ValueTask.CompletedTask; + } + + return WriteAndAwaitFlushAsync(ReadOnlyMemory.Empty, cancellationToken); + } + + /// Begins a multi-line response and writes the initial status line. + public async ValueTask WriteMultilineStartAsync( + int code, + string text, + CancellationToken cancellationToken = default) + { + await WriteLineAsync(code, text, cancellationToken).ConfigureAwait(false); + } + + /// Writes one multi-line body line (applies dot-stuffing) without terminating the block. + public ValueTask WriteMultilineDataAsync(string line, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(line); + if (line.StartsWith('.')) + { + line = "." + line; + } + + return WriteAndAwaitFlushAsync(EncodeAscii(line + "\r\n"), cancellationToken); + } + + /// Terminates a multi-line response with .CRLF and flushes. + public ValueTask WriteMultilineEndAsync(CancellationToken cancellationToken = default) => + WriteAndAwaitFlushAsync(DotCrlf, cancellationToken); + + /// + /// Writes a precomputed byte payload to the session output pipe and flushes (honouring pipe backpressure). + /// + /// + /// Does not copy . The memory must remain unchanged until this + /// write has been flushed. Intended for BENCHIT reuse of an immutable wire buffer. Still + /// uses the production ordered writer path; does not bypass transport pumps. + /// + public ValueTask WriteBytesAndFlushAsync( + ReadOnlyMemory payload, + CancellationToken cancellationToken = default) => + WriteAndAwaitFlushAsync(payload, cancellationToken); + + /// + /// Writes one framed article through the shared bounded article TX path (default 64 KiB chunks). + /// + /// + /// Destuffed article payload bytes supplied by the caller (no terminating .\r\n); + /// see . Not an NNTPD storage API. + /// + /// Mode-independent wire framing (e.g. 220 ARTICLE, 222 BODY, TAKETHIS). + /// Cancellation token. + /// + /// Materialized restuff → owned chunks → existing ordered Channel. Does not inspect session mode. + /// Awaits each chunk's flush into the outbound Pipe (not peer socket delivery). + /// + public ValueTask WriteArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedArticle, + framing, + NntpArticleTxChunkBudget.DefaultBytes, + cancellationToken); + + /// + /// Writes one framed article through the shared bounded article TX path with an explicit chunk budget. + /// + /// Caller-supplied destuffed article payload bytes. + /// Mode-independent wire framing. + /// Production chunk budget (64–256 KiB). + /// Cancellation token. + public ValueTask WriteArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + NntpArticleTxChunkBudget.ThrowIfNotProductionRange(chunkBytes); + return WriteArticleCoreAsync(storedDestuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// + /// Test-only article TX allowing non-production chunk sizes for boundary coverage. + /// + internal ValueTask WriteArticleForTestsAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + return WriteArticleCoreAsync(storedDestuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// + /// Convenience: ARTICLE-style (220) response from caller-supplied full destuffed article bytes. + /// + /// Full destuffed article (headers + body); external input. + /// Message-id for the 220 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + /// + /// Name reflects RFC 3977 ARTICLE wire shape, not an NNTPD customer repository. + /// + public ValueTask WriteCustomerArticleAsync( + ReadOnlyMemory storedDestuffedArticle, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedArticle, + NntpArticleTxFraming.CustomerArticle(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: BODY-style (222) response; strips headers at the first \r\n\r\n. + /// + /// + /// Full destuffed article; headers stripped at the first blank line. + /// When no blank line exists, an empty body is transmitted (still with 222 + terminator). + /// + /// Message-id for the 222 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + /// + /// Same Channel → pump → PipeWriter path as ARTICLE. Does not create a BODY-specific TX stack. + /// Name reflects RFC 3977 BODY wire shape, not storage ownership. + /// + public ValueTask WriteCustomerBodyAsync( + ReadOnlyMemory storedDestuffedArticle, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) + { + _ = ArticleWireReconstructor.TrySplitHeadersAndBody( + storedDestuffedArticle.Span, + out _, + out var bodySpan); + // Copy body span to owned memory before await (span cannot cross await). + var body = bodySpan.IsEmpty ? ReadOnlyMemory.Empty : bodySpan.ToArray(); + return WriteArticleAsync( + body, + NntpArticleTxFraming.CustomerBody(messageId, articleNumber), + cancellationToken); + } + + /// + /// Convenience: BODY-style (222) response from an already-separated destuffed body payload. + /// + /// Destuffed body bytes only (no headers; no terminator). + /// Message-id for the 222 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerBodyFromBodyAsync( + ReadOnlyMemory storedDestuffedBody, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedBody, + NntpArticleTxFraming.CustomerBody(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: HEAD-style (221) response from destuffed header bytes (including blank-line separator when present). + /// + /// Header block only; no body; no terminator. + /// Message-id for the 221 status line. + /// Article number for the status line, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerHeadAsync( + ReadOnlyMemory storedDestuffedHeaders, + string messageId, + long articleNumber = 0, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + storedDestuffedHeaders, + NntpArticleTxFraming.CustomerHead(messageId, articleNumber), + cancellationToken); + + /// + /// Convenience: STAT-style (223) single-line response (no multiline body). + /// + /// Message-id octets including angle brackets. + /// Article number, or 0 when mid-selected. + /// Cancellation token. + public ValueTask WriteCustomerStatAsync( + ReadOnlySpan messageIdBytes, + long articleNumber = 0, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(articleNumber); + var number = articleNumber.ToString(System.Globalization.CultureInfo.InvariantCulture); + var prefix = System.Text.Encoding.ASCII.GetBytes("223 " + number + " "); + var owned = NntpResponseCompose.Concat(prefix, messageIdBytes, NntpResponses.Crlf.Span); + return WriteLineAsync(owned, cancellationToken); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _channel.Writer.TryComplete(); + await _pumpCts.CancelAsync().ConfigureAwait(false); + try + { + await _pump.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on dispose. + } + finally + { + _pumpCts.Dispose(); + } + } + + private async ValueTask WriteArticleCoreAsync( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + var chunks = PrepareArticleChunks(storedDestuffedArticle, framing, chunkBytes); + foreach (var chunk in chunks) + { + // Ownership of `chunk` transfers to the Channel; do not mutate after enqueue. + await WriteAndAwaitFlushAsync(chunk, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Builds owned framed wire chunks (header + restuff) without touching the TX Channel. + /// + /// + /// Used by so preparation can run outside the + /// article-atomic enqueue gate. Caller owns the returned arrays until each is enqueued. + /// + internal static List PrepareArticleChunks( + ReadOnlyMemory storedDestuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(storedDestuffedArticle.Span); + var totalLength = header.Length + body.Length; + if (totalLength == 0) + { + return []; + } + + var chunks = new List((totalLength + chunkBytes - 1) / chunkBytes); + var offset = 0; + while (offset < totalLength) + { + var take = Math.Min(chunkBytes, totalLength - offset); + var chunk = new byte[take]; + CopyFramedWire(header, body, offset, chunk.AsSpan()); + chunks.Add(chunk); + offset += take; + } + + return chunks; + } + + /// + /// Enqueues an owned payload into the ordered TX Channel and returns a task that completes + /// when the pump has flushed that item into the outbound Pipe (not peer delivery). + /// + /// + /// Does not await flush before returning. Used with a short article-atomic enqueue gate so all + /// chunks of one article can be admitted contiguously before awaiting backpressure. + /// + internal async ValueTask EnqueueOwnedPayloadAsync( + byte[] payload, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(payload); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await EnqueueAsync(new WriteRequest(payload, completed), cancellationToken).ConfigureAwait(false); + return WaitForFlushAsync(completed, cancellationToken); + } + + private static async Task WaitForFlushAsync(TaskCompletionSource completed, CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register( + static state => ((TaskCompletionSource)state!).TrySetCanceled(), + completed); + await completed.Task.ConfigureAwait(false); + } + + private static void CopyFramedWire(byte[] header, byte[] body, int absoluteOffset, Span destination) + { + var written = 0; + while (written < destination.Length) + { + if (absoluteOffset < header.Length) + { + var fromHeader = Math.Min(header.Length - absoluteOffset, destination.Length - written); + header.AsSpan(absoluteOffset, fromHeader).CopyTo(destination[written..]); + absoluteOffset += fromHeader; + written += fromHeader; + continue; + } + + var bodyOffset = absoluteOffset - header.Length; + var fromBody = Math.Min(body.Length - bodyOffset, destination.Length - written); + body.AsSpan(bodyOffset, fromBody).CopyTo(destination[written..]); + absoluteOffset += fromBody; + written += fromBody; + } + } + + private async ValueTask WriteAndAwaitFlushAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await EnqueueAsync(new WriteRequest(payload, completed), cancellationToken).ConfigureAwait(false); + using var registration = cancellationToken.Register(static state => + { + ((TaskCompletionSource)state!).TrySetCanceled(); + }, completed); + await completed.Task.ConfigureAwait(false); + } + + /// + /// DIAGNOSTIC-ONLY. Parks Channel/pump production and returns a lease that writes the + /// existing TX . SPEEDTEST is the only caller. + /// + internal async ValueTask AcquireDirectTxPipeAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + await FlushCoalescedAsync(cancellationToken).ConfigureAwait(false); + if (Interlocked.CompareExchange(ref _directExclusive, 1, 0) != 0) + { + throw new InvalidOperationException( + "TX PipeWriter is already exclusively leased; ResponseWriter and SPEEDTEST cannot produce concurrently."); + } + + return new DirectTxPipeLease(this); + } + + internal void ReleaseDirectTxPipe() => Volatile.Write(ref _directExclusive, 0); + + internal async ValueTask WriteDirectToOutputAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + if (Volatile.Read(ref _directExclusive) != 1) + { + throw new InvalidOperationException("Direct TX PipeWriter write requires an exclusive SPEEDTEST lease."); + } + + var remaining = payload; + while (!remaining.IsEmpty) + { + var memory = _output.GetMemory(Math.Min(remaining.Length, 64 * 1024)); + var toCopy = Math.Min(memory.Length, remaining.Length); + remaining.Span[..toCopy].CopyTo(memory.Span); + _output.Advance(toCopy); + Volatile.Read(ref _byteSink).ObserveCopied(toCopy); + remaining = remaining[toCopy..]; + + if (_output.UnflushedBytes >= 64 * 1024) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _directPipeFlushCount); + } + } + + if (_output.UnflushedBytes > 0) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _directPipeFlushCount); + } + } + + private async ValueTask EnqueueAsync(WriteRequest request, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + if (Volatile.Read(ref _directExclusive) == 1) + { + throw new InvalidOperationException( + "TX PipeWriter is exclusively leased to SPEEDTEST; ResponseWriter Channel cannot write concurrently."); + } + + try + { + await _channel.Writer.WriteAsync(request, cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _channelEnqueueCount); + Interlocked.Add(ref _ownedPayloadBytesEnqueued, request.Payload.Length); + if (request.Completed is null) + { + Interlocked.Increment(ref _coalescedUnflushed); + } + } + catch (ChannelClosedException ex) + { + throw new ObjectDisposedException(nameof(NntpResponseWriter), ex); + } + } + + private async Task PumpAsync(CancellationToken cancellationToken) + { + var batch = new WriteRequest[CoalesceResponseBatchSize + 1]; + var count = 0; + var coalesceCount = 0; + + void CancelHeld() + { + var coalesced = 0; + for (var i = 0; i < count; i++) + { + if (batch[i].Completed is null) + { + coalesced++; + } + + batch[i].Completed?.TrySetCanceled(); + } + + if (coalesced > 0) + { + Interlocked.Add(ref _coalescedUnflushed, -coalesced); + } + + count = 0; + coalesceCount = 0; + } + + try + { + await foreach (var request in _channel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + if (request.Completed is not null) + { + batch[count++] = request; + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + continue; + } + + batch[count++] = request; + coalesceCount++; + + if (request.FlushAfter) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + continue; + } + + while (coalesceCount < CoalesceResponseBatchSize && + _channel.Reader.TryRead(out var queued)) + { + if (queued.Completed is not null || queued.FlushAfter) + { + batch[count++] = queued; + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + break; + } + + batch[count++] = queued; + coalesceCount++; + } + + if (coalesceCount >= CoalesceResponseBatchSize) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + for (var i = 0; i < count; i++) + { + batch[i].Completed?.TrySetException(ex); + } + + count = 0; + coalesceCount = 0; + if (cancellationToken.IsCancellationRequested) + { + break; + } + } + } + + if (count > 0 && !cancellationToken.IsCancellationRequested) + { + await FlushHeldAsync(batch, count, cancellationToken).ConfigureAwait(false); + count = 0; + coalesceCount = 0; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Dispose path. + } + finally + { + CancelHeld(); + while (_channel.Reader.TryRead(out var leftover)) + { + if (leftover.Completed is null) + { + Interlocked.Decrement(ref _coalescedUnflushed); + } + + leftover.Completed?.TrySetCanceled(); + } + } + } + + private async ValueTask FlushHeldAsync( + WriteRequest[] batch, + int count, + CancellationToken cancellationToken) + { + var coalesced = 0; + for (var i = 0; i < count; i++) + { + if (batch[i].Completed is null) + { + coalesced++; + } + } + + try + { + for (var i = 0; i < count; i++) + { + await CopyPayloadToPipeAsync(batch[i].Payload, cancellationToken).ConfigureAwait(false); + } + + if (_output.UnflushedBytes > 0) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + } + + for (var i = 0; i < count; i++) + { + batch[i].Completed?.TrySetResult(); + } + } + finally + { + if (coalesced > 0) + { + Interlocked.Add(ref _coalescedUnflushed, -coalesced); + } + } + } + + private async ValueTask CopyPayloadToPipeAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + { + var remaining = payload; + while (!remaining.IsEmpty) + { + var memory = _output.GetMemory(Math.Min(remaining.Length, 64 * 1024)); + var toCopy = Math.Min(memory.Length, remaining.Length); + remaining.Span[..toCopy].CopyTo(memory.Span); + _output.Advance(toCopy); + Volatile.Read(ref _byteSink).ObserveCopied(toCopy); + remaining = remaining[toCopy..]; + + if (_output.UnflushedBytes >= 64 * 1024) + { + await FlushOutputAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + private async ValueTask FlushOutputAsync(CancellationToken cancellationToken) + { + var flush = _output.FlushAsync(cancellationToken); + var result = flush.IsCompletedSuccessfully + ? flush.Result + : await flush.ConfigureAwait(false); + + Interlocked.Increment(ref _pipeFlushCount); + + if (result.IsCanceled) + { + cancellationToken.ThrowIfCancellationRequested(); + } + + if (result.IsCompleted) + { + throw new InvalidOperationException("NNTP output pipe completed while writing."); + } + } + + private static byte[] EncodeLine(int code, string text) => + EncodeAscii($"{code} {text}\r\n"); + + private static byte[] EncodeAscii(string text) + { + var byteCount = Encoding.ASCII.GetByteCount(text); + var bytes = new byte[byteCount]; + Encoding.ASCII.GetBytes(text, bytes); + return bytes; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponses.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponses.cs new file mode 100644 index 00000000..a5bad825 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpResponses.cs @@ -0,0 +1,440 @@ +using System.Text; + +using VectorNNTPD.TransitService.Session.Commands; + +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Immortal pre-encoded NNTP wire responses. Each value is ASCII, CRLF-terminated, and + /// encoded once at type initialization. The TX pump may read the same backing array + /// repeatedly; callers must not mutate it. + /// + /// + /// Invariant replies are complete wire buffers. CHECK/TAKETHIS/DATE use the prefix/suffix + /// fields with (one owned copy of dynamic octets). + /// + internal static class NntpResponses + { + internal static readonly ReadOnlyMemory GreetingPostingPermitted = + Line("200 VectorNNTPD.TransitService ready, posting permitted\r\n"u8); + + internal static readonly ReadOnlyMemory GreetingPostingProhibited = + Line("201 VectorNNTPD.TransitService ready, posting prohibited\r\n"u8); + + internal static readonly ReadOnlyMemory ReaderModePostingPermitted = + Line("200 Reader mode, posting permitted\r\n"u8); + + internal static readonly ReadOnlyMemory ReaderModePostingProhibited = + Line("201 Reader mode, posting prohibited\r\n"u8); + + internal static readonly ReadOnlyMemory StreamingPermitted = + Line("203 Streaming permitted\r\n"u8); + + internal static readonly ReadOnlyMemory ConnectionClosing = + Line("205 Connection closing\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionActive = + Line("206 Compression active\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationAccepted = + Line("281 Authentication accepted\r\n"u8); + + internal static readonly ReadOnlyMemory PasswordRequired = + Line("381 Password required\r\n"u8); + + internal static readonly ReadOnlyMemory ContinueWithTls = + Line("382 Continue with TLS negotiation\r\n"u8); + + internal static readonly ReadOnlyMemory ServiceTemporarilyUnavailable = + Line("400 Service temporarily unavailable\r\n"u8); + + internal static readonly ReadOnlyMemory CommandFailed = + Line("403 Command failed\r\n"u8); + + internal static readonly ReadOnlyMemory UnableToActivateCompression = + Line("403 Unable to activate compression\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationRequired = + Line("480 Authentication required\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationFailed = + Line("481 Authentication failed\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationCancelled = + Line("481 Authentication cancelled\r\n"u8); + + internal static readonly ReadOnlyMemory TooManySessions = + Line("481 Too many sessions\r\n"u8); + + internal static readonly ReadOnlyMemory TooManySourceAddresses = + Line("481 Too many source addresses\r\n"u8); + + internal static readonly ReadOnlyMemory AuthenticationOutOfSequence = + Line("482 Authentication commands issued out of sequence\r\n"u8); + + internal static readonly ReadOnlyMemory PrivacyRequired = + Line("483 Encryption or stronger authentication required\r\n"u8); + + internal static readonly ReadOnlyMemory UnknownCommand = + Line("500 Unknown command\r\n"u8); + + internal static readonly ReadOnlyMemory CommandNotImplemented = + Line("500 Command not implemented\r\n"u8); + + internal static readonly ReadOnlyMemory SyntaxError = + Line("501 Syntax error\r\n"u8); + + internal static readonly ReadOnlyMemory UnknownCommandVariant = + Line("501 Unknown command variant\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoUserRequiresUsername = + Line("501 AUTHINFO USER requires a username\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoPassRequiresPassword = + Line("501 AUTHINFO PASS requires a password\r\n"u8); + + internal static readonly ReadOnlyMemory CompressRequiresAlgorithm = + Line("501 COMPRESS requires a single algorithm argument\r\n"u8); + + internal static readonly ReadOnlyMemory CompressAlgorithmSyntaxInvalid = + Line("501 Syntactically incorrect compression algorithm\r\n"u8); + + internal static readonly ReadOnlyMemory AuthinfoSaslNotImplemented = + Line("501 AUTHINFO SASL not implemented\r\n"u8); + + internal static readonly ReadOnlyMemory SaslMechanismRequired = + Line("501 SASL mechanism required\r\n"u8); + + internal static readonly ReadOnlyMemory SaslInvalidBase64 = + Line("504 Base64 encoding error\r\n"u8); + + internal static readonly ReadOnlyMemory SaslLoginUsernameChallenge = + Line("383 VXNlcm5hbWU6\r\n"u8); + + internal static readonly ReadOnlyMemory SaslLoginPasswordChallenge = + Line("383 UGFzc3dvcmQ6\r\n"u8); + + internal static readonly ReadOnlyMemory SaslEmptyChallenge = + Line("383 =\r\n"u8); + + internal static readonly ReadOnlyMemory SaslProtocolError = + Line("482 SASL protocol error\r\n"u8); + + internal static readonly ReadOnlyMemory SaslMechanismNotSupported = + Line("503 Mechanism not supported\r\n"u8); + + internal static readonly ReadOnlyMemory SaslContinuationNotSupported = + Line("503 SASL continuation not supported\r\n"u8); + + internal static readonly ReadOnlyMemory NoSaslExchangeInProgress = + Line("503 No SASL exchange in progress\r\n"u8); + + internal static readonly ReadOnlyMemory TemporaryAuthenticationFailure = + Line("503 Temporary authentication failure\r\n"u8); + + internal static readonly ReadOnlyMemory SaslScramUnavailable = + Line("503 SCRAM not available\r\n"u8); + + internal static readonly ReadOnlyMemory SaslCramUnavailable = + Line("503 CRAM-MD5 not available\r\n"u8); + + internal static readonly ReadOnlyMemory AlreadyAuthenticated = + Line("502 Already authenticated\r\n"u8); + + internal static readonly ReadOnlyMemory CommandUnavailableAfterAuthentication = + Line("502 Command unavailable after authentication\r\n"u8); + + internal static readonly ReadOnlyMemory CommandUnavailableAfterCompression = + Line("502 Command unavailable after compression\r\n"u8); + + internal static readonly ReadOnlyMemory DeflateAlreadyActive = + Line("502 DEFLATE compression already active\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionAlreadyActive = + Line("502 Compression already active\r\n"u8); + + internal static readonly ReadOnlyMemory TlsAlreadyActive = + Line("502 TLS already active\r\n"u8); + + internal static readonly ReadOnlyMemory TlsProviderUnavailable = + Line("502 TLS provider unavailable\r\n"u8); + + internal static readonly ReadOnlyMemory PostingNotPermitted = + Line("502 Posting not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory PermissionDenied = + Line("502 Permission denied\r\n"u8); + + internal static readonly ReadOnlyMemory CommandNotAvailable = + Line("502 Command not available on this server\r\n"u8); + + internal static readonly ReadOnlyMemory StreamingNotPermitted = + Line("502 Streaming not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory NotInReaderMode = + Line("502 Not in reader mode\r\n"u8); + + internal static readonly ReadOnlyMemory NotInStreamMode = + Line("502 Not in stream mode\r\n"u8); + + internal static readonly ReadOnlyMemory CompressionAlgorithmNotSupported = + Line("503 Compression algorithm not supported\r\n"u8); + + /// RFC 3977 §7.6.1.2: recognized LIST keyword whose information is not maintained. + internal static readonly ReadOnlyMemory ListDataItemNotStored = + Line("503 Data item not stored\r\n"u8); + + internal static readonly ReadOnlyMemory HelpTextFollows = + Line("100 Help text follows\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityListFollows = + Line("101 Capability list:\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityVersion2 = + Multiline("VERSION 2\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityImplementation = + Multiline("IMPLEMENTATION VectorNNTPD.TransitService\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityReader = + Multiline("READER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityModeReader = + Multiline("MODE-READER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityPost = + Multiline("POST\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfo = + Multiline("AUTHINFO\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfoUser = + Multiline("AUTHINFO USER\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityAuthinfoUserSasl = + Multiline("AUTHINFO USER SASL\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilitySasl = + Multiline("SASL PLAIN LOGIN SCRAM-SHA-256 CRAM-MD5\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityStartTls = + Multiline("STARTTLS\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityCompressDeflate = + Multiline("COMPRESS DEFLATE\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityStreaming = + Multiline("STREAMING\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilitySpeedTest = + Multiline("SPEEDTEST\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityList = + Multiline("LIST ACTIVE COUNTS HEADERS NEWSGROUPS OVERVIEW.FMT\r\n"u8); + + internal static readonly ReadOnlyMemory ListOfNewsgroupsFollows = + Line("215 list of newsgroups follows\r\n"u8); + + internal static readonly ReadOnlyMemory ListOverviewFmtFollows = + Line("215 Order of fields in overview database.\r\n"u8); + + internal static readonly ReadOnlyMemory ListHeadersFollows = + Line("215 headers and metadata items supported:\r\n"u8); + + internal static readonly ReadOnlyMemory NoSuchNewsgroup = + Line("411 No such newsgroup\r\n"u8); + + internal static readonly ReadOnlyMemory NoNewsgroupSelected = + Line("412 No newsgroup selected\r\n"u8); + + internal static readonly ReadOnlyMemory CurrentArticleNumberInvalid = + Line("420 Current article number is invalid\r\n"u8); + + internal static readonly ReadOnlyMemory NoArticleWithNumber = + Line("423 No article with that number\r\n"u8); + + internal static readonly ReadOnlyMemory NoArticleWithMessageId = + Line("430 No article with that message-id\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestUnknownPeer = + Line("502 UNKNOWN SPEEDTEST PEER\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestPeerMismatch = + Line("502 SPEEDTEST PEER MISMATCH\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestBusy = + Line("400 SPEEDTEST BUSY\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestNotSupported = + Line("503 SPEEDTEST NOT SUPPORTED\r\n"u8); + + internal static readonly ReadOnlyMemory SpeedTestReadyPrefix = Prefix("290 SPEEDTEST "u8); + + internal static readonly ReadOnlyMemory SpeedTestReadySuffix = Suffix(" TX\r\n"u8); + + internal static readonly ReadOnlyMemory MultilineTerminator = + Line(".\r\n"u8); + + internal static readonly ReadOnlyMemory CheckPrefix = Prefix("238 "u8); + + internal static readonly ReadOnlyMemory CheckSuffix = + Suffix(" send article to be transferred\r\n"u8); + + internal static readonly ReadOnlyMemory CheckNotWantedPrefix = Prefix("438 "u8); + + internal static readonly ReadOnlyMemory CheckNotWantedSuffix = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory CheckTryLaterPrefix = Prefix("431 "u8); + + internal static readonly ReadOnlyMemory CheckTryLaterSuffix = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveSendArticle = + Line("335 Send article to be transferred\r\n"u8); + + internal static readonly ReadOnlyMemory ArticleReceivedOk = + Line("240 Article received OK\r\n"u8); + + internal static readonly ReadOnlyMemory PostSendArticle = + Line("340 Input article; end with .\r\n"u8); + + internal static readonly ReadOnlyMemory PostingProhibited = + Line("440 Posting not permitted\r\n"u8); + + internal static readonly ReadOnlyMemory PostingFailed = + Line("441 Posting failed\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTransferredOk = + Line("235 Article transferred OK\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveNotWanted = + Line("435 Article not wanted\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTryLater = + Line("436 Transfer not possible; try again later\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveTransferFailed = + Line("436 Transfer failed; try again later\r\n"u8); + + internal static readonly ReadOnlyMemory IhaveRejected = + Line("437 Transfer rejected; do not retry\r\n"u8); + + internal static readonly ReadOnlyMemory CapabilityIhave = + Multiline("IHAVE\r\n"u8); + + internal static readonly ReadOnlyMemory ArticleTransferredOkPrefix = Prefix("239 "u8); + + internal static readonly ReadOnlyMemory TransferRejectedPrefix = Prefix("439 "u8); + + internal static readonly ReadOnlyMemory Crlf = Suffix("\r\n"u8); + + internal static readonly ReadOnlyMemory DatePrefix = Prefix("111 "u8); + + internal static readonly ReadOnlyMemory[] HelpBodyLines = EncodeHelpBody(); + + /// Complete HELP multiline wire (status + body + terminator). Immortal. + internal static readonly ReadOnlyMemory HelpComplete = EncodeHelpComplete(); + + /// Complete LIST OVERVIEW.FMT multiline wire. Immortal; RFC 3977 §8.4 compatibility form. + internal static readonly ReadOnlyMemory ListOverviewFmtComplete = EncodeListOverviewFmtComplete(); + + /// Complete LIST HEADERS multiline wire. Immortal; same for MSGID and RANGE. + internal static readonly ReadOnlyMemory ListHeadersComplete = EncodeListHeadersComplete(); + + /// + /// Copies onto an immortal heap array once. + /// The UTF-8 literal span must not be retained. + /// + private static ReadOnlyMemory Line(ReadOnlySpan asciiIncludingCrlf) + { + if (asciiIncludingCrlf.Length < 2 + || asciiIncludingCrlf[^2] != (byte)'\r' + || asciiIncludingCrlf[^1] != (byte)'\n') + { + throw new ArgumentException("Static NNTP responses must end with a single CRLF."); + } + + var copy = asciiIncludingCrlf.ToArray(); + if (CrlfCount(copy) != 1) + { + throw new ArgumentException("Static single-line NNTP responses must contain exactly one CRLF."); + } + + return copy; + } + + private static ReadOnlyMemory Multiline(ReadOnlySpan asciiIncludingCrlf) => + Line(asciiIncludingCrlf); + + private static ReadOnlyMemory[] EncodeHelpBody() + { + var source = Help.BodyLines; + var encoded = new ReadOnlyMemory[source.Count]; + for (var i = 0; i < source.Count; i++) + { + encoded[i] = Line(Encoding.ASCII.GetBytes(source[i] + "\r\n")); + } + + return encoded; + } + + private static ReadOnlyMemory EncodeHelpComplete() + { + var parts = new ReadOnlyMemory[HelpBodyLines.Length + 2]; + parts[0] = HelpTextFollows; + HelpBodyLines.CopyTo(parts, 1); + parts[^1] = MultilineTerminator; + return NntpResponseCompose.Concatenate(parts); + } + + private static ReadOnlyMemory EncodeListOverviewFmtComplete() => + EncodeStaticMultiline( + ListOverviewFmtFollows, + Multiline("Subject:\r\n"u8), + Multiline("From:\r\n"u8), + Multiline("Date:\r\n"u8), + Multiline("Message-ID:\r\n"u8), + Multiline("References:\r\n"u8), + Multiline("Bytes:\r\n"u8), + Multiline("Lines:\r\n"u8)); + + private static ReadOnlyMemory EncodeListHeadersComplete() => + EncodeStaticMultiline( + ListHeadersFollows, + Multiline("Subject\r\n"u8), + Multiline("From\r\n"u8), + Multiline("Date\r\n"u8), + Multiline("Message-ID\r\n"u8), + Multiline("References\r\n"u8), + Multiline(":bytes\r\n"u8), + Multiline(":lines\r\n"u8)); + + private static ReadOnlyMemory EncodeStaticMultiline( + ReadOnlyMemory status, + params ReadOnlyMemory[] bodyLines) + { + var parts = new ReadOnlyMemory[bodyLines.Length + 2]; + parts[0] = status; + bodyLines.CopyTo(parts, 1); + parts[^1] = MultilineTerminator; + return NntpResponseCompose.Concatenate(parts); + } + + private static ReadOnlyMemory Prefix(ReadOnlySpan ascii) => ascii.ToArray(); + + private static ReadOnlyMemory Suffix(ReadOnlySpan ascii) => ascii.ToArray(); + + private static int CrlfCount(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs new file mode 100644 index 00000000..fd9868f7 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpStreamArticleTxScheduler.cs @@ -0,0 +1,263 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Bounded STREAM outstanding article TX scheduler above the shared + /// TX path. + /// + /// + /// + /// Limits how many STREAM article operations may be admitted concurrently (default 8; valid 4–16). + /// Does not replace TX Channel capacity, Pipe thresholds, or transport backpressure. + /// + /// + /// Producer model (Phase 3B; evidence: + /// docs/STREAM-TX-PRODUCER-CONCURRENCY-EXPERIMENT.md): + /// + /// + /// Acquire STREAM depth slot. + /// Prepare framing/restuff/owned chunks outside the enqueue gate. + /// Acquire short FIFO enqueue gate; admit all chunks of this article + /// contiguously into the existing Channel; release gate. + /// Await existing per-chunk Pipe flush completion (backpressure) outside the gate. + /// Release depth slot in finally. + /// + /// + /// Callers must start operations in protocol order. The enqueue gate is FIFO among waiters and + /// prevents chunk interleaving (A1 B1 A2…). There is no second TX Channel or pump. + /// + /// + /// This type does not load or look up articles. Callers supply destuffed bytes (bench, peer feed, + /// or other external source). Article storage/catalogue is out of scope for NNTPD. + /// + /// + public sealed class NntpStreamArticleTxScheduler : IAsyncDisposable + { + /// Default STREAM outstanding article depth. + public const int DefaultDepth = 8; + + /// Minimum allowed STREAM outstanding article depth. + public const int MinDepth = 4; + + /// Maximum allowed STREAM outstanding article depth. + public const int MaxDepth = 16; + + private readonly SemaphoreSlim _depth; + private readonly SemaphoreSlim _enqueueGate = new(1, 1); + private int _outstanding; + private int _preparing; + private int _enqueuing; + private int _peakPreparing; + private int _disposed; + + /// Initializes a scheduler with . + public NntpStreamArticleTxScheduler() + : this(DefaultDepth) + { + } + + /// Initializes a scheduler with an explicit depth. + /// Outstanding article depth; must be in –. + public NntpStreamArticleTxScheduler(int depth) + { + ThrowIfNotValidDepth(depth); + Depth = depth; + _depth = new SemaphoreSlim(depth, depth); + } + + /// Gets the configured outstanding depth. + public int Depth { get; } + + /// Gets the number of currently admitted (not yet completed) operations. + public int Outstanding => Volatile.Read(ref _outstanding); + + /// Gets the peak number of concurrent preparation phases observed (diagnostics / tests). + internal int PeakPreparing => Volatile.Read(ref _peakPreparing); + + /// Gets the number of operations currently holding the enqueue gate (0 or 1). + internal int Enqueuing => Volatile.Read(ref _enqueuing); + + /// Test hook: invoked after preparation, before the enqueue gate (deterministic cancel tests). + internal Func? TestAfterPrepare { get; set; } + + /// Test hook: invoked while holding the enqueue gate, before chunk admission. + internal Func? TestWhileHoldingEnqueueGate { get; set; } + + /// Test hook: invoked after releasing the enqueue gate, before awaiting Pipe flush. + internal Func? TestBeforeAwaitFlush { get; set; } + + /// Returns when is in the valid range. + public static bool IsValidDepth(int depth) => depth is >= MinDepth and <= MaxDepth; + + /// Throws if is outside –. + public static void ThrowIfNotValidDepth(int depth) + { + if (!IsValidDepth(depth)) + { + throw new ArgumentOutOfRangeException( + nameof(depth), + depth, + $"Stream outstanding article depth must be between {MinDepth} and {MaxDepth}."); + } + } + + /// + /// Acquires one outstanding slot, runs , then releases the slot + /// (including on cancellation or fault). + /// + public async ValueTask ExecuteAsync( + Func operation, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + + await _depth.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _outstanding); + try + { + await operation(cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _outstanding); + _depth.Release(); + } + } + + /// + /// Writes one framed article: prepare outside enqueue gate, admit all chunks under the gate, + /// await Pipe flush outside the gate. + /// + public ValueTask WriteArticleAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + CancellationToken cancellationToken = default) => + WriteArticleAsync( + writer, + destuffedArticle, + framing, + NntpArticleTxChunkBudget.DefaultBytes, + cancellationToken); + + /// + /// Writes one framed article with an explicit production chunk budget. + /// + public ValueTask WriteArticleAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(writer); + NntpArticleTxChunkBudget.ThrowIfNotProductionRange(chunkBytes); + return WriteArticleCoreAsync(writer, destuffedArticle, framing, chunkBytes, cancellationToken); + } + + /// Test-only write allowing non-production chunk sizes. + internal ValueTask WriteArticleForTestsAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentOutOfRangeException.ThrowIfLessThan(chunkBytes, 1); + return WriteArticleCoreAsync(writer, destuffedArticle, framing, chunkBytes, cancellationToken); + } + + private ValueTask WriteArticleCoreAsync( + NntpResponseWriter writer, + ReadOnlyMemory destuffedArticle, + NntpArticleTxFraming framing, + int chunkBytes, + CancellationToken cancellationToken) + { + return ExecuteAsync( + async token => + { + // 1) PREPARATION — outside enqueue gate (bounded by depth). + var preparing = Interlocked.Increment(ref _preparing); + UpdatePeakPreparing(preparing); + List chunks; + try + { + chunks = NntpResponseWriter.PrepareArticleChunks(destuffedArticle, framing, chunkBytes); + if (TestAfterPrepare is not null) + { + await TestAfterPrepare(token).ConfigureAwait(false); + } + } + finally + { + Interlocked.Decrement(ref _preparing); + } + + if (chunks.Count == 0) + { + return; + } + + // 2) ARTICLE-ATOMIC ENQUEUE — all chunks contiguous; gate may wait on Channel. + var flushTasks = new Task[chunks.Count]; + await _enqueueGate.WaitAsync(token).ConfigureAwait(false); + Interlocked.Increment(ref _enqueuing); + try + { + if (TestWhileHoldingEnqueueGate is not null) + { + await TestWhileHoldingEnqueueGate(token).ConfigureAwait(false); + } + + for (var i = 0; i < chunks.Count; i++) + { + flushTasks[i] = await writer + .EnqueueOwnedPayloadAsync(chunks[i], token) + .ConfigureAwait(false); + } + } + finally + { + Interlocked.Decrement(ref _enqueuing); + _enqueueGate.Release(); + } + + // 3) PIPE / TX BACKPRESSURE — outside enqueue gate. + if (TestBeforeAwaitFlush is not null) + { + await TestBeforeAwaitFlush(token).ConfigureAwait(false); + } + + await Task.WhenAll(flushTasks).ConfigureAwait(false); + }, + cancellationToken); + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return ValueTask.CompletedTask; + } + + _depth.Dispose(); + _enqueueGate.Dispose(); + return ValueTask.CompletedTask; + } + + private void UpdatePeakPreparing(int current) + { + int snap; + while (current > (snap = Volatile.Read(ref _peakPreparing))) + { + if (Interlocked.CompareExchange(ref _peakPreparing, current, snap) == snap) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerb.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerb.cs new file mode 100644 index 00000000..b82aedcd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerb.cs @@ -0,0 +1,122 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Canonical NNTP verb or qualifier identity. Built-in dispatch uses this enum, not strings. + /// + public enum NntpVerb : byte + { + /// No verb (empty line). + None = 0, + + /// Unrecognized verb or qualifier. + Unknown, + + /// QUIT. + Quit, + + /// HELP. + Help, + + /// DATE. + Date, + + /// CAPABILITIES. + Capabilities, + + /// MODE. + Mode, + + /// STARTTLS. + StartTls, + + /// COMPRESS. + Compress, + + /// LIST. + List, + + /// LISTGROUP. + ListGroup, + + /// HDR. + Hdr, + + /// OVER. + Over, + + /// GROUP. + Group, + + /// ARTICLE. + Article, + + /// HEAD. + Head, + + /// BODY. + Body, + + /// STAT. + Stat, + + /// NEXT. + Next, + + /// LAST. + Last, + + /// POST. + Post, + + /// CHECK. + Check, + + /// IHAVE. + Ihave, + + /// TAKETHIS. + TakeThis, + + /// AUTHINFO. + AuthInfo, + + /// Internal BENCHIT facility. + BenchIt, + + /// VectorNNTP SPEEDTEST diagnostic extension. + SpeedTest, + + /// MODE READER qualifier. + Reader, + + /// MODE STREAM qualifier. + Stream, + + /// AUTHINFO USER qualifier. + User, + + /// AUTHINFO PASS qualifier. + Pass, + + /// AUTHINFO SASL qualifier. + Sasl, + + /// LIST ACTIVE qualifier. + Active, + + /// LIST COUNTS qualifier. + Counts, + + /// LIST HEADERS qualifier. + Headers, + + /// LIST MOTD qualifier. + Motd, + + /// LIST NEWSGROUPS qualifier. + Newsgroups, + + /// LIST OVERVIEW.FMT qualifier. + OverviewFmt, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerbClassifier.cs b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerbClassifier.cs new file mode 100644 index 00000000..c405a27c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/CommandProcessor/NntpVerbClassifier.cs @@ -0,0 +1,109 @@ +namespace VectorNNTPD.TransitService.Session.CommandProcessor +{ + /// + /// Zero-allocation verb classifier: length, then first byte, then remaining ASCII-folded compare. + /// + internal static class NntpVerbClassifier + { + public static NntpVerb Classify(ReadOnlySpan line, out int verbStart, out int verbEnd) + { + verbStart = NntpAscii.SkipWhitespace(line, 0); + if (verbStart >= line.Length) + { + verbEnd = verbStart; + return NntpVerb.None; + } + + verbEnd = NntpAscii.SkipToken(line, verbStart); + return ClassifyVerb(line.Slice(verbStart, verbEnd - verbStart)); + } + + private static NntpVerb ClassifyVerb(ReadOnlySpan verb) + { + if (verb.IsEmpty) + { + return NntpVerb.Unknown; + } + + byte first = NntpAscii.FoldUpper(verb[0]); + return verb.Length switch + { + 3 => first == (byte)'H' && NntpAscii.EqualsFolded(verb, "HDR"u8) ? NntpVerb.Hdr : NntpVerb.Unknown, + 4 => ClassifyLen4(verb, first), + 5 => ClassifyLen5(verb, first), + 7 => ClassifyLen7(verb, first), + 8 => ClassifyLen8(verb, first), + 9 => ClassifyLen9(verb, first), + 12 => first == (byte)'C' && NntpAscii.EqualsFolded(verb, "CAPABILITIES"u8) + ? NntpVerb.Capabilities + : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen4(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'B' => NntpAscii.EqualsFolded(verb, "BODY"u8) ? NntpVerb.Body : NntpVerb.Unknown, + (byte)'D' => NntpAscii.EqualsFolded(verb, "DATE"u8) ? NntpVerb.Date : NntpVerb.Unknown, + (byte)'H' => NntpAscii.EqualsFolded(verb, "HEAD"u8) ? NntpVerb.Head + : NntpAscii.EqualsFolded(verb, "HELP"u8) ? NntpVerb.Help + : NntpVerb.Unknown, + (byte)'L' => NntpAscii.EqualsFolded(verb, "LIST"u8) ? NntpVerb.List + : NntpAscii.EqualsFolded(verb, "LAST"u8) ? NntpVerb.Last + : NntpVerb.Unknown, + (byte)'M' => NntpAscii.EqualsFolded(verb, "MODE"u8) ? NntpVerb.Mode : NntpVerb.Unknown, + (byte)'N' => NntpAscii.EqualsFolded(verb, "NEXT"u8) ? NntpVerb.Next : NntpVerb.Unknown, + (byte)'O' => NntpAscii.EqualsFolded(verb, "OVER"u8) ? NntpVerb.Over : NntpVerb.Unknown, + (byte)'P' => NntpAscii.EqualsFolded(verb, "POST"u8) ? NntpVerb.Post : NntpVerb.Unknown, + (byte)'Q' => NntpAscii.EqualsFolded(verb, "QUIT"u8) ? NntpVerb.Quit : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "STAT"u8) ? NntpVerb.Stat : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen5(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'C' => NntpAscii.EqualsFolded(verb, "CHECK"u8) ? NntpVerb.Check : NntpVerb.Unknown, + (byte)'G' => NntpAscii.EqualsFolded(verb, "GROUP"u8) ? NntpVerb.Group : NntpVerb.Unknown, + (byte)'I' => NntpAscii.EqualsFolded(verb, "IHAVE"u8) ? NntpVerb.Ihave : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen7(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'A' => NntpAscii.EqualsFolded(verb, "ARTICLE"u8) ? NntpVerb.Article : NntpVerb.Unknown, + (byte)'B' => NntpAscii.EqualsFolded(verb, "BENCHIT"u8) ? NntpVerb.BenchIt : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen8(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'A' => NntpAscii.EqualsFolded(verb, "AUTHINFO"u8) ? NntpVerb.AuthInfo : NntpVerb.Unknown, + (byte)'C' => NntpAscii.EqualsFolded(verb, "COMPRESS"u8) ? NntpVerb.Compress : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "STARTTLS"u8) ? NntpVerb.StartTls : NntpVerb.Unknown, + (byte)'T' => NntpAscii.EqualsFolded(verb, "TAKETHIS"u8) ? NntpVerb.TakeThis : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + + private static NntpVerb ClassifyLen9(ReadOnlySpan verb, byte first) + { + return first switch + { + (byte)'L' => NntpAscii.EqualsFolded(verb, "LISTGROUP"u8) ? NntpVerb.ListGroup : NntpVerb.Unknown, + (byte)'S' => NntpAscii.EqualsFolded(verb, "SPEEDTEST"u8) ? NntpVerb.SpeedTest : NntpVerb.Unknown, + _ => NntpVerb.Unknown, + }; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Article.cs b/src/VectorNNTPD.TransitService/Session/Commands/Article.cs new file mode 100644 index 00000000..91152fba --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Article.cs @@ -0,0 +1,796 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// ARTICLE, HEAD, BODY, and STAT as defined by RFC 3977, Sections 6.2.1–6.2.4. + /// + /// + /// + /// Message-id form asks first. A positive + /// StorageServer response is fetched with the existing VATP client + /// (). Only a completed lookup with no positive + /// response calls ArticleWork. When has no + /// entries, that client does not publish a StorageServer request and the same miss + /// continues to ArticleWork. An empty registry is not an article-not-found result. + /// Transport, malformed, and invalid StorageServer results stay on that path. A validated CanonicalV1 + /// is required before any 220/221/222/223 + /// reply. A StorageServer success is served without ingestion. A BackFiller success + /// admits a Path traversal whose current hop is this NNTPD, and the reply sends + /// that record. A StorageServer success traverses once on ingress and the reply + /// sends that record. A failed traversal still sends the fetched bytes. + /// + /// + /// Numeric article lookup and omitted current-article forms are unchanged: + /// message-id unavailable → 430, article number → 412/423, + /// omitted current article → 412/420. Temporary transfer failures use + /// 400 without closing the connection. Error replies are single-line. + /// + /// + /// An unsuccessful lookup MUST NOT change the selected group or current article + /// (RFC 3977 §6.2.1.2). There is no current-article pointer yet, so the omitted form + /// after a successful GROUP is 420. + /// + /// + internal static class Article + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Article)); + + private enum RetrievalKind + { + Article, + Head, + Body, + Stat, + } + + /// Handles ARTICLE (RFC 3977, Section 6.2.1). + public static ValueTask HandleArticleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "ARTICLE", static (c, ct) => ExecuteAsync(c, RetrievalKind.Article, ct), cancellationToken); + + /// Handles HEAD (RFC 3977, Section 6.2.2). + public static ValueTask HandleHeadAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "HEAD", static (c, ct) => ExecuteAsync(c, RetrievalKind.Head, ct), cancellationToken); + + /// Handles BODY (RFC 3977, Section 6.2.3). + public static ValueTask HandleBodyAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "BODY", static (c, ct) => ExecuteAsync(c, RetrievalKind.Body, ct), cancellationToken); + + /// Handles STAT (RFC 3977, Section 6.2.4). + public static ValueTask HandleStatAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "STAT", static (c, ct) => ExecuteAsync(c, RetrievalKind.Stat, ct), cancellationToken); + + private static async ValueTask ExecuteAsync( + NntpCommandContext context, + RetrievalKind kind, + CancellationToken cancellationToken) + { + var argument = context.ArgumentSpan; + if (argument.IsEmpty) + { + await WriteCurrentArticleFailureAsync(context, cancellationToken).ConfigureAwait(false); + return; + } + + if (argument[0] == (byte)'<') + { + await ExecuteMessageIdAsync(context, kind, cancellationToken).ConfigureAwait(false); + return; + } + + await WriteNumberLookupFailureAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ExecuteMessageIdAsync( + NntpCommandContext context, + RetrievalKind kind, + CancellationToken cancellationToken) + { + var resolved = await ResolveByMessageIdAsync(context, cancellationToken).ConfigureAwait(false); + switch (resolved.Kind) + { + case ResolveKind.NotFound: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoArticleWithMessageId, + NntpResponseStatus.NoArticleWithMessageId, + cancellationToken).ConfigureAwait(false); + return; + case ResolveKind.TemporaryFailure: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + case ResolveKind.Success: + break; + default: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + } + + var record = resolved.Record; + if (resolved.Source == RetrievalSource.BackFiller + && TryAdmitBackFiller(context, in record, out var admitted)) + { + record = admitted; + } + else if (resolved.Source == RetrievalSource.StorageServer + && TryTraverseStorageIngress(context, in record, out var entered)) + { + record = entered; + } + + var messageIdText = Encoding.ASCII.GetString(record.MessageId); + switch (kind) + { + case RetrievalKind.Article: + await context.Response.WriteCustomerArticleAsync( + record.ArtData, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "220 0 " + messageIdText); + return; + case RetrievalKind.Head: + if (!ArticleWireReconstructor.TrySplitHeadersAndBody(record.ArtData.Span, out var headersSpan, out _)) + { + headersSpan = record.ArtData.Span; + } + + var headers = record.ArtData[..headersSpan.Length]; + await context.Response.WriteCustomerHeadAsync( + headers, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "221 0 " + messageIdText); + return; + case RetrievalKind.Body: + _ = ArticleWireReconstructor.TrySplitHeadersAndBody(record.ArtData.Span, out var headerSpan, out var bodySpan); + var body = bodySpan.IsEmpty + ? ReadOnlyMemory.Empty + : record.ArtData.Slice(headerSpan.Length, bodySpan.Length); + await context.Response.WriteCustomerBodyFromBodyAsync( + body, + messageIdText, + articleNumber: 0, + cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "222 0 " + messageIdText); + return; + case RetrievalKind.Stat: + await context.Response.WriteCustomerStatAsync(record.MessageId, articleNumber: 0, cancellationToken) + .ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, "223 0 " + messageIdText); + return; + default: + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ServiceTemporarilyUnavailable, + NntpResponseStatus.ServiceTemporarilyUnavailable, + cancellationToken).ConfigureAwait(false); + return; + } + } + + private static async ValueTask ResolveByMessageIdAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var storage = context.Session.StorageArticleLookup; + if (storage is null) + { + return ResolveResult.NotFound(); + } + + var articleId = ArticleId.FromMessageId(messageId.Span); + var logger = Logger; + StorageArticleLookupResult fleet; + try + { + fleet = await storage.LookupAsync(articleId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "unavailable", + articleIdText, + Guid.Empty); + } + return ResolveResult.Temporary(); + } + + if (fleet.Outcome == StorageArticleLookupOutcome.Found) + { + return await FetchPositiveStorageAsync(context, messageId, articleId, fleet, cancellationToken) + .ConfigureAwait(false); + } + + var silence = string.Equals(fleet.Error, StorageLookupSilence, StringComparison.Ordinal); + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + silence ? "miss" : "unavailable", + articleIdText, + fleet.RequestId); + } + if (!silence) + { + return ResolveResult.Temporary(); + } + + return await ResolveFromArticleWorkAsync(context, messageId, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ResolveFromArticleWorkAsync( + NntpCommandContext context, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var rpc = context.Session.ArticleWorkRpc; + if (rpc is null) + { + return ResolveResult.NotFound(); + } + + ArticleWorkRpcResult lookup; + try + { + lookup = await rpc.LookupByMessageIdAsync(messageId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleWorkRpcLogMessages.ArticleLookupFailed( + Logger, + ex, + Encoding.ASCII.GetString(messageId.Span)); + return ResolveResult.Temporary(); + } + + if (lookup.Outcome is ArticleWorkOutcome.ArticleNotFound or ArticleWorkOutcome.InvalidArticle) + { + return ResolveResult.NotFound(); + } + + if (lookup.Outcome != ArticleWorkOutcome.Success + || lookup.ArticleId is not { } expectedArtId + || !VatpEndpointFields.IsCanonicalFqdn(lookup.Fqdn) + || lookup.VatpPort is not int vatpPort + || !VatpEndpointFields.IsCanonicalPort(vatpPort)) + { + ArticleRetrievalLogMessages.TransferUnavailable( + Logger, + lookup.Outcome.ToString(), + lookup.Error); + return ResolveResult.Temporary(); + } + + var vatp = context.Session.VatpArticleClient; + if (vatp is null) + { + ArticleRetrievalLogMessages.TransferUnavailable( + Logger, + "VatpClientMissing", + EndpointDetail(lookup.Fqdn, vatpPort)); + return ResolveResult.Temporary(); + } + + VatpFetchResult fetch; + try + { + fetch = await vatp.FetchArticleAsync( + lookup.Fqdn, + vatpPort, + lookup.RequestId, + expectedArtId, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleRetrievalLogMessages.VatpFetchFailed(Logger, ex, lookup.Fqdn, vatpPort, lookup.RequestId); + return ResolveResult.Temporary(); + } + + return WithSource(MapFetch(fetch, messageId.Span, expectedArtId), RetrievalSource.BackFiller); + } + + /// + /// Silence from the fleet lookup. Matches StorageArticleLookupService when no + /// positive response arrives before the existing lookup budget. + /// + private const string StorageLookupSilence = "Storage article lookup timed out with no positive response."; + + private static async ValueTask FetchPositiveStorageAsync( + NntpCommandContext context, + ReadOnlyMemory messageId, + ArticleId articleId, + StorageArticleLookupResult fleet, + CancellationToken cancellationToken) + { + var logger = Logger; + if (fleet.ArticleId != articleId || !TryBindStorageEndpoint(fleet.Fqdn, fleet.VatpPort, out var host, out var port)) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "unavailable", + articleIdText, + fleet.RequestId); + } + + ArticleRetrievalLogMessages.TransferUnavailable(Logger, "StorageEndpoint", EndpointDetail(fleet.Fqdn, fleet.VatpPort)); + return ResolveResult.Temporary(); + } + + var vatp = context.Session.VatpArticleClient; + if (vatp is null) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "found", + articleIdText, + fleet.RequestId); + } + + ArticleRetrievalLogMessages.TransferUnavailable(Logger, "VatpClientMissing", EndpointDetail(host, port)); + return ResolveResult.Temporary(); + } + + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageLookupCompleted( + logger, + "found", + articleIdText, + fleet.RequestId); + } + + var serverId = fleet.ServerId ?? 0; + VatpFetchResult fetch; + try + { + fetch = await FetchStorageCandidateAsync( + vatp, + host, + port, + fleet.RequestId, + articleId, + serverId, + attempt: 1, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return ResolveResult.Temporary(); + } + + if (IsAlternateEligible(fetch) && fleet.Alternates is not null) + { + // Caller cancellation is terminal even when an alternate is already retained + // or the same cancel has already closed the lookup window. + cancellationToken.ThrowIfCancellationRequested(); + var alternate = await fleet.Alternates.WaitForAlternateAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (alternate is { } next) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + fetch = await FetchStorageCandidateAsync( + vatp, + next.Fqdn, + next.VatpPort, + fleet.RequestId, + articleId, + next.ServerId, + attempt: 2, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + return ResolveResult.Temporary(); + } + } + else + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageCandidateExhausted( + logger, + articleIdText, + fleet.RequestId, + 1); + } + } + } + + return WithSource(MapFetch(fetch, messageId.Span, articleId), RetrievalSource.StorageServer); + } + + private static async ValueTask FetchStorageCandidateAsync( + IVatpArticleClient vatp, + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + int serverId, + int attempt, + CancellationToken cancellationToken) + { + var logger = Logger; + if (logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + ArticleRetrievalLogMessages.StorageCandidateSelected( + logger, + articleIdText, + requestId, + serverId, + attempt); + } + try + { + var fetch = await vatp.FetchArticleAsync(fqdn, vatpPort, requestId, articleId, cancellationToken) + .ConfigureAwait(false); + if (fetch.Kind != VatpFetchKind.Success && logger.IsEnabled(LogLevel.Debug)) + { + var articleIdText = articleId.ToLowerHexString(); + var kind = fetch.Kind.ToString(); + var failoverEligible = IsAlternateEligible(fetch); + ArticleRetrievalLogMessages.StorageCandidateFailed( + logger, + articleIdText, + requestId, + serverId, + attempt, + kind, + fetch.AcceptedDataBytes, + failoverEligible); + } + + return fetch; + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + ArticleRetrievalLogMessages.VatpFetchFailed(Logger, ex, fqdn, vatpPort, requestId); + throw; + } + } + + /// + /// A second candidate is allowed only when no DATA payload was copied and the failure + /// is connection, remote transfer, or protocol. Every other result is terminal. + /// + private static bool IsAlternateEligible(VatpFetchResult fetch) => + fetch.AcceptedDataBytes == 0 + && fetch.Kind is VatpFetchKind.ConnectionFailure + or VatpFetchKind.RemoteTransferFailure + or VatpFetchKind.ProtocolFailure; + + private static bool TryBindStorageEndpoint(string? fqdn, int? vatpPort, out string host, out int port) + { + host = string.Empty; + port = 0; + if (!VatpEndpointFields.IsCanonicalFqdn(fqdn) || vatpPort is not int parsed || !VatpEndpointFields.IsCanonicalPort(parsed)) + { + return false; + } + + host = fqdn; + port = parsed; + return true; + } + + private static string EndpointDetail(string? fqdn, int? vatpPort) => + FormattableString.Invariant($"fqdn={fqdn} port={vatpPort}"); + + private static ResolveResult MapFetch( + VatpFetchResult fetch, + ReadOnlySpan requestedMessageId, + ArticleId expectedArtId) + { + switch (fetch.Kind) + { + case VatpFetchKind.Success: + break; + case VatpFetchKind.Cancelled: + throw new OperationCanceledException(); + default: + ArticleRetrievalLogMessages.VatpFetchUnsuccessful( + Logger, + fetch.Kind.ToString(), + fetch.Error, + fetch.ErrorCode?.ToString()); + return ResolveResult.Temporary(); + } + + var record = fetch.Record; + if (record.ParseStatus != ArticleParseStatus.CanonicalV1 || record.ArtSize <= 0) + { + ArticleRetrievalLogMessages.VatpFetchUnsuccessful( + Logger, + "IncompleteOrMalformedArticle", + fetch.Error, + fetch.ErrorCode?.ToString()); + return ResolveResult.Temporary(); + } + + if (!record.ArtId.Equals(expectedArtId) + || !NntpArticleIdentity.MatchesRequest(record.MessageId, requestedMessageId)) + { + ArticleRetrievalLogMessages.IdentityMismatch(Logger, fetch.Error); + return ResolveResult.NotFound(); + } + + return ResolveResult.Success(record, RetrievalSource.None); + } + + /// + /// Keeps a failure result unchanged and stamps only on success. + /// + private static ResolveResult WithSource(ResolveResult result, RetrievalSource source) => + result.Kind == ResolveKind.Success + ? ResolveResult.Success(result.Record, source) + : result; + + /// + /// Records this NNTPD as the receiving hop for a StorageServer article. + /// + /// when traversal fails. The caller keeps the fetched record. + private static bool TryTraverseStorageIngress( + NntpCommandContext context, + in ArticleRecord record, + out ArticleRecord entered) + { + entered = default; + NntpArticlePolicyCapture.Capture( + context.Session, + ArticleResourceLimits.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + var created = siteNameUtf8 is null + ? ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse) + : ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted || created.Record.ArtId != record.ArtId) + { + return false; + } + + entered = created.Record; + return true; + } + + /// + /// Queues the Path-canonical copy and returns that same record for the current reply. + /// + /// when canonicalization or queue admission fails. + private static bool TryAdmitBackFiller( + NntpCommandContext context, + in ArticleRecord record, + out ArticleRecord admitted) + { + admitted = default; + try + { + var logger = Logger; + NntpArticlePolicyCapture.Capture( + context.Session, + ArticleResourceLimits.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + var created = siteNameUtf8 is null + ? ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse) + : ArticleRecordFactory.TryCreate( + context.Session.ArticleParser, + record.ArtData, + ArticlePathMode.Traverse, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted || created.Record.ArtId != record.ArtId) + { + if (logger.IsEnabled(LogLevel.Debug)) + { + var reason = created.ParseFailure != NntpArticleParseFailureCode.None + ? created.ParseFailure.ToString() + : created.MaterializeFailure.ToString(); + var rejectedMessageId = Encoding.ASCII.GetString(record.MessageId); + ArticleRetrievalLogMessages.IngestNotAdmitted( + logger, + reason, + rejectedMessageId); + } + return false; + } + + admitted = created.Record; + var messageIdText = Encoding.ASCII.GetString(admitted.MessageId); + var inbound = ArticleRecordIngress.CreateQueued( + messageIdText, + in admitted, + context.Session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.BackFiller); + + var enqueue = context.Session.ArticleIngestion.TryAdmit(inbound); + if (enqueue == ArticleEnqueueResult.Accepted) + { + // Copy Message-ID octets once for History; do not invent a second History path. + var midBytes = record.MessageId.ToArray(); + context.Session.HistoryDb?.Remember(midBytes); + return true; + } + + admitted = default; + if (logger.IsEnabled(LogLevel.Debug)) + { + var result = enqueue.ToString(); + ArticleRetrievalLogMessages.IngestNotAdmitted(logger, result, messageIdText); + } + return false; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + admitted = default; + ArticleRetrievalLogMessages.IngestFailed(Logger, ex); + return false; + } + } + + private static ValueTask WriteNumberLookupFailureAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!context.Session.HasSelectedGroup) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoNewsgroupSelected, + NntpResponseStatus.NoNewsgroupSelected, + cancellationToken); + } + + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoArticleWithNumber, + NntpResponseStatus.NoArticleWithNumber, + cancellationToken); + } + + private static ValueTask WriteCurrentArticleFailureAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!context.Session.HasSelectedGroup) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoNewsgroupSelected, + NntpResponseStatus.NoNewsgroupSelected, + cancellationToken); + } + + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CurrentArticleNumberInvalid, + NntpResponseStatus.CurrentArticleNumberInvalid, + cancellationToken); + } + + private enum RetrievalSource + { + None = 0, + StorageServer = 1, + BackFiller = 2, + } + + private enum ResolveKind + { + NotFound, + TemporaryFailure, + Success, + } + + private readonly struct ResolveResult + { + private ResolveResult(ResolveKind kind, ArticleRecord record, RetrievalSource source) + { + Kind = kind; + Record = record; + Source = source; + } + + public ResolveKind Kind { get; } + + public ArticleRecord Record { get; } + + public RetrievalSource Source { get; } + + public static ResolveResult NotFound() => new(ResolveKind.NotFound, default, RetrievalSource.None); + + public static ResolveResult Temporary() => new(ResolveKind.TemporaryFailure, default, RetrievalSource.None); + + public static ResolveResult Success(ArticleRecord record, RetrievalSource source) => + new(ResolveKind.Success, record, source); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/ArticleRetrievalLogMessages.cs b/src/VectorNNTPD.TransitService/Session/Commands/ArticleRetrievalLogMessages.cs new file mode 100644 index 00000000..69d58e0b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/ArticleRetrievalLogMessages.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// Source-generated log messages for ARTICLE/HEAD/BODY/STAT BackFiller retrieval. + internal static partial class ArticleRetrievalLogMessages + { + [LoggerMessage( + EventId = 2610, + Level = LogLevel.Warning, + Message = "Article retrieval transfer unavailable ({Outcome}): {Detail}")] + public static partial void TransferUnavailable(ILogger logger, string Outcome, string? Detail); + + [LoggerMessage( + EventId = 2611, + Level = LogLevel.Warning, + Message = "VATP article fetch failed for {Fqdn}:{VatpPort} request {RequestId}")] + public static partial void VatpFetchFailed(ILogger logger, Exception ex, string Fqdn, int VatpPort, Guid RequestId); + + [LoggerMessage( + EventId = 2612, + Level = LogLevel.Warning, + Message = "VATP article fetch unsuccessful ({Kind}): {Detail} ({ErrorCode})")] + public static partial void VatpFetchUnsuccessful(ILogger logger, string Kind, string? Detail, string? ErrorCode); + + [LoggerMessage( + EventId = 2613, + Level = LogLevel.Warning, + Message = "Fetched ArticleRecord identity mismatch: {Detail}")] + public static partial void IdentityMismatch(ILogger logger, string? Detail); + + [LoggerMessage( + EventId = 2614, + Level = LogLevel.Debug, + Message = "BackFiller article ingest not admitted ({Result}) for {MessageId}")] + public static partial void IngestNotAdmitted(ILogger logger, string Result, string MessageId); + + [LoggerMessage( + EventId = 2615, + Level = LogLevel.Warning, + Message = "BackFiller article ingest failed after successful retrieval")] + public static partial void IngestFailed(ILogger logger, Exception ex); + + [LoggerMessage( + EventId = 2616, + Level = LogLevel.Debug, + Message = "Storage article lookup {Outcome} articleId={ArticleId} requestId={RequestId}")] + public static partial void StorageLookupCompleted( + ILogger logger, + string Outcome, + string ArticleId, + Guid RequestId); + + [LoggerMessage( + EventId = 2617, + Level = LogLevel.Debug, + Message = "Storage candidate selected articleId={ArticleId} requestId={RequestId} serverId={ServerId} attempt={Attempt}")] + public static partial void StorageCandidateSelected( + ILogger logger, + string ArticleId, + Guid RequestId, + int ServerId, + int Attempt); + + [LoggerMessage( + EventId = 2618, + Level = LogLevel.Debug, + Message = "Storage candidate failed articleId={ArticleId} requestId={RequestId} serverId={ServerId} attempt={Attempt} kind={Kind} acceptedDataBytes={AcceptedDataBytes} failoverEligible={FailoverEligible}")] + public static partial void StorageCandidateFailed( + ILogger logger, + string ArticleId, + Guid RequestId, + int ServerId, + int Attempt, + string Kind, + int AcceptedDataBytes, + bool FailoverEligible); + + [LoggerMessage( + EventId = 2619, + Level = LogLevel.Debug, + Message = "Storage candidate exhausted articleId={ArticleId} requestId={RequestId} attempt={Attempt}")] + public static partial void StorageCandidateExhausted( + ILogger logger, + string ArticleId, + Guid RequestId, + int Attempt); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/AuthInfo.cs b/src/VectorNNTPD.TransitService/Session/Commands/AuthInfo.cs new file mode 100644 index 00000000..a19b36aa --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/AuthInfo.cs @@ -0,0 +1,420 @@ +using System.Text; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// AUTHINFO USER / PASS (RFC 4643, Section 2.3) and AUTHINFO SASL (RFC 4643, Section 2.4). + /// + /// + /// USER caches a username. PASS selects the credential authority from + /// (MODE), never from source IP. + /// Transit authority uses only. + /// Reader authority uses only (newsmaster then MySQL). + /// There is no Transit ↔ Reader fall-through. SASL is reader-authority only. + /// + internal static class AuthInfo + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(AuthInfo)); + + /// Handles AUTHINFO USER. + public static ValueTask HandleUserAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO USER", + (ctx, ct) => ExecuteUserAsync(ctx, authenticationProvider, ct), + cancellationToken); + + /// Handles AUTHINFO PASS. + public static ValueTask HandlePassAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO PASS", + (ctx, ct) => ExecutePassAsync(ctx, authenticationProvider, ct), + cancellationToken); + + /// Handles AUTHINFO SASL. + public static ValueTask HandleSaslAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "AUTHINFO SASL", + ExecuteSaslAsync, + cancellationToken); + + private static async ValueTask ExecuteUserAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) + { + _ = authenticationProvider; + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + context.Session.AbandonSaslExchange(); + var username = Encoding.ASCII.GetString(context.ArgumentSpan); + context.Session.SetPendingAuthUsername(username); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PasswordRequired, + NntpResponseStatus.PasswordRequired, + cancellationToken) + .ConfigureAwait(false); + } + + private static async ValueTask ExecutePassAsync( + NntpCommandContext context, + INntpAuthenticationProvider authenticationProvider, + CancellationToken cancellationToken) + { + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + var pending = context.Session.PendingAuthUsername; + if (pending is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthenticationOutOfSequence, + NntpResponseStatus.AuthenticationOutOfSequence, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var password = Encoding.ASCII.GetString(context.ArgumentSpan); + NntpAuthenticationResult result; + try + { + result = await AuthenticatePassAsync( + context.Session, + authenticationProvider, + pending, + password, + cancellationToken) + .ConfigureAwait(false); + } + finally + { + // codeql[cs/useless-assignment-to-local] Clear password reference after use; assignment is intentional. + password = null!; + } + + await CompleteAuthenticationAsync(context, result, cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ExecuteSaslAsync( + NntpCommandContext context, + CancellationToken cancellationToken) + { + if (!TryBeginAuthCommand(context, cancellationToken, out var blocked)) + { + await blocked.ConfigureAwait(false); + return; + } + + if (context.Session.AuthenticationAuthority == NntpAuthenticationAuthority.Transit) + { + context.Session.ApplyFailedAuthentication(); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthenticationFailed, + NntpResponseStatus.AuthenticationFailed, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var sasl = context.Session.SaslService; + if (sasl is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AuthinfoSaslNotImplemented, + NntpResponseStatus.AuthinfoSaslNotImplemented, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var argument = context.ArgumentSpan; + if (argument.IsEmpty) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SaslMechanismRequired, + NntpResponseStatus.SaslMechanismRequired, + cancellationToken) + .ConfigureAwait(false); + return; + } + + SplitMechanism(argument, out var mechanism, out var initial); + context.Session.AbandonSaslExchange(); + var reply = await sasl + .StartAsync(mechanism, Encoding.ASCII.GetString(initial), context.Session.ClientAddress, cancellationToken) + .ConfigureAwait(false); + await ApplySaslReplyAsync(context, reply, cancellationToken).ConfigureAwait(false); + } + + /// Handles a SASL continuation line (base64, =, or *). + internal static async ValueTask ContinueSaslAsync( + NntpSession session, + NntpResponseWriter response, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + var sasl = session.SaslService; + var exchange = session.CurrentSaslExchange; + if (sasl is null || exchange is null) + { + session.AbandonSaslExchange(); + await response.WriteLineAsync(NntpResponses.NoSaslExchangeInProgress, cancellationToken).ConfigureAwait(false); + return; + } + + var reply = await sasl + .ContinueAsync(exchange, Encoding.ASCII.GetString(line.Span), session.ClientAddress, cancellationToken) + .ConfigureAwait(false); + await ApplySaslReplyCoreAsync(session, response, context: null, reply, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask AuthenticatePassAsync( + NntpSession session, + INntpAuthenticationProvider authenticationProvider, + string username, + string password, + CancellationToken cancellationToken) + { + if (session.AuthenticationAuthority == NntpAuthenticationAuthority.Transit) + { + return ValueTask.FromResult( + session.TransitAuthenticator.Authenticate( + session.Authorization.TransitPeerPolicy, + username, + password, + session.Authorization)); + } + + return authenticationProvider.AuthenticateAsync( + username, + password, + session.ClientAddress, + cancellationToken); + } + + private static bool TryBeginAuthCommand( + NntpCommandContext context, + CancellationToken cancellationToken, + out ValueTask blocked) + { + if (context.Session.Authentication.IsAuthenticated) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.AlreadyAuthenticated, + NntpResponseStatus.AlreadyAuthenticated, + cancellationToken); + return false; + } + + if (context.Connection.IsCompressed) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.DeflateAlreadyActive, + NntpResponseStatus.DeflateAlreadyActive, + cancellationToken); + return false; + } + + if (!context.Session.IsAuthinfoPassPermitted) + { + blocked = NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PrivacyRequired, + NntpResponseStatus.PrivacyRequired, + cancellationToken); + return false; + } + + blocked = default; + return true; + } + + private static ValueTask ApplySaslReplyAsync( + NntpCommandContext context, + NntpSaslReply reply, + CancellationToken cancellationToken) => + ApplySaslReplyCoreAsync(context.Session, context.Response, context, reply, cancellationToken); + + private static async ValueTask ApplySaslReplyCoreAsync( + NntpSession session, + NntpResponseWriter response, + NntpCommandContext? context, + NntpSaslReply reply, + CancellationToken cancellationToken) + { + if (!reply.Completed && reply.Exchange is not null) + { + session.SetSaslExchange(reply.Exchange); + await WriteAsync(context, response, reply.Wire, reply.StatusLine, cancellationToken).ConfigureAwait(false); + return; + } + + session.AbandonSaslExchange(); + if (reply.Authentication is { Succeeded: true } result) + { + await CompleteAuthenticationCoreAsync( + session, + response, + context, + result, + reply.Wire, + reply.StatusLine, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (reply.Authentication is { } failed) + { + await CompleteAuthenticationCoreAsync( + session, + response, + context, + failed, + successWire: default, + successStatus: null, + cancellationToken) + .ConfigureAwait(false); + return; + } + + session.ApplyFailedAuthentication(); + await WriteAsync(context, response, reply.Wire, reply.StatusLine, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask CompleteAuthenticationAsync( + NntpCommandContext context, + NntpAuthenticationResult result, + CancellationToken cancellationToken) => + CompleteAuthenticationCoreAsync( + context.Session, + context.Response, + context, + result, + NntpResponses.AuthenticationAccepted, + NntpResponseStatus.AuthenticationAccepted, + cancellationToken); + + private static async ValueTask CompleteAuthenticationCoreAsync( + NntpSession session, + NntpResponseWriter response, + NntpCommandContext? context, + NntpAuthenticationResult result, + ReadOnlyMemory successWire, + string? successStatus, + CancellationToken cancellationToken) + { + if (!result.Succeeded || result.Username is null || result.Authorization is null) + { + session.ApplyFailedAuthentication(); + var (wire, status) = MapFailure(result.Failure); + await WriteAsync(context, response, wire, status, cancellationToken).ConfigureAwait(false); + return; + } + + var admitted = await session.AdmitAuthenticatedSessionAsync(result.Policy, cancellationToken) + .ConfigureAwait(false); + if (!admitted.Succeeded) + { + session.ApplyFailedAuthentication(); + var (wire, status) = MapFailure(admitted.Failure); + if (admitted.Failure is NntpAuthenticationFailureKind.TooManySessions + or NntpAuthenticationFailureKind.TooManySourceAddresses + or NntpAuthenticationFailureKind.TransientFailure) + { + AuthenticationLogMessages.AdmissionRejected( + Logger, + result.Username, + admitted.Failure.ToString(), + session.ClientAddress.ToString()); + } + + await WriteAsync(context, response, wire, status, cancellationToken).ConfigureAwait(false); + return; + } + + session.ApplySuccessfulAuthentication(result.Username, result.Authorization, result.Policy); + await session.ReleaseTransitAdmissionWhenNotTransitAsync().ConfigureAwait(false); + await session.AttachByteAccountingAsync(response, cancellationToken).ConfigureAwait(false); + var wireSuccess = successWire.IsEmpty ? NntpResponses.AuthenticationAccepted : successWire; + var statusSuccess = successStatus ?? NntpResponseStatus.AuthenticationAccepted; + await WriteAsync(context, response, wireSuccess, statusSuccess, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteAsync( + NntpCommandContext? context, + NntpResponseWriter response, + ReadOnlyMemory wire, + string status, + CancellationToken cancellationToken) + { + if (context is not null) + { + return NntpCommandReply.WriteAsync(context, Logger, wire, status, cancellationToken); + } + + return response.WriteLineAsync(wire, cancellationToken); + } + + private static (ReadOnlyMemory Wire, string Status) MapFailure(NntpAuthenticationFailureKind failure) => + failure switch + { + NntpAuthenticationFailureKind.TransientFailure => + (NntpResponses.TemporaryAuthenticationFailure, NntpResponseStatus.TemporaryAuthenticationFailure), + NntpAuthenticationFailureKind.TooManySessions => + (NntpResponses.TooManySessions, NntpResponseStatus.TooManySessions), + NntpAuthenticationFailureKind.TooManySourceAddresses => + (NntpResponses.TooManySourceAddresses, NntpResponseStatus.TooManySourceAddresses), + _ => (NntpResponses.AuthenticationFailed, NntpResponseStatus.AuthenticationFailed), + }; + + private static void SplitMechanism(ReadOnlySpan argument, out string mechanism, out ReadOnlySpan initial) + { + var end = NntpAscii.SkipToken(argument, 0); + mechanism = Encoding.ASCII.GetString(argument[..end]); + var next = NntpAscii.SkipWhitespace(argument, end); + initial = next >= argument.Length ? ReadOnlySpan.Empty : argument[next..]; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/BenchIt.cs b/src/VectorNNTPD.TransitService/Session/Commands/BenchIt.cs new file mode 100644 index 00000000..683b2bf0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/BenchIt.cs @@ -0,0 +1,163 @@ +using System.Text; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// Internal benchmark facility: unadvertised BENCHIT command that serves one static + /// ~750 KiB ARTICLE-style multiline response over the production transport path. + /// + /// + /// + /// Not a production protocol feature and not advertised via CAPABILITIES, HELP, or the public + /// command inventory. Retained intentionally for transport performance baselines and regressions. + /// Registered so the real dispatcher/session stack can be exercised against actual TCP sockets. + /// + /// + /// The complete NNTP multiline wire representation (status line, CRLF article lines with dot-stuffing, + /// and terminating .CRLF) is generated exactly once at type initialization and reused immutably + /// for every request. Per-request work is: dispatch → write precomputed bytes → pipe/transport flush. + /// + /// + /// Logging: no per-request INFO TX completion (handler does not use ). + /// Session RX logging for BENCHIT is suppressed separately. + /// + /// + internal static class BenchIt + { + /// Target article content size before NNTP multiline framing (750 KiB). + public const int TargetArticleContentBytes = 750 * 1024; + + /// Stable Message-ID used in the static article and 220 status line. + private const string MessageId = ""; + + /// Immutable ASCII article bytes (headers + body), length . + public static readonly ReadOnlyMemory ArticleContent; + + /// Immutable complete multiline wire response including 220 status and terminating .CRLF. + public static readonly ReadOnlyMemory WireResponse; + + /// Exact article content length in bytes (before framing). + public static readonly int ArticleContentBytes; + + /// Exact wire response length in bytes (status + framed article + terminator). + public static readonly int WireResponseBytes; + + static BenchIt() + { + var article = BuildArticleContent(); + ArticleContent = article; + ArticleContentBytes = article.Length; + + var wire = BuildWireResponse(article); + WireResponse = wire; + WireResponseBytes = wire.Length; + } + + /// Handles BENCHIT (internal; public access; non-pipelined via normal session loop). + internal static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + // Intentionally skip NntpCommandExecution TX INFO logging — benchmark must not measure Serilog. + return context.Response.WriteBytesAndFlushAsync(WireResponse, cancellationToken); + } + + private static byte[] BuildArticleContent() + { + // Deterministic mandatory-style headers + blank line, then a fixed body pattern. + // One body line begins with '.' so the static wire form exercises dot-stuffing. + var headerText = + "From: benchit@vectornntp.local\r\n" + + "Newsgroups: vectornntp.bench\r\n" + + "Subject: VectorNNTP BENCHIT transport payload\r\n" + + "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" + + "Message-ID: " + MessageId + "\r\n" + + "Path: vectornntp.bench\r\n" + + "\r\n"; + + var headerBytes = Encoding.ASCII.GetBytes(headerText); + if (headerBytes.Length >= TargetArticleContentBytes) + { + throw new InvalidOperationException("BENCHIT headers alone exceed the target article size."); + } + + var article = new byte[TargetArticleContentBytes]; + headerBytes.CopyTo(article.AsSpan()); + + // 76-octet body lines (common Usenet width) of a repeating digit pattern, CRLF-terminated. + // First body line is ".DOTSTUFF-CHECK" so framing must emit "..DOTSTUFF-CHECK\r\n". + var offset = headerBytes.Length; + var firstLine = Encoding.ASCII.GetBytes(".DOTSTUFF-CHECK\r\n"); + if (offset + firstLine.Length > article.Length) + { + throw new InvalidOperationException("BENCHIT target size too small for required body line."); + } + + firstLine.CopyTo(article.AsSpan(offset)); + offset += firstLine.Length; + + // Pattern line without leading '.': 74 payload chars + CRLF = 76 bytes. + Span patternLine = stackalloc byte[76]; + for (var i = 0; i < 74; i++) + { + patternLine[i] = (byte)('0' + (i % 10)); + } + + patternLine[74] = (byte)'\r'; + patternLine[75] = (byte)'\n'; + + while (offset + patternLine.Length <= article.Length) + { + patternLine.CopyTo(article.AsSpan(offset)); + offset += patternLine.Length; + } + + // Fill any remainder with ASCII 'X' (no CR/LF) so content length is exact. + article.AsSpan(offset).Fill((byte)'X'); + return article; + } + + private static byte[] BuildWireResponse(ReadOnlySpan article) + { + var status = Encoding.ASCII.GetBytes($"220 0 {MessageId}\r\n"); + using var buffer = new MemoryStream(status.Length + article.Length + (article.Length / 64) + 16); + buffer.Write(status); + + var lineStart = 0; + for (var i = 0; i < article.Length; i++) + { + if (article[i] != (byte)'\n') + { + continue; + } + + // Line includes optional trailing CR before LF. + var lineEnd = i + 1; + var line = article[lineStart..lineEnd]; + if (line.Length > 0 && line[0] == (byte)'.') + { + buffer.WriteByte((byte)'.'); + } + + buffer.Write(line); + lineStart = lineEnd; + } + + if (lineStart < article.Length) + { + // Final unterminated fragment: emit as a line with CRLF (RFC multiline writer style). + var fragment = article[lineStart..]; + if (fragment.Length > 0 && fragment[0] == (byte)'.') + { + buffer.WriteByte((byte)'.'); + } + + buffer.Write(fragment); + buffer.Write("\r\n"u8); + } + + buffer.Write(".\r\n"u8); + return buffer.ToArray(); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Capabilities.cs b/src/VectorNNTPD.TransitService/Session/Commands/Capabilities.cs new file mode 100644 index 00000000..da97b7b9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Capabilities.cs @@ -0,0 +1,131 @@ +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// CAPABILITIES command as defined by RFC 3977, Section 5.2. + /// + /// + /// Returns the server's capability list (VERSION, READER, AUTHINFO, STARTTLS, COMPRESS, STREAMING, SPEEDTEST, and related labels). + /// Advertisement for AUTHINFO and MODE-READER follows RFC 4643 rules after authentication. + /// COMPRESS / STARTTLS / MODE-READER / AUTHINFO arguments follow RFC 8054 once a compression layer is active. + /// STREAMING (RFC 4644) is advertised when TAKETHIS/CHECK streaming transfer is implemented. + /// Individual lines are pre-encoded and immortal. Which lines appear depends on session state, + /// so the complete response is composed into one owned buffer and written once. + /// + internal static class Capabilities + { + private const int MaxCapabilityParts = 16; + + private static ILogger Logger => NntpCommandLoggers.For(typeof(Capabilities)); + + /// Handles CAPABILITIES. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "CAPABILITIES", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var parts = new ReadOnlyMemory[MaxCapabilityParts]; + var count = CollectLines(context, parts); + var owned = NntpResponseCompose.Concatenate(parts.AsSpan(0, count)); + return NntpCommandReply.WriteAsync( + context, + Logger, + owned, + NntpResponseStatus.CapabilityListFollows, + cancellationToken); + } + + private static int CollectLines(NntpCommandContext context, Span> parts) + { + var n = 0; + parts[n++] = NntpResponses.CapabilityListFollows; + parts[n++] = NntpResponses.CapabilityVersion2; + parts[n++] = NntpResponses.CapabilityImplementation; + + var authenticated = context.Session.Authentication.IsAuthenticated; + var compressed = context.Connection.IsCompressed; + var role = context.Session.ServerRole; + var advertiseReader = false /* reader not hosted */; + var advertiseTransit = role is NntpServerRole.Transit; + + if (advertiseReader && + context.Session.Mode is NntpSessionMode.Unspecified or NntpSessionMode.Reader) + { + parts[n++] = NntpResponses.CapabilityReader; + // RFC 3977 §7.6: LIST keywords that are actually implemented. + parts[n++] = NntpResponses.CapabilityList; + // RFC 4643: MUST NOT advertise MODE-READER after authentication. + // RFC 8054 §2.2.2: MUST NOT advertise MODE-READER once a compression layer is active. + if (!authenticated && !compressed) + { + parts[n++] = NntpResponses.CapabilityModeReader; + } + } + + if (advertiseReader && + context.Session.Authorization.PostingPermitted && + context.Session.Mode is NntpSessionMode.Unspecified or NntpSessionMode.Reader) + { + parts[n++] = NntpResponses.CapabilityPost; + } + + // RFC 4643: MUST NOT return AUTHINFO after successful authentication. + // RFC 8054 §2.2.2 / §7: after COMPRESS, advertise AUTHINFO with no arguments (or omit). + // SASL is advertised only when NntpSaslService is registered and AUTHINFO is permitted. + if (!authenticated) + { + var saslAvailable = context.Session.SaslService is not null; + if (compressed) + { + parts[n++] = NntpResponses.CapabilityAuthinfo; + } + else if (context.Session.IsAuthinfoPassPermitted) + { + if (saslAvailable) + { + parts[n++] = NntpResponses.CapabilityAuthinfoUserSasl; + parts[n++] = NntpResponses.CapabilitySasl; + } + else + { + parts[n++] = NntpResponses.CapabilityAuthinfoUser; + } + } + else + { + // Policy forbids cleartext AUTHINFO and TLS is inactive — do not advertise USER or SASL. + parts[n++] = NntpResponses.CapabilityAuthinfo; + } + } + + // RFC 8054 §2.2.2: MUST NOT advertise STARTTLS once a compression layer is active. + if (!context.Connection.IsTls && !compressed) + { + parts[n++] = NntpResponses.CapabilityStartTls; + } + + // RFC 8054 §2.1: advertise COMPRESS DEFLATE when available; MUST NOT once compression is active. + if (!compressed) + { + parts[n++] = NntpResponses.CapabilityCompressDeflate; + } + + if (advertiseTransit) + { + // RFC 3977 §3.3.2: advertise IHAVE when the command is implemented. + parts[n++] = NntpResponses.CapabilityIhave; + + // RFC 4644 §2.2: STREAMING capability for CHECK/TAKETHIS (MODE STREAM is legacy discovery). + parts[n++] = NntpResponses.CapabilityStreaming; + + // VectorNNTP private SPEEDTEST diagnostic (current-session TX; no outbound initiator). + parts[n++] = NntpResponses.CapabilitySpeedTest; + } + + parts[n++] = NntpResponses.MultilineTerminator; + return n; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Check.cs b/src/VectorNNTPD.TransitService/Session/Commands/Check.cs new file mode 100644 index 00000000..5ff2f600 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Check.cs @@ -0,0 +1,132 @@ +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// CHECK command as defined by RFC 4644, Section 2.4. + /// + /// + /// Advises whether the server wants an article (STREAMING) using HistoryDB + /// (no miss reservation): + /// local memory hit and Redis hit are and do not + /// record the identifier. A double miss is 238. Redis infrastructure failure returns 431 + /// and is never treated as a 238 miss. A result is + /// 431 when the connection's TransitPeerPolicy.DeferOnDuplicate is true, and 438 + /// otherwise. TAKETHIS does not read that flag. + /// The Message-ID is copied from session scratch into one owned wire buffer; it is not + /// converted to a string for response construction. A completed local-hit lookup + /// writes that CHECK reply without an extra async state machine. + /// Consecutive authorized CHECK commands may overlap Redis lookups in the per-session + /// (fixed depth ). + /// Every response passes through the pipeline emit gate and is enqueued in send order. + /// A slot stays occupied until the writer accepts the line. + /// + internal static class Check + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Check)); + + /// Handles CHECK when dispatched serially (gates / non-pipeline fallback). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "CHECK", ExecuteAsync, cancellationToken); + + /// + /// Looks up without composing a response or recording a miss. + /// + internal static ValueTask LookupAsync( + NntpSession session, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + if (session.HistoryDb is not { } history) + { + return new ValueTask(HistoryLookupResult.Unseen); + } + + return history.PeekAsync(messageId, cancellationToken); + } + + /// Composes the RFC 4644 CHECK wire line for . + /// History lookup outcome. is not rewritten. + /// Message-ID octets copied into the response. + /// + /// Connection TransitPeerPolicy.DeferOnDuplicate. When , a seen + /// identifier uses the existing 431 line. Misses and infrastructure failures ignore it. + /// + internal static ReadOnlyMemory Compose( + HistoryLookupResult result, + ReadOnlySpan messageId, + bool deferOnDuplicate) + { + ReadOnlyMemory prefix; + ReadOnlyMemory suffix; + switch (result) + { + case HistoryLookupResult.Seen when deferOnDuplicate: + prefix = NntpResponses.CheckTryLaterPrefix; + suffix = NntpResponses.CheckTryLaterSuffix; + break; + case HistoryLookupResult.Seen: + prefix = NntpResponses.CheckNotWantedPrefix; + suffix = NntpResponses.CheckNotWantedSuffix; + break; + case HistoryLookupResult.Unavailable: + prefix = NntpResponses.CheckTryLaterPrefix; + suffix = NntpResponses.CheckTryLaterSuffix; + break; + default: + prefix = NntpResponses.CheckPrefix; + suffix = NntpResponses.CheckSuffix; + break; + } + + return NntpResponseCompose.Concat(prefix.Span, messageId, suffix.Span); + } + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var lookup = LookupAsync(context.Session, messageId, cancellationToken); + if (lookup.IsCompletedSuccessfully) + { + return WriteResultAsync(context, lookup.Result, messageId, cancellationToken); + } + + return AwaitLookupAndWriteAsync(context, lookup, messageId, cancellationToken); + } + + private static async ValueTask AwaitLookupAndWriteAsync( + NntpCommandContext context, + ValueTask lookup, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var result = await lookup.ConfigureAwait(false); + await WriteResultAsync(context, result, messageId, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteResultAsync( + NntpCommandContext context, + HistoryLookupResult result, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + var deferOnDuplicate = DeferOnDuplicate(context.Session); + var owned = Compose(result, messageId.Span, deferOnDuplicate); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= NntpCommandStatusText.FormatCheck(result, messageId.Span, deferOnDuplicate); + } + + return context.Response.WriteLineAsync(owned, cancellationToken); + } + + /// + /// Reads the connection's captured Receive DeferOnDuplicate flag. + /// A session without a peer policy does not defer. + /// + internal static bool DeferOnDuplicate(NntpSession session) => + session.Authorization.TransitPeerPolicy is { DeferOnDuplicate: true }; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Compress.cs b/src/VectorNNTPD.TransitService/Session/Commands/Compress.cs new file mode 100644 index 00000000..456a7acf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Compress.cs @@ -0,0 +1,143 @@ +using System.IO.Pipelines; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// COMPRESS command as defined by RFC 8054, Section 2.2. + /// + /// + /// + /// Activates the NNTP compression layer using the mandatory DEFLATE algorithm (RFC 8054 §2.2 / §4). + /// The command MUST NOT be pipelined (RFC 8054 §2.2.1). Success reply 206 is the final + /// uncompressed NNTP response; compression takes effect immediately after its CRLF. + /// + /// + /// Transport owns raw DEFLATE wrapping and quiescence via UpgradeToDeflateAsync. This module + /// owns algorithm validation, capability-related session sequencing (discard pipelined input, pause + /// reads before 206, wait for outbound delivery, upgrade), and the TX completion record. + /// Layering follows RFC 8054 §2.2.2: application → DEFLATE → (TLS) → network. + /// + /// + internal static class Compress + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Compress)); + + /// Handles COMPRESS (algorithm argument validated per RFC 8054). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "COMPRESS", + ExecuteAsync, + cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionAlreadyActive, + NntpResponseStatus.CompressionAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §5.3: algorithm names are case-sensitive. Parser already rejected + // lowercase/illegal octets (501). This is the semantic DEFLATE check — no string. + if (!context.ArgumentSpan.SequenceEqual("DEFLATE"u8)) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionAlgorithmNotSupported, + NntpResponseStatus.CompressionAlgorithmNotSupported, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // Client MUST NOT pipeline (RFC 8054 §2.2.1). Discard any already-buffered application + // octets so they cannot be misinterpreted after the compression layer activates. + DiscardBufferedApplicationInput(context.Connection.Input); + + // Claim read ownership BEFORE 206 can reach the peer so the first compressed octets + // cannot enter application Input as plaintext. + try + { + await context.Connection.PauseReadsAsync(cancellationToken).ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + CommandLogMessages.CompressRefusedBeforeActivation( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.UnableToActivateCompression, + NntpResponseStatus.UnableToActivateCompression, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "failed"; + return; + } + + var idleVersionBefore206 = context.Connection.OutboundIdleVersion; + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CompressionActive, + NntpResponseStatus.CompressionActive, + cancellationToken) + .ConfigureAwait(false); + + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore206, cancellationToken) + .ConfigureAwait(false); + + try + { + await context.Connection.UpgradeToDeflateAsync(cancellationToken).ConfigureAwait(false); + context.CompletionDetail = "DEFLATE active"; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // 206 already committed the transition. Do not write another NNTP status — the peer + // expects compressed traffic. Terminate so we never leave a half-compressed session. + CommandLogMessages.CompressActivationFailed( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + if (!context.Connection.IsCompleted) + { + await context.Connection.CompleteAsync(ex).ConfigureAwait(false); + } + + context.Session.RequestClose(); + } + } + + /// + /// Consumes any octets already buffered in application without waiting for more. + /// + private static void DiscardBufferedApplicationInput(PipeReader input) + { + while (input.TryRead(out var result)) + { + var isCompleted = result.IsCompleted; + var isEmpty = result.Buffer.IsEmpty; + input.AdvanceTo(result.Buffer.End); + if (isEmpty || isCompleted) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Date.cs b/src/VectorNNTPD.TransitService/Session/Commands/Date.cs new file mode 100644 index 00000000..58512194 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Date.cs @@ -0,0 +1,71 @@ +using System.Globalization; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// DATE command as defined by RFC 3977, Section 7.1. + /// + /// + /// Returns the current UTC server date and time as YYYYMMDDhhmmss. + /// The stamp is formatted into an owned 20-octet wire line + /// (111 + 14 digits + CRLF) without a protocol string round-trip. + /// + internal static class Date + { + private const int DateWireLength = 20; + private const int StampLength = 14; + + private static ILogger Logger => NntpCommandLoggers.For(typeof(Date)); + + /// Handles DATE. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "DATE", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var utc = DateTime.UtcNow; + var owned = FormatUtcDateLine(utc, Logger, out var statusLine); + if (statusLine is not null) + { + context.StatusLine ??= statusLine; + } + + return context.Response.WriteLineAsync(owned, cancellationToken); + } + + /// + /// Builds 111 YYYYMMDDhhmmss\r\n as one owned buffer using + /// DateTime.TryFormat so UTC stamp semantics stay on the framework formatter. + /// + internal static byte[] FormatUtcDateLine(DateTime utc) => + FormatUtcDateLine(utc, logger: null, out _); + + /// + internal static byte[] FormatUtcDateLine(DateTime utc, ILogger? logger, out string? statusLine) + { + statusLine = null; + Span stamp = stackalloc char[StampLength]; + if (!utc.TryFormat(stamp, out var written, "yyyyMMddHHmmss", CultureInfo.InvariantCulture) + || written != StampLength) + { + throw new InvalidOperationException("DATE stamp formatting failed."); + } + + if (logger is not null && logger.IsEnabled(LogLevel.Debug)) + { + statusLine = NntpCommandStatusText.FormatDate(stamp); + } + + var wire = new byte[DateWireLength]; + NntpResponses.DatePrefix.Span.CopyTo(wire); + for (var i = 0; i < StampLength; i++) + { + wire[NntpResponses.DatePrefix.Length + i] = (byte)stamp[i]; + } + + NntpResponses.Crlf.Span.CopyTo(wire.AsSpan(NntpResponses.DatePrefix.Length + StampLength)); + return wire; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Group.cs b/src/VectorNNTPD.TransitService/Session/Commands/Group.cs new file mode 100644 index 00000000..029edcaf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Group.cs @@ -0,0 +1,49 @@ +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// GROUP command as defined by RFC 3977, Section 6.1.1. + /// + /// + /// Group existence and water marks come from one captured + /// . The command path does not query MySQL. + /// Article-number sequences are not invented from the water marks. + /// + internal static class Group + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Group)); + + /// Handles GROUP. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "GROUP", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = context.Session.NewsgroupCatalogue?.Current; + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + if (!snapshot.TryGet(context.ArgumentSpan, out var group)) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.NoSuchNewsgroup, + NntpResponseStatus.NoSuchNewsgroup, + cancellationToken); + } + + context.Session.SelectGroup(group.GroupNameBytes); + return NntpCommandReply.WriteAsync( + context, + Logger, + group.GroupSelectedLine, + NntpResponseStatus.GroupSelected, + cancellationToken); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Hdr.cs b/src/VectorNNTPD.TransitService/Session/Commands/Hdr.cs new file mode 100644 index 00000000..1369ba44 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Hdr.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// HDR command as defined by RFC 3977, Section 8.5. + /// + /// + /// Returns a specific header field for articles. Deliberate placeholder until header/overview access is implemented. + /// + internal static class Hdr + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Hdr)); + + /// Handles HDR. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "HDR", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Help.cs b/src/VectorNNTPD.TransitService/Session/Commands/Help.cs new file mode 100644 index 00000000..deddbc58 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Help.cs @@ -0,0 +1,141 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// HELP command as defined by RFC 3977, Section 7.2. + /// + /// + /// + /// Syntax: HELP (no arguments). Returns a multiline 100 static syntax reference + /// for registered NNTP commands, followed by concise RANGE and WILDMAT explanations + /// (RFC 3977 §6.1.2 / §8.3 / §8.5 and §4). Mandatory; not a substitute for CAPABILITIES. + /// + /// + /// The body is identical for every session state (TLS, auth, COMPRESS, mode). The complete + /// multiline wire (100 + body + terminator) is one immortal buffer + /// () written with a single awaited + /// . + /// Trailing arguments are a syntax error (501). Internal commands such as + /// BENCHIT are not listed. + /// + /// + /// Compact HELP syntax uses ABNF-style notation where applicable: + /// [element] denotes an optional element and / denotes alternatives. + /// Required arguments appear without brackets. These lines are compact human-readable + /// summaries of the command forms a client should use, not complete standalone ABNF productions. + /// + /// + internal static class Help + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Help)); + + /// + /// Static HELP syntax lines for the supported public command surface. + /// + /// + /// LIST keywords ACTIVE.TIMES, COUNTS, DISTRIB.PATS, DISTRIBUTIONS, MODERATORS, and + /// SUBSCRIPTIONS are intentionally omitted — VectorNNTPD.TransitService does not support them. + /// + internal static readonly IReadOnlyList SyntaxLines = + [ + "ARTICLE [message-id / article-number]", + "AUTHINFO PASS password", + "AUTHINFO SASL mechanism [initial-response]", + "AUTHINFO USER username", + "BODY [message-id / article-number]", + "CAPABILITIES [keyword]", + "CHECK message-id", + "COMPRESS DEFLATE", + "DATE", + "GROUP newsgroup", + "HDR header [range / message-id]", + "HEAD [message-id / article-number]", + "HELP", + "IHAVE message-id", + "LAST", + "LIST", + "LIST ACTIVE [wildmat]", + "LIST COUNTS [wildmat]", + "LIST HEADERS [MSGID / RANGE]", + "LIST MOTD", + "LIST NEWSGROUPS [wildmat]", + "LIST OVERVIEW.FMT", + "LISTGROUP [newsgroup [range]]", + "MODE READER", + "MODE STREAM", + "NEXT", + "OVER [range / message-id]", + "POST", + "QUIT", + "SPEEDTEST ", + "STARTTLS", + "STAT [message-id / article-number]", + "TAKETHIS message-id", + "XHDR header [range / message-id]", + "XOVER [range]", + ]; + + /// + /// Static RANGE explanation (RFC 3977 §6.1.2, §8.3, §8.5; ABNF range in §9.2). + /// + internal static readonly IReadOnlyList RangeHelpLines = + [ + "Range formats:", + "Where HELP shows \"range\" (LISTGROUP, HDR, OVER, XOVER):", + "123 a single article number", + "123-456 articles from 123 through 456 inclusive", + "123- article 123 and all following article numbers", + "If the high number is less than the low number, the range contains no articles.", + ]; + + /// + /// Static WILDMAT explanation (RFC 3977 §4; LIST ACTIVE wildmat example in §7.6.3). + /// + /// + /// Character classes and backslash escapes are intentionally omitted: RFC 3977 §4.1 + /// excludes ,, \, [, and ] from wildmats. + /// + internal static readonly IReadOnlyList WildmatHelpLines = + [ + "Wildmat formats:", + "Where HELP shows \"wildmat\" (e.g. LIST ACTIVE, LIST NEWSGROUPS); matches a complete name:", + "* matches zero or more characters", + "? matches exactly one character", + "Other permitted characters match themselves (case-sensitive).", + "Patterns may be comma-separated; a leading \"!\" negates that pattern.", + "The rightmost matching pattern decides whether the wildmat matches.", + "The characters comma, backslash, and square brackets are not allowed in wildmats.", + "Examples: a* ; a*,!*b ; *.recovery", + ]; + + /// + /// Full HELP multiline body: command syntax, then RANGE and WILDMAT sections + /// separated by blank lines. + /// + internal static readonly IReadOnlyList BodyLines = CreateBodyLines(); + + private static IReadOnlyList CreateBodyLines() + { + var lines = new List(SyntaxLines.Count + RangeHelpLines.Count + WildmatHelpLines.Count + 2); + lines.AddRange(SyntaxLines); + lines.Add(string.Empty); + lines.AddRange(RangeHelpLines); + lines.Add(string.Empty); + lines.AddRange(WildmatHelpLines); + return lines; + } + + /// Handles HELP (RFC 3977 §7.2). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "HELP", ExecuteAsync, cancellationToken); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/IHave.cs b/src/VectorNNTPD.TransitService/Session/Commands/IHave.cs new file mode 100644 index 00000000..ba405742 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/IHave.cs @@ -0,0 +1,295 @@ +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// IHAVE command as defined by RFC 3977, Section 6.3.2. + /// + /// + /// Not pipelined. Serial two-stage exchange: HistoryDB peek → non-blocking + /// Transit queue probe → 335/435/436 → raw article receive (frame terminator, + /// own stuffed wire) → destuff + ArticleRecordFactory → non-blocking + /// → 235/436/437. IHAVE never + /// waits for queue memory. TAKETHIS is not used and is not modified. Common + /// owns article parse/materialize. IHAVE-specific behaviour is History peek, + /// non-blocking probe/admit, and 335/235/435/436/437 timing. After CanonicalV1, + /// the connection's Receive article-type mask and newsgroup expression are applied. + /// The command + /// Message-ID is used for History and is not matched against the article + /// Message-ID (RFC 3977 §6.3.2 permits a mismatch). A History hit before + /// 335 is 436 Transfer not possible; try again later when the + /// connection's DeferOnDuplicate is true, and 435 Article not wanted + /// otherwise. Neither response reads an article. TAKETHIS does not read that flag. + /// Incomplete articles that cannot become CanonicalV1 are rejected with 437 after 335. + /// + internal static class IHave + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(IHave)); + + /// Handles IHAVE (RFC 3977 §6.3.2). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "IHAVE", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var messageId = context.ArgumentMemory; + var history = context.Session.HistoryDb; + var peek = history is null + ? HistoryLookupResult.Unseen + : await history.PeekAsync(messageId, cancellationToken).ConfigureAwait(false); + + if (peek == HistoryLookupResult.Seen) + { + var defer = Check.DeferOnDuplicate(context.Session); + await NntpCommandReply.WriteAsync( + context, + Logger, + defer ? NntpResponses.IhaveTryLater : NntpResponses.IhaveNotWanted, + defer ? NntpResponseStatus.IhaveTryLater : NntpResponseStatus.IhaveNotWanted, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = defer ? "deferred duplicate" : "not wanted"; + return; + } + + if (peek == HistoryLookupResult.Unavailable) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTryLater, + NntpResponseStatus.IhaveTryLater, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "try later"; + return; + } + + var queue = context.Session.ArticleIngestion; + if (!queue.TryProbeCapacity()) + { + IHaveAdmissionLog.BudgetExhausted(Logger, queue); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTryLater, + NntpResponseStatus.IhaveTryLater, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "queue full"; + return; + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveSendArticle, + NntpResponseStatus.IhaveSendArticle, + cancellationToken) + .ConfigureAwait(false); + + NntpArticlePolicyCapture.Capture( + context.Session, + queue.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + IHaveArticleReadResult read; + try + { + read = await IHaveArticleReader + .ReadAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ArticleTooLarge, + read.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = tooLarge; + return; + } + + var created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromStuffedWire( + context.Session.ArticleParser, + read.Payload, + maxArticleBytes) + : ArticleRecordIngress.TryCreateFromStuffedWire( + context.Session.ArticleParser, + read.Payload, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted) + { + var recordReject = created.ParseFailure != VectorNNTPD.Common.Articles.Parsing.NntpArticleParseFailureCode.None + ? "rejected article record " + created.ParseFailure + : "rejected article record " + created.MaterializeFailure; + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ForArticleRecord(in created), + read.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = recordReject; + return; + } + + if (TransitReceivePolicy.TryReject( + context.Session.Authorization.TransitPeerPolicy, + created.Record, + out var policyReject)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + CommandMessageId(messageId), + 437, + IngressNewsReasons.ForExistingRejectDetail(policyReject), + created.Record.ArtSize); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = policyReject; + return; + } + + var messageIdText = System.Text.Encoding.ASCII.GetString(messageId.Span); + var inbound = ArticleRecordIngress.CreateQueued( + messageIdText, + created.Record, + context.Session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.IHave, + IngressNewsEvents.SnapshotInboundFeed(context.Session)); + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + context.Session.Transit, + context.Session.NewsgroupCatalogue, + out var uncarriedReason)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageIdText, + 437, + uncarriedReason, + inbound.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = IngressNewsEvents.NewsgroupNotCarried; + return; + } + + var enqueue = queue.TryAdmit(inbound); + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + context.Session, + messageIdText, + 437, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveRejected, + NntpResponseStatus.IhaveRejected, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = budget; + return; + } + + if (enqueue != ArticleEnqueueResult.Accepted) + { + if (enqueue == ArticleEnqueueResult.Full) + { + IHaveAdmissionLog.BudgetExhausted(Logger, queue); + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTransferFailed, + NntpResponseStatus.IhaveTransferFailed, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = enqueue == ArticleEnqueueResult.Full + ? "queue full" + : "queue unavailable"; + return; + } + + history?.Remember(messageId); + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + IHaveLogMessages.Received( + logger, + client, + messageIdText, + read.Metrics.ArticleSize, + read.Metrics.PipeReads, + read.Metrics.ReceiveElapsed.TotalMilliseconds, + Queued: true); + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.IhaveTransferredOk, + NntpResponseStatus.IhaveTransferredOk, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "accepted"; + } + + private static string CommandMessageId(ReadOnlyMemory messageId) => + System.Text.Encoding.ASCII.GetString(messageId.Span); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Last.cs b/src/VectorNNTPD.TransitService/Session/Commands/Last.cs new file mode 100644 index 00000000..5ed1138b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Last.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// LAST command as defined by RFC 3977, Section 6.1.3. + /// + /// + /// Moves the current article pointer backward. Deliberate placeholder until article selection state is implemented. + /// + internal static class Last + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Last)); + + /// Handles LAST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "LAST", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/List.cs b/src/VectorNNTPD.TransitService/Session/Commands/List.cs new file mode 100644 index 00000000..e79ecdef --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/List.cs @@ -0,0 +1,192 @@ +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// LIST command as defined by RFC 3977, Section 7.6.1 (see also RFC 6048). + /// + /// + /// + /// Implemented keywords: default/ACTIVE, COUNTS, and NEWSGROUPS (catalogue + /// snapshot), plus static OVERVIEW.FMT and HEADERS field lists. LIST ACTIVE + /// and LIST COUNTS status is the stored y/n/m/x/j + /// octet; the RFC 6048 =<newsgroup> form is not supported. LIST + /// COUNTS estimated counts reuse the GROUP watermark estimate + /// (RFC 3977 §6.1.1 / RFC 6048 §2.2). The command path does not query MySQL. + /// + /// + /// LIST OVERVIEW.FMT and LIST HEADERS write immortal static 215 multiline + /// responses (RFC 3977 §8.4 / §8.6). They do not read articles or the + /// catalogue. LIST HEADERS MSGID and RANGE return the same list because HDR + /// forms are not distinguished. LIST MOTD remains recognized without stored + /// information and returns 503. Unsupported keywords (ACTIVE.TIMES, + /// DISTRIB.PATS, DISTRIBUTIONS, MODERATORS, SUBSCRIPTIONS) are not registered + /// and resolve as unknown variants. + /// + /// + internal static class List + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(List)); + + /// Handles LIST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, DisplayName(context), ExecuteAsync, cancellationToken); + + private static string DisplayName(NntpCommandContext context) => + DefaultNntpCommandCatalog.DisplayName(context.Command.Verb, context.Command.Qualifier); + + private static ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + return context.Command.Qualifier switch + { + NntpVerb.None or NntpVerb.Active => WriteActiveAsync(context, cancellationToken), + NntpVerb.Counts => WriteCountsAsync(context, cancellationToken), + NntpVerb.Newsgroups => WriteNewsgroupsAsync(context, cancellationToken), + NntpVerb.OverviewFmt => WriteOverviewFmtAsync(context, cancellationToken), + NntpVerb.Headers => WriteHeadersAsync(context, cancellationToken), + NntpVerb.Motd => WriteDataItemNotStoredAsync(context, cancellationToken), + _ => NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken), + }; + } + + private static ValueTask WriteOverviewFmtAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListOverviewFmtComplete, + NntpResponseStatus.ListOverviewFmtFollows, + cancellationToken); + + private static ValueTask WriteHeadersAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListHeadersComplete, + NntpResponseStatus.ListHeadersFollows, + cancellationToken); + + private static ValueTask WriteDataItemNotStoredAsync( + NntpCommandContext context, + CancellationToken cancellationToken) => + NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ListDataItemNotStored, + NntpResponseStatus.ListDataItemNotStored, + cancellationToken); + + private static ValueTask WriteActiveAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListActiveComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Active, cancellationToken); + } + + private static ValueTask WriteCountsAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListCountsComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Counts, cancellationToken); + } + + private static ValueTask WriteNewsgroupsAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var snapshot = Capture(context); + if (snapshot is null) + { + return NntpCommandNotImplemented.HandleAsync(context, Logger, cancellationToken); + } + + var wildmat = context.ArgumentMemory; + if (wildmat.IsEmpty) + { + return NntpCommandReply.WriteAsync( + context, + Logger, + snapshot.ListNewsgroupsComplete, + NntpResponseStatus.ListOfNewsgroupsFollows, + cancellationToken); + } + + return WriteFilteredAsync(context, snapshot, wildmat, ListLineKind.Newsgroups, cancellationToken); + } + + private static async ValueTask WriteFilteredAsync( + NntpCommandContext context, + NewsgroupSnapshot snapshot, + ReadOnlyMemory pattern, + ListLineKind kind, + CancellationToken cancellationToken) + { + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ListOfNewsgroupsFollows); + await context.Response + .WriteLineAsync(NntpResponses.ListOfNewsgroupsFollows, cancellationToken) + .ConfigureAwait(false); + + var groups = snapshot.Groups; + for (var i = 0; i < groups.Count; i++) + { + var group = groups[i]; + if (!NntpWildmat.IsMatchValidated(group.GroupNameBytes.Span, pattern.Span)) + { + continue; + } + + var line = kind switch + { + ListLineKind.Active => group.ListActiveLine, + ListLineKind.Counts => group.ListCountsLine, + _ => group.ListNewsgroupsLine, + }; + await context.Response.WriteLineAsync(line, cancellationToken).ConfigureAwait(false); + } + + await context.Response.WriteMultilineEndAsync(cancellationToken).ConfigureAwait(false); + } + + private static NewsgroupSnapshot? Capture(NntpCommandContext context) => + context.Session.NewsgroupCatalogue?.Current; + + private enum ListLineKind + { + Active, + Counts, + Newsgroups, + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/ListGroup.cs b/src/VectorNNTPD.TransitService/Session/Commands/ListGroup.cs new file mode 100644 index 00000000..cfc76acb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/ListGroup.cs @@ -0,0 +1,26 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// LISTGROUP command as defined by RFC 3977, Section 6.1.2. + /// + /// + /// Lists article numbers in a newsgroup. Deliberate placeholder until an article-number + /// source exists. Do not invent article numbers from catalogue water marks. + /// + internal static class ListGroup + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(ListGroup)); + + /// Handles LISTGROUP. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "LISTGROUP", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Mode.cs b/src/VectorNNTPD.TransitService/Session/Commands/Mode.cs new file mode 100644 index 00000000..5482521e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Mode.cs @@ -0,0 +1,124 @@ +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// MODE READER (RFC 3977, Section 5.3) and MODE STREAM (RFC 4644, Section 2.3). + /// + /// + /// + /// MODE READER selects reader mode and reports posting permission. After authentication, + /// MODE READER is rejected (RFC 4643). + /// + /// + /// MODE STREAM (RFC 4644 §2.3) is retained for legacy clients. It returns 203 and + /// MUST NOT change receive/capability session mode. It does select Transit AUTHINFO + /// authority (no MySQL). Authorization is + /// (peer or authenticated streaming privilege). CHECK/TAKETHIS are gated by transit authorization + /// and the advertised STREAMING capability — not by issuing MODE STREAM first. + /// + /// + internal static class Mode + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Mode)); + + /// Handles MODE READER. + public static ValueTask HandleReaderAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "MODE READER", ExecuteReaderAsync, cancellationToken); + + /// Handles MODE STREAM. + public static ValueTask HandleStreamAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "MODE STREAM", ExecuteStreamAsync, cancellationToken); + + private static async ValueTask ExecuteReaderAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Session.ServerRole == NntpServerRole.Transit) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 4643: client MUST NOT issue MODE READER after authentication; reject consistently. + if (context.Session.Authentication.IsAuthenticated) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandUnavailableAfterAuthentication, + NntpResponseStatus.CommandUnavailableAfterAuthentication, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §2.2.2: client MUST NOT issue MODE READER after a compression layer is active. + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandUnavailableAfterCompression, + NntpResponseStatus.CommandUnavailableAfterCompression, + cancellationToken) + .ConfigureAwait(false); + return; + } + + context.Session.SetMode(NntpSessionMode.Reader); + if (context.Session.Authorization.PostingPermitted) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ReaderModePostingPermitted, + NntpResponseStatus.ReaderModePostingPermitted, + cancellationToken) + .ConfigureAwait(false); + } + else + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ReaderModePostingProhibited, + NntpResponseStatus.ReaderModePostingProhibited, + cancellationToken) + .ConfigureAwait(false); + } + } + + private static async ValueTask ExecuteStreamAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (context.Session.ServerRole == NntpServerRole.Reader) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.CommandNotAvailable, + NntpResponseStatus.CommandNotAvailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 4644 §2.3.2: MUST return 203 and MUST NOT change receive/capability mode. + // AUTHINFO authority is selected here so STREAM credentials are Transit-only. + // StreamingPermitted is enforced by the dispatcher (RequiresStreaming). + context.Session.SetAuthenticationAuthority(NntpAuthenticationAuthority.Transit); + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.StreamingPermitted, + NntpResponseStatus.StreamingPermitted, + cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Next.cs b/src/VectorNNTPD.TransitService/Session/Commands/Next.cs new file mode 100644 index 00000000..4d1d2787 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Next.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// NEXT command as defined by RFC 3977, Section 6.1.4. + /// + /// + /// Moves the current article pointer forward. Deliberate placeholder until article selection state is implemented. + /// + internal static class Next + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Next)); + + /// Handles NEXT. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "NEXT", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Over.cs b/src/VectorNNTPD.TransitService/Session/Commands/Over.cs new file mode 100644 index 00000000..f91276b0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Over.cs @@ -0,0 +1,25 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// OVER command as defined by RFC 3977, Section 8.3. + /// + /// + /// Returns overview information for articles. Deliberate placeholder until overview database is implemented. + /// + internal static class Over + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Over)); + + /// Handles OVER. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "OVER", + static (ctx, ct) => NntpCommandNotImplemented.HandleAsync(ctx, Logger, ct), + cancellationToken, + successDetail: "not implemented"); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Post.cs b/src/VectorNNTPD.TransitService/Session/Commands/Post.cs new file mode 100644 index 00000000..23bb00f2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Post.cs @@ -0,0 +1,528 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.PostFilter.Quota; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// POST command as defined by RFC 3977, Section 6.3.1. + /// + /// + /// Not pipelined. First-stage 440 does not consume an article. After 340 + /// the article is streamed from the session PipeReader into one destuffed output + /// buffer (client headers destuffed write-through, server-owned headers at the + /// header/body boundary for ordinary injection, body destuffed). Unapproved + /// moderated proto-articles are submitted through + /// without Injection-Info/Injection-Date and without History Peek / TryAdmit / Remember. + /// Authorized moderator reinjection follows the ordinary injection path. History Peek, + /// , Remember, and 240 stay + /// after the terminator and after injection authorization. Downstream spool/worker + /// processing is not awaited. + /// + internal static class Post + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Post)); + + /// Handles POST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "POST", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + if (!context.Session.Authorization.PostingPermitted) + { + await WriteStatusAsync( + context, + NntpResponses.PostingProhibited, + NntpResponseStatus.PostingProhibited, + "posting not permitted", + cancellationToken) + .ConfigureAwait(false); + return; + } + + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.PostSendArticle, + NntpResponseStatus.PostSendArticle, + cancellationToken) + .ConfigureAwait(false); + + NntpArticlePolicyCapture.Capture( + context.Session, + context.Session.MaxArticleSize, + out var maxArticleBytes, + out var siteNameUtf8); + StreamingPostReadResult read; + try + { + read = await StreamingPostArticleReader + .ReadAsync( + context.Connection.Input, + new StreamingPostReadOptions + { + MaxArticleSize = maxArticleBytes, + Time = context.Session.Time, + NewsgroupPolicy = context.Session.NewsgroupPostingPolicy, + InjectionIdentity = context.Session.InjectionIdentity, + ClientIdentity = context.Session.ClientIdentity, + MailComplaintsTo = context.Session.MailComplaintsTo, + TraceProtector = context.Session.PostingTraceProtector, + ControlCancelPermitted = context.Session.Authorization.ControlCancelPermitted, + AuthenticatedUsername = context.Session.Authentication.IsAuthenticated + ? context.Session.Authentication.Username + : null, + ModeratorAuthorization = context.Session.CaptureModeratorAuthorization(), + }, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (read.Status == StreamingPostReadStatus.Incomplete) + { + return; + } + + if (read.Status != StreamingPostReadStatus.Completed) + { + var failure = read.Status == StreamingPostReadStatus.TooLarge + ? new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded") + : read.Failure; + await RejectAsync( + context, + failure, + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (read.Disposition == StreamingPostDisposition.SubmitForModeration) + { + await SubmitForModerationAsync(context, read, cancellationToken).ConfigureAwait(false); + return; + } + + var messageIdBytes = Encoding.ASCII.GetBytes(read.MessageId!); + var history = context.Session.HistoryDb; + if (history is not null) + { + var peek = await history.PeekAsync(messageIdBytes, cancellationToken).ConfigureAwait(false); + if (peek == HistoryLookupResult.Seen) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.DuplicateArticle, "Message-ID already known"), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (peek == HistoryLookupResult.Unavailable) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PersistenceFailure, "history unavailable"), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + } + + if (cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + var created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromDestuffed( + context.Session.ArticleParser, + read.Wire) + : ArticleRecordIngress.TryCreateFromDestuffed( + context.Session.ArticleParser, + read.Wire, + maxArticleBytes, + siteNameUtf8); + if (!created.IsAccepted) + { + await RejectAsync( + context, + MapArticleRecordFailure(in created), + read.MessageId, + FormatGroups(read.Newsgroups), + read.Wire.Length, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (!ArticleTypeAccessPolicy.CanPostArticleType(context.Session, created.Record.ArtType)) + { + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PolicyRejected, "arttype-capability"), + read.MessageId, + FormatGroups(read.Newsgroups), + created.Record.ArtSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var filterRequest = new PostFilterRequest( + created.Record, + context.Session.Authentication.Username, + context.Session.ClientIdentity, + context.Session.AccountPolicy, + read.InjectionUtc, + read.Newsgroups, + context.Session.Time.GetUtcNow(), + context.Session.InjectionIdentity); + + PostFilterResult filter; + try + { + filter = await context.Session.PostFilter + .EvaluateAsync(filterRequest, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (filter.Decision == PostFilterDecision.Reject) + { + EnqueueRejectionEvidence( + context, + PostFilterRejectionEvidence.FromEvaluation( + filterRequest, + filter, + context.Session.Time.GetUtcNow())); + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PolicyRejected, filter.Reason), + read.MessageId, + FormatGroups(read.Newsgroups), + created.Record.ArtSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var lease = filter.Lease; + if (cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + await ReleaseLeaseAsync(context, lease).ConfigureAwait(false); + return; + } + + var inbound = ArticleRecordIngress.CreateQueued( + read.MessageId!, + created.Record, + context.Session.ClientIdentity, + read.InjectionUtc, + InboundArticleProducer.Post, + IngressNewsEvents.SnapshotInboundFeed(context.Session)); + + var enqueue = context.Session.ArticleIngestion.TryAdmit(inbound); + if (enqueue != ArticleEnqueueResult.Accepted) + { + await ReleaseLeaseAsync(context, lease).ConfigureAwait(false); + EnqueueRejectionEvidence( + context, + PostFilterRejectionEvidence.FromAdmissionFailure( + filterRequest, + filter, + enqueue.ToString(), + context.Session.Time.GetUtcNow())); + await RejectAsync( + context, + new PostingFailure(PostingFailureCategory.PersistenceFailure, enqueue.ToString()), + read.MessageId, + FormatGroups(read.Newsgroups), + inbound.Payload.Length, + cancellationToken) + .ConfigureAwait(false); + return; + } + + await CommitLeaseAsync(context, lease, read.MessageId!).ConfigureAwait(false); + + history?.Remember(messageIdBytes); + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + var groups = FormatGroups(read.Newsgroups); + PostLogMessages.Accepted( + logger, + client, + read.MessageId!, + groups, + read.Wire.Length); + } + + await WriteStatusAsync( + context, + NntpResponses.ArticleReceivedOk, + NntpResponseStatus.ArticleReceivedOk, + "accepted", + cancellationToken) + .ConfigureAwait(false); + } + + private static void EnqueueRejectionEvidence( + NntpCommandContext context, + PostFilterRejectionEvidence evidence) + { + if (context.Session.PostFilterEvidence.TryEnqueue(evidence)) + { + return; + } + + PostFilterLogMessages.EvidenceQueueFull(Logger, evidence.Stage, evidence.Reason); + } + + private static async ValueTask ReleaseLeaseAsync(NntpCommandContext context, PostFilterLease? lease) + { + if (lease is null) + { + return; + } + + var status = await lease + .ReleaseAsync(context.Session.Time.GetUtcNow(), CancellationToken.None) + .ConfigureAwait(false); + PostFilterLogMessages.Released( + Logger, + status, + lease.AccountName, + lease.Token); + } + + private static async ValueTask CommitLeaseAsync( + NntpCommandContext context, + PostFilterLease? lease, + string messageId) + { + if (lease is null) + { + return; + } + + var status = await lease + .CommitAsync(context.Session.Time.GetUtcNow(), CancellationToken.None) + .ConfigureAwait(false); + if (status == PostFilterQuotaCommitStatus.Committed) + { + return; + } + + if (status == PostFilterQuotaCommitStatus.Noop) + { + context.Session.PostFilterMetrics.RecordCommitNoop(); + PostFilterLogMessages.CommitNoop( + Logger, + lease.AccountName, + lease.Token, + messageId); + return; + } + + context.Session.PostFilterMetrics.RecordCommitUnavailable(); + PostFilterLogMessages.CommitUnavailable( + Logger, + lease.AccountName, + lease.Token, + messageId); + } + + private static async ValueTask SubmitForModerationAsync( + NntpCommandContext context, + StreamingPostReadResult read, + CancellationToken cancellationToken) + { + var submission = new ModerationSubmission + { + ProtoArticle = ArticleWireReconstructor.RestuffArticle(read.Wire.Span, includeTerminator: false), + MessageId = read.MessageId ?? string.Empty, + Newsgroups = read.Newsgroups, + TargetModeratedGroup = read.TargetModeratedGroup ?? string.Empty, + ModeratorAddress = read.ModeratorAddress ?? string.Empty, + AuthenticatedUsername = context.Session.Authentication.IsAuthenticated + ? context.Session.Authentication.Username + : null, + Sender = context.Session.ClientIdentity, + }; + + ModerationSubmissionResult result; + try + { + result = await context.Session.ModerationSubmission + .SubmitAsync(submission, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (result.Status != ModerationSubmissionStatus.Accepted) + { + await RejectAsync( + context, + new PostingFailure( + PostingFailureCategory.ModerationForwardingFailed, + result.Detail), + read.MessageId, + FormatGroups(read.Newsgroups), + read.DestuffedSize, + cancellationToken) + .ConfigureAwait(false); + return; + } + + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + var groups = FormatGroups(read.Newsgroups); + PostLogMessages.SubmittedForModeration( + logger, + client, + read.MessageId ?? "-", + groups, + read.TargetModeratedGroup ?? "-", + read.ModeratorAddress ?? "-", + context.Session.Authentication.Username ?? "-", + read.DestuffedSize); + } + + IngressNewsEvents.TryWriteModerated(context.Session, read.MessageId, read.DestuffedSize); + await WriteStatusAsync( + context, + NntpResponses.ArticleReceivedOk, + NntpResponseStatus.ArticleReceivedOk, + "accepted for moderation", + cancellationToken) + .ConfigureAwait(false); + } + + private static async ValueTask RejectAsync( + NntpCommandContext context, + PostingFailure failure, + string? messageId, + string? newsgroups, + int size, + CancellationToken cancellationToken) + { + var logger = Logger; + if (logger.IsEnabled(LogLevel.Information)) + { + var client = NntpCommandLogFormat.Client(context.Session); + PostLogMessages.Rejected( + logger, + client, + failure.Category, + messageId ?? "-", + newsgroups ?? "-", + size, + failure.Detail); + } + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId ?? "-", + 441, + IngressNewsReasons.ForPostingFailure(failure), + size); + await WriteFailedAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static ValueTask WriteFailedAsync(NntpCommandContext context, CancellationToken cancellationToken) => + WriteStatusAsync( + context, + NntpResponses.PostingFailed, + NntpResponseStatus.PostingFailed, + "posting failed", + cancellationToken); + + private static ValueTask WriteStatusAsync( + NntpCommandContext context, + ReadOnlyMemory wire, + string statusLine, + string detail, + CancellationToken cancellationToken) + { + context.StatusLine = statusLine; + context.CompletionDetail = detail; + return context.Response.WriteLineAsync(wire, cancellationToken); + } + + private static string FormatGroups(string[] groups) => + groups.Length == 0 ? "-" : string.Join(',', groups); + + private static PostingFailure MapArticleRecordFailure(in ArticleRecordCreateResult created) + { + if (created.ParseFailure != NntpArticleParseFailureCode.None) + { + return new PostingFailure(MapParseFailure(created.ParseFailure), created.ParseFailure.ToString()); + } + + return new PostingFailure(PostingFailureCategory.InvalidPath, created.MaterializeFailure.ToString()); + } + + private static PostingFailureCategory MapParseFailure(NntpArticleParseFailureCode code) => + code switch + { + NntpArticleParseFailureCode.ArticleTooLarge => PostingFailureCategory.ArticleTooLarge, + NntpArticleParseFailureCode.MalformedHeader + or NntpArticleParseFailureCode.MalformedHeaderContinuation + or NntpArticleParseFailureCode.MissingHeaderBodySeparator + or NntpArticleParseFailureCode.ContainsNul + or NntpArticleParseFailureCode.ContainsIllegalControlByte => PostingFailureCategory.MalformedHeader, + NntpArticleParseFailureCode.MissingMessageId + or NntpArticleParseFailureCode.MissingNewsgroups + or NntpArticleParseFailureCode.EmptyArticle => PostingFailureCategory.MissingRequiredHeader, + NntpArticleParseFailureCode.DuplicateMessageId + or NntpArticleParseFailureCode.DuplicateNewsgroups + or NntpArticleParseFailureCode.DuplicatePath => PostingFailureCategory.DuplicateHeader, + NntpArticleParseFailureCode.InvalidMessageId => PostingFailureCategory.InvalidMessageId, + NntpArticleParseFailureCode.MissingOrInvalidDate => PostingFailureCategory.InvalidDate, + NntpArticleParseFailureCode.InvalidNewsgroups => PostingFailureCategory.InvalidNewsgroups, + NntpArticleParseFailureCode.InvalidPath => PostingFailureCategory.InvalidPath, + _ => PostingFailureCategory.PolicyRejected, + }; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs new file mode 100644 index 00000000..271f33fd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/AesGcmPostingTraceProtector.cs @@ -0,0 +1,351 @@ +using System.Buffers.Text; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// AES-256-GCM protector for POST X-Trace (envelope version v1). + /// + /// + /// Token envelope: v1. followed by Base64url of nonce, ciphertext, and tag. + /// Envelope version is independent of the inner payload version and of key rotation. + /// Inner payload version 1 is the pre-username layout (still decryptable). + /// Inner payload version 2 appends a big-endian u16 length and UTF-8 username. + /// always writes payload version 2 with the current key. + /// tries the current key, then the optional previous key. + /// This type does not log keys, tokens, or decrypted payloads. + /// + public sealed class AesGcmPostingTraceProtector : IPostingTraceProtector + { + /// Header-safe envelope prefix for the current token version. + public const string TokenPrefix = "v1."; + + private const int NonceLength = 12; + private const int TagLength = 16; + private const byte LegacyPayloadVersion = 1; + private const byte CurrentPayloadVersion = 2; + internal const int MaxAuthenticatedUsernameBytes = 256; + private static readonly byte[] AssociatedData = "VectorNNTP.XTrace.v1"u8.ToArray(); + private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + private readonly byte[] _currentKey; + private readonly byte[]? _previousKey; + + /// Creates a protector from already-decoded AES-256 keys. + public AesGcmPostingTraceProtector(byte[] currentKey, byte[]? previousKey = null) + { + ArgumentNullException.ThrowIfNull(currentKey); + if (currentKey.Length != XTraceKeyParser.KeyLength) + { + throw new ArgumentException("X-Trace key must be 32 bytes.", nameof(currentKey)); + } + + if (previousKey is not null && previousKey.Length != XTraceKeyParser.KeyLength) + { + throw new ArgumentException("X-Trace previous key must be 32 bytes.", nameof(previousKey)); + } + + _currentKey = currentKey; + _previousKey = previousKey; + } + + /// Creates a protector from validated . + public static AesGcmPostingTraceProtector Create(NntpdOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!XTraceKeyParser.TryDecode(options.XTraceKey, out var current)) + { + throw new InvalidOperationException("XTraceKey is missing or is not a 32-byte AES-256 key."); + } + + byte[]? previous = null; + if (!string.IsNullOrWhiteSpace(options.XTracePreviousKey) + && !XTraceKeyParser.TryDecode(options.XTracePreviousKey, out previous)) + { + throw new InvalidOperationException("XTracePreviousKey is not a 32-byte AES-256 key."); + } + + return new AesGcmPostingTraceProtector(current, previous); + } + + /// Creates a protector from options monitored at resolve time. + public static AesGcmPostingTraceProtector Create(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return Create(options.Value); + } + + /// + public string Protect(PostingTracePayload payload) => EncryptPacked(EncodePayload(payload)); + + /// + public bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload) + { + payload = default; + if (!token.StartsWith(TokenPrefix, StringComparison.Ordinal)) + { + return false; + } + + var encoded = token[TokenPrefix.Length..]; + var max = Base64Url.GetMaxDecodedLength(encoded.Length); + if (max < NonceLength + TagLength + 1) + { + return false; + } + + Span packed = max <= 256 ? stackalloc byte[max] : new byte[max]; + int written; + try + { + if (!Base64Url.TryDecodeFromChars(encoded, packed, out written) + || written < NonceLength + TagLength + 1) + { + return false; + } + } + catch (FormatException) + { + return false; + } + + packed = packed[..written]; + var nonce = packed[..NonceLength]; + var tag = packed[^TagLength..]; + var ciphertext = packed[NonceLength..^TagLength]; + return TryDecrypt(nonce, ciphertext, tag, _currentKey, out payload) + || (_previousKey is not null && TryDecrypt(nonce, ciphertext, tag, _previousKey, out payload)); + } + + private static bool TryDecrypt( + ReadOnlySpan nonce, + ReadOnlySpan ciphertext, + ReadOnlySpan tag, + byte[] key, + out PostingTracePayload payload) + { + payload = default; + Span plaintext = stackalloc byte[ciphertext.Length]; + try + { + using var aes = new AesGcm(key, TagLength); + aes.Decrypt(nonce, ciphertext, tag, plaintext, AssociatedData); + } + catch (CryptographicException) + { + return false; + } + + return TryDecodePayload(plaintext, out payload); + } + + /// + /// Protects a pre-username payload (inner version 1) for backward-compatibility tests. + /// Production always writes inner version 2. + /// + internal string ProtectLegacyV1(PostingTracePayload payload) => + EncryptPacked(EncodeLegacyV1(payload)); + + /// Encrypts caller-supplied plaintext for malformed-payload tests. + internal string ProtectRawForTests(ReadOnlySpan plaintext) => EncryptPacked(plaintext); + + private string EncryptPacked(ReadOnlySpan plaintext) + { + Span nonce = stackalloc byte[NonceLength]; + RandomNumberGenerator.Fill(nonce); + Span ciphertext = stackalloc byte[plaintext.Length]; + Span tag = stackalloc byte[TagLength]; + using (var aes = new AesGcm(_currentKey, TagLength)) + { + aes.Encrypt(nonce, plaintext, ciphertext, tag, AssociatedData); + } + + Span packed = stackalloc byte[NonceLength + plaintext.Length + TagLength]; + nonce.CopyTo(packed); + ciphertext.CopyTo(packed[NonceLength..]); + tag.CopyTo(packed[(NonceLength + plaintext.Length)..]); + return TokenPrefix + Base64Url.EncodeToString(packed); + } + + private static byte[] EncodePayload(PostingTracePayload payload) + { + var usernameBytes = EncodeUsername(payload.AuthenticatedUsername); + var address = payload.Address.GetAddressBytes(); + var length = FixedFieldLength(address.Length) + 2 + usernameBytes.Length; + var buffer = new byte[length]; + var offset = WriteFixedFields(buffer, CurrentPayloadVersion, payload, address); + System.Buffers.Binary.BinaryPrimitives.WriteUInt16BigEndian( + buffer.AsSpan(offset, 2), + (ushort)usernameBytes.Length); + offset += 2; + usernameBytes.CopyTo(buffer.AsSpan(offset)); + return buffer; + } + + private static byte[] EncodeLegacyV1(PostingTracePayload payload) + { + var address = payload.Address.GetAddressBytes(); + var buffer = new byte[FixedFieldLength(address.Length)]; + WriteFixedFields(buffer, LegacyPayloadVersion, payload, address); + return buffer; + } + + private static int FixedFieldLength(int addressLength) => 1 + 1 + addressLength + 2 + 8 + 16; + + private static int WriteFixedFields( + Span buffer, + byte version, + PostingTracePayload payload, + ReadOnlySpan address) + { + var offset = 0; + buffer[offset++] = version; + buffer[offset++] = (byte)address.Length; + address.CopyTo(buffer[offset..]); + offset += address.Length; + System.Buffers.Binary.BinaryPrimitives.WriteUInt16BigEndian(buffer[offset..(offset + 2)], (ushort)payload.Port); + offset += 2; + System.Buffers.Binary.BinaryPrimitives.WriteInt64BigEndian( + buffer[offset..(offset + 8)], + payload.InjectedAtUtc.ToUnixTimeSeconds()); + offset += 8; + payload.TraceId.TryWriteBytes(buffer[offset..(offset + 16)]); + return offset + 16; + } + + private static byte[] EncodeUsername(string? username) + { + if (string.IsNullOrEmpty(username)) + { + return []; + } + + var bytes = Utf8Strict.GetBytes(username); + if (bytes.Length > MaxAuthenticatedUsernameBytes || ContainsNul(bytes)) + { + throw new ArgumentException("Authenticated username is not encodable in X-Trace.", nameof(username)); + } + + return bytes; + } + + private static bool ContainsNul(ReadOnlySpan value) + { + foreach (var b in value) + { + if (b == 0) + { + return true; + } + } + + return false; + } + + private static bool TryDecodePayload(ReadOnlySpan plaintext, out PostingTracePayload payload) + { + payload = default; + if (plaintext.Length < 1 + 1 + 4 + 2 + 8 + 16) + { + return false; + } + + var version = plaintext[0]; + var addressLength = plaintext[1]; + if (addressLength is not (4 or 16)) + { + return false; + } + + var fixedLength = FixedFieldLength(addressLength); + if (plaintext.Length < fixedLength) + { + return false; + } + + var offset = 2; + IPAddress address; + try + { + address = new IPAddress(plaintext.Slice(offset, addressLength)); + } + catch (ArgumentException) + { + return false; + } + + offset += addressLength; + var port = System.Buffers.Binary.BinaryPrimitives.ReadUInt16BigEndian(plaintext.Slice(offset, 2)); + offset += 2; + var unix = System.Buffers.Binary.BinaryPrimitives.ReadInt64BigEndian(plaintext.Slice(offset, 8)); + offset += 8; + var traceId = new Guid(plaintext.Slice(offset, 16)); + offset += 16; + DateTimeOffset injected; + try + { + injected = DateTimeOffset.FromUnixTimeSeconds(unix); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + + string? username = null; + if (version == LegacyPayloadVersion) + { + if (plaintext.Length != fixedLength) + { + return false; + } + } + else if (version == CurrentPayloadVersion) + { + if (plaintext.Length < fixedLength + 2) + { + return false; + } + + var userLength = System.Buffers.Binary.BinaryPrimitives.ReadUInt16BigEndian(plaintext.Slice(offset, 2)); + offset += 2; + if (userLength > MaxAuthenticatedUsernameBytes || plaintext.Length != fixedLength + 2 + userLength) + { + return false; + } + + if (userLength > 0) + { + var userBytes = plaintext.Slice(offset, userLength); + if (ContainsNul(userBytes)) + { + return false; + } + + try + { + username = Utf8Strict.GetString(userBytes); + } + catch (DecoderFallbackException) + { + return false; + } + catch (ArgumentException) + { + return false; + } + } + } + else + { + return false; + } + + payload = new PostingTracePayload(address, port, injected, traceId, username); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/ApprovedHeaderParser.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/ApprovedHeaderParser.cs new file mode 100644 index 00000000..b337d021 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/ApprovedHeaderParser.cs @@ -0,0 +1,240 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// Parses Approved: as RFC 5536 mailbox-list identities. + /// + /// + /// Header names compare case-insensitively. Values are mailbox identities, not booleans. + /// Multiple fields or mailbox-list entries are collected; identical identities collapse. + /// Conflicting identities are rejected rather than silently choosing one. + /// Comparison of the extracted mailbox is ASCII case-insensitive. + /// + internal static class ApprovedHeaderParser + { + /// + /// Collects every Approved: identity from . + /// + /// + /// when there is no Approved field, or when every field is a + /// well-formed mailbox-list whose identities do not conflict. + /// + public static bool TryRead( + ParsedPostArticle article, + out string[] identities, + out PostingFailure failure) + { + ArgumentNullException.ThrowIfNull(article); + identities = []; + failure = default; + + List? collected = null; + foreach (var header in article.Headers) + { + if (!PostFieldSyntax.EqualsFolded(header.Name.Span, "APPROVED"u8)) + { + continue; + } + + if (!TryParseMailboxList(header.UnfoldedValue.Span, out var parsed, out failure)) + { + identities = []; + return false; + } + + collected ??= new List(parsed.Length); + foreach (var identity in parsed) + { + if (ContainsIdentity(collected, identity)) + { + continue; + } + + collected.Add(identity); + } + } + + identities = collected is null ? [] : collected.ToArray(); + return true; + } + + internal static bool TryParseMailboxList( + ReadOnlySpan value, + out string[] identities, + out PostingFailure failure) + { + identities = []; + failure = default; + var collected = new List(2); + var i = 0; + var sawToken = false; + while (i < value.Length) + { + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "malformed Approved"); + return false; + } + + var start = i; + var angle = 0; + while (i < value.Length) + { + var b = value[i]; + if (b == (byte)'<') + { + angle++; + } + else if (b == (byte)'>') + { + angle--; + } + else if (b == (byte)',' && angle == 0) + { + break; + } + + i++; + } + + if (angle != 0 || !TryExtractMailbox(value[start..i], out var mailbox)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "malformed Approved"); + return false; + } + + if (!ContainsIdentity(collected, mailbox)) + { + collected.Add(mailbox); + } + + sawToken = true; + if (i >= value.Length) + { + break; + } + + i++; + } + + if (!sawToken) + { + failure = new PostingFailure(PostingFailureCategory.InvalidApproved, "empty Approved"); + return false; + } + + identities = collected.ToArray(); + return true; + } + + internal static bool TryExtractMailbox(ReadOnlySpan value, out string mailbox) + { + mailbox = string.Empty; + var trimmed = Trim(value); + if (trimmed.IsEmpty) + { + return false; + } + + var open = LastIndexOf(trimmed, (byte)'<'); + var close = LastIndexOf(trimmed, (byte)'>'); + ReadOnlySpan candidate; + if (open >= 0 || close >= 0) + { + if (open < 0 || close <= open) + { + return false; + } + + candidate = Trim(trimmed[(open + 1)..close]); + } + else + { + candidate = trimmed; + } + + if (!PostFieldSyntax.IsMailbox(candidate)) + { + return false; + } + + mailbox = Encoding.ASCII.GetString(candidate); + return true; + } + + internal static bool IdentitiesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (Fold(left[i]) != Fold(right[i])) + { + return false; + } + } + + return true; + } + + private static bool ContainsIdentity(List identities, string candidate) + { + foreach (var existing in identities) + { + if (IdentitiesEqual(existing, candidate)) + { + return true; + } + } + + return false; + } + + private static ReadOnlySpan Trim(ReadOnlySpan value) + { + var start = 0; + var end = value.Length; + while (start < end && PostFieldSyntax.IsWsp(value[start])) + { + start++; + } + + while (end > start && PostFieldSyntax.IsWsp(value[end - 1])) + { + end--; + } + + return value[start..end]; + } + + private static int LastIndexOf(ReadOnlySpan value, byte needle) + { + for (var i = value.Length - 1; i >= 0; i--) + { + if (value[i] == needle) + { + return i; + } + } + + return -1; + } + + private static char Fold(char c) => + c is >= 'a' and <= 'z' ? (char)(c - 32) : c; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs new file mode 100644 index 00000000..0faa1346 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/CatalogueNewsgroupPostingPolicy.cs @@ -0,0 +1,94 @@ +using System.Text; +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// POST newsgroup policy backed by one captured snapshot. + /// + /// + /// + /// Every syntax-validated Newsgroups: target must exist in the snapshot. + /// Local POST is rejected for unknown groups and for n/x/j + /// (RFC 3977 §7.6.3, RFC 6048 §3.1). Status y is ordinary local posting. + /// Status m is classified as ; + /// this policy does not authorize Approved: and does not forward to a moderator. + /// + /// + /// The catalogue snapshot is captured once per evaluation. Lookups do not query MySQL. + /// + /// + public sealed class CatalogueNewsgroupPostingPolicy : INewsgroupPostingPolicy + { + private readonly INewsgroupCatalogue _catalogue; + + /// Initializes a new instance of the class. + /// Process-wide catalogue whose is captured once per evaluation. + public CatalogueNewsgroupPostingPolicy(INewsgroupCatalogue catalogue) + { + ArgumentNullException.ThrowIfNull(catalogue); + _catalogue = catalogue; + } + + /// + public NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups) + { + if (groups.IsEmpty) + { + return Reject(PostingFailureCategory.InvalidNewsgroups, "empty Newsgroups"); + } + + var snapshot = _catalogue.Current; + Span nameBytes = stackalloc byte[128]; + string[]? moderated = null; + var moderatedCount = 0; + for (var i = 0; i < groups.Length; i++) + { + var group = groups[i]; + if (group.Length is < 1 or > 128) + { + return Reject(PostingFailureCategory.InvalidNewsgroups, "malformed newsgroup name"); + } + + var written = Encoding.ASCII.GetBytes(group, nameBytes); + if (!snapshot.TryGet(nameBytes[..written], out var record)) + { + return Reject(PostingFailureCategory.PolicyRejected, "unknown newsgroup"); + } + + switch (record.PostingStatus) + { + case NewsgroupPostingStatus.Allowed: + continue; + case NewsgroupPostingStatus.Prohibited: + return Reject(PostingFailureCategory.PolicyRejected, "posting prohibited"); + case NewsgroupPostingStatus.Moderated: + moderated ??= new string[groups.Length]; + moderated[moderatedCount++] = group; + continue; + case NewsgroupPostingStatus.NoPostingOrPeerArticles: + return Reject(PostingFailureCategory.PolicyRejected, "newsgroup closed"); + case NewsgroupPostingStatus.PeerOnly: + return Reject(PostingFailureCategory.PolicyRejected, "local posting not accepted"); + default: + return Reject(PostingFailureCategory.PolicyRejected, "unsupported posting status"); + } + } + + if (moderatedCount == 0) + { + return new NewsgroupPostingEvaluation(NewsgroupCatalogStatus.Accepted, Failure: null); + } + + var names = new string[moderatedCount]; + Array.Copy(moderated!, names, moderatedCount); + return new NewsgroupPostingEvaluation( + NewsgroupCatalogStatus.RequiresModeration, + Failure: null, + names); + } + + private static NewsgroupPostingEvaluation Reject(PostingFailureCategory category, string detail) => + new(NewsgroupCatalogStatus.Rejected, new PostingFailure(category, detail)); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/INewsgroupPostingPolicy.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/INewsgroupPostingPolicy.cs new file mode 100644 index 00000000..71714d40 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/INewsgroupPostingPolicy.cs @@ -0,0 +1,87 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// Newsgroup existence, posting-authorization, and moderation boundary used by POST. + /// + /// + /// Production sessions that have an + /// use . + /// remains the stand-in when no catalogue is attached (tests). + /// + public interface INewsgroupPostingPolicy + { + /// + /// Evaluates posting authorization for already syntax-validated newsgroup names. + /// + /// Distinct, syntactically valid newsgroup names. + /// + /// Catalogue classification. Does not re-validate group-name syntax and does not + /// treat Approved: as authorization. + /// + NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups); + } + + /// Result of . + /// Whether a catalogue decision was available. + /// Set when posting is rejected by policy. + /// + /// Moderated targets in Newsgroups: order when + /// ; otherwise empty. + /// + public readonly record struct NewsgroupPostingEvaluation( + NewsgroupCatalogStatus Status, + PostingFailure? Failure, + string[]? ModeratedGroups = null) + { + /// Gets moderated targets, or an empty array when none were classified. + public string[] ModeratedGroupNames => ModeratedGroups ?? []; + } + + /// Availability of newsgroup existence/moderation state. + public enum NewsgroupCatalogStatus + { + /// + /// Group existence and moderation state are not available. This is not a confirmed + /// existence match and must not be logged or treated as such. + /// + CatalogUnavailable = 0, + + /// Policy rejected posting to one or more groups. + Rejected = 1, + + /// + /// Every target exists in the captured snapshot and permits local POST + /// (y). This is a confirmed catalogue decision. + /// + Accepted = 2, + + /// + /// At least one target is moderated (m) and every other target permits + /// local POST (y). Approval authorization or moderator forwarding is required. + /// + RequiresModeration = 3, + } + + /// + /// POST newsgroup policy used when no catalogue is attached: syntax-valid groups + /// are accepted without asserting that the groups exist or that moderation state + /// is known. + /// + /// + /// Production sessions with a catalogue use . + /// This stand-in does not consult the catalogue and does not treat Approved: + /// as authorization to post to a moderated group. + /// + public sealed class SyntaxOnlyNewsgroupPostingPolicy : INewsgroupPostingPolicy + { + /// Shared instance. + public static SyntaxOnlyNewsgroupPostingPolicy Instance { get; } = new(); + + /// + public NewsgroupPostingEvaluation Evaluate(ReadOnlySpan groups) + { + _ = groups; + return new NewsgroupPostingEvaluation(NewsgroupCatalogStatus.CatalogUnavailable, Failure: null); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/IPostingTraceProtector.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/IPostingTraceProtector.cs new file mode 100644 index 00000000..9f35d03d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/IPostingTraceProtector.cs @@ -0,0 +1,26 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// Authenticated-encryption boundary for server-owned POST X-Trace values. + /// + /// + /// Implementations must provide confidentiality, integrity, and tamper detection. + /// Do not log keys or decrypted payloads. + /// + public interface IPostingTraceProtector + { + /// + /// Protects as an opaque header-safe token. + /// + string Protect(PostingTracePayload payload); + + /// + /// Attempts to recover a payload from a previously protected token. + /// + /// + /// when authentication succeeds and the payload is well-formed; + /// otherwise (tamper, unknown version, or wrong key). + /// + bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/ParsedPostArticle.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/ParsedPostArticle.cs new file mode 100644 index 00000000..90cedf35 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/ParsedPostArticle.cs @@ -0,0 +1,92 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// One structurally parsed header field from a destuffed POST article. + /// Original field-name octets (case preserved). + /// Logical field body with folding CRLF removed. + /// Original field octets including folding and the terminating CRLF. + internal readonly record struct ParsedPostHeader( + ReadOnlyMemory Name, + ReadOnlyMemory UnfoldedValue, + ReadOnlyMemory RawField); + + /// Destuffed POST article after one structural header parse. + internal sealed class ParsedPostArticle + { + /// Initializes a new instance of the class. + public ParsedPostArticle( + ReadOnlyMemory payload, + IReadOnlyList headers, + ReadOnlyMemory body, + int headerBlockBytes) + { + Payload = payload; + Headers = headers; + Body = body; + HeaderBlockBytes = headerBlockBytes; + } + + /// Gets the destuffed received payload (headers + separator + body). + public ReadOnlyMemory Payload { get; } + + /// Gets parsed headers in wire order. + public IReadOnlyList Headers { get; } + + /// Gets the destuffed body (may be empty). + public ReadOnlyMemory Body { get; } + + /// Gets destuffed header-block length including the blank separator CRLF. + public int HeaderBlockBytes { get; } + + /// Gets destuffed article size (payload length; terminator excluded). + public int ArticleSize => Payload.Length; + + /// Gets or sets the article Message-ID (supplied or synthesized). + public string? MessageId { get; set; } + + /// Gets or sets whether was synthesized by the server. + public bool MessageIdSynthesized { get; set; } + + /// Gets or sets syntax-validated newsgroup names. + public string[] Newsgroups { get; set; } = []; + + /// Gets or sets the parsed author Date:, or the injection time when synthesized. + public DateTimeOffset AuthorDate { get; set; } + + /// Gets or sets whether was generated because the client omitted Date:. + public bool DateSynthesized { get; set; } + + /// Gets or sets whether a structurally valid Approved: is present. + public bool ApprovedPresent { get; set; } + + /// + /// Gets or sets the distinct mailbox identities parsed from Approved:. + /// Empty when the header is absent. + /// + public string[] ApprovedIdentities { get; set; } = []; + + /// Gets or sets the catalogue classification for this article's newsgroups. + public NewsgroupCatalogStatus CatalogStatus { get; set; } + + /// + /// Gets or sets moderated Newsgroups: targets in header order when + /// is . + /// + public string[] ModeratedGroups { get; set; } = []; + + /// Returns the first header whose name matches . + public bool TryGetHeader(ReadOnlySpan upperName, out ParsedPostHeader header) + { + foreach (var candidate in Headers) + { + if (PostFieldSyntax.EqualsFolded(candidate.Name.Span, upperName)) + { + header = candidate; + return true; + } + } + + header = default; + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostArticleValidator.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostArticleValidator.cs new file mode 100644 index 00000000..2356af65 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostArticleValidator.cs @@ -0,0 +1,423 @@ +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Strict POST header-field validation after one structural parse. + internal static class PostArticleValidator + { + private static readonly byte[][] SingletonNames = + [ + "DATE"u8.ToArray(), + "FROM"u8.ToArray(), + "MESSAGE-ID"u8.ToArray(), + "NEWSGROUPS"u8.ToArray(), + "PATH"u8.ToArray(), + "SUBJECT"u8.ToArray(), + "CONTROL"u8.ToArray(), + "DISTRIBUTION"u8.ToArray(), + "FOLLOWUP-TO"u8.ToArray(), + "INJECTION-DATE"u8.ToArray(), + "INJECTION-INFO"u8.ToArray(), + "LINES"u8.ToArray(), + "ORGANIZATION"u8.ToArray(), + "REFERENCES"u8.ToArray(), + "SUMMARY"u8.ToArray(), + "SUPERSEDES"u8.ToArray(), + "USER-AGENT"u8.ToArray(), + "XREF"u8.ToArray(), + "NNTP-POSTING-DATE"u8.ToArray(), + "NNTP-POSTING-HOST"u8.ToArray(), + "X-TRACE"u8.ToArray(), + "IN-REPLY-TO"u8.ToArray(), + ]; + + /// Validates parsed headers and fills Message-ID, Newsgroups, and Date on . + public static bool TryValidate( + ParsedPostArticle article, + DateTimeOffset injectionUtc, + INewsgroupPostingPolicy newsgroupPolicy, + out PostingFailure failure, + bool controlCancelPermitted = false) + { + ArgumentNullException.ThrowIfNull(article); + ArgumentNullException.ThrowIfNull(newsgroupPolicy); + + if (!TryCheckSingletonsAndForbidden(article, out failure)) + { + return false; + } + + if (!TryRequire(article, "FROM"u8, "From", out var fromHeader, out failure) + || !TryRequire(article, "NEWSGROUPS"u8, "Newsgroups", out var groupsHeader, out failure) + || !TryRequire(article, "SUBJECT"u8, "Subject", out _, out failure)) + { + return false; + } + + if (!PostFieldSyntax.IsMailbox(fromHeader.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "invalid From"); + return false; + } + + if (article.TryGetHeader("DATE"u8, out var dateHeader)) + { + if (!PostRfcDate.TryParse(dateHeader.UnfoldedValue.Span, out var authorDate)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDate, "malformed Date"); + return false; + } + + if (!PostRfcDate.IsWithinPolicy(authorDate, injectionUtc)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDate, "Date outside policy window"); + return false; + } + + article.AuthorDate = authorDate; + article.DateSynthesized = false; + } + else + { + article.AuthorDate = injectionUtc; + article.DateSynthesized = true; + } + + var groups = new List(4); + if (!PostFieldSyntax.TryParseNewsgroupList( + groupsHeader.UnfoldedValue.Span, + groups, + PostingLimits.MaxNewsgroups, + allowPoster: false)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidNewsgroups, "malformed Newsgroups"); + return false; + } + + article.Newsgroups = groups.ToArray(); + + if (article.TryGetHeader("MESSAGE-ID"u8, out var messageIdHeader)) + { + if (!NntpMessageIdValidation.IsValidMessageId(messageIdHeader.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidMessageId, "malformed Message-ID"); + return false; + } + + article.MessageId = System.Text.Encoding.ASCII.GetString(messageIdHeader.UnfoldedValue.Span); + article.MessageIdSynthesized = false; + } + else + { + article.MessageId = PostMessageIdFactory.Create(); + article.MessageIdSynthesized = true; + } + + if (!TryValidateOptionalFields(article, controlCancelPermitted, out failure)) + { + return false; + } + + var evaluation = newsgroupPolicy.Evaluate(article.Newsgroups); + article.CatalogStatus = evaluation.Status; + article.ModeratedGroups = evaluation.ModeratedGroupNames; + if (evaluation.Status == NewsgroupCatalogStatus.Rejected) + { + failure = evaluation.Failure + ?? new PostingFailure(PostingFailureCategory.PolicyRejected, "newsgroup policy rejected"); + return false; + } + + return true; + } + + private static bool TryCheckSingletonsAndForbidden(ParsedPostArticle article, out PostingFailure failure) + { + failure = default; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var header in article.Headers) + { + var name = System.Text.Encoding.ASCII.GetString(header.Name.Span); + if (IsForbiddenMailRouting(header.Name.Span)) + { + failure = new PostingFailure(PostingFailureCategory.PolicyRejected, "mail routing header"); + return false; + } + + if (!IsSingleton(header.Name.Span)) + { + continue; + } + + if (!seen.Add(name)) + { + failure = new PostingFailure(PostingFailureCategory.DuplicateHeader, name); + return false; + } + } + + return true; + } + + private static bool TryValidateOptionalFields( + ParsedPostArticle article, + bool controlCancelPermitted, + out PostingFailure failure) + { + failure = default; + var ids = new List>(8); + + if (article.TryGetHeader("FOLLOWUP-TO"u8, out var followup)) + { + var followGroups = new List(4); + if (!PostFieldSyntax.TryParseNewsgroupList( + followup.UnfoldedValue.Span, + followGroups, + PostingLimits.MaxFollowupToGroups, + allowPoster: true)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidFollowupTo, "malformed Followup-To"); + return false; + } + } + + if (article.TryGetHeader("REFERENCES"u8, out var references) + && !PostFieldSyntax.TryParseMessageIdList(references.UnfoldedValue.Span, ids, maxCount: 32)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidReferences, "malformed References"); + return false; + } + + var referenceIds = ids.ToArray(); + if (article.TryGetHeader("IN-REPLY-TO"u8, out var inReplyTo)) + { + if (!PostFieldSyntax.TryParseMessageIdList(inReplyTo.UnfoldedValue.Span, ids, maxCount: 8)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidReferences, "malformed In-Reply-To"); + return false; + } + + if (referenceIds.Length > 0 && !ContainsMessageId(referenceIds, ids[^1].Span)) + { + failure = new PostingFailure( + PostingFailureCategory.InvalidReferences, + "In-Reply-To is not in References"); + return false; + } + } + + if (article.TryGetHeader("DISTRIBUTION"u8, out var distribution) + && !TryValidateDistribution(distribution.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidDistribution, "malformed Distribution"); + return false; + } + + if (!ApprovedHeaderParser.TryRead(article, out var approvedIdentities, out failure)) + { + return false; + } + + article.ApprovedIdentities = approvedIdentities; + article.ApprovedPresent = approvedIdentities.Length > 0; + + if (article.TryGetHeader("CONTROL"u8, out var control) + && (!controlCancelPermitted + || !IsAuthorizedCancelControl(control.UnfoldedValue.Span))) + { + failure = new PostingFailure(PostingFailureCategory.InvalidControl, "unsupported Control"); + return false; + } + + if (article.TryGetHeader("SUPERSEDES"u8, out var supersedes) + && !PostFieldSyntax.TryParseMessageIdList(supersedes.UnfoldedValue.Span, ids, maxCount: 8)) + { + failure = new PostingFailure(PostingFailureCategory.InvalidControl, "malformed Supersedes"); + return false; + } + + if (article.TryGetHeader("LINES"u8, out var lines) + && !IsUnsignedInteger(lines.UnfoldedValue.Span)) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "malformed Lines"); + return false; + } + + return true; + } + + private static bool TryValidateDistribution(ReadOnlySpan value) + { + var i = 0; + var count = 0; + while (i < value.Length) + { + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + return false; + } + + var start = i; + while (i < value.Length && value[i] != (byte)',' && !PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (!PostFieldSyntax.IsDistributionToken(value[start..i])) + { + return false; + } + + count++; + if (count > 16) + { + return false; + } + + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)',') + { + return false; + } + + i++; + } + + return count > 0; + } + + /// + /// Accepts only cancel <message-id> (optional surrounding WSP). + /// Other Control verbs stay rejected even for the newsmaster. + /// + internal static bool IsAuthorizedCancelControl(ReadOnlySpan value) + // codeql[cs/complex-block] Protocol validation clarity for cancel Control parsing. + { + var i = 0; + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + if (i + 6 > value.Length || !PostFieldSyntax.EqualsFolded(value.Slice(i, 6), "CANCEL"u8)) + { + return false; + } + + i += 6; + if (i >= value.Length || !PostFieldSyntax.IsWsp(value[i])) + { + return false; + } + + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + var idStart = i; + while (i < value.Length && !PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + var messageId = value[idStart..i]; + while (i < value.Length && PostFieldSyntax.IsWsp(value[i])) + { + i++; + } + + return i == value.Length && NntpMessageIdValidation.IsValidMessageId(messageId); + } + + private static bool TryRequire( + ParsedPostArticle article, + ReadOnlySpan upperName, + string display, + out ParsedPostHeader header, + out PostingFailure failure) + { + if (article.TryGetHeader(upperName, out header)) + { + failure = default; + return true; + } + + header = default; + failure = new PostingFailure(PostingFailureCategory.MissingRequiredHeader, display); + return false; + } + + private static bool IsSingleton(ReadOnlySpan name) + { + foreach (var candidate in SingletonNames) + { + if (PostFieldSyntax.EqualsFolded(name, candidate)) + { + return true; + } + } + + return false; + } + + private static bool IsForbiddenMailRouting(ReadOnlySpan name) => + PostFieldSyntax.EqualsFolded(name, "RECEIVED"u8) + || PostFieldSyntax.EqualsFolded(name, "DELIVERED-TO"u8) + || PostFieldSyntax.EqualsFolded(name, "BCC"u8) + || PostFieldSyntax.EqualsFolded(name, "CC"u8) + || PostFieldSyntax.EqualsFolded(name, "TO"u8) + || PostFieldSyntax.EqualsFolded(name, "X-FORWARDED-FOR"u8) + || PostFieldSyntax.EqualsFolded(name, "FORWARDED"u8); + + private static bool ContainsMessageId(ReadOnlyMemory[] haystack, ReadOnlySpan needle) + { + foreach (var item in haystack) + { + if (item.Span.SequenceEqual(needle)) + { + return true; + } + } + + return false; + } + + private static bool IsUnsignedInteger(ReadOnlySpan value) + { + if (value.IsEmpty) + { + return false; + } + + foreach (var b in value) + { + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostFieldSyntax.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostFieldSyntax.cs new file mode 100644 index 00000000..c0216d51 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostFieldSyntax.cs @@ -0,0 +1,287 @@ +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Structural syntax checks for Netnews header field values (byte-oriented). + internal static class PostFieldSyntax + { + /// RFC 5536 newsgroup-name: component *( "." component ). + public static bool IsNewsgroupName(ReadOnlySpan value) + { + if (value.IsEmpty || value.Length > 128) + { + return false; + } + + var componentLength = 0; + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b == (byte)'.') + { + if (componentLength == 0) + { + return false; + } + + componentLength = 0; + continue; + } + + if (!IsComponentChar(b)) + { + return false; + } + + componentLength++; + } + + return componentLength > 0; + } + + /// One or more Message-IDs separated by WSP. + public static bool TryParseMessageIdList( + ReadOnlySpan value, + List> ids, + int maxCount) + { + ids.Clear(); + var i = 0; + while (i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)'<') + { + return false; + } + + var end = value[i..].IndexOf((byte)'>'); + if (end < 0) + { + return false; + } + + var token = value.Slice(i, end + 1); + if (!NntpMessageIdValidation.IsValidMessageId(token)) + { + return false; + } + + if (ids.Count >= maxCount) + { + return false; + } + + ids.Add(token.ToArray()); + i += end + 1; + } + + return ids.Count > 0; + } + + /// Comma-separated newsgroup names with optional WSP around commas. + public static bool TryParseNewsgroupList( + ReadOnlySpan value, + List groups, + int maxCount, + bool allowPoster) + { + groups.Clear(); + var i = 0; + var sawToken = false; + while (i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] == (byte)',') + { + return false; + } + + var start = i; + while (i < value.Length && value[i] != (byte)',' && !IsWsp(value[i])) + { + i++; + } + + var token = value[start..i]; + if (token.IsEmpty) + { + return false; + } + + if (allowPoster && EqualsFolded(token, "POSTER"u8)) + { + if (sawToken || i < value.Length) + { + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i < value.Length) + { + return false; + } + } + + groups.Add("poster"); + return groups.Count == 1; + } + + if (!IsNewsgroupName(token)) + { + return false; + } + + var name = System.Text.Encoding.ASCII.GetString(token); + foreach (var existing in groups) + { + if (existing.Equals(name, StringComparison.Ordinal)) + { + return false; + } + } + + if (groups.Count >= maxCount) + { + return false; + } + + groups.Add(name); + sawToken = true; + + while (i < value.Length && IsWsp(value[i])) + { + i++; + } + + if (i >= value.Length) + { + break; + } + + if (value[i] != (byte)',') + { + return false; + } + + i++; + if (i >= value.Length) + { + return false; + } + } + + return groups.Count > 0; + } + + /// Requires a mailbox-shaped value containing exactly one @ in a printable region. + public static bool IsMailbox(ReadOnlySpan value) + { + if (value.IsEmpty || value.Length > 512) + { + return false; + } + + var at = -1; + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b == 0 || (b < 32 && b != (byte)'\t')) + { + return false; + } + + if (b == (byte)'@') + { + if (at >= 0) + { + return false; + } + + at = i; + } + } + + return at > 0 && at < value.Length - 1; + } + + /// Distribution token: world, local, or a newsgroup-name. + public static bool IsDistributionToken(ReadOnlySpan value) => + EqualsFolded(value, "WORLD"u8) + || EqualsFolded(value, "LOCAL"u8) + || IsNewsgroupName(value); + + public static bool EqualsFolded(ReadOnlySpan value, ReadOnlySpan upperAscii) + { + if (value.Length != upperAscii.Length) + { + return false; + } + + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b >= (byte)'a' && b <= (byte)'z') + { + b = (byte)(b - 32); + } + + if (b != upperAscii[i]) + { + return false; + } + } + + return true; + } + + public static bool HeaderNamesEqual(ReadOnlySpan left, ReadOnlySpan right) + { + if (left.Length != right.Length) + { + return false; + } + + for (var i = 0; i < left.Length; i++) + { + if (FoldUpper(left[i]) != FoldUpper(right[i])) + { + return false; + } + } + + return true; + } + + public static byte FoldUpper(byte b) => + b is >= (byte)'a' and <= (byte)'z' ? (byte)(b - 32) : b; + + public static bool IsWsp(byte b) => b is (byte)' ' or (byte)'\t'; + + private static bool IsComponentChar(byte b) => + // codeql[cs/complex-condition] Protocol validation clarity for component character class. + (b >= (byte)'A' && b <= (byte)'Z') + || (b >= (byte)'a' && b <= (byte)'z') + || (b >= (byte)'0' && b <= (byte)'9') + || b is (byte)'+' or (byte)'-' or (byte)'_'; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderNormalizer.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderNormalizer.cs new file mode 100644 index 00000000..e5256338 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderNormalizer.cs @@ -0,0 +1,173 @@ +using System.Buffers; +using System.Net; +using System.Text; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Replaces server-owned POST headers and emits one destuffed article buffer. + internal static class PostHeaderNormalizer + { + /// Server-controlled Path for locally POSTed articles. + public const string PostedPath = ".POSTED"; + + /// + /// Builds the accepted article: client Path / injection metadata / Xref discarded, + /// server-owned values written once, a valid client Date preserved, and a Date + /// written from the injection time only when the client omitted Date. + /// + public static ReadOnlyMemory Normalize( + ParsedPostArticle article, + DateTimeOffset injectionUtc, + string injectionIdentity, + ConnectionClientIdentity clientIdentity, + string mailComplaintsTo, + IPostingTraceProtector traceProtector, + string? authenticatedUsername = null) + { + ArgumentNullException.ThrowIfNull(article); + ArgumentException.ThrowIfNullOrWhiteSpace(article.MessageId); + ArgumentException.ThrowIfNullOrWhiteSpace(injectionIdentity); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(mailComplaintsTo); + ArgumentNullException.ThrowIfNull(traceProtector); + + var writer = new ArrayBufferWriter(article.Payload.Length + 256); + foreach (var header in article.Headers) + { + if (IsServerOwned(header.Name.Span)) + { + continue; + } + + writer.Write(header.RawField.Span); + } + + WriteServerOwnedHeaders( + writer, + article.MessageIdSynthesized, + article.MessageId, + article.DateSynthesized, + injectionUtc, + injectionIdentity, + clientIdentity, + mailComplaintsTo, + traceProtector, + authenticatedUsername); + if (!article.Body.IsEmpty) + { + writer.Write(article.Body.Span); + } + + return writer.WrittenMemory.ToArray(); + } + + /// Formats the transport peer for trusted diagnostics (never written as a header). + public static string FormatPostingHost(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return address.IsIPv4MappedToIPv6 ? address.MapToIPv4().ToString() : address.ToString(); + } + + internal static bool IsServerOwned(ReadOnlySpan name) => + PostFieldSyntax.EqualsFolded(name, "PATH"u8) + || PostFieldSyntax.EqualsFolded(name, "INJECTION-DATE"u8) + || PostFieldSyntax.EqualsFolded(name, "INJECTION-INFO"u8) + || PostFieldSyntax.EqualsFolded(name, "NNTP-POSTING-DATE"u8) + || PostFieldSyntax.EqualsFolded(name, "NNTP-POSTING-HOST"u8) + || PostFieldSyntax.EqualsFolded(name, "X-TRACE"u8) + || PostFieldSyntax.EqualsFolded(name, "XREF"u8); + + /// + /// Writes synthesized Message-ID and Date (when needed), Path, Injection-Date, Injection-Info, + /// X-Trace, and the header/body separator into . + /// + /// + /// writes of + /// . A client Date already copied into the article is not rewritten. + /// + internal static void WriteServerOwnedHeaders( + IBufferWriter writer, + bool writeMessageId, + string messageId, + bool writeDate, + DateTimeOffset injectionUtc, + string injectionIdentity, + ConnectionClientIdentity clientIdentity, + string mailComplaintsTo, + IPostingTraceProtector traceProtector, + string? authenticatedUsername = null) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentException.ThrowIfNullOrWhiteSpace(injectionIdentity); + ArgumentNullException.ThrowIfNull(clientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(mailComplaintsTo); + ArgumentNullException.ThrowIfNull(traceProtector); + + if (writeMessageId) + { + WriteHeader(writer, "Message-ID: "u8, messageId); + } + + if (writeDate) + { + WriteHeader(writer, "Date: "u8, PostRfcDate.Format(injectionUtc)); + } + + WriteHeader(writer, "Path: "u8, PostedPath); + WriteHeader(writer, "Injection-Date: "u8, PostRfcDate.Format(injectionUtc)); + WriteHeader(writer, "Injection-Info: "u8, BuildInjectionInfo(injectionIdentity, messageId, mailComplaintsTo)); + WriteHeader( + writer, + "X-Trace: "u8, + traceProtector.Protect( + new PostingTracePayload( + clientIdentity.ClientAddress, + clientIdentity.ClientPort, + injectionUtc, + Guid.NewGuid(), + authenticatedUsername))); + writer.Write("\r\n"u8); + } + + /// + /// Writes a synthesized Message-ID and Date when required, then the header/body separator. + /// Does not write Path, Injection-Date, Injection-Info, or X-Trace (RFC 5537 §3.5 step 7). + /// + internal static void WriteProtoArticleBoundary( + IBufferWriter writer, + bool writeMessageId, + string messageId, + bool writeDate, + DateTimeOffset injectionUtc) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + if (writeMessageId) + { + WriteHeader(writer, "Message-ID: "u8, messageId); + } + + if (writeDate) + { + WriteHeader(writer, "Date: "u8, PostRfcDate.Format(injectionUtc)); + } + + writer.Write("\r\n"u8); + } + + private static void WriteHeader(IBufferWriter writer, ReadOnlySpan prefix, string value) + { + writer.Write(prefix); + Encoding.ASCII.GetBytes(value, writer); + writer.Write("\r\n"u8); + } + + private static string BuildInjectionInfo( + string identity, + string messageId, + string mailComplaintsTo) => + $"{identity}; logging-data=\"{messageId}\"; mail-complaints-to=\"{mailComplaintsTo}\""; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderParser.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderParser.cs new file mode 100644 index 00000000..c5ba99f1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostHeaderParser.cs @@ -0,0 +1,301 @@ +using System.Buffers; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Structural destuffed-article header parser. Does not use string.Split. + internal static class PostHeaderParser + { + /// + /// Parses destuffed article bytes (headers + blank separator + body). + /// Size has already been enforced by the multiline reader. + /// + public static bool TryParse( + ReadOnlyMemory payload, + out ParsedPostArticle? article, + out PostingFailure failure) + { + article = null; + var span = payload.Span; + if (span.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty article"); + return false; + } + + for (var i = 0; i < span.Length; i++) + { + if (span[i] == 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "embedded NUL"); + return false; + } + } + + if (!TryFindSeparator(span, out var separator, out failure)) + { + return false; + } + + if (separator + 2 > PostingLimits.MaxHeaderBlockBytes) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + return false; + } + + var headerBlock = span[..separator]; + var body = payload[(separator + 2)..]; + if (!TryParseHeaderBlock(payload, headerBlock, out var headers, out failure)) + { + return false; + } + + article = new ParsedPostArticle(payload, headers, body, separator + 2); + return true; + } + + private static bool TryFindSeparator( + ReadOnlySpan span, + out int separator, + out PostingFailure failure) + { + separator = -1; + var i = 0; + while (i < span.Length) + { + if (span[i] == (byte)'\n' && (i == 0 || span[i - 1] != (byte)'\r')) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "LF without CR"); + return false; + } + + if (span[i] == (byte)'\r') + { + if (i + 1 >= span.Length || span[i + 1] != (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "CR without LF"); + return false; + } + + if (i + 3 < span.Length && span[i + 2] == (byte)'\r' && span[i + 3] == (byte)'\n') + { + separator = i + 2; + failure = default; + return true; + } + + i += 2; + continue; + } + + i++; + } + + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "missing header/body separator"); + return false; + } + + private static bool TryParseHeaderBlock( + ReadOnlyMemory payload, + ReadOnlySpan headerBlock, + out List headers, + out PostingFailure failure) + { + headers = []; + failure = default; + if (headerBlock.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header block"); + return false; + } + + if (headerBlock.Length < 2 + || headerBlock[^2] != (byte)'\r' + || headerBlock[^1] != (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block is not CRLF terminated"); + return false; + } + + var unfolded = new ArrayBufferWriter(256); + var fieldStart = 0; + var nameStart = -1; + var nameLength = 0; + var hasField = false; + var lineStart = 0; + var i = 0; + while (i < headerBlock.Length) + { + if (i + 1 >= headerBlock.Length + || headerBlock[i] != (byte)'\r' + || headerBlock[i + 1] != (byte)'\n') + { + i++; + continue; + } + + var line = headerBlock[lineStart..i]; + var isFold = line.Length > 0 && PostFieldSyntax.IsWsp(line[0]); + if (isFold) + { + if (!hasField) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "leading fold"); + return false; + } + + if (line.Length < 2) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty fold"); + return false; + } + + unfolded.Write(line); + } + else + { + if (hasField + && !FlushHeader(payload, fieldStart, lineStart, nameStart, nameLength, unfolded, headers, out failure)) + { + return false; + } + + if (line.IsEmpty) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header line"); + return false; + } + + if (!TryReadFieldName(line, out nameLength, out failure)) + { + return false; + } + + fieldStart = lineStart; + nameStart = lineStart; + hasField = true; + unfolded.Clear(); + unfolded.Write(line[(nameLength + 2)..]); + } + + i += 2; + lineStart = i; + } + + if (lineStart != headerBlock.Length) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "truncated header line"); + return false; + } + + if (!hasField) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "no header fields"); + return false; + } + + if (!FlushHeader(payload, fieldStart, headerBlock.Length, nameStart, nameLength, unfolded, headers, out failure)) + { + return false; + } + + if (headers.Count > PostingLimits.MaxHeaderCount) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "too many headers"); + return false; + } + + return true; + } + + internal static bool TryReadFieldName( + ReadOnlySpan line, + out int nameLength, + out PostingFailure failure) + { + nameLength = 0; + var colon = line.IndexOf((byte)':'); + if (colon <= 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header line missing field name"); + return false; + } + + if (colon + 1 >= line.Length || line[colon + 1] != (byte)' ') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header requires colon-space"); + return false; + } + + for (var i = 0; i < colon; i++) + { + var b = line[i]; + if (b is < 33 or > 126 || b == (byte)':') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "malformed field name"); + return false; + } + } + + nameLength = colon; + failure = default; + return true; + } + + private static bool FlushHeader( + ReadOnlyMemory payload, + int fieldStart, + int fieldEnd, + int nameStart, + int nameLength, + ArrayBufferWriter unfolded, + List headers, + out PostingFailure failure) + { + failure = default; + var raw = payload.Slice(fieldStart, fieldEnd - fieldStart); + var name = payload.Slice(nameStart, nameLength); + var value = unfolded.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : unfolded.WrittenMemory.ToArray(); + + if (!TryCreateHeader(name, value, raw, out var header, out failure)) + { + return false; + } + + headers.Add(header); + return true; + } + + /// Validates unfolded field length/control characters and builds a parsed header. + internal static bool TryCreateHeader( + ReadOnlyMemory name, + ReadOnlyMemory unfoldedValue, + ReadOnlyMemory rawField, + out ParsedPostHeader header, + out PostingFailure failure) + { + header = default; + if (name.Length + 2 + unfoldedValue.Length > PostingLimits.MaxSingleHeaderBytes) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header too long"); + return false; + } + + var valueSpan = unfoldedValue.Span; + for (var i = 0; i < valueSpan.Length; i++) + { + var b = valueSpan[i]; + if (b == 0 || (b < 32 && b != (byte)'\t')) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "forbidden control character"); + return false; + } + } + + header = new ParsedPostHeader(name, unfoldedValue, rawField); + failure = default; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostLogMessages.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostLogMessages.cs new file mode 100644 index 00000000..3002ab9b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostLogMessages.cs @@ -0,0 +1,44 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Source-generated POST accept/reject diagnostics. Does not log article bodies. + internal static partial class PostLogMessages + { + [LoggerMessage( + EventId = 1720, + Level = LogLevel.Information, + Message = "[{Client}] POST accepted Message-ID={MessageId} newsgroups={Newsgroups} size={Size}")] + public static partial void Accepted( + ILogger logger, + string Client, + string MessageId, + string Newsgroups, + int Size); + + [LoggerMessage( + EventId = 1722, + Level = LogLevel.Information, + Message = "[{Client}] POST submitted for moderation Message-ID={MessageId} newsgroups={Newsgroups} target={TargetGroup} moderator={ModeratorAddress} user={Username} size={Size}")] + public static partial void SubmittedForModeration( + ILogger logger, + string Client, + string MessageId, + string Newsgroups, + string TargetGroup, + string ModeratorAddress, + string Username, + int Size); + + [LoggerMessage( + EventId = 1721, + Level = LogLevel.Information, + Message = "[{Client}] POST rejected category={Category} Message-ID={MessageId} newsgroups={Newsgroups} size={Size} detail={Detail}")] + public static partial void Rejected( + ILogger logger, + string Client, + PostingFailureCategory Category, + string MessageId, + string Newsgroups, + int Size, + string Detail); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostMessageIdFactory.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostMessageIdFactory.cs new file mode 100644 index 00000000..f34f2e08 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostMessageIdFactory.cs @@ -0,0 +1,31 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Synthesizes a POST Message-ID as <MD5(UUID())@usenet.ninja>. + internal static class PostMessageIdFactory + { + /// Right-hand side of a synthesized Message-ID. + public const string Domain = "usenet.ninja"; + + /// + /// Creates a Message-ID for one posting attempt. The UUID is generated once; + /// MD5 is taken over its canonical D-format text (not the raw UUID). + /// + public static string Create() + { + var uuid = Guid.NewGuid().ToString("D"); + var hash = MD5.HashData(Encoding.ASCII.GetBytes(uuid)); + var hex = Convert.ToHexStringLower(hash); + return string.Create(hex.Length + Domain.Length + 3, hex, static (span, id) => + { + span[0] = '<'; + id.CopyTo(span[1..]); + span[1 + id.Length] = '@'; + Domain.CopyTo(span[(2 + id.Length)..]); + span[^1] = '>'; + }); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostRfcDate.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostRfcDate.cs new file mode 100644 index 00000000..403cc143 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostRfcDate.cs @@ -0,0 +1,255 @@ +using System.Globalization; +using System.Text; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// RFC 5322 / Netnews date-time format used for POST Date: and injection timestamps. + internal static class PostRfcDate + { + private static readonly string[] DayNames = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + private static readonly string[] MonthNames = + ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + + /// Formats as Wed, 25 Sep 2026 10:49:00 +0000. + public static string Format(DateTimeOffset utc) + { + var value = utc.ToUniversalTime(); + return string.Create( + CultureInfo.InvariantCulture, + $"{DayNames[(int)value.DayOfWeek]}, {value.Day:00} {MonthNames[value.Month - 1]} {value.Year:0000} {value.Hour:00}:{value.Minute:00}:{value.Second:00} +0000"); + } + + /// Parses an RFC-compatible date (numeric zone, GMT, UT, or UTC). + public static bool TryParse(ReadOnlySpan value, out DateTimeOffset date) + { + date = default; + if (value.IsEmpty || value.Length > 128) + { + return false; + } + + Span chars = stackalloc char[value.Length]; + if (Encoding.ASCII.GetChars(value, chars) != value.Length) + { + return false; + } + + return TryParse(chars, out date); + } + + /// Parses an RFC-compatible date from already-decoded ASCII. + public static bool TryParse(ReadOnlySpan value, out DateTimeOffset date) + { + date = default; + var span = value.Trim(); + if (span.IsEmpty) + { + return false; + } + + if (span.Length >= 5 && span[3] == ',') + { + if (!IsDayName(span[..3])) + { + return false; + } + + span = span[4..].TrimStart(); + } + + if (!TryReadNumber(ref span, minDigits: 1, maxDigits: 2, out var day) || day is < 1 or > 31) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadMonth(ref span, out var month)) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadNumber(ref span, minDigits: 4, maxDigits: 4, out var year) + || year < 1900) + { + return false; + } + + if (!TrySkipWsp(ref span) || !TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out var hour) + || hour > 23) + { + return false; + } + + if (span.IsEmpty || span[0] != ':') + { + return false; + } + + span = span[1..]; + if (!TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out var minute) || minute > 59) + { + return false; + } + + var second = 0; + if (!span.IsEmpty && span[0] == ':') + { + span = span[1..]; + if (!TryReadNumber(ref span, minDigits: 2, maxDigits: 2, out second) || second > 60) + { + return false; + } + } + + if (!TrySkipWsp(ref span) || !TryReadZone(ref span, out var offset)) + { + return false; + } + + if (!span.IsEmpty) + { + return false; + } + + try + { + date = new DateTimeOffset(year, month, day, hour, minute, Math.Min(second, 59), offset); + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + } + + /// Returns whether is inside the POST date window relative to . + public static bool IsWithinPolicy(DateTimeOffset date, DateTimeOffset nowUtc) + { + var utc = date.ToUniversalTime(); + var now = nowUtc.ToUniversalTime(); + if (utc > now + PostingLimits.MaxDateSkewFuture) + { + return false; + } + + return utc >= now - PostingLimits.MaxDateAge; + } + + private static bool IsDayName(ReadOnlySpan value) + { + foreach (var name in DayNames) + { + if (value.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static bool TryReadMonth(ref ReadOnlySpan span, out int month) + { + month = 0; + if (span.Length < 3) + { + return false; + } + + var token = span[..3]; + for (var i = 0; i < MonthNames.Length; i++) + { + if (token.Equals(MonthNames[i], StringComparison.OrdinalIgnoreCase)) + { + month = i + 1; + span = span[3..]; + return true; + } + } + + return false; + } + + private static bool TryReadZone(ref ReadOnlySpan span, out TimeSpan offset) + { + offset = TimeSpan.Zero; + if (span.IsEmpty) + { + return false; + } + + if (span.StartsWith("GMT", StringComparison.OrdinalIgnoreCase) + || span.StartsWith("UTC", StringComparison.OrdinalIgnoreCase)) + { + span = span[3..]; + return true; + } + + if (span.StartsWith("UT", StringComparison.OrdinalIgnoreCase) + && (span.Length == 2 || !char.IsAsciiLetter(span[2]))) + { + span = span[2..]; + return true; + } + + var sign = span[0]; + if (sign is not ('+' or '-') || span.Length < 5) + { + return false; + } + + if (!char.IsAsciiDigit(span[1]) || !char.IsAsciiDigit(span[2]) + || !char.IsAsciiDigit(span[3]) || !char.IsAsciiDigit(span[4])) + { + return false; + } + + var hours = ((span[1] - '0') * 10) + (span[2] - '0'); + var minutes = ((span[3] - '0') * 10) + (span[4] - '0'); + if (hours > 14 || minutes > 59) + { + return false; + } + + var ticks = TimeSpan.FromHours(hours) + TimeSpan.FromMinutes(minutes); + offset = sign == '-' ? -ticks : ticks; + span = span[5..]; + return true; + } + + private static bool TryReadNumber(ref ReadOnlySpan span, int minDigits, int maxDigits, out int value) + { + value = 0; + var taken = 0; + while (taken < maxDigits && taken < span.Length && char.IsAsciiDigit(span[taken])) + { + value = (value * 10) + (span[taken] - '0'); + taken++; + } + + if (taken < minDigits) + { + return false; + } + + span = span[taken..]; + return true; + } + + private static bool TrySkipWsp(ref ReadOnlySpan span) + { + var i = 0; + while (i < span.Length && (span[i] == ' ' || span[i] == '\t')) + { + i++; + } + + if (i == 0) + { + return false; + } + + span = span[i..]; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingFailureCategory.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingFailureCategory.cs new file mode 100644 index 00000000..c36cb361 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingFailureCategory.cs @@ -0,0 +1,71 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Internal POST validation/persistence failure category for structured logs. + /// Never copied onto the NNTP wire. Clients always receive 441 Posting failed. + public enum PostingFailureCategory + { + /// Destuffed article exceeded Nntpd:MaxArticleSize during receive. + ArticleTooLarge = 0, + + /// Header line, folding, field-name, or CR/LF structure is invalid. + MalformedHeader = 1, + + /// A required header is absent. + MissingRequiredHeader = 2, + + /// A singleton header occurred more than once. + DuplicateHeader = 3, + + /// Message-ID syntax is invalid. + InvalidMessageId = 4, + + /// Date syntax is invalid or outside server date policy. + InvalidDate = 5, + + /// Newsgroups syntax is invalid. + InvalidNewsgroups = 6, + + /// Path generation/normalization failed. + InvalidPath = 7, + + /// Followup-To syntax is invalid. + InvalidFollowupTo = 8, + + /// References / In-Reply-To syntax or consistency is invalid. + InvalidReferences = 9, + + /// Distribution syntax or policy is invalid. + InvalidDistribution = 10, + + /// Control / unsupported control semantics. + InvalidControl = 11, + + /// Approved syntax is invalid. + InvalidApproved = 12, + + /// HistoryDB already knows this Message-ID. + DuplicateArticle = 13, + + /// Session or isolated policy rejected the article. + PolicyRejected = 14, + + /// Ingestion/history persistence did not accept the article. + PersistenceFailure = 15, + + /// + /// An Approved: identity was present but the authenticated principal is not + /// authorized to approve every moderated target. + /// + UnauthorizedApproval = 16, + + /// + /// The proto-article required moderator forwarding and submission was unavailable or failed. + /// + ModerationForwardingFailed = 17, + } + + /// A POST validation or persistence failure. + /// Structured failure category. + /// Short internal detail (not sent to the client). + public readonly record struct PostingFailure(PostingFailureCategory Category, string Detail); +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingLimits.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingLimits.cs new file mode 100644 index 00000000..43a8169d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingLimits.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// Fixed POST header and newsgroup bounds (distinct from Nntpd:MaxArticleSize). + internal static class PostingLimits + { + /// Maximum destuffed header block including the blank separator line. + public const int MaxHeaderBlockBytes = 64 * 1024; + + /// Maximum unfolded single header (name + colon + value). + public const int MaxSingleHeaderBytes = 8192; + + /// Maximum header fields in one article. + public const int MaxHeaderCount = 256; + + /// Maximum distinct newsgroups in Newsgroups:. + public const int MaxNewsgroups = 12; + + /// Maximum distinct groups in Followup-To:. + public const int MaxFollowupToGroups = 12; + + /// Articles whose Date: is more than this far in the future are rejected. + public static readonly TimeSpan MaxDateSkewFuture = TimeSpan.FromHours(24); + + /// Articles whose Date: is older than this relative to injection time are rejected. + public static readonly TimeSpan MaxDateAge = TimeSpan.FromDays(14); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingTracePayload.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingTracePayload.cs new file mode 100644 index 00000000..f1f51893 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/PostingTracePayload.cs @@ -0,0 +1,67 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// Transport and authentication identity recovered from a server-owned POST X-Trace token. + /// + /// + /// Intended for trusted server-side diagnostics only. Do not log this value. + /// is the AUTHINFO identity captured at POST admission; + /// it is never taken from article headers. Null or empty means the session was unauthenticated. + /// + public readonly struct PostingTracePayload + { + /// Initializes a payload without an authenticated username (unauthenticated POST). + public PostingTracePayload(IPAddress address, int port, DateTimeOffset injectedAtUtc, Guid traceId) + : this(address, port, injectedAtUtc, traceId, authenticatedUsername: null) + { + } + + /// Initializes a payload, optionally including the authenticated session username. + public PostingTracePayload( + IPAddress address, + int port, + DateTimeOffset injectedAtUtc, + Guid traceId, + string? authenticatedUsername) + { + ArgumentNullException.ThrowIfNull(address); + ArgumentOutOfRangeException.ThrowIfLessThan(port, 0); + ArgumentOutOfRangeException.ThrowIfGreaterThan(port, 65535); + Address = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + Port = port; + InjectedAtUtc = injectedAtUtc.ToUniversalTime(); + TraceId = traceId; + AuthenticatedUsername = NormalizeUsername(authenticatedUsername); + } + + /// Gets the effective transport peer address. + public IPAddress Address { get; } + + /// Gets the effective transport peer port. + public int Port { get; } + + /// Gets the injection-boundary UTC timestamp. + public DateTimeOffset InjectedAtUtc { get; } + + /// Gets the unique identifier for this POST attempt. + public Guid TraceId { get; } + + /// + /// Gets the AUTHINFO username captured at POST admission, or + /// when the session was unauthenticated or the token is a pre-username v1 payload. + /// + public string? AuthenticatedUsername { get; } + + internal static string? NormalizeUsername(string? username) + { + if (string.IsNullOrWhiteSpace(username)) + { + return null; + } + + return username; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostArticleReader.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostArticleReader.cs new file mode 100644 index 00000000..9b90c1bd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostArticleReader.cs @@ -0,0 +1,694 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// + /// Streams a POST multiline article from a into one destuffed + /// output buffer for . + /// + /// + /// Header lines are destuffed for validation and written destuffed (client Path / + /// injection metadata discarded). Body lines drop one NNTP stuffing layer + /// (RFC 3977 §3.1.1). Destuffed byte counting enforces Nntpd:MaxArticleSize. + /// History Peek is not performed here; the caller peeks after the terminator. + /// + internal static class StreamingPostArticleReader + { + /// Initial capacity of the single destuffed output buffer. + public const int InitialCapacity = 64 * 1024; + + /// + /// Reads one POST article from into one owned destuffed buffer. + /// + public static async ValueTask ReadAsync( + PipeReader reader, + StreamingPostReadOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(options); + ArgumentOutOfRangeException.ThrowIfLessThan(options.MaxArticleSize, 1); + ArgumentNullException.ThrowIfNull(options.Time); + ArgumentNullException.ThrowIfNull(options.NewsgroupPolicy); + ArgumentException.ThrowIfNullOrWhiteSpace(options.InjectionIdentity); + ArgumentNullException.ThrowIfNull(options.ClientIdentity); + ArgumentException.ThrowIfNullOrWhiteSpace(options.MailComplaintsTo); + + var session = new ReadSession(options); + return await session.RunAsync(reader, cancellationToken).ConfigureAwait(false); + } + + private sealed class ReadSession(StreamingPostReadOptions options) + { + private readonly GrowingArticleBuffer _output = new(InitialCapacity); + private readonly List _headers = []; + private readonly ArrayBufferWriter _unfolded = new(256); + private bool _inHeaders = true; + private bool _hasField; + private bool _writeCurrentField; + private bool _drain; + private byte[]? _currentName; + private int _destuffedBytes; + private int _headerBlockBytes; + // Incomplete until HandleTerminator assigns Completed/Rejected/TooLarge. Defaulting to + // Completed would make Finish()'s success path look reachable with a null _parsed. + private StreamingPostReadStatus _status = StreamingPostReadStatus.Incomplete; + private PostingFailure _failure; + private ParsedPostArticle? _parsed; + private DateTimeOffset _injectionUtc; + private StreamingPostDisposition _disposition = StreamingPostDisposition.Inject; + private string? _moderatorAddress; + private string? _targetModeratedGroup; + + public async ValueTask RunAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examined = buffer.End; + + while (NntpDelimiterSearch.TryReadLine(ref buffer, out var line)) + { + var finished = HandleLine(line); + if (finished) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return Finish(); + } + } + + reader.AdvanceTo(buffer.Start, examined); + if (result.IsCompleted) + { + return Incomplete(); + } + } + } + + private bool HandleLine(ReadOnlySequence stuffedLine) + { + if (NntpDelimiterSearch.IsTerminatorLine(stuffedLine)) + { + return HandleTerminator(); + } + + if (_drain) + { + return false; + } + + var leadingDot = NntpDelimiterSearch.FirstByte(stuffedLine) == (byte)'.'; + var destuffedLength = (int)stuffedLine.Length - (leadingDot ? 1 : 0); + var destuffedWithCrlf = destuffedLength + 2; + if (_destuffedBytes + destuffedWithCrlf > options.MaxArticleSize) + { + MarkTooLarge(); + return false; + } + + if (_inHeaders) + { + return HandleHeaderLine(stuffedLine, destuffedLength, destuffedWithCrlf); + } + + _destuffedBytes += destuffedWithCrlf; + WriteDestuffedLine(_output, stuffedLine, destuffedLength); + return false; + } + + private bool HandleHeaderLine( + ReadOnlySequence stuffedLine, + int destuffedLength, + int destuffedWithCrlf) + { + var rented = ArrayPool.Shared.Rent(Math.Max(destuffedLength, 1)); + try + { + CopyDestuffed(stuffedLine, rented.AsSpan(0, destuffedLength)); + var destuffed = rented.AsSpan(0, destuffedLength); + if (!TryValidateLineOctets(destuffed, out _failure)) + { + MarkRejected(); + return false; + } + + if (destuffed.IsEmpty) + { + return CompleteHeaderSection(destuffedWithCrlf); + } + + if (_headerBlockBytes + destuffedWithCrlf > PostingLimits.MaxHeaderBlockBytes) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + MarkRejected(); + return false; + } + + _destuffedBytes += destuffedWithCrlf; + _headerBlockBytes += destuffedWithCrlf; + + var isFold = destuffed.Length > 0 && PostFieldSyntax.IsWsp(destuffed[0]); + if (isFold) + { + return HandleFold(destuffed); + } + + if (_hasField && !FlushCurrentField()) + { + MarkRejected(); + return false; + } + + if (!PostHeaderParser.TryReadFieldName(destuffed, out var nameLength, out _failure)) + { + MarkRejected(); + return false; + } + + _currentName = destuffed[..nameLength].ToArray(); + _writeCurrentField = !PostHeaderNormalizer.IsServerOwned(_currentName); + _hasField = true; + _unfolded.Clear(); + _unfolded.Write(destuffed[(nameLength + 2)..]); + if (_writeCurrentField) + { + _output.Write(destuffed); + _output.Write(NntpDelimiterSearch.Crlf); + } + + return false; + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + private bool HandleFold(ReadOnlySpan destuffed) + { + if (!_hasField) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "leading fold"); + MarkRejected(); + return false; + } + + if (destuffed.Length < 2) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty fold"); + MarkRejected(); + return false; + } + + _unfolded.Write(destuffed); + if (_writeCurrentField) + { + _output.Write(destuffed); + _output.Write(NntpDelimiterSearch.Crlf); + } + + return false; + } + + private bool CompleteHeaderSection(int separatorBytes) + { + if (_headerBlockBytes + separatorBytes > PostingLimits.MaxHeaderBlockBytes) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "header block too large"); + MarkRejected(); + return false; + } + + if (!_hasField) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "empty header block"); + MarkRejected(); + return false; + } + + if (!FlushCurrentField()) + { + MarkRejected(); + return false; + } + + if (_headers.Count > PostingLimits.MaxHeaderCount) + { + _failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "too many headers"); + MarkRejected(); + return false; + } + + _destuffedBytes += separatorBytes; + _headerBlockBytes += separatorBytes; + + _injectionUtc = options.Time.GetUtcNow(); + _parsed = new ParsedPostArticle( + ReadOnlyMemory.Empty, + _headers, + ReadOnlyMemory.Empty, + _headerBlockBytes); + + if (!PostArticleValidator.TryValidate( + _parsed, + _injectionUtc, + options.NewsgroupPolicy, + out _failure, + options.ControlCancelPermitted)) + { + MarkRejected(); + return false; + } + + if (!TryCompletePostingDecision(_parsed)) + { + MarkRejected(); + return false; + } + + _inHeaders = false; + return false; + } + + private bool TryCompletePostingDecision(ParsedPostArticle article) + { + if (article.CatalogStatus == NewsgroupCatalogStatus.RequiresModeration) + { + if (!article.ApprovedPresent) + { + return TryBeginModerationSubmission(article); + } + + var authorization = options.ModeratorAuthorization ?? EmptyModeratorAuthorization.Instance; + if (!authorization.TryAuthorizeApproval( + options.AuthenticatedUsername, + article.ApprovedIdentities, + article.ModeratedGroups, + out var detail)) + { + _failure = new PostingFailure( + detail == "unresolved moderator" + ? PostingFailureCategory.ModerationForwardingFailed + : PostingFailureCategory.UnauthorizedApproval, + detail ?? "unauthorized approval"); + return false; + } + } + + return TryWriteInjectionHeaders(article); + } + + private bool TryBeginModerationSubmission(ParsedPostArticle article) + { + if (article.ModeratedGroups.Length == 0) + { + _failure = new PostingFailure(PostingFailureCategory.PolicyRejected, "moderated newsgroup"); + return false; + } + + var target = article.ModeratedGroups[0]; + if (target.Length is < 1 or > 128) + { + _failure = new PostingFailure(PostingFailureCategory.InvalidNewsgroups, "malformed newsgroup name"); + return false; + } + + Span nameBytes = stackalloc byte[128]; + var written = Encoding.ASCII.GetBytes(target, nameBytes); + var authorization = options.ModeratorAuthorization ?? EmptyModeratorAuthorization.Instance; + if (!authorization.TryResolve(nameBytes[..written], out var identity)) + { + _failure = new PostingFailure( + PostingFailureCategory.ModerationForwardingFailed, + "unresolved moderator"); + return false; + } + + try + { + PostHeaderNormalizer.WriteProtoArticleBoundary( + _output, + article.MessageIdSynthesized, + article.MessageId!, + article.DateSynthesized, + _injectionUtc); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "proto-article header generation failed"); + return false; + } + + _disposition = StreamingPostDisposition.SubmitForModeration; + _targetModeratedGroup = target; + _moderatorAddress = identity.Address; + return true; + } + + private bool TryWriteInjectionHeaders(ParsedPostArticle article) + { + if (options.TraceProtector is null) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "trace protector unavailable"); + return false; + } + + try + { + PostHeaderNormalizer.WriteServerOwnedHeaders( + _output, + article.MessageIdSynthesized, + article.MessageId!, + article.DateSynthesized, + _injectionUtc, + options.InjectionIdentity, + options.ClientIdentity, + options.MailComplaintsTo, + options.TraceProtector, + options.AuthenticatedUsername); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + _failure = new PostingFailure(PostingFailureCategory.PersistenceFailure, "server header generation failed"); + return false; + } + + _disposition = StreamingPostDisposition.Inject; + return true; + } + + private bool HandleTerminator() + { + if (_drain) + { + return true; + } + + if (_inHeaders) + { + _failure = _destuffedBytes == 0 + ? new PostingFailure(PostingFailureCategory.MalformedHeader, "empty article") + : new PostingFailure(PostingFailureCategory.MalformedHeader, "missing header/body separator"); + _status = StreamingPostReadStatus.Rejected; + return true; + } + + _status = StreamingPostReadStatus.Completed; + return true; + } + + private bool FlushCurrentField() + { + var name = _currentName is null + ? ReadOnlyMemory.Empty + : _currentName.AsMemory(); + var value = _unfolded.WrittenCount == 0 + ? ReadOnlyMemory.Empty + : _unfolded.WrittenMemory.ToArray(); + if (!PostHeaderParser.TryCreateHeader(name, value, ReadOnlyMemory.Empty, out var header, out _failure)) + { + return false; + } + + _headers.Add(header); + _hasField = false; + _currentName = null; + _unfolded.Clear(); + return true; + } + + private void MarkTooLarge() + { + _status = StreamingPostReadStatus.TooLarge; + _failure = new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded"); + _drain = true; + } + + private void MarkRejected() + { + _status = StreamingPostReadStatus.Rejected; + _drain = true; + } + + private StreamingPostReadResult Finish() + { + if (_status != StreamingPostReadStatus.Completed) + { + return new StreamingPostReadResult( + _status, + _failure, + ReadOnlyMemory.Empty, + _parsed?.MessageId, + _parsed?.Newsgroups ?? [], + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + _parsed?.ApprovedIdentities); + } + + // Completed is assigned only after CompleteHeaderSection materializes _parsed. + var parsed = _parsed + ?? throw new InvalidOperationException( + "Streaming POST completed without a parsed article; header completion must precede the terminator."); + + return new StreamingPostReadResult( + StreamingPostReadStatus.Completed, + default, + _output.Take(), + parsed.MessageId, + parsed.Newsgroups, + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + parsed.ApprovedIdentities); + } + + private StreamingPostReadResult Incomplete() => + new( + StreamingPostReadStatus.Incomplete, + default, + ReadOnlyMemory.Empty, + _parsed?.MessageId, + _parsed?.Newsgroups ?? [], + _destuffedBytes, + _injectionUtc, + _disposition, + _moderatorAddress, + _targetModeratedGroup, + _parsed?.ApprovedIdentities); + } + + private static bool TryValidateLineOctets(ReadOnlySpan destuffed, out PostingFailure failure) + { + for (var i = 0; i < destuffed.Length; i++) + { + var b = destuffed[i]; + if (b == 0) + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "embedded NUL"); + return false; + } + + if (b == (byte)'\n') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "LF without CR"); + return false; + } + + if (b == (byte)'\r') + { + failure = new PostingFailure(PostingFailureCategory.MalformedHeader, "CR without LF"); + return false; + } + } + + failure = default; + return true; + } + + private static void CopyDestuffed(ReadOnlySequence stuffed, Span destuffed) + { + if (destuffed.IsEmpty) + { + return; + } + + var leadingDot = NntpDelimiterSearch.FirstByte(stuffed) == (byte)'.'; + if (stuffed.IsSingleSegment) + { + var src = stuffed.FirstSpan; + if (leadingDot) + { + src = src[1..]; + } + + src.CopyTo(destuffed); + return; + } + + var rented = ArrayPool.Shared.Rent((int)stuffed.Length); + try + { + stuffed.CopyTo(rented); + var src = leadingDot + ? rented.AsSpan(1, destuffed.Length) + : rented.AsSpan(0, destuffed.Length); + src.CopyTo(destuffed); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + /// + /// Writes one destuffed content line plus CRLF (one NNTP stuffing layer removed). + /// + private static void WriteDestuffedLine( + GrowingArticleBuffer output, + ReadOnlySequence stuffedLine, + int destuffedLength) + { + if (destuffedLength > 0) + { + if (stuffedLine.IsSingleSegment) + { + var src = stuffedLine.FirstSpan; + if (src.Length > 0 && src[0] == (byte)'.') + { + src = src[1..]; + } + + output.Write(src); + } + else + { + var rented = ArrayPool.Shared.Rent(destuffedLength); + try + { + CopyDestuffed(stuffedLine, rented.AsSpan(0, destuffedLength)); + output.Write(rented.AsSpan(0, destuffedLength)); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + } + + output.Write(NntpDelimiterSearch.Crlf); + } + + /// One owned destuffed output buffer transferred to the caller on success. + internal sealed class GrowingArticleBuffer : IBufferWriter + { + private byte[] _buffer; + private int _written; + + public GrowingArticleBuffer(int initialCapacity) + { + _buffer = new byte[initialCapacity]; + } + + public int Written => _written; + + public void Advance(int count) + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + if (_written + count > _buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + _written += count; + } + + public Memory GetMemory(int sizeHint = 0) + { + Ensure(_written + Math.Max(sizeHint, 1)); + return _buffer.AsMemory(_written); + } + + public Span GetSpan(int sizeHint = 0) + { + Ensure(_written + Math.Max(sizeHint, 1)); + return _buffer.AsSpan(_written); + } + + public void Write(ReadOnlySpan bytes) + { + if (bytes.IsEmpty) + { + return; + } + + Ensure(_written + bytes.Length); + bytes.CopyTo(_buffer.AsSpan(_written)); + _written += bytes.Length; + } + + public void Write(ReadOnlySequence bytes) + { + if (bytes.IsEmpty) + { + return; + } + + if (bytes.IsSingleSegment) + { + Write(bytes.FirstSpan); + return; + } + + foreach (var segment in bytes) + { + Write(segment.Span); + } + } + + public ReadOnlyMemory Take() + { + var buffer = _buffer; + var length = _written; + _buffer = []; + _written = 0; + return buffer.AsMemory(0, length); + } + + private void Ensure(int needed) + { + if (_buffer.Length >= needed) + { + return; + } + + var next = _buffer.Length < 256 ? 256 : _buffer.Length * 2; + if (next < needed) + { + next = needed; + } + + var grown = new byte[next]; + if (_written > 0) + { + _buffer.AsSpan(0, _written).CopyTo(grown); + } + + _buffer = grown; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostReadResult.cs b/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostReadResult.cs new file mode 100644 index 00000000..2d0c0a9c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Posting/StreamingPostReadResult.cs @@ -0,0 +1,99 @@ +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Session.Commands.Posting +{ + /// What the POST command should do with a completed receive. + internal enum StreamingPostDisposition + { + /// Article is ready for normal History Peek / TryAdmit injection. + Inject = 0, + + /// + /// Unapproved moderated proto-article; caller must submit for moderation and must + /// not Peek, TryAdmit, or Remember. + /// + SubmitForModeration = 1, + } + + /// Outcome of one streaming POST article receive. + internal enum StreamingPostReadStatus + { + /// Terminator seen; destuffed article is ready for the caller-selected disposition. + Completed = 0, + + /// Peer disconnected or cancelled before the terminating dot line. + Incomplete = 1, + + /// Destuffed article exceeded Nntpd:MaxArticleSize; remaining bytes were drained. + TooLarge = 2, + + /// Header parse or POST validation failed; remaining bytes were drained when needed. + Rejected = 3, + } + + /// Inputs required to stream one POST article into a destuffed article buffer. + internal sealed class StreamingPostReadOptions + { + /// Gets the destuffed POST size limit (Nntpd:MaxArticleSize). + public required int MaxArticleSize { get; init; } + + /// Gets the clock used for the single injection timestamp. + public required TimeProvider Time { get; init; } + + /// Gets the newsgroup posting policy. + public required INewsgroupPostingPolicy NewsgroupPolicy { get; init; } + + /// Gets the server injection identity (nntpd{ServerId:00}.{DnsSuffix}). + public required string InjectionIdentity { get; init; } + + /// Gets the session client identity used only inside protected X-Trace. + public required ConnectionClientIdentity ClientIdentity { get; init; } + + /// Gets the configured mail-complaints-to mailbox. + public required string MailComplaintsTo { get; init; } + + /// Gets the X-Trace protector, or when unset. + public IPostingTraceProtector? TraceProtector { get; init; } + + /// + /// Gets whether this receive may accept a well-formed Control: cancel header. + /// Ordinary posters remain . + /// + public bool ControlCancelPermitted { get; init; } + + /// + /// Gets the AUTHINFO username captured at POST admission, or + /// when the session is unauthenticated. Never taken from article headers. + /// + public string? AuthenticatedUsername { get; init; } + + /// Gets the moderator authorization table used for Approved: decisions. + public IModeratorAuthorization ModeratorAuthorization { get; init; } = EmptyModeratorAuthorization.Instance; + } + + /// Result of . + /// Receive / validation status. + /// Set when is or TooLarge. + /// Destuffed article (terminator omitted) when Completed; otherwise empty. + /// Article Message-ID when known. + /// Validated newsgroup names when known. + /// Logical destuffed client article size (stuffing dots excluded). + /// Single captured UTC timestamp written into server-owned headers. + /// Inject or submit-for-moderation once the terminator is seen. + /// Resolved leftmost moderator mailbox on the moderation path. + /// Leftmost moderated group on the moderation path. + /// Parsed Approved: identities when present. + internal readonly record struct StreamingPostReadResult( + StreamingPostReadStatus Status, + PostingFailure Failure, + ReadOnlyMemory Wire, + string? MessageId, + string[] Newsgroups, + int DestuffedSize, + DateTimeOffset InjectionUtc, + StreamingPostDisposition Disposition = StreamingPostDisposition.Inject, + string? ModeratorAddress = null, + string? TargetModeratedGroup = null, + string[]? ApprovedIdentities = null); +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/Quit.cs b/src/VectorNNTPD.TransitService/Session/Commands/Quit.cs new file mode 100644 index 00000000..c766a9c0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/Quit.cs @@ -0,0 +1,93 @@ +using System.Net.Sockets; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// QUIT command as defined by RFC 3977, Section 5.4. + /// + /// + /// + /// Syntax: QUIT (no arguments). Returns 205 Connection closing then ends the session. + /// Trailing arguments are a syntax error (501) and do not close the connection. + /// + /// + /// Real clients often close the TCP socket immediately after sending QUIT, before observing 205. + /// After a valid QUIT is accepted, peer disappearance during response write/delivery or close is + /// treated as a normal termination race — not an unexpected application failure. Unrelated + /// transport failures are still logged and rethrown. + /// + /// + internal static class Quit + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(Quit)); + + /// Handles QUIT (RFC 3977 §5.4). + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "QUIT", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + // Valid QUIT accepted: after this point the session must become terminal and no further + // NNTP commands/responses should be processed (aside from finishing this 205 write). + var responseWritten = false; + var idleVersionBefore205 = context.Connection.OutboundIdleVersion; + try + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ConnectionClosing, + NntpResponseStatus.ConnectionClosing, + cancellationToken) + .ConfigureAwait(false); + responseWritten = true; + + // Prefer observing send-pump idle so 205 has left the outbound pipe before teardown, + // matching STARTTLS/COMPRESS delivery waits. Peer-gone during this wait is expected. + if (context.Connection is { IsCompleted: false, ConnectionClosed.IsCancellationRequested: false }) + { + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore205, cancellationToken) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (context.Connection.ConnectionClosed.IsCancellationRequested) + { + // Connection became terminal (often peer FIN/RST) while writing or waiting for 205. + context.CompletionDetail = "peer disconnected"; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (NntpPeerDisconnect.IsPeerDisconnect(ex, context.Connection)) + { + context.CompletionDetail = "peer disconnected"; + CommandLogMessages.QuitPeerDisconnected( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + // TEMP DIAG — remove after capture + CommandLogMessages.QuitUncaught( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session), + ex.GetType().FullName, + (ex as SocketException)?.SocketErrorCode + ?? (ex.InnerException as SocketException)?.SocketErrorCode); + throw; + } + + // Always terminate after a valid QUIT — even when the peer vanished mid-205. + context.Session.RequestClose(); + + // Do not claim a clean delivery when we never successfully wrote the status line. + if (!responseWritten && context.CompletionDetail is null) + { + context.CompletionDetail = "peer disconnected"; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/SpeedTest.cs b/src/VectorNNTPD.TransitService/Session/Commands/SpeedTest.cs new file mode 100644 index 00000000..4f202641 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/SpeedTest.cs @@ -0,0 +1,273 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.SpeedTest; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// VectorNNTP private extension: SPEEDTEST identifier. + /// + /// + /// + /// Syntax: SPEEDTEST identifier. identifier is one NNTP token and must be an + /// existing Transit dictionary key (exact ordinal match). It is not PeerName, a + /// hostname, IP, port, URL, or socket endpoint. Multi-token display names are not accepted. + /// + /// + /// Access: transit-authorized sessions only (dispatcher 480/502). When the + /// session was identified as a named Transit peer, the argument must match that name. + /// + /// + /// V1 measures TX on the current NNTP session only (this host writes a synthetic multiline + /// payload to the connected client). Outbound Transit initiation is not implemented; the + /// command never connects to ConnectTo and never reports a measurement that was not + /// performed. RX, RTT, and packet loss are NOT-MEASURED. + /// + /// + /// Responses use private-extension x9x codes (RFC 3977 §3.2): 290 starts the + /// TX payload multiline; 291 is followed by machine-readable result fields. + /// Unknown peers use the project 502 convention. The command is serial (not pipelined) + /// and does not use article ingestion, HistoryDB, or Redis. + /// + /// + internal static class SpeedTest + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(SpeedTest)); + + /// Handles SPEEDTEST. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "SPEEDTEST", ExecuteAsync, cancellationToken); + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var coordinator = context.Session.SpeedTest; + if (coordinator is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestNotSupported, + NntpResponseStatus.SpeedTestNotSupported, + cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = "not supported"; + return; + } + + var peerToken = context.ArgumentMemory; + if (!coordinator.TryResolvePeer(peerToken.Span, out var peer) || peer is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestUnknownPeer, + NntpResponseStatus.SpeedTestUnknownPeer, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + FormatPeerForLog(peerToken.Span), + string.Empty, + "unknown peer"); + context.CompletionDetail = "unknown peer"; + return; + } + + var sessionPeer = context.Session.Authorization.TransitPeerName; + if (sessionPeer is not null + && !SpeedTestCoordinator.NameEquals(sessionPeer, peerToken.Span)) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestPeerMismatch, + NntpResponseStatus.SpeedTestPeerMismatch, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "peer mismatch"); + context.CompletionDetail = "peer mismatch"; + return; + } + + if (!coordinator.TryAcquire(peer.Identifier, out var lease) || lease is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.SpeedTestBusy, + NntpResponseStatus.SpeedTestBusy, + cancellationToken) + .ConfigureAwait(false); + CommandLogMessages.SpeedTestRejected( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "busy"); + context.CompletionDetail = "busy"; + return; + } + + using (lease) + using (var workCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + context.Session.Connection.ConnectionClosed)) + { + CommandLogMessages.SpeedTestStarted( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName); + + var ready = NntpResponseCompose.Concat( + NntpResponses.SpeedTestReadyPrefix.Span, + peerToken.Span, + NntpResponses.SpeedTestReadySuffix.Span); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= + "290 SPEEDTEST " + System.Text.Encoding.ASCII.GetString(peerToken.Span) + " TX"; + } + + await context.Response.WriteLineAsync(ready, workCts.Token).ConfigureAwait(false); + + if (!ReferenceEquals(context.Response.UnderlyingOutput, context.Session.Connection.Output)) + { + throw new InvalidOperationException( + "SPEEDTEST direct path must use the existing session TX PipeWriter."); + } + + long bytes; + TimeSpan elapsed; + long directFlushes; + try + { + (bytes, elapsed, directFlushes) = await WritePayloadAsync( + context.Response, + lease.Limits, + workCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Session.Connection.ConnectionClosed.IsCancellationRequested) + { + CommandLogMessages.SpeedTestCancelled( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + "cancelled"); + context.CompletionDetail = "cancelled"; + return; + } + + var result = FormatResult(peer.Identifier, peer.PeerName, bytes, elapsed, directFlushes); + await context.Response.WriteLineAsync(result, workCts.Token).ConfigureAwait(false); + + CommandLogMessages.SpeedTestCompleted( + Logger, + NntpCommandLogFormat.Client(context.Session), + peer.Identifier, + peer.PeerName, + bytes, + elapsed.TotalMilliseconds); + context.CompletionDetail = "tx complete"; + } + } + + internal static async ValueTask<(long Bytes, TimeSpan Elapsed, long DirectFlushes)> WritePayloadAsync( + NntpResponseWriter response, + Configuration.SpeedTestLimits limits, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + await using var pipeLease = await response.AcquireDirectTxPipeAsync(cancellationToken).ConfigureAwait(false); + var started = Stopwatch.GetTimestamp(); + long bytes = 0; + while (bytes < limits.MaxBytes) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Stopwatch.GetElapsedTime(started) >= limits.MaxDuration) + { + break; + } + + var slice = SpeedTestPayload.Take(limits.MaxBytes - bytes); + if (slice.IsEmpty) + { + break; + } + + await pipeLease.WriteAndFlushAsync(slice, cancellationToken).ConfigureAwait(false); + bytes += slice.Length; + } + + await pipeLease.WriteAndFlushAsync(NntpResponses.MultilineTerminator, cancellationToken).ConfigureAwait(false); + return (bytes, Stopwatch.GetElapsedTime(started), pipeLease.FlushCount); + } + + /// Builds the 291 multiline result (status + fields + terminator). + internal static byte[] FormatResult( + string identifier, + string peerName, + long bytes, + TimeSpan elapsed, + long directPipeFlushes = 0) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + var durationMs = Math.Max(0, elapsed.TotalMilliseconds); + var seconds = elapsed.TotalSeconds; + var measured = bytes > 0 && seconds > 0; + var mbps = measured + ? (bytes * 8d / seconds / 1_000_000d).ToString("F3", CultureInfo.InvariantCulture) + : "NOT-MEASURED"; + var gbit = measured + ? (bytes * 8d / seconds / 1_000_000_000d).ToString("F6", CultureInfo.InvariantCulture) + : "NOT-MEASURED"; + + var text = + "291 SPEEDTEST COMPLETE\r\n" + + "PEER=" + identifier + "\r\n" + + "PEERNAME=" + peerName + "\r\n" + + "IMPLEMENTATION=VectorNNTP\r\n" + + "DIRECTION=TX\r\n" + + "BYTES=" + bytes.ToString(CultureInfo.InvariantCulture) + "\r\n" + + "DURATION_MS=" + durationMs.ToString("F3", CultureInfo.InvariantCulture) + "\r\n" + + "THROUGHPUT_MBPS=" + mbps + "\r\n" + + "THROUGHPUT_GBIT=" + gbit + "\r\n" + + "RTT_US=NOT-MEASURED\r\n" + + "RX=NOT-MEASURED\r\n" + + "OUTBOUND=NOT-AVAILABLE\r\n" + + "PATH=DIRECT-PIPE\r\n" + + "DIRECT_PIPE_FLUSHES=" + directPipeFlushes.ToString(CultureInfo.InvariantCulture) + "\r\n" + + "COMPLETE\r\n" + + ".\r\n"; + return Encoding.UTF8.GetBytes(text); + } + + private static string FormatPeerForLog(ReadOnlySpan peerToken) + { + var unsafeControl = false; + for (var i = 0; i < peerToken.Length; i++) + { + if (peerToken[i] < 0x20 || peerToken[i] > 0x7E) + { + unsafeControl = true; + break; + } + } + + return unsafeControl ? "" : Encoding.ASCII.GetString(peerToken); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/StartTls.cs b/src/VectorNNTPD.TransitService/Session/Commands/StartTls.cs new file mode 100644 index 00000000..46a70518 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/StartTls.cs @@ -0,0 +1,138 @@ +using System.IO.Pipelines; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// STARTTLS command as defined by RFC 4642, Section 2.2 (updated by RFC 8143). + /// + /// + /// Negotiates TLS on an existing plaintext NNTP connection. Transport owns the handshake and + /// quiescence; this module owns command sequencing (discard pipelined input, pause reads before + /// 382, wait for outbound delivery, upgrade) and the TX completion record. + /// + internal static class StartTls + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(StartTls)); + + /// Handles STARTTLS. + public static ValueTask HandleAsync( + NntpCommandContext context, + ITlsCertificateContextProvider? certificateProvider, + CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync( + Logger, + context, + "STARTTLS", + (ctx, ct) => ExecuteAsync(ctx, certificateProvider, ct), + cancellationToken); + + private static async ValueTask ExecuteAsync( + NntpCommandContext context, + ITlsCertificateContextProvider? certificateProvider, + CancellationToken cancellationToken) + { + if (context.Connection.IsTls) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.TlsAlreadyActive, + NntpResponseStatus.TlsAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8054 §2.2.2: MUST reply 502 to STARTTLS while a compression layer is already active. + if (context.Connection.IsCompressed) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.DeflateAlreadyActive, + NntpResponseStatus.DeflateAlreadyActive, + cancellationToken) + .ConfigureAwait(false); + return; + } + + if (certificateProvider is null) + { + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.TlsProviderUnavailable, + NntpResponseStatus.TlsProviderUnavailable, + cancellationToken) + .ConfigureAwait(false); + return; + } + + // RFC 8143 §4: discard any NNTP command pipelined between STARTTLS and TLS negotiation. + DiscardBufferedApplicationInput(context.Connection.Input); + + // Claim read ownership BEFORE 382 can reach the peer. Writes remain allowed while reads + // are paused, so ClientHello completions stash for PrefixedStream instead of Input. + await context.Connection.PauseReadsAsync(cancellationToken).ConfigureAwait(false); + + var idleVersionBefore382 = context.Connection.OutboundIdleVersion; + await NntpCommandReply.WriteAsync( + context, + Logger, + NntpResponses.ContinueWithTls, + NntpResponseStatus.ContinueWithTls, + cancellationToken) + .ConfigureAwait(false); + + await context.Connection + .WaitForOutboundDeliveryAsync(idleVersionBefore382, cancellationToken) + .ConfigureAwait(false); + + try + { + await context.Connection.UpgradeToTlsAsync(certificateProvider, cancellationToken).ConfigureAwait(false); + if (context.Connection.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)) + { + context.CompletionDetail = TlsNegotiationLogging.FormatDetail(tlsVersion, cipher); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + CommandLogMessages.StartTlsHandshakeFailed( + Logger, + ex, + NntpCommandLogFormat.Client(context.Session)); + context.Session.RequestClose(); + // Handshake failures complete the connection; do not rethrow into the dispatcher error path. + // Precondition failures leave the connection usable — rethrow so the dispatcher can respond. + if (context.Connection.IsCompleted) + { + return; + } + + throw; + } + } + + /// + /// Consumes any octets already buffered in application without waiting for more. + /// + private static void DiscardBufferedApplicationInput(PipeReader input) + { + while (input.TryRead(out var result)) + { + var isCompleted = result.IsCompleted; + var isEmpty = result.Buffer.IsEmpty; + input.AdvanceTo(result.Buffer.End); + if (isEmpty || isCompleted) + { + break; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Commands/TakeThis.cs b/src/VectorNNTPD.TransitService/Session/Commands/TakeThis.cs new file mode 100644 index 00000000..adc9a4b6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Commands/TakeThis.cs @@ -0,0 +1,437 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.Commands +{ + /// + /// TAKETHIS command as defined by RFC 4644, Section 2.5. + /// + /// + /// + /// Syntax: TAKETHIS message-id followed immediately by an NNTP multiline article. + /// Responses: 239 message-id (accepted into ingestion) or 439 message-id + /// (permanent reject). Temporary infrastructure failure uses 400 and closes the connection. + /// + /// + /// STREAM path: the session RX task starts HistoryDB peek at the Message-ID, frames + /// the article with (one owned stuffed-wire buffer, + /// terminator omitted, no destuff on RX), attaches that buffer to + /// , and returns. Peek and destuff/ArticleRecord + /// run concurrently per occupied slot. Enqueue / Remember / ordered 239/439 stay + /// on the emit gate so publication remains command-ordered. Depth bounds how + /// many owned articles stay in flight. TAKETHIS responses flush immediately + /// (no coalesce batch). ArticleRecord construction failure is a permanent + /// 439. After CanonicalV1, the connection's Receive article-type mask and + /// newsgroup expression are applied. CHECK does not use them. + /// + /// + /// MODE READER fallback still destuffs via and + /// overlaps HistoryDB peek with that receive. + /// + /// + internal static class TakeThis + { + private static ILogger Logger => NntpCommandLoggers.For(typeof(TakeThis)); + + /// Handles TAKETHIS (RFC 4644 §2.5) on the serial (non-pipeline) path. + public static ValueTask HandleAsync(NntpCommandContext context, CancellationToken cancellationToken) => + NntpCommandExecution.RunAsync(Logger, context, "TAKETHIS", ExecuteAsync, cancellationToken); + + /// HistoryDB peek without miss reservation (same contract as CHECK and IHAVE). + internal static ValueTask PeekAsync( + NntpSession session, + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + if (session.HistoryDb is not { } history) + { + return new ValueTask(HistoryLookupResult.Unseen); + } + + return history.PeekAsync(messageId, cancellationToken); + } + + internal static void WriteCompletion( + ILogger logger, + NntpSession session, + long startedTimestamp, + string? detail = null, + string? statusLine = null) => + NntpCommandExecution.WriteCompletion( + logger, + session, + "TAKETHIS", + System.Diagnostics.Stopwatch.GetElapsedTime(startedTimestamp), + detail, + statusLine); + + internal static async ValueTask FailTemporaryAsync( + NntpSession session, + NntpResponseWriter response, + CancellationToken cancellationToken) + { + try + { + await response + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when ( + cancellationToken.IsCancellationRequested + || session.Connection.ConnectionClosed.IsCancellationRequested + || ex is ObjectDisposedException or InvalidOperationException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + session.RequestClose(); + CommandLogMessages.TakeThisTemporaryFailure( + Logger, + NntpCommandLogFormat.Client(session)); + } + + private static async ValueTask ExecuteAsync(NntpCommandContext context, CancellationToken cancellationToken) + { + var probe = context.Session.FeedProbe; + probe?.RecordCommand(); + context.Session.SetActivityState(FeedSessionState.Receiving); + var messageIdBytes = context.ArgumentSpan.ToArray(); + var lookup = PeekAsync(context.Session, messageIdBytes, cancellationToken); + var queue = context.Session.ArticleIngestion; + + NntpArticlePolicyCapture.Capture( + context.Session, + queue.MaxArticleBytes, + out var maxArticleBytes, + out var siteNameUtf8); + NntpMultilineReadStatus status; + ReadOnlyMemory payload; + var receiveStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + if (context.PreReadArticle is { } preRead) + { + status = preRead.Status; + payload = preRead.Payload; + } + else if (context.Session.ReceiveStrategy == NntpReceiveStrategy.StreamDataPlane) + { + var read = await IHaveArticleReader + .ReadAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + status = read.Status; + payload = read.Payload; + } + else + { + var article = await NntpMultilineDataReader + .ReadArticleAsync(context.Connection.Input, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + status = article.Status; + payload = article.Payload; + } + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + probe?.RecordArticleReceived( + payload.Length, + System.Diagnostics.Stopwatch.GetTimestamp() - receiveStart); + context.Session.RecordPeerArticleReceived(payload.Length); + context.Session.SetActivityState(FeedSessionState.WaitingHistory); + + HistoryLookupResult peek; + var peekStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + peek = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + peek = HistoryLookupResult.Unavailable; + } + + probe?.RecordHistory(peek, System.Diagnostics.Stopwatch.GetTimestamp() - peekStart); + + if (status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + context.Session, + System.Text.Encoding.ASCII.GetString(messageIdBytes), + 439, + IngressNewsReasons.ArticleTooLarge, + payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = tooLarge; + return; + } + + if (peek == HistoryLookupResult.Unavailable || !queue.IsAccepting) + { + await FailTemporaryAsync(context.Session, context.Response, cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ServiceTemporarilyUnavailable); + context.CompletionDetail = "temporary failure"; + return; + } + + if (peek == HistoryLookupResult.Seen) + { + context.Session.SetActivityState(FeedSessionState.Completing); + var seenStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueTransferReplyAsync(context, rejected: false, cancellationToken) + .ConfigureAwait(false); + probe?.RecordArticleCompleted( + duplicate: true, + System.Diagnostics.Stopwatch.GetTimestamp() - seenStart); + context.Session.SetActivityState(FeedSessionState.Idle); + context.CompletionDetail = "accepted duplicate"; + return; + } + + var messageId = System.Text.Encoding.ASCII.GetString(messageIdBytes); + var stuffed = context.Session.ReceiveStrategy == NntpReceiveStrategy.StreamDataPlane + && context.PreReadArticle is null; + if (!TryCreateQueuedRecord( + context.Session, + payload, + stuffed, + messageId, + maxArticleBytes, + siteNameUtf8, + out var inbound, + out var recordReject, + out var canonicalSize)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + IngressNewsReasons.ForExistingRejectDetail(recordReject), + canonicalSize != 0 ? canonicalSize : payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = recordReject; + return; + } + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + context.Session.Transit, + context.Session.NewsgroupCatalogue, + out var uncarriedReason)) + { + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + uncarriedReason, + inbound.Payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = IngressNewsEvents.NewsgroupNotCarried; + return; + } + + ArticleEnqueueResult enqueue; + try + { + context.Session.SetActivityState(FeedSessionState.WaitingQueue); + var queueStart = System.Diagnostics.Stopwatch.GetTimestamp(); + enqueue = await queue.EnqueueAsync(inbound, cancellationToken).ConfigureAwait(false); + probe?.RecordQueue(enqueue, System.Diagnostics.Stopwatch.GetTimestamp() - queueStart, payload.Length); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || context.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (enqueue == ArticleEnqueueResult.Unavailable) + { + await FailTemporaryAsync(context.Session, context.Response, cancellationToken).ConfigureAwait(false); + NntpCommandReply.TryNote(context, Logger, NntpResponseStatus.ServiceTemporarilyUnavailable); + context.CompletionDetail = "temporary failure"; + return; + } + + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + context.Session, + messageId, + 439, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + await EnqueueTransferReplyAsync(context, rejected: true, cancellationToken) + .ConfigureAwait(false); + context.CompletionDetail = budget; + return; + } + + context.Session.HistoryDb?.Remember(messageIdBytes); + context.Session.SetActivityState(FeedSessionState.Completing); + var doneStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueTransferReplyAsync(context, rejected: false, cancellationToken) + .ConfigureAwait(false); + probe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - doneStart); + context.Session.SetActivityState(FeedSessionState.Idle); + context.CompletionDetail = "accepted"; + } + + private static ValueTask EnqueueTransferReplyAsync( + NntpCommandContext context, + bool rejected, + CancellationToken cancellationToken) + { + var prefix = rejected + ? NntpResponses.TransferRejectedPrefix + : NntpResponses.ArticleTransferredOkPrefix; + var owned = NntpResponseCompose.Concat( + prefix.Span, + context.ArgumentSpan, + NntpResponses.Crlf.Span); + if (Logger.IsEnabled(LogLevel.Debug)) + { + context.StatusLine ??= rejected + ? NntpCommandStatusText.FormatTakeThisRejected(context.ArgumentSpan) + : NntpCommandStatusText.FormatTakeThisAccepted(context.ArgumentSpan); + } + + return context.Response.EnqueueLineImmediateAsync(owned, cancellationToken); + } + + /// + /// Destuffs when required, builds , and returns a queue item + /// that references ArtData. Does not enqueue. + /// + internal static bool TryCreateQueuedRecord( + NntpSession session, + ReadOnlyMemory payload, + bool stuffed, + string messageId, + out InboundArticle inbound, + out string rejectDetail, + out int canonicalSize) + => TryCreateQueuedRecord( + session, + payload, + stuffed, + messageId, + session.ArticleIngestion.MaxArticleBytes, + siteNameUtf8: null, + out inbound, + out rejectDetail, + out canonicalSize); + + internal static bool TryCreateQueuedRecord( + NntpSession session, + ReadOnlyMemory payload, + bool stuffed, + string messageId, + int maxArticleBytes, + byte[]? siteNameUtf8, + out InboundArticle inbound, + out string rejectDetail, + out int canonicalSize) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + canonicalSize = 0; + + ArticleRecordCreateResult created; + if (stuffed) + { + created = siteNameUtf8 is null + ? ArticleRecordIngress.TryCreateFromStuffedWire( + session.ArticleParser, + payload, + maxArticleBytes) + : ArticleRecordIngress.TryCreateFromStuffedWire( + session.ArticleParser, + payload, + maxArticleBytes, + siteNameUtf8); + } + else if (siteNameUtf8 is null) + { + created = ArticleRecordIngress.TryCreateFromDestuffed( + session.ArticleParser, + payload, + maxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + } + else + { + created = ArticleRecordIngress.TryCreateFromDestuffed( + session.ArticleParser, + payload, + maxArticleBytes, + siteNameUtf8); + } + + if (!created.IsAccepted) + { + inbound = null!; + rejectDetail = created.ParseFailure != NntpArticleParseFailureCode.None + ? "rejected article record " + created.ParseFailure + : "rejected article record " + created.MaterializeFailure; + return false; + } + + canonicalSize = created.Record.ArtSize; + if (!ArticleTypeAccessPolicy.CanPostArticleType(session, created.Record.ArtType)) + { + inbound = null!; + rejectDetail = TransitReceivePolicy.ArticleTypeRejectDetail; + return false; + } + + if (TransitReceivePolicy.TryReject(session.Authorization.TransitPeerPolicy, created.Record, out rejectDetail)) + { + inbound = null!; + return false; + } + + inbound = ArticleRecordIngress.CreateQueued( + messageId, + created.Record, + session.ClientIdentity, + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis, + IngressNewsEvents.SnapshotInboundFeed(session)); + rejectDetail = string.Empty; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/ArticleWireReconstructor.cs b/src/VectorNNTPD.TransitService/Session/Framing/ArticleWireReconstructor.cs new file mode 100644 index 00000000..ef3f099c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/ArticleWireReconstructor.cs @@ -0,0 +1,155 @@ +using System.Buffers; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// + /// Reconstructs NNTP multiline wire bytes from a destuffed article payload (wire-framing utility). + /// + /// + /// + /// Destuffed payload convention (aligned with ingest output). + /// This type does not own article storage or catalogues; callers supply bytes (bench corpus, spool + /// snapshot, peer feed, etc.). + /// + /// + /// Terminator .\r\n is not part of the destuffed payload. + /// Dot-stuffing is removed on ingest: a wire line starting with .. is kept with one leading . removed. + /// Payload lines use \r\n (CRLF). A final line without CRLF is still emitted with CRLF on the wire. + /// + /// + /// Re-stuffing (RFC 3977 §3.1.1): any destuffed line whose content begins with . gets an extra + /// . prepended on the wire, then \r\n, then the article is terminated with .\r\n. + /// + /// + public static class ArticleWireReconstructor + { + private static readonly byte[] Crlf = "\r\n"u8.ToArray(); + private static readonly byte[] Terminator = ".\r\n"u8.ToArray(); + + /// + /// Re-stuffs a de-stuffed stored article into multiline wire form (no status line). + /// + /// Destuffed article bytes (terminator not present). + /// + /// When , appends the NNTP multiline terminator .CRLF + /// (TX / client transfer). When , omits it so the result matches + /// the IHAVE/TAKETHIS ingestion-queue contract. + /// + public static byte[] RestuffArticle(ReadOnlySpan storedDestuffed, bool includeTerminator = true) + { + if (storedDestuffed.IsEmpty) + { + return includeTerminator ? Terminator.ToArray() : []; + } + + var estimated = EstimateRestuffedWireBytes(storedDestuffed, includeTerminator); + var writer = new ArrayBufferWriter((int)Math.Min(estimated, int.MaxValue)); + var offset = 0; + while (offset < storedDestuffed.Length) + { + var remaining = storedDestuffed[offset..]; + var crlfAt = remaining.IndexOf(Crlf); + ReadOnlySpan line; + if (crlfAt < 0) + { + line = remaining; + offset = storedDestuffed.Length; + } + else + { + line = remaining[..crlfAt]; + offset += crlfAt + 2; + } + + if (line.Length > 0 && line[0] == (byte)'.') + { + writer.Write([(byte)'.']); + } + + writer.Write(line); + writer.Write(Crlf); + } + + if (includeTerminator) + { + writer.Write(Terminator); + } + + return writer.WrittenSpan.ToArray(); + } + + /// + /// Estimates wire article bytes (restuffed body + terminator) without allocating the full buffer. + /// + public static long EstimateRestuffedWireBytes( + ReadOnlySpan storedDestuffed, + bool includeTerminator = true) + { + if (storedDestuffed.IsEmpty) + { + return includeTerminator ? Terminator.Length : 0; + } + + long extraDots = 0; + var offset = 0; + while (offset < storedDestuffed.Length) + { + var remaining = storedDestuffed[offset..]; + var crlfAt = remaining.IndexOf(Crlf); + ReadOnlySpan line; + if (crlfAt < 0) + { + line = remaining; + // Missing trailing CRLF will be added on the wire. + extraDots += line.Length > 0 && line[0] == (byte)'.' ? 1 : 0; + var withoutTerminator = storedDestuffed.Length + 2 + extraDots; + return includeTerminator ? withoutTerminator + Terminator.Length : withoutTerminator; + } + + line = remaining[..crlfAt]; + if (line.Length > 0 && line[0] == (byte)'.') + { + extraDots++; + } + + offset += crlfAt + 2; + } + + var complete = storedDestuffed.Length + extraDots; + return includeTerminator ? complete + Terminator.Length : complete; + } + + /// + /// Splits a destuffed article into header block and body at the first blank line (\r\n\r\n). + /// + /// Full destuffed article (headers + body; no NNTP terminator). + /// Header bytes including the blank-line separator when present. + /// Body bytes after the blank line (maybe empty). + /// + /// when a blank line was found; otherwise and + /// is empty (entire payload treated as headers-only / no body). + /// + /// + /// Used by BODY TX to transmit only the body portion. Does not invent alternate separators; + /// matches CRLF blank-line semantics of the destuffed payload convention. + /// + public static bool TrySplitHeadersAndBody( + ReadOnlySpan storedDestuffedArticle, + out ReadOnlySpan headers, + out ReadOnlySpan body) + { + var separator = "\r\n\r\n"u8; + var at = storedDestuffedArticle.IndexOf(separator); + if (at < 0) + { + headers = storedDestuffedArticle; + body = ReadOnlySpan.Empty; + return false; + } + + headers = storedDestuffedArticle[..(at + separator.Length)]; + body = storedDestuffedArticle[(at + separator.Length)..]; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReadResult.cs b/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReadResult.cs new file mode 100644 index 00000000..a2172dd4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReadResult.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// Outcome of . + /// + /// is owned NNTP wire bytes (dot-stuffing preserved, terminator omitted) + /// or empty when is not . + /// + public readonly record struct IHaveArticleReadResult( + NntpMultilineReadStatus Status, + ReadOnlyMemory Payload, + IHaveReceiveMetrics Metrics); +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReader.cs b/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReader.cs new file mode 100644 index 00000000..c67d11ef --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/IHaveArticleReader.cs @@ -0,0 +1,285 @@ +using System.Buffers; +using System.Diagnostics; +using System.IO.Pipelines; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// + /// IHAVE article receive: locate CRLF . CRLF and copy stuffed wire octets into one owned buffer. + /// + /// + /// IHAVE is not pipelined (RFC 3977 §6.3.2). TAKETHIS STREAM receive reuses this reader + /// on the session RX task so the queued payload is one owned stuffed-wire copy + /// (OwnedWireBuffer.Take()). After that take, Pipe sequences are not retained + /// and the buffer is handed to . + /// This reader does not build an article record. The retained buffer is stuffed + /// wire with the terminator omitted. The size ceiling is the destuffed article: + /// one leading stuffing dot removed per line, content CRLF included, terminator + /// excluded, using . + /// A line that would exceed that ceiling stops further retention; the reader still + /// consumes through the terminator and returns + /// with an empty payload. IHAVE destuff and ArticleRecordFactory run after + /// this read, before queue admission. Pipeline workers never call this reader. + /// + public static class IHaveArticleReader + { + /// Initial owned-buffer capacity when the incoming article size is unknown. + public const int InitialCapacity = 64 * 1024; + + /// + /// Reads one IHAVE article from as owned stuffed wire. + /// + /// Connection input. Advanced through the article terminator. + /// + /// Destuffed article ceiling. Stuffing dots and the terminator are not counted. + /// + /// Cancels the pipe read. + /// + /// Stuffed wire without the terminator when the destuffed size fits; otherwise + /// or + /// with an empty payload. + /// + public static async ValueTask ReadAsync( + PipeReader reader, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var session = new ReadSession(maxArticleBytes); + return await session.RunAsync(reader, cancellationToken).ConfigureAwait(false); + } + + private sealed class ReadSession(int maxArticleBytes) + { + private readonly OwnedWireBuffer _article = new(InitialCapacity); + private int _pipeReads; + private bool _exceeded; + private int _destuffed; + private int _lineStart; + + public async ValueTask RunAsync( + PipeReader reader, + CancellationToken cancellationToken) + { + var started = Stopwatch.GetTimestamp(); + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + _pipeReads++; + var unread = result.Buffer; + var examined = unread.End; + var atStart = _article.Written == 0; + + if (NntpDelimiterSearch.TryFindArticleTerminator( + unread, + atStart, + out var payloadBytes, + out var consumedBytes)) + { + if (payloadBytes > 0) + { + Append(unread.Slice(0, payloadBytes)); + } + + unread = unread.Slice(consumedBytes); + reader.AdvanceTo(unread.Start, unread.Start); + return Finish( + _exceeded ? NntpMultilineReadStatus.TooLarge : NntpMultilineReadStatus.Completed, + started); + } + + var hold = (int)Math.Min(NntpDelimiterSearch.ArticleTerminatorLookbehind, unread.Length); + var copyBytes = (int)unread.Length - hold; + if (copyBytes > 0) + { + Append(unread.Slice(0, copyBytes)); + unread = unread.Slice(copyBytes); + } + + reader.AdvanceTo(unread.Start, examined); + if (result.IsCompleted) + { + return Finish(NntpMultilineReadStatus.Incomplete, started); + } + } + } + + private void Append(ReadOnlySequence bytes) + { + if (_exceeded || bytes.IsEmpty) + { + return; + } + + foreach (var segment in bytes) + { + AppendSpan(segment.Span); + if (_exceeded) + { + return; + } + } + } + + private void AppendSpan(ReadOnlySpan span) + { + var offset = 0; + while (offset < span.Length && !_exceeded) + { + if (_article.Written > _lineStart + && _article.EndsWithCr + && span[offset] == (byte)'\n') + { + _article.AppendRaw(span.Slice(offset, 1)); + offset++; + TryCommitLine(); + continue; + } + + var slice = span[offset..]; + var crlf = slice.IndexOf("\r\n"u8); + if (crlf < 0) + { + TryAppendOpenLine(slice); + return; + } + + _article.AppendRaw(slice[..(crlf + 2)]); + offset += crlf + 2; + TryCommitLine(); + } + } + + private void TryAppendOpenLine(ReadOnlySpan slice) + { + if (slice.IsEmpty) + { + return; + } + + var lineLength = (_article.Written - _lineStart) + slice.Length; + var prospective = NntpArticleDestuffer.DestuffedLineBytes(lineLength, LineStartsWithDot(slice)); + if (_destuffed + prospective > maxArticleBytes) + { + RejectOpenLine(); + return; + } + + _article.AppendRaw(slice); + } + + private void TryCommitLine() + { + var contentLength = _article.Written - _lineStart - 2; + var add = NntpArticleDestuffer.DestuffedLineBytes(_article.LineAt(_lineStart, contentLength)); + if (_destuffed + add > maxArticleBytes) + { + RejectOpenLine(); + return; + } + + _destuffed += add; + _lineStart = _article.Written; + } + + private void RejectOpenLine() + { + _article.Truncate(_lineStart); + _exceeded = true; + } + + private bool LineStartsWithDot(ReadOnlySpan upcoming) + { + if (_article.Written > _lineStart) + { + return _article.FirstAt(_lineStart) == (byte)'.'; + } + + return upcoming.Length > 0 && upcoming[0] == (byte)'.'; + } + + private IHaveArticleReadResult Finish(NntpMultilineReadStatus status, long started) + { + var elapsed = Stopwatch.GetElapsedTime(started); + if (status != NntpMultilineReadStatus.Completed) + { + return new IHaveArticleReadResult( + status, + ReadOnlyMemory.Empty, + new IHaveReceiveMetrics(_pipeReads, 0, elapsed)); + } + + var payload = _article.Take(); + return new IHaveArticleReadResult( + status, + payload, + new IHaveReceiveMetrics(_pipeReads, payload.Length, elapsed)); + } + } + + private sealed class OwnedWireBuffer + { + private byte[] _buffer; + private int _written; + + public OwnedWireBuffer(int initialCapacity) + { + _buffer = new byte[initialCapacity]; + } + + public int Written => _written; + + public bool EndsWithCr => _written > 0 && _buffer[_written - 1] == (byte)'\r'; + + public byte FirstAt(int index) => _buffer[index]; + + public ReadOnlySpan LineAt(int start, int length) => _buffer.AsSpan(start, length); + + public void Truncate(int written) => _written = written; + + public void AppendRaw(ReadOnlySpan bytes) + { + if (bytes.IsEmpty) + { + return; + } + + Ensure(_written + bytes.Length); + bytes.CopyTo(_buffer.AsSpan(_written, bytes.Length)); + _written += bytes.Length; + } + + public ReadOnlyMemory Take() + { + var buffer = _buffer; + var length = _written; + _buffer = []; + _written = 0; + return buffer.AsMemory(0, length); + } + + private void Ensure(int needed) + { + if (_buffer.Length >= needed) + { + return; + } + + var next = _buffer.Length < 256 ? 256 : _buffer.Length * 2; + if (next < needed) + { + next = needed; + } + + var grown = new byte[next]; + if (_written > 0) + { + _buffer.AsSpan(0, _written).CopyTo(grown); + } + + _buffer = grown; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/IHaveReceiveMetrics.cs b/src/VectorNNTPD.TransitService/Session/Framing/IHaveReceiveMetrics.cs new file mode 100644 index 00000000..0c34267f --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/IHaveReceiveMetrics.cs @@ -0,0 +1,8 @@ +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// IHAVE receive instrumentation (frame + own; no destuff). + public readonly record struct IHaveReceiveMetrics( + int PipeReads, + int ArticleSize, + TimeSpan ReceiveElapsed); +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/NntpArticleDestuffer.cs b/src/VectorNNTPD.TransitService/Session/Framing/NntpArticleDestuffer.cs new file mode 100644 index 00000000..d64eb149 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/NntpArticleDestuffer.cs @@ -0,0 +1,191 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// + /// Dot-unstuffs NNTP multiline article lines into an owned buffer (RFC 3977 §3.1.1). + /// + /// + /// Shared by and the continuous TAKETHIS scanner so destuff + /// semantics stay identical. Pipe sequences must not be retained after AdvanceTo; callers + /// copy into before releasing Pipe memory. + /// + public static class NntpArticleDestuffer + { + /// + /// Appends one destuffed content line plus CRLF, or marks when + /// the destuffed article would exceed . + /// + public static void AppendUnstuffedLine( + ArrayBufferWriter output, + ReadOnlySequence lineBytes, + int maxArticleBytes, + ref bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return; + } + + var length = (int)lineBytes.Length; + var leadingDot = length > 0 && NntpDelimiterSearch.FirstByte(lineBytes) == (byte)'.'; + var needed = DestuffedLineBytes(length, leadingDot); + var contentLength = needed - 2; + if (output.WrittenCount + needed > maxArticleBytes) + { + exceeded = true; + return; + } + + var span = output.GetSpan(needed); + if (lineBytes.IsSingleSegment) + { + var src = lineBytes.FirstSpan; + if (leadingDot) + { + src = src[1..]; + } + + src.CopyTo(span); + } + else + { + var rented = ArrayPool.Shared.Rent(length); + try + { + lineBytes.CopyTo(rented); + var src = leadingDot ? rented.AsSpan(1, contentLength) : rented.AsSpan(0, contentLength); + src.CopyTo(span); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + span[contentLength] = (byte)'\r'; + span[contentLength + 1] = (byte)'\n'; + output.Advance(needed); + } + + /// + /// Appends one destuffed content line plus CRLF from a contiguous span. + /// + public static void AppendUnstuffedLine( + ArrayBufferWriter output, + ReadOnlySpan lineBytes, + int maxArticleBytes, + ref bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return; + } + + var needed = DestuffedLineBytes(lineBytes); + var content = lineBytes.Length > 0 && lineBytes[0] == (byte)'.' ? lineBytes[1..] : lineBytes; + if (output.WrittenCount + needed > maxArticleBytes) + { + exceeded = true; + return; + } + + var span = output.GetSpan(needed); + content.CopyTo(span); + span[content.Length] = (byte)'\r'; + span[content.Length + 1] = (byte)'\n'; + output.Advance(needed); + } + + /// + /// Returns the destuffed size of one content line plus CRLF. + /// + /// One stuffed line with the CRLF already removed. + /// + /// Content bytes after removing a single leading stuffing dot, plus the two CRLF octets. + /// A line that does not begin with . contributes its full length plus CRLF. + /// + public static int DestuffedLineBytes(ReadOnlySpan lineWithoutCrlf) => + DestuffedLineBytes( + lineWithoutCrlf.Length, + lineWithoutCrlf.Length > 0 && lineWithoutCrlf[0] == (byte)'.'); + + /// + /// Returns the destuffed size of one content line plus CRLF. + /// + /// Stuffed line length with the CRLF already removed. + /// when the line begins with a stuffing dot. + /// Destuffed content length plus two CRLF octets. + public static int DestuffedLineBytes(int lineLength, bool leadingDot) + { + var contentLength = leadingDot ? lineLength - 1 : lineLength; + return contentLength + 2; + } + + /// + /// Completes a destuffed article: with an empty + /// payload, or with an owned copy. + /// + public static NntpMultilineReadResult Complete(ArrayBufferWriter output, bool exceeded) + { + ArgumentNullException.ThrowIfNull(output); + if (exceeded) + { + return new NntpMultilineReadResult(NntpMultilineReadStatus.TooLarge, ReadOnlyMemory.Empty); + } + + return new NntpMultilineReadResult( + NntpMultilineReadStatus.Completed, + output.WrittenMemory.ToArray()); + } + + /// + /// Destuffs terminator-omitted stuffed wire into one owned destuffed article buffer. + /// + /// + /// Uses the same line destuff as . + /// Does not classify, parse, or rewrite Date/Path. A line without a CRLF terminator is + /// ignored, matching . + /// + /// Stuffed article bytes without the NNTP terminator. + /// Destuffed size ceiling. + /// Owned destuffed bytes when the method returns . + /// when destuff completed within the limit. + public static bool TryDestuffStuffedWire( + ReadOnlySpan stuffedWire, + int maxArticleBytes, + [NotNullWhen(true)] out byte[]? destuffed) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + var output = new ArrayBufferWriter(Math.Min(Math.Max(stuffedWire.Length, 1), maxArticleBytes)); + var exceeded = false; + var offset = 0; + while (offset < stuffedWire.Length && !exceeded) + { + var remaining = stuffedWire[offset..]; + var crlf = remaining.IndexOf(NntpDelimiterSearch.Crlf); + if (crlf < 0) + { + break; + } + + AppendUnstuffedLine(output, remaining[..crlf], maxArticleBytes, ref exceeded); + offset += crlf + 2; + } + + if (exceeded) + { + destuffed = null; + return false; + } + + destuffed = output.WrittenCount == 0 + ? [] + : output.WrittenMemory.ToArray(); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxParser.cs b/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxParser.cs new file mode 100644 index 00000000..b355bd30 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxParser.cs @@ -0,0 +1,256 @@ +using System.Buffers; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// Scan mode for . + public enum NntpContinuousRxMode + { + /// Waiting for a command line terminated by CRLF. + Command = 0, + + /// Inside a TAKETHIS multiline article; only the terminator ends the unit. + Article = 1, + } + + /// Kind of protocol unit produced by the continuous RX parser. + public enum NntpContinuousRxKind + { + /// Need more Pipe bytes (incomplete command or article). + NeedMore = 0, + + /// A complete non-article command line (without CRLF). + Command = 1, + + /// A complete TAKETHIS command plus framed article bytes (no destuff). + TakeThis = 2, + } + + /// One consumed protocol unit. Pipe memory is not retained. + public readonly struct NntpContinuousRxUnit + { + /// Initializes a new instance of the struct. + public NntpContinuousRxUnit( + NntpContinuousRxKind kind, + NntpCommand command = default, + NntpMultilineReadResult article = default) + { + Kind = kind; + Command = command; + Article = article; + } + + /// Gets the unit kind. + public NntpContinuousRxKind Kind { get; } + + /// Gets the parsed command. Indexes refer to . + public NntpCommand Command { get; } + + /// + /// Gets the article result for . + /// STREAM framing copies wire bytes without destuffing; the terminator is omitted. + /// + public NntpMultilineReadResult Article { get; } + + /// Need-more sentinel. + public static NntpContinuousRxUnit NeedMore { get; } = new(NntpContinuousRxKind.NeedMore); + } + + /// + /// Stateful command/article scanner over . + /// + /// + /// Command CRLF ends a command. Inside an article, the terminator is the five-octet + /// \r\n.\r\n (or a leading .\r\n for an empty article). CHECK/QUIT/TAKETHIS + /// text in the body is payload. Incomplete tails of at most four octets stay in the Pipe. + /// STREAM article bytes are copied as received (no destuff) into an owned buffer before the + /// caller advances the Pipe. The terminator is framing, not payload. MODE READER destuff + /// remains line-oriented in . + /// + public sealed class NntpContinuousRxParser + { + private readonly ArrayBufferWriter _article = new(64 * 1024); + private readonly byte[] _commandScratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + private int _commandLength; + private bool _articleExceeded; + private int _maxArticleBytes = int.MaxValue; + private NntpCommand _pendingCommand; + + /// Gets the current scan mode. + public NntpContinuousRxMode Mode { get; private set; } + + /// + /// Gets the current command-line bytes. Valid until the next command line is consumed. + /// + public ReadOnlyMemory CurrentCommandLine => _commandScratch.AsMemory(0, _commandLength); + + /// Resets command/article state (does not release writer capacity). + public void Reset() + { + Mode = NntpContinuousRxMode.Command; + _pendingCommand = default; + // Keep _commandLength/_commandScratch until the next command is copied so + // CurrentCommandLine remains valid for dispatch after a TAKETHIS article. + _articleExceeded = false; + _maxArticleBytes = int.MaxValue; + _article.ResetWrittenCount(); + } + + /// + /// Attempts to consume one complete protocol unit from . + /// Slices to the unconsumed remainder. + /// + /// Unread Pipe bytes. Sliced to the unconsumed tail. + /// + /// When , a TAKETHIS verb consumes the following article in this + /// parser. When , TAKETHIS is returned as a command line so session + /// gates can reject it without consuming a body (existing 480/502 behaviour). + /// + /// Framed (non-destuffed) payload cap for TAKETHIS articles. + public NntpContinuousRxUnit TryConsume( + ref ReadOnlySequence buffer, + bool consumeTakeThisArticle, + int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (Mode == NntpContinuousRxMode.Article) + { + return TryConsumeArticle(ref buffer); + } + + if (!NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + return NntpContinuousRxUnit.NeedMore; + } + + if (!TryCopyCommandLine(lineBytes)) + { + return new NntpContinuousRxUnit( + NntpContinuousRxKind.Command, + NntpCommand.OverlongLine); + } + + var command = NntpCommandParser.Parse(CurrentCommandLine.Span); + + if (consumeTakeThisArticle && command.Verb == NntpVerb.TakeThis) + { + _pendingCommand = command; + Mode = NntpContinuousRxMode.Article; + _maxArticleBytes = maxArticleBytes; + _articleExceeded = false; + _article.ResetWrittenCount(); + return TryConsumeArticle(ref buffer); + } + + return new NntpContinuousRxUnit(NntpContinuousRxKind.Command, command); + } + + private NntpContinuousRxUnit TryConsumeArticle(ref ReadOnlySequence buffer) + { + var atArticleStart = _article.WrittenCount == 0 && !_articleExceeded; + if (NntpDelimiterSearch.TryFindArticleTerminator( + buffer, + atArticleStart, + out var payloadBytes, + out var consumedBytes)) + { + if (payloadBytes > 0) + { + AppendFramedWire(buffer.Slice(0, payloadBytes)); + } + + buffer = buffer.Slice(consumedBytes); + var article = NntpArticleDestuffer.Complete(_article, _articleExceeded); + var unit = new NntpContinuousRxUnit( + NntpContinuousRxKind.TakeThis, + _pendingCommand, + article); + Reset(); + return unit; + } + + // Incomplete terminator: copy the prefix that cannot be a delimiter lookbehind + // so Pipe memory can be released. The last 0–4 octets stay unconsumed. + var hold = (int)Math.Min(NntpDelimiterSearch.ArticleTerminatorLookbehind, buffer.Length); + var copyBytes = (int)buffer.Length - hold; + if (copyBytes > 0) + { + AppendFramedWire(buffer.Slice(0, copyBytes)); + buffer = buffer.Slice(copyBytes); + } + + return NntpContinuousRxUnit.NeedMore; + } + + /// + /// Destuffs article lines from up to . + /// Used by (MODE READER multiline semantics). + /// + public NntpContinuousRxUnit TryConsumeArticleOnly( + ref ReadOnlySequence buffer, + int maxArticleBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + if (Mode != NntpContinuousRxMode.Article) + { + Mode = NntpContinuousRxMode.Article; + _maxArticleBytes = maxArticleBytes; + _articleExceeded = false; + _article.ResetWrittenCount(); + } + + while (NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + if (NntpDelimiterSearch.IsTerminatorLine(lineBytes)) + { + var article = NntpArticleDestuffer.Complete(_article, _articleExceeded); + Reset(); + return new NntpContinuousRxUnit(NntpContinuousRxKind.TakeThis, article: article); + } + + NntpArticleDestuffer.AppendUnstuffedLine(_article, lineBytes, maxArticleBytes, ref _articleExceeded); + } + + return NntpContinuousRxUnit.NeedMore; + } + + /// + /// Copies framed STREAM wire octets (already including content CRLFs) without destuffing. + /// + private void AppendFramedWire(ReadOnlySequence wireBytes) + { + if (_articleExceeded || wireBytes.IsEmpty) + { + return; + } + + var needed = checked((int)wireBytes.Length); + if (_article.WrittenCount + needed > _maxArticleBytes) + { + _articleExceeded = true; + return; + } + + var span = _article.GetSpan(needed); + wireBytes.CopyTo(span); + _article.Advance(needed); + } + + /// + /// Copies a command line that fits. An overlong line is consumed by the caller and is not copied. + /// + private bool TryCopyCommandLine(ReadOnlySequence lineBytes) + { + if (lineBytes.Length > _commandScratch.Length) + { + _commandLength = 0; + return false; + } + + var length = checked((int)lineBytes.Length); + lineBytes.CopyTo(_commandScratch); + _commandLength = length; + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxReader.cs b/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxReader.cs new file mode 100644 index 00000000..8beee637 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/NntpContinuousRxReader.cs @@ -0,0 +1,106 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// + /// Long-lived loop over . + /// + /// + /// + /// Consumed/examined positions follow parser progress: complete units are consumed; incomplete + /// tails are examined but not consumed. AdvanceTo(buffer.End) is never used as a generic + /// progress mechanism. + /// + /// + /// Article payloads are owned byte[] copies taken before AdvanceTo. + /// STREAM units copy framed wire bytes (no destuff). destuffs + /// for . Pipe sequences are not enqueued. + /// + /// + public static class NntpContinuousRxReader + { + /// + /// Reads one protocol unit. Returns only when + /// the pipe completes mid-command (treated as end of stream by the caller) or mid-article. + /// + public static async ValueTask ReadUnitAsync( + PipeReader reader, + NntpContinuousRxParser parser, + bool consumeTakeThisArticle, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examinedOrigin = buffer.End; + + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle, maxArticleBytes); + if (unit.Kind != NntpContinuousRxKind.NeedMore) + { + // Consume the unit; remaining bytes stay in the Pipe for the next call. + reader.AdvanceTo(buffer.Start, buffer.Start); + return unit; + } + + // Incomplete command or article: consume copied/complete prefix (buffer.Start), + // examine everything so the next read waits for new bytes. + reader.AdvanceTo(buffer.Start, examinedOrigin); + if (result.IsCompleted) + { + if (parser.Mode == NntpContinuousRxMode.Article) + { + parser.Reset(); + return new NntpContinuousRxUnit( + NntpContinuousRxKind.TakeThis, + article: new NntpMultilineReadResult( + NntpMultilineReadStatus.Incomplete, + ReadOnlyMemory.Empty)); + } + + parser.Reset(); + return new NntpContinuousRxUnit(NntpContinuousRxKind.NeedMore); + } + } + } + + /// + /// Reads one destuffed article from the current Pipe position (MODE READER / multiline body). + /// + public static async ValueTask ReadArticleAsync( + PipeReader reader, + NntpContinuousRxParser parser, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(parser); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + while (true) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + var examinedOrigin = buffer.End; + var unit = parser.TryConsumeArticleOnly(ref buffer, maxArticleBytes); + if (unit.Kind == NntpContinuousRxKind.TakeThis) + { + reader.AdvanceTo(buffer.Start, buffer.Start); + return unit.Article; + } + + reader.AdvanceTo(buffer.Start, examinedOrigin); + if (result.IsCompleted) + { + parser.Reset(); + return new NntpMultilineReadResult(NntpMultilineReadStatus.Incomplete, ReadOnlyMemory.Empty); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/NntpDelimiterSearch.cs b/src/VectorNNTPD.TransitService/Session/Framing/NntpDelimiterSearch.cs new file mode 100644 index 00000000..631a77e6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/NntpDelimiterSearch.cs @@ -0,0 +1,193 @@ +using System.Buffers; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// + /// Runtime-vectorized delimiter search for NNTP CRLF framing and multiline terminators. + /// + /// + /// Uses runtime ReadOnlySpan<byte>.IndexOf so hardware acceleration is provided + /// by the runtime. A scalar fallback is implicit when vectorization is unavailable. + /// Correctness does not depend on SIMD. + /// + public static class NntpDelimiterSearch + { + /// CRLF sequence that terminates an NNTP command or article line. + public static ReadOnlySpan Crlf => "\r\n"u8; + + /// Five-byte article terminator delimiter after a preceding content line. + public static ReadOnlySpan FiveByteTerminator => "\r\n.\r\n"u8; + + /// Empty-article terminator at the start of a multiline block. + public static ReadOnlySpan EmptyTerminator => ".\r\n"u8; + + /// Finds the first CRLF in , or -1. + public static int IndexOfCrlf(ReadOnlySpan span) => span.IndexOf(Crlf); + + /// Finds the first \r\n.\r\n in , or -1. + public static int IndexOfFiveByteTerminator(ReadOnlySpan span) => span.IndexOf(FiveByteTerminator); + + /// + /// Bytes that must remain unconsumed when a STREAM article terminator is incomplete. + /// + /// + /// \r\n.\r\n is five octets; four may already be present at the tail of a Pipe read. + /// + public const int ArticleTerminatorLookbehind = 4; + + /// + /// Locates the STREAM article terminator in without walking CRLF lines. + /// + /// Unread article octets (not previously copied into the parser buffer). + /// + /// when no article payload has been accepted yet, so a leading + /// .\r\n is the empty-article terminator. + /// + /// + /// Count of octets that are article payload (includes the last + /// content line's CRLF; excludes the terminator). + /// + /// + /// Count of octets to consume (payload plus terminator). + /// + /// when a complete terminator is present. + /// + /// RFC 3977 §3.1.1: a non-empty multiline block ends with the five octets CRLF "." CRLF; + /// an empty block is "." CRLF. Payload excludes the terminating line. STREAM copies those + /// payload octets as received (no destuff). + /// + public static bool TryFindArticleTerminator( + ReadOnlySequence buffer, + bool atArticleStart, + out int payloadBytes, + out int consumedBytes) + { + payloadBytes = 0; + consumedBytes = 0; + + if (atArticleStart && StartsWithEmptyTerminator(buffer)) + { + consumedBytes = EmptyTerminator.Length; + return true; + } + + if (buffer.Length < FiveByteTerminator.Length) + { + return false; + } + + if (buffer.IsSingleSegment) + { + var index = IndexOfFiveByteTerminator(buffer.FirstSpan); + if (index < 0) + { + return false; + } + + payloadBytes = index + Crlf.Length; + consumedBytes = index + FiveByteTerminator.Length; + return true; + } + + var reader = new SequenceReader(buffer); + if (!reader.TryReadTo(out ReadOnlySequence before, FiveByteTerminator, advancePastDelimiter: true)) + { + return false; + } + + payloadBytes = checked((int)before.Length + Crlf.Length); + consumedBytes = checked((int)reader.Consumed); + return true; + } + + /// Returns whether begins with the empty-article terminator. + public static bool StartsWithEmptyTerminator(ReadOnlySequence buffer) + { + if (buffer.Length < EmptyTerminator.Length) + { + return false; + } + + if (buffer.IsSingleSegment) + { + return buffer.FirstSpan.StartsWith(EmptyTerminator); + } + + Span prefix = stackalloc byte[3]; + buffer.Slice(0, 3).CopyTo(prefix); + return prefix.SequenceEqual(EmptyTerminator); + } + + /// + /// Reads one CRLF-terminated line from (CRLF not included). + /// + /// when a complete line was found. + public static bool TryReadLine(ref ReadOnlySequence buffer, out ReadOnlySequence lineBytes) + { + if (buffer.IsSingleSegment) + { + var span = buffer.FirstSpan; + var crlf = IndexOfCrlf(span); + if (crlf < 0) + { + lineBytes = default; + return false; + } + + lineBytes = buffer.Slice(0, crlf); + buffer = buffer.Slice(crlf + 2); + return true; + } + + var seqReader = new SequenceReader(buffer); + if (!seqReader.TryReadTo(out lineBytes, Crlf)) + { + lineBytes = default; + return false; + } + + buffer = buffer.Slice(seqReader.Position); + return true; + } + + /// Returns whether is the NNTP multiline terminator line. + public static bool IsTerminatorLine(ReadOnlySequence lineBytes) + { + if (lineBytes.Length != 1) + { + return false; + } + + return FirstByte(lineBytes) == (byte)'.'; + } + + /// Returns whether is the NNTP multiline terminator line. + public static bool IsTerminatorLine(ReadOnlySpan line) => + line.Length == 1 && line[0] == (byte)'.'; + + /// Returns the first byte of , or -1 when empty. + public static int FirstByte(ReadOnlySequence sequence) + { + if (sequence.IsEmpty) + { + return -1; + } + + var first = sequence.FirstSpan; + if (!first.IsEmpty) + { + return first[0]; + } + + foreach (var segment in sequence) + { + if (!segment.IsEmpty) + { + return segment.Span[0]; + } + } + + return -1; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/Framing/NntpMultilineDataReader.cs b/src/VectorNNTPD.TransitService/Session/Framing/NntpMultilineDataReader.cs new file mode 100644 index 00000000..cacb0e67 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/Framing/NntpMultilineDataReader.cs @@ -0,0 +1,62 @@ +using System.IO.Pipelines; + +namespace VectorNNTPD.TransitService.Session.Framing +{ + /// Result of reading one NNTP multiline data block from a . + public enum NntpMultilineReadStatus + { + /// + /// Complete article consumed (terminator seen). Payload meaning depends on the reader: + /// STREAM framing copies wire bytes; destuffs. + /// + Completed = 0, + + /// Peer disconnected before the terminating dot line; nothing should be enqueued. + Incomplete = 1, + + /// + /// Article exceeded the configured size limit; remaining bytes through the terminator were discarded. + /// + TooLarge = 2, + } + + /// Outcome of . + /// Read status. + /// + /// Article bytes when (STREAM: framed wire + /// without terminator; multiline reader: destuffed); otherwise empty. + /// + public readonly record struct NntpMultilineReadResult( + NntpMultilineReadStatus Status, + ReadOnlyMemory Payload); + + /// + /// Reads an NNTP multiline data block (RFC 3977 §3.1.1) with dot-unstuffing. + /// + /// + /// Terminator is a line containing only . (i.e. .CRLF). + /// A leading doubled dot on a content line is reduced to a single leading dot. + /// The terminator line is not included in the payload. + /// Line boundaries are located with (runtime-vectorized + /// IndexOf on contiguous spans, sequence scan across segments). + /// + public static class NntpMultilineDataReader + { + /// + /// Reads one multiline article from up to . + /// + public static async ValueTask ReadArticleAsync( + PipeReader reader, + int maxArticleBytes, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + var parser = new NntpContinuousRxParser(); + return await NntpContinuousRxReader + .ReadArticleAsync(reader, parser, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/INntpSessionCensus.cs b/src/VectorNNTPD.TransitService/Session/INntpSessionCensus.cs new file mode 100644 index 00000000..00a882ee --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/INntpSessionCensus.cs @@ -0,0 +1,99 @@ +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.Session +{ + /// Point-in-time counts of every established NNTP session on this process. + public readonly struct NntpSessionCensusSnapshot + { + /// Initializes a census snapshot. + public NntpSessionCensusSnapshot( + int active, + int established, + int idle, + int receiving, + int waitingHistory, + int waitingQueue, + int waitingWindow, + int completing, + int takeThisOccupied = 0, + int takeThisOccupiedMax = 0, + int takeThisFullSessions = 0, + int takeThisProcessing = 0, + int takeThisProcessingMax = 0) + { + Active = active; + Established = established; + Idle = idle; + Receiving = receiving; + WaitingHistory = waitingHistory; + WaitingQueue = waitingQueue; + WaitingWindow = waitingWindow; + Completing = completing; + TakeThisOccupied = takeThisOccupied; + TakeThisOccupiedMax = takeThisOccupiedMax; + TakeThisFullSessions = takeThisFullSessions; + TakeThisProcessing = takeThisProcessing; + TakeThisProcessingMax = takeThisProcessingMax; + } + + /// Gets established session count (complete server population). + public int Active { get; } + + /// Gets established session count (same population as ). + public int Established { get; } + + /// Gets sessions in . + public int Idle { get; } + + /// Gets sessions in . + public int Receiving { get; } + + /// Gets sessions in . + public int WaitingHistory { get; } + + /// Gets sessions in . + public int WaitingQueue { get; } + + /// Gets sessions in . + public int WaitingWindow { get; } + + /// Gets sessions in . + public int Completing { get; } + + /// Gets the sum of TAKETHIS Occupied slots across sessions. + public int TakeThisOccupied { get; } + + /// Gets the highest TAKETHIS Occupied count on any one session. + public int TakeThisOccupiedMax { get; } + + /// Gets how many sessions currently have Occupied equal to TAKETHIS Depth (16). + public int TakeThisFullSessions { get; } + + /// Gets the sum of concurrent ProcessArticleAsync tasks across sessions. + public int TakeThisProcessing { get; } + + /// Gets the highest concurrent ProcessArticleAsync count on any one session. + public int TakeThisProcessingMax { get; } + } + + /// + /// Authoritative registry of established NNTP sessions (readers, streamers, and transit). + /// + public interface INntpSessionCensus + { + /// Registers a session that has entered . + void Register(NntpSession session); + + /// Unregisters a session that is leaving . + void Unregister(NntpSession session); + + /// + /// Stops attributing to Transit peer telemetry after + /// it authenticates as a reader. Does not remove it from the process census. + /// + void ReleasePeerAttribution(NntpSession session); + + /// Captures the current complete session population. + NntpSessionCensusSnapshot Capture(); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpArticleNumber.cs b/src/VectorNNTPD.TransitService/Session/NntpArticleNumber.cs new file mode 100644 index 00000000..c4147013 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpArticleNumber.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// Validates RFC 3977 article-number tokens (1*16DIGIT). + public static class NntpArticleNumber + { + /// Maximum article-number length in octets (RFC 3977 §9.2). + public const int MaxDigits = 16; + + /// + /// Returns whether is a syntactically valid article number. + /// Does not imply that an article exists. + /// + public static bool IsSyntax(ReadOnlySpan value) + { + if (value.Length is < 1 or > MaxDigits) + { + return false; + } + + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b is < (byte)'0' or > (byte)'9') + { + return false; + } + } + + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpAuthenticationAuthority.cs b/src/VectorNNTPD.TransitService/Session/NntpAuthenticationAuthority.cs new file mode 100644 index 00000000..95c5f01c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpAuthenticationAuthority.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Selects which AUTHINFO credential authority is active. + /// Independent of receive/capability state. + /// + /// + /// Source IP never selects this value. MODE READER and the unspecified default + /// use . A successful MODE STREAM (or an explicit + /// ) uses . + /// There is no Transit ↔ Reader fallback. + /// + public enum NntpAuthenticationAuthority + { + /// Newsmaster then MySQL nntpusers. Never Transit peer credentials. + Reader = 0, + + /// Configured Transit peer credentials only. Never MySQL or newsmaster. + Transit = 1, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpAuthenticationState.cs b/src/VectorNNTPD.TransitService/Session/NntpAuthenticationState.cs new file mode 100644 index 00000000..840a6b45 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpAuthenticationState.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// Immutable authentication identity for an NNTP session (distinct from authorization). + public sealed class NntpAuthenticationState + { + /// Unauthenticated session (no identity). + public static NntpAuthenticationState Unauthenticated { get; } = new(isAuthenticated: false, username: null); + + private NntpAuthenticationState(bool isAuthenticated, string? username) + { + IsAuthenticated = isAuthenticated; + Username = username; + } + + /// Gets a value indicating whether the peer completed AUTHINFO successfully. + public bool IsAuthenticated { get; } + + /// Gets the authenticated username when is . + public string? Username { get; } + + /// Creates an authenticated identity snapshot. + public static NntpAuthenticationState ForUser(string username) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + return new(isAuthenticated: true, username: username); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpAuthorization.cs b/src/VectorNNTPD.TransitService/Session/NntpAuthorization.cs new file mode 100644 index 00000000..2030300a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpAuthorization.cs @@ -0,0 +1,134 @@ +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Immutable authorization snapshot for an NNTP session. + /// + /// + /// Authentication and authorization are distinct: a session may be authenticated without + /// reader, transit, posting, streaming, or control-cancel privileges. Production default for + /// new sessions is unauthenticated with false. A Transit + /// AllowFrom match retains (the Transit identifier) and + /// . + /// + public sealed class NntpAuthorization + { + /// Default privileges before authentication (no reader/transit/posting/streaming/control-cancel). + public static NntpAuthorization Unauthenticated { get; } = new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + /// + /// Unauthenticated transit/streaming peer privileges (ACL match). + /// Does not grant reader access, posting, or . + /// Nameless; production identification uses . + /// + public static NntpAuthorization TrustedTransitPeer { get; } = new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + /// Initializes a new instance of the class. + public NntpAuthorization( + bool isAuthenticated, + bool authorizedReader, + bool authorizedTransit, + bool postingPermitted, + bool streamingPermitted, + string? transitPeerName = null, + TransitPeerPolicy? transitPeerPolicy = null, + bool controlCancelPermitted = false) + { + IsAuthenticated = isAuthenticated; + AuthorizedReader = authorizedReader; + AuthorizedTransit = authorizedTransit; + PostingPermitted = postingPermitted; + StreamingPermitted = streamingPermitted; + TransitPeerName = transitPeerName; + TransitPeerPolicy = transitPeerPolicy; + ControlCancelPermitted = controlCancelPermitted; + } + + /// + /// Creates streaming privileges bound to a named peer policy. + /// + /// + /// is always set so MODE STREAM can select Transit AUTHINFO. + /// is set only when receive authentication is disabled + /// ( is false: username or password empty). + /// When both are non-empty, feed commands stay unauthorized until AUTHINFO matches that pair. + /// The peer name and policy are retained either way, including for inbound connection limits. + /// + public static NntpAuthorization ForTransitPeer(TransitPeerPolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + return new( + isAuthenticated: false, + authorizedReader: false, + authorizedTransit: !policy.HasPeerCredentials, + postingPermitted: false, + streamingPermitted: true, + transitPeerName: policy.Identifier, + transitPeerPolicy: policy); + } + + /// Gets a value indicating whether the peer has completed authentication. + public bool IsAuthenticated { get; } + + /// Gets a value indicating whether reader commands are authorized. + public bool AuthorizedReader { get; } + + /// Gets a value indicating whether transit/streaming feed commands are authorized. + public bool AuthorizedTransit { get; } + + /// Gets a value indicating whether POST is permitted (affects greeting / MODE READER). + public bool PostingPermitted { get; } + + /// Gets a value indicating whether streaming feed mode may be entered. + public bool StreamingPermitted { get; } + + /// + /// Gets a value indicating whether this session may POST a well-formed + /// Control: cancel <message-id> article. + /// + /// + /// Ordinary posting remains forbidden from supplying Control. This flag is granted + /// only by an authentication provider (the configured newsmaster account). It is not a + /// generic control-header bypass. + /// + public bool ControlCancelPermitted { get; } + + /// + /// Gets the Transit identifier when this session was identified as a named peer. + /// + /// This is the dictionary-key identifier, not . + public string? TransitPeerName { get; } + + /// Gets the identified Transit peer policy, or when not a named peer. + public TransitPeerPolicy? TransitPeerPolicy { get; } + + /// Returns a copy with updated flags, preserving Transit peer identity. + public NntpAuthorization With( + bool? isAuthenticated = null, + bool? authorizedReader = null, + bool? authorizedTransit = null, + bool? postingPermitted = null, + bool? streamingPermitted = null, + bool? controlCancelPermitted = null) => + new( + isAuthenticated ?? IsAuthenticated, + authorizedReader ?? AuthorizedReader, + authorizedTransit ?? AuthorizedTransit, + postingPermitted ?? PostingPermitted, + streamingPermitted ?? StreamingPermitted, + TransitPeerName, + TransitPeerPolicy, + controlCancelPermitted ?? ControlCancelPermitted); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpReaderCommandRx.cs b/src/VectorNNTPD.TransitService/Session/NntpReaderCommandRx.cs new file mode 100644 index 00000000..9ab05b0e --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpReaderCommandRx.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// MODE READER RX: one command line at a time via . + /// + /// + /// Does not use the continuous TAKETHIS scanner. TAKETHIS on this path is dispatched as a + /// normal command; the handler reads the article after authorization gates. + /// + internal sealed class NntpReaderCommandRx + { + private readonly NntpSession _session; + private readonly NntpCommandDispatcher _dispatcher; + private readonly ILogger _logger; + private readonly byte[] _commandScratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + /// Initializes a new instance of the class. + public NntpReaderCommandRx(NntpSession session, NntpCommandDispatcher dispatcher, ILogger logger) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(logger); + _session = session; + _dispatcher = dispatcher; + _logger = logger; + } + + /// + /// Reads and dispatches one command. Returns on EOF. + /// + public async ValueTask ProcessOneAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + await _session.WaitForCheckCapacityAsync(cancellationToken).ConfigureAwait(false); + var length = await NntpCommandLineReader + .ReadLineBytesAsync(_session.Connection.Input, _commandScratch, cancellationToken) + .ConfigureAwait(false); + if (length == NntpCommandLineReader.OverlongLine) + { + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + NntpCommand.OverlongLine, + ReadOnlyMemory.Empty, + preReadArticle: null, + cancellationToken) + .ConfigureAwait(false); + return true; + } + + if (length < 0) + { + return false; + } + + var line = _commandScratch.AsMemory(0, length); + if (_session.HasSaslExchange) + { + _session.BeginCommandWork(); + try + { + if (_session.TakeThisWindow is not null) + { + await _session.TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + } + + if (_session.Pipeline is not null) + { + await _session.Pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + } + + await Commands.AuthInfo + .ContinueSaslAsync(_session, response, line, cancellationToken) + .ConfigureAwait(false); + } + finally + { + _session.EndCommandWork(); + } + + return true; + } + + var command = NntpCommandParser.Parse(line.Span); + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + command, + line, + preReadArticle: null, + cancellationToken) + .ConfigureAwait(false); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpReceiveStrategy.cs b/src/VectorNNTPD.TransitService/Session/NntpReceiveStrategy.cs new file mode 100644 index 00000000..6de5fa4c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpReceiveStrategy.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Per-connection RX strategy selected from existing . + /// + /// + /// RFC 4644 §2.3: MODE STREAM MUST NOT change server state, so this is not set by that + /// command. is the default (unspecified) path used by transit/feed + /// peers. is selected only after a successful MODE READER. + /// + internal enum NntpReceiveStrategy + { + /// Command-oriented RX: ReadLineAsync → parser → dispatcher. + ReaderCommand = 0, + + /// Continuous CHECK/TAKETHIS data-plane RX. + StreamDataPlane = 1, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpReplyCodes.cs b/src/VectorNNTPD.TransitService/Session/NntpReplyCodes.cs new file mode 100644 index 00000000..0288c80c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpReplyCodes.cs @@ -0,0 +1,117 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// Common NNTP reply codes used by the session and command layer (RFC 3977 / 4643). + public static class NntpReplyCodes + { + /// Help text follows (multiline). + public const int HelpTextFollows = 100; + + /// Capability list follows (multiline). + public const int CapabilityListFollows = 101; + + /// Server date stamp. + public const int ServerDate = 111; + + /// Posting allowed (greeting / MODE READER). + public const int PostingAllowed = 200; + + /// Posting prohibited (greeting / MODE READER). + public const int PostingProhibited = 201; + + /// Connection closing (QUIT). + public const int ConnectionClosing = 205; + + /// Compression layer activated (COMPRESS; RFC 8054). + public const int CompressionActive = 206; + + /// Streaming permitted (MODE STREAM; RFC 4644 §2.3). + public const int StreamingPermitted = 203; + + /// Article follows (ARTICLE; RFC 3977 §6.2.1). + public const int ArticleFollows = 220; + + /// Headers follow (HEAD; RFC 3977 §6.2.2). + public const int HeadFollows = 221; + + /// Body follows (BODY; RFC 3977 §6.2.3). + public const int BodyFollows = 222; + + /// Article exists / selected (STAT; RFC 3977 §6.2.4). + public const int ArticleExists = 223; + + /// Article received OK (POST second stage; RFC 3977 §6.3.1). + public const int ArticleReceivedOk = 240; + + /// Send article to be posted (POST first stage; RFC 3977 §6.3.1). + public const int PostSendArticle = 340; + + /// Posting not permitted (POST first stage; RFC 3977 §6.3.1). + public const int PostingNotPermitted = 440; + + /// Posting failed (POST second stage; RFC 3977 §6.3.1). + public const int PostingFailed = 441; + + /// Article transferred OK (IHAVE second stage; RFC 3977 §6.3.2). + public const int IhaveTransferredOk = 235; + + /// Send article to be transferred (IHAVE first stage; RFC 3977 §6.3.2). + public const int IhaveSendArticle = 335; + + /// Article not wanted (IHAVE first stage; RFC 3977 §6.3.2). + public const int IhaveNotWanted = 435; + + /// Transfer not possible or failed; try again later (IHAVE; RFC 3977 §6.3.2). + public const int IhaveTryLater = 436; + + /// Transfer rejected; do not retry (IHAVE second stage; RFC 3977 §6.3.2). + public const int IhaveRejected = 437; + + /// Send article to be transferred (CHECK; RFC 4644 §2.4). + public const int SendArticleToBeTransferred = 238; + + /// Article transferred OK (TAKETHIS; RFC 4644 §2.5). + public const int ArticleTransferredOk = 239; + + /// Service temporarily unavailable (close connection; RFC 4644 §2.5). + public const int ServiceTemporarilyUnavailable = 400; + + /// Transfer rejected; do not retry (TAKETHIS; RFC 4644 §2.5). + public const int TransferRejected = 439; + + /// Authentication accepted (AUTHINFO). + public const int AuthenticationAccepted = 281; + + /// Continue with TLS negotiation (STARTTLS). + public const int ContinueWithTls = 382; + + /// Password required to complete AUTHINFO USER/PASS. + public const int PasswordRequired = 381; + + /// Command failed due to temporary/internal fault. + public const int CommandFailed = 403; + + /// Feature exists but is unavailable or not supported in this form (e.g. unsupported COMPRESS algorithm). + public const int FeatureUnavailable = 503; + + /// Authentication required before the facility can be used. + public const int AuthenticationRequired = 480; + + /// Authentication failed or rejected. + public const int AuthenticationRejected = 481; + + /// Authentication commands issued out of sequence. + public const int AuthenticationOutOfSequence = 482; + + /// Privacy/encryption required for the command. + public const int PrivacyRequired = 483; + + /// Unknown or unimplemented command. + public const int UnknownCommand = 500; + + /// Syntax error or unknown command variant. + public const int SyntaxError = 501; + + /// Command not permitted in the current state / permanently unavailable. + public const int CommandUnavailable = 502; + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpSession.cs b/src/VectorNNTPD.TransitService/Session/NntpSession.cs new file mode 100644 index 00000000..9d406fcf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpSession.cs @@ -0,0 +1,1361 @@ +using System.IO.Pipelines; +using System.Net; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState.RateLimiting; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Session +{ + using VectorNNTPD.Common.Core; + /// + /// NNTP session boundary: greeting, command loop, mode, authentication, and authorization state over an + /// . + /// + /// + /// Transport owns sockets, TLS, DEFLATE, PROXY identity, and connection lifecycle. + /// This type owns NNTP protocol sequencing and does not touch sockets or stream wrappers directly. + /// Authentication and authorization are distinct; AUTHINFO success applies only provider-returned privileges. + /// Connection-time may grant transit/streaming peer privileges + /// without authentication and retains the named Transit peer policy. + /// An established session that executes no NNTP command for Nntpd:IdleTime is disconnected + /// via the existing close path (reason IdleTimeout). In-flight commands including + /// pipelined CHECK and TAKETHIS keep the session non-idle. + /// Any command-loop exit (QUIT, EOF, reset, timeout, cancel, exception, shutdown) + /// takes the same one-shot finalization path and releases distributed admission at most once. + /// + public sealed class NntpSession + { + private readonly ILogger _logger; + private readonly NntpCommandDispatcher _dispatcher; + private readonly NntpAuthorization _connectionAuthorization; + private NntpResponseWriter? _response; + private NntpAuthorization _authorization; + private NntpAuthenticationState _authentication; + private string? _pendingAuthUsername; + private NntpAccountPolicy? _accountPolicy; + private string? _admittedAccountName; + private TransitInboundConnectionLease? _transitAdmission; + private NntpSaslExchange? _saslExchange; + private NntpSessionMode _mode; + private NntpAuthenticationAuthority _authenticationAuthority; + private int _closeRequested; + private int _lifetime; + private int _activityState; + private int _commandWork; + private long _lastCommandTimestamp; + /// + /// Low 63 bits: activity epoch, incremented when commits. + /// High bit set: idle waiter has committed . + /// + private long _idleState; + private readonly TimeSpan _commandIdleTimeout; + private readonly TimeProvider _timeProvider; + private readonly TaskCompletionSource _idleWatchArmed = new(TaskCreationOptions.RunContinuationsAsynchronously); + // codeql[cs/missed-using-statement] Owned by NntpSession until Dispose; a using at this declaration would cancel session lifetime when the constructor returns. + private readonly CancellationTokenSource _closeCts = new(); + private Task? _idleWatchTask; + private int _closeReason; + private ReadOnlyMemory _selectedGroupName; + + /// Initializes a new instance of the class. + public NntpSession( + INntpConnection connection, + ILogger logger, + ITlsCertificateContextProvider? certificateProvider = null, + INntpAuthenticationProvider? authenticationProvider = null, + bool allowCleartextAuth = true, + ILoggerFactory? loggerFactory = null, + IArticleIngestionQueue? articleIngestion = null, + ITransitPeerAuthorization? transitPeerAuthorization = null, + int streamOutstandingArticleDepth = NntpStreamArticleTxScheduler.DefaultDepth, + IHistoryDb? historyDb = null, + ISpeedTestCoordinator? speedTest = null, + INntpSessionCensus? sessionCensus = null, + ITransitPeerMetrics? peerMetrics = null, + TimeSpan? commandIdleTimeout = null, + TimeProvider? timeProvider = null, + int? maxArticleSize = null, + string? injectionIdentity = null, + Commands.Posting.INewsgroupPostingPolicy? newsgroupPostingPolicy = null, + string? mailComplaintsTo = null, + Commands.Posting.IPostingTraceProtector? postingTraceProtector = null, + INewsgroupCatalogue? newsgroupCatalogue = null, + IModeratorCatalogue? moderatorCatalogue = null, + IModeratorAuthorization? moderatorAuthorization = null, + IModerationSubmissionService? moderationSubmission = null, + ISessionStateTracker? sessionAdmission = null, + NntpSaslService? saslService = null, + ITransitPeerAuthenticator? transitAuthenticator = null, + IAccountByteAccountant? accountBytes = null, + IAccountRateAllocator? accountRates = null, + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatpArticleClient = null, + IStorageArticleLookupClient? storageArticleLookup = null, + IPostFilter? postFilter = null, + PostFilterMetrics? postFilterMetrics = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null, + TransitOptions? transit = null, + NntpServerRole serverRole = NntpServerRole.Transit) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(logger); + ServerRole = serverRole; + Connection = connection; + ClientIdentity = connection.ClientIdentity; + _logger = logger; + CertificateProvider = certificateProvider; + _connectionAuthorization = (transitPeerAuthorization ?? TransitPeerAuthorization.Disabled) + .Resolve(ClientIdentity.ClientAddress); + _authorization = _connectionAuthorization; + _authentication = NntpAuthenticationState.Unauthenticated; + _mode = NntpSessionMode.Unspecified; + _authenticationAuthority = NntpAuthenticationAuthority.Reader; + AllowCleartextAuth = allowCleartextAuth; + AuthenticationProvider = authenticationProvider ?? DenyAllNntpAuthenticationProvider.Instance; + TransitAuthenticator = transitAuthenticator ?? TransitPeerAuthenticator.Instance; + ArticleIngestion = articleIngestion ?? DisabledArticleIngestionQueue.Instance; + HistoryDb = historyDb; + SpeedTest = speedTest; + SessionCensus = sessionCensus; + PeerMetrics = peerMetrics; + StreamArticleTx = new NntpStreamArticleTxScheduler(streamOutstandingArticleDepth); + _dispatcher = new NntpCommandDispatcher(loggerFactory); + _timeProvider = timeProvider ?? TimeProvider.System; + _commandIdleTimeout = commandIdleTimeout ?? TimeSpan.FromSeconds(NntpdOptions.DefaultIdleTime); + if (_commandIdleTimeout < TimeSpan.FromSeconds(NntpdOptions.MinIdleTime) + || _commandIdleTimeout > TimeSpan.FromSeconds(NntpdOptions.MaxIdleTime)) + { + throw new ArgumentOutOfRangeException( + nameof(commandIdleTimeout), + commandIdleTimeout, + $"Idle timeout must be between {NntpdOptions.MinIdleTime} and {NntpdOptions.MaxIdleTime} seconds."); + } + + MaxArticleSize = maxArticleSize ?? NntpdOptions.DefaultMaxArticleSize; + if (MaxArticleSize < NntpdOptions.MinMaxArticleSize + || MaxArticleSize > NntpdOptions.MaxMaxArticleSize) + { + throw new ArgumentOutOfRangeException( + nameof(maxArticleSize), + maxArticleSize, + $"Max article size must be between {NntpdOptions.MinMaxArticleSize} and {NntpdOptions.MaxMaxArticleSize} bytes."); + } + + InjectionIdentity = string.IsNullOrWhiteSpace(injectionIdentity) + ? NntpdOptions.FormatFqdn(1, "usenet.ninja") + : injectionIdentity.Trim(); + ArticleParser = new NntpArticleParser(InjectionIdentity); + NewsgroupPostingPolicy = newsgroupPostingPolicy + ?? (newsgroupCatalogue is not null + ? new Commands.Posting.CatalogueNewsgroupPostingPolicy(newsgroupCatalogue) + : Commands.Posting.SyntaxOnlyNewsgroupPostingPolicy.Instance); + MailComplaintsTo = string.IsNullOrWhiteSpace(mailComplaintsTo) + ? NntpdOptions.DefaultMailComplaintsTo + : mailComplaintsTo.Trim(); + PostingTraceProtector = postingTraceProtector; + NewsgroupCatalogue = newsgroupCatalogue; + ModeratorCatalogue = moderatorCatalogue; + ModeratorAuthorization = moderatorAuthorization + ?? (IModeratorAuthorization?)moderatorCatalogue + ?? EmptyModeratorAuthorization.Instance; + ModerationSubmission = moderationSubmission ?? UnavailableModerationSubmissionService.Instance; + SessionAdmission = sessionAdmission; + SaslService = saslService; + AccountBytes = accountBytes ?? NullAccountByteAccountant.Instance; + AccountRates = accountRates ?? NullAccountRateAllocator.Instance; + ArticleWorkRpc = articleWorkRpc; + VatpArticleClient = vatpArticleClient; + StorageArticleLookup = storageArticleLookup; + PostFilter = postFilter ?? DisabledPostFilter.Instance; + PostFilterMetrics = postFilterMetrics ?? new PostFilterMetrics(); + PostFilterEvidence = postFilterEvidence ?? DisabledPostFilterRejectionEvidenceQueue.Instance; + NewsLog = newsLog ?? NullNewsLogWriter.Instance; + Transit = transit ?? new TransitOptions(); + SessionId = Guid.NewGuid().ToString("N"); + } + + /// Gets the underlying transport connection. + public INntpConnection Connection { get; } + + /// Gets the immutable client identity established at connection start. + public ConnectionClientIdentity ClientIdentity { get; } + + /// + /// Gets the article ingestion queue used by transfer commands (TAKETHIS, IHAVE, POST). + /// + public IArticleIngestionQueue ArticleIngestion { get; } + + /// + /// Gets the article-work RPC client used by ARTICLE/HEAD/BODY/STAT message-id lookup, or + /// when tests construct a session without RabbitMQ. + /// + internal IArticleWorkRpcClient? ArticleWorkRpc { get; } + + /// + /// Gets the VATP article client used after ArticleWork Success, or + /// when tests construct a session without VATP. + /// + internal IVatpArticleClient? VatpArticleClient { get; } + + /// + /// Gets the StorageServer fleet lookup used after a definitive ArticleWork miss, or + /// when tests construct a session without fleet lookup. + /// + internal IStorageArticleLookupClient? StorageArticleLookup { get; } + + /// Gets the POST-only PostFilter. IHAVE/TAKETHIS must not call this. + internal IPostFilter PostFilter { get; } + + /// Gets process counters for PostFilter COMMIT outcomes. + internal PostFilterMetrics PostFilterMetrics { get; } + + /// Gets the bounded PostFilter rejection-evidence queue. + internal IPostFilterRejectionEvidenceQueue PostFilterEvidence { get; } + + /// Gets the dedicated INN news writer. + internal INewsLogWriter NewsLog { get; } + + /// Gets site-wide transit options including WantTrash/LogTrash. + internal TransitOptions Transit { get; } + + /// Gets the destuffed POST article size limit (Nntpd:MaxArticleSize). + public int MaxArticleSize { get; } + + /// + /// Published nntpsharedconfig. Null when the session was constructed without the catalogue. + /// Article commands capture one policy for the whole operation. + /// + internal INntpArticlePolicySource? SharedConfiguration { get; set; } + + /// + /// Gets the server injection identity used for POST metadata + /// (nntpd{ServerId:00}.{DnsSuffix}). + /// + public string InjectionIdentity { get; } + + /// + /// Gets the session-scoped Common article parser (local identity is + /// ). Used by TAKETHIS and POST before queue admission. + /// + internal NntpArticleParser ArticleParser { get; } + + /// + /// Gets the newsgroup existence/posting-authorization boundary used by POST. + /// Catalogue-backed when is set and no policy was injected. + /// + public Commands.Posting.INewsgroupPostingPolicy NewsgroupPostingPolicy { get; } + + /// Gets the configured POST mail-complaints-to mailbox. + public string MailComplaintsTo { get; } + + /// Gets the POST X-Trace protector, or when unset (tests). + public Commands.Posting.IPostingTraceProtector? PostingTraceProtector { get; } + + /// + /// Gets the in-memory newsgroup catalogue, or when unset (tests). + /// + public INewsgroupCatalogue? NewsgroupCatalogue { get; } + + /// + /// Gets the process-wide moderator catalogue, or when unset (tests). + /// + public IModeratorCatalogue? ModeratorCatalogue { get; } + + /// + /// Gets the moderator authorization table used by POST for Approved: decisions. + /// + public IModeratorAuthorization ModeratorAuthorization { get; } + + /// Captures the current moderator snapshot once for a POST command. + public IModeratorAuthorization CaptureModeratorAuthorization() => + ModeratorCatalogue?.Current ?? ModeratorAuthorization; + + /// + /// Gets the moderation submission boundary used when an unapproved moderated + /// proto-article must be forwarded. + /// + public IModerationSubmissionService ModerationSubmission { get; } + + /// Gets whether a newsgroup is currently selected. + internal bool HasSelectedGroup => !_selectedGroupName.IsEmpty; + + /// Gets the original stored name of the currently selected newsgroup. + internal ReadOnlySpan SelectedGroupName => _selectedGroupName.Span; + + /// Gets the clock used for idle accounting and POST injection timestamps. + public TimeProvider Time => _timeProvider; + + /// Gets the HistoryDB used by CHECK, IHAVE, TAKETHIS, and POST, or when unset (tests). + public IHistoryDb? HistoryDb { get; } + + /// + /// Gets the SPEEDTEST coordinator, or when the diagnostic is not registered. + /// + public ISpeedTestCoordinator? SpeedTest { get; } + + /// + /// Gets the opt-in feed-diagnostics probe for this session, or when disabled. + /// + public FeedSessionProbe? FeedProbe { get; set; } + + /// Gets the process-wide session census, or when unset (tests). + internal INntpSessionCensus? SessionCensus { get; } + + /// Gets always-on Transit peer counters, or when unset (tests). + internal ITransitPeerMetrics? PeerMetrics { get; } + + /// Gets the existing article-ingest activity state for this session. + public FeedSessionState ActivityState => (FeedSessionState)Volatile.Read(ref _activityState); + + /// Gets the per-session CHECK pipeline once has started. + internal CheckPipeline? Pipeline { get; private set; } + + /// Gets the per-session TAKETHIS pipeline once has started. + internal TakeThisPipeline? TakeThisWindow { get; private set; } + + /// + /// Gets the bounded STREAM article TX scheduler (depth gate above + /// ). + /// + /// + /// Available for outbound STREAM article producers that already hold destuffed bytes. + /// No production command currently supplies those bytes (storage/catalogue is out of scope). + /// + public NntpStreamArticleTxScheduler StreamArticleTx { get; } + + /// Gets the effective client IP address for this session. + public IPAddress ClientAddress => ClientIdentity.ClientAddress; + + /// Gets the effective client TCP source port for this session. + public int ClientPort => ClientIdentity.ClientPort; + + /// Gets the actual TCP peer endpoint of the accepted socket. + public IPEndPoint TcpPeer => ClientIdentity.TcpPeer; + + /// Gets the reader AUTHINFO provider (newsmaster / MySQL). Never used in Transit authority. + public INntpAuthenticationProvider AuthenticationProvider { get; } + + /// Gets the Transit AUTHINFO authenticator. Never used in Reader authority. + public ITransitPeerAuthenticator TransitAuthenticator { get; } + + /// + /// Gets the AUTHINFO credential authority selected by MODE, not by source IP. + /// + public NntpAuthenticationAuthority AuthenticationAuthority => _authenticationAuthority; + + /// Gets the authenticated-session admission tracker, if registered. + public ISessionStateTracker? SessionAdmission { get; } + + /// Gets the AUTHINFO SASL service, if registered. + public NntpSaslService? SaslService { get; } + + /// Gets the account byte-quota accountant. No-op when unauthenticated. + public IAccountByteAccountant AccountBytes { get; } + + /// Gets the account rate allocator. No-op when the rate is unlimited. + public IAccountRateAllocator AccountRates { get; } + + /// Gets the unique id used for admission tracking. + public string SessionId { get; } + + /// Gets the authenticated account policy, or when none applies. + public NntpAccountPolicy? AccountPolicy => _accountPolicy; + + /// Gets whether an AUTHINFO SASL exchange is waiting for a continuation line. + internal bool HasSaslExchange => _saslExchange is not null; + + /// Gets the in-progress SASL exchange, or . + internal NntpSaslExchange? CurrentSaslExchange => _saslExchange; + + /// Gets the TLS certificate provider used by STARTTLS, if configured. + public ITlsCertificateContextProvider? CertificateProvider { get; } + + /// + /// Gets a value indicating whether AUTHINFO USER/PASS is permitted without TLS + /// (Nntpd:AllowCleartextAuth). TLS connections always permit AUTHINFO USER/PASS. + /// + public bool AllowCleartextAuth { get; } + + /// + /// Gets a value indicating whether AUTHINFO USER/PASS may proceed on this connection + /// given TLS state and . + /// + public bool IsAuthinfoPassPermitted => Connection.IsTls || AllowCleartextAuth; + + /// Gets the current authentication identity snapshot. + public NntpAuthenticationState Authentication => _authentication; + + /// Gets the current authorization snapshot. + public NntpAuthorization Authorization => _authorization; + + /// Gets the current NNTP operating mode. + public NntpSessionMode Mode => _mode; + + /// Gets the host process role that created this session. + public NntpServerRole ServerRole { get; } + + /// + /// Gets the RX strategy implied by . + /// + /// + /// only after MODE READER. + /// MODE STREAM does not change this (RFC 4644 §2.3). + /// + internal NntpReceiveStrategy ReceiveStrategy => + _mode == NntpSessionMode.Reader + ? NntpReceiveStrategy.ReaderCommand + : NntpReceiveStrategy.StreamDataPlane; + + /// + /// Gets the pending username from AUTHINFO USER awaiting AUTHINFO PASS, + /// or when no USER is cached. + /// + public string? PendingAuthUsername => _pendingAuthUsername; + + /// Selects as the current newsgroup (immortal snapshot bytes). + internal void SelectGroup(ReadOnlyMemory groupName) => _selectedGroupName = groupName; + + /// Replaces the authorization snapshot (tests / advanced handlers). + public void SetAuthorization(NntpAuthorization authorization) + { + ArgumentNullException.ThrowIfNull(authorization); + _authorization = authorization; + if (!authorization.IsAuthenticated) + { + _authentication = NntpAuthenticationState.Unauthenticated; + _accountPolicy = null; + _saslExchange = null; + } + } + + /// + /// Sets the session operating mode and the AUTHINFO authority that accompanies it. + /// selects Transit credentials; any other mode + /// selects reader (newsmaster / MySQL) credentials. + /// + public void SetMode(NntpSessionMode mode) + { + _mode = mode; + _authenticationAuthority = mode == NntpSessionMode.Stream + ? NntpAuthenticationAuthority.Transit + : NntpAuthenticationAuthority.Reader; + } + + /// + /// Selects the AUTHINFO authority without changing . + /// Used by MODE STREAM, which must not change RFC 4644 receive state. + /// + public void SetAuthenticationAuthority(NntpAuthenticationAuthority authority) => + _authenticationAuthority = authority; + + /// Caches the username from AUTHINFO USER (does not authenticate). + public void SetPendingAuthUsername(string username) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + _pendingAuthUsername = username; + } + + /// Clears any pending AUTHINFO USER username. + public void ClearPendingAuthUsername() => _pendingAuthUsername = null; + + /// + /// Applies a successful authentication atomically: identity + authorization, pending USER cleared. + /// Forces to . + /// + public void ApplySuccessfulAuthentication( + string username, + NntpAuthorization authorization, + NntpAccountPolicy? policy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(authorization); + var previousPeer = _authorization.TransitPeerName; + _pendingAuthUsername = null; + _saslExchange = null; + _authentication = NntpAuthenticationState.ForUser(username); + _authorization = authorization.With(isAuthenticated: true); + _accountPolicy = policy; + if (previousPeer is not null && _authorization.TransitPeerName is null) + { + SessionCensus?.ReleasePeerAttribution(this); + PeerMetrics?.RetractAccepted(previousPeer); + } + } + + /// + /// Attaches the inbound TransitPeerState lease taken at connection identification. + /// The listener still disposes the same lease on disconnect; the lease is one-shot. + /// + internal void AttachTransitAdmission(TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + _transitAdmission = lease; + } + + /// + /// Releases the connection-time TransitPeerState slot when AUTHINFO replaced + /// named-peer identity with reader privileges. No-op when the session is still + /// a named Transit peer or no slot was taken. + /// + internal ValueTask ReleaseTransitAdmissionWhenNotTransitAsync() + { + if (_authorization.TransitPeerName is not null) + { + return ValueTask.CompletedTask; + } + + return _transitAdmission?.DisposeAsync() ?? ValueTask.CompletedTask; + } + + /// + /// Attaches output accounting after AUTHINFO success and before the 281. + /// Live remaining is observed from Redis/MySQL, not from the cached policy snapshot. + /// Every authenticated account participates. + /// + internal async ValueTask AttachByteAccountingAsync( + NntpResponseWriter response, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(response); + var policy = _accountPolicy; + if (policy is null) + { + return; + } + + long? remaining; + try + { + remaining = await AccountBytes.ObserveRemainingAsync(policy.Username, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + remaining = null; + } + + if (remaining is <= 0) + { + AccountBytes.MarkExhausted(policy.Username); + } + else if (remaining is > 0) + { + AccountBytes.ClearExhausted(policy.Username); + } + + response.SetByteSink(AccountBytes.CreateSink(policy.Username)); + } + + /// Gets whether this authenticated session must reject the next command. + internal bool IsByteQuotaExhausted + { + get + { + var policy = _accountPolicy; + return policy is not null && AccountBytes.IsExhausted(policy.Username); + } + } + + /// + /// Records a failed authentication attempt: session remains unauthenticated. + /// Connection-time peer privileges (if any) are restored; pending USER is retained so the + /// client may retry AUTHINFO PASS or issue a new AUTHINFO USER. + /// + public void ApplyFailedAuthentication() + { + _authentication = NntpAuthenticationState.Unauthenticated; + _authorization = _connectionAuthorization; + _accountPolicy = null; + _saslExchange = null; + ReleaseAdmission(); + } + + /// Abandons an in-progress SASL exchange without changing identity. + internal void AbandonSaslExchange() => _saslExchange = null; + + /// Stores in-progress SASL state for the next continuation line. + internal void SetSaslExchange(NntpSaslExchange exchange) + { + ArgumentNullException.ThrowIfNull(exchange); + _saslExchange = exchange; + } + + /// Admits this session for after credential success. + internal async ValueTask AdmitAuthenticatedSessionAsync( + NntpAccountPolicy? policy, + CancellationToken cancellationToken = default) + { + if (policy is null + || !policy.RequiresAdmission + || SessionAdmission is null + || !string.Equals(_admittedAccountName, policy.Username, StringComparison.Ordinal)) + { + await ReleaseAdmissionAsync().ConfigureAwait(false); + } + + if (policy is null || !policy.RequiresAdmission || SessionAdmission is null) + { + return NntpAuthenticationResult.Success( + policy?.Username ?? Authentication.Username ?? "unknown", + Authorization, + policy); + } + + var outcome = await SessionAdmission.TryAdmitAsync( + policy.Username, + SessionId, + ClientAddress, + policy.SessionLimit, + policy.SrcIpLimit, + policy.RequiresRateTracking ? policy.RateLimitBps : 0, + cancellationToken).ConfigureAwait(false); + switch (outcome) + { + case SessionAdmissionResult.Success: + _admittedAccountName = policy.Username; + if (policy.RequiresRateTracking && Connection.OutboundRate is { } cap) + { + AccountRates.Register(policy.Username, SessionId, cap, policy.RateLimitBps); + } + + return NntpAuthenticationResult.Success(policy.Username, Authorization, policy); + case SessionAdmissionResult.SessionLimitExceeded: + return NntpAuthenticationResult.TooManySessions; + case SessionAdmissionResult.SourceAddressLimitExceeded: + return NntpAuthenticationResult.TooManySourceAddresses; + case SessionAdmissionResult.Unavailable: + return NntpAuthenticationResult.TransientFailure; + default: + return NntpAuthenticationResult.TransientFailure; + } + } + + /// Gets the current connection lifetime (tests and diagnostics). + internal NntpSessionLifetime Lifetime => (NntpSessionLifetime)Volatile.Read(ref _lifetime); + + /// + /// Claims the one-shot → + /// transition. + /// + internal bool TryBeginFinalization() + { + var current = Volatile.Read(ref _lifetime); + if (current is (int)NntpSessionLifetime.Finalizing or (int)NntpSessionLifetime.Finalized) + { + return false; + } + + if (current == (int)NntpSessionLifetime.Running + && Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Finalizing, + (int)NntpSessionLifetime.Running) == (int)NntpSessionLifetime.Running) + { + return true; + } + + return Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Finalizing, + (int)NntpSessionLifetime.Created) == (int)NntpSessionLifetime.Created; + } + + /// + /// Releases distributed admission at most once and marks the session finalized. + /// Safe to call from QUIT, cancellation, timeout, and exception paths. + /// + internal async ValueTask FinalizeAdmissionAsync() + { + if (!TryBeginFinalization()) + { + return; + } + + try + { + await ReleaseAdmissionAsync().ConfigureAwait(false); + } + finally + { + Volatile.Write(ref _lifetime, (int)NntpSessionLifetime.Finalized); + } + } + + /// Releases any admission slot held by this session without awaiting Redis. + internal void ReleaseAdmission() + { + var pending = ReleaseAdmissionAsync(); + if (pending.IsCompletedSuccessfully) + { + return; + } + + if (pending.IsCompleted) + { + _ = pending.AsTask().Exception; + return; + } + + _ = ObserveAdmissionReleaseAsync(pending); + } + + /// Releases any admission slot held by this session. + internal async ValueTask ReleaseAdmissionAsync() + { + var account = Interlocked.Exchange(ref _admittedAccountName, null); + if (account is not null) + { + AccountRates.Unregister(account, SessionId); + if (SessionAdmission is not null) + { + await SessionAdmission.ReleaseAsync(account, SessionId).ConfigureAwait(false); + } + } + } + + private async Task ObserveAdmissionReleaseAsync(ValueTask pending) + { + try + { + await pending.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed( + _logger, + ex, + "session", + ClientAddress.ToString()); + } + } + + /// Records a successful inbound admission for this session's Transit peer, if any. + public void RecordPeerAccepted() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordAccepted(peerId); + } + } + + /// Records an inbound admission rejection for this session's Transit peer, if any. + public void RecordPeerRejected() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordRejected(peerId); + } + } + + /// Records one CHECK for this session's Transit peer, if any. + public void RecordPeerCheck() + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordCheck(peerId); + } + } + + /// Records one framed article received from this session's Transit peer, if any. + public void RecordPeerArticleReceived(int bytes) + { + if (Authorization.TransitPeerName is { } peerId) + { + PeerMetrics?.RecordArticleReceived(peerId, bytes); + } + } + + /// + /// Updates using the existing feed-session state machine + /// and forwards to when attached. + /// + public void SetActivityState(FeedSessionState state) + { + Volatile.Write(ref _activityState, (int)state); + FeedProbe?.SetState(state); + } + + /// + /// Requests the command loop to exit after the current response (e.g. QUIT / TAKETHIS 400). + /// Cancels a pending RX ReadAsync so a close requested off the RX stack still ends + /// the session. + /// + public void RequestClose() => RequestCloseWithReason(TcpDisconnectReason.ProtocolClose); + + /// + /// Notes (first-wins on the transport) and cancels the session loop. + /// + internal void RequestCloseWithReason(TcpDisconnectReason reason) + { + if (Connection is NntpConnection nntpConnection) + { + nntpConnection.NoteDisconnectReason(reason); + } + + Interlocked.CompareExchange( + ref _closeReason, + (int)reason, + (int)TcpDisconnectReason.Unspecified); + + Interlocked.Exchange(ref _closeRequested, 1); + try + { + _closeCts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + /// Marks that an NNTP command has been accepted for processing. + /// Timestamp and work count are published before the idle-epoch CAS so a waiter + /// that has not yet committed close observes activity and cannot close on stale idle. + /// + /// + /// Ownership: if this method increments the epoch while the high bit of + /// is still clear, idle-timeout close must not win. + /// If the high bit is already set, this command loses and runs on the closing session. + /// + internal void BeginCommandWork() + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + Interlocked.Increment(ref _commandWork); + CommitCommandActivityEpoch(); + } + + /// + /// Increments the idle activity epoch unless idle-timeout close has already committed. + /// + private void CommitCommandActivityEpoch() + { + while (true) + { + var current = Interlocked.Read(ref _idleState); + if ((current & IdleClosedFlag) != 0) + { + return; + } + + var next = current + 1; + if (Interlocked.CompareExchange(ref _idleState, next, current) == current) + { + return; + } + } + } + + /// High bit of : idle waiter committed close. + private const long IdleClosedFlag = unchecked((long)0x8000_0000_0000_0000); + + /// + /// Marks that one accepted NNTP command (or pipeline slot) has finished. + /// Timestamp is published before the work count drops to zero so a waiter + /// cannot treat a just-finished long command as already idle. + /// + internal void EndCommandWork() + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + Interlocked.Decrement(ref _commandWork); + } + + /// Signaled once the per-session idle waiter is armed after the greeting. + internal Task IdleWatchArmed => _idleWatchArmed.Task; + + /// The idle-watch task, or before starts it. + internal Task? IdleWatchTaskForTests => _idleWatchTask; + + /// Session-requested close reason (first-wins); tests and idle timeout. + internal TcpDisconnectReason CloseReasonForTests => + (TcpDisconnectReason)Volatile.Read(ref _closeReason); + + /// In-flight accepted command / pipeline-slot count (tests). + internal int CommandWorkForTests => Volatile.Read(ref _commandWork); + + /// Gets whether the idle waiter has committed . + internal bool IdleCloseCommittedForTests => + (Interlocked.Read(ref _idleState) & IdleClosedFlag) != 0; + + /// + /// Test-only: set when the idle waiter has observed a due deadline and is about to + /// CAS-commit close. Production never sets this. + /// + internal TaskCompletionSource? IdleAboutToCommit { get; set; } + + /// + /// Test-only: when set, the idle waiter awaits this after + /// and before the close CAS. Production never sets this. + /// + internal Task? IdleCommitHold { get; set; } + + /// + /// Sends the initial greeting and runs the command loop until QUIT, EOF, or cancellation. + /// + public async Task RunAsync(CancellationToken cancellationToken = default) + { + if (Connection is NntpConnection nntpConnection) + { + nntpConnection.AttachInputReaderConsumer(); + } + + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + Connection.ConnectionClosed, + _closeCts.Token); + var token = linked.Token; + + try + { + Interlocked.CompareExchange( + ref _lifetime, + (int)NntpSessionLifetime.Running, + (int)NntpSessionLifetime.Created); + SessionCensus?.Register(this); + var response = _response ??= new NntpResponseWriter(Connection.Output); + Pipeline = new CheckPipeline(this, response); + TakeThisWindow = new TakeThisPipeline(this, response); + await SendGreetingAsync(response, token).ConfigureAwait(false); + _idleWatchTask = WatchIdleAsync(token); + + var readerRx = new NntpReaderCommandRx(this, _dispatcher, _logger); + var streamRx = new NntpStreamDataPlaneRx(this, _dispatcher, _logger); + + while (!token.IsCancellationRequested && Volatile.Read(ref _closeRequested) == 0) + { + var progressed = ReceiveStrategy == NntpReceiveStrategy.ReaderCommand + ? await readerRx.ProcessOneAsync(response, token).ConfigureAwait(false) + : await streamRx.ProcessOneAsync(response, token).ConfigureAwait(false); + if (!progressed) + { + break; + } + + // Coalesced fire-and-forget lines (CHECK EnqueueLineAsync) flush when the + // inbound pipe has nothing ready. TAKETHIS uses EnqueueLineImmediateAsync + // and does not wait for this idle flush. STREAM leftover stays unconsumed + // so CHECK can still reach its coalesce batch. + if (response.HasCoalescedUnflushed && !HasUnconsumedInput(Connection.Input)) + { + await response.FlushCoalescedAsync(token).ConfigureAwait(false); + } + } + } + catch (OperationCanceledException ex) when (token.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected on shutdown or connection close. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (NntpPeerDisconnect.IsPeerDisconnect(ex, Connection)) + { + SessionLogMessages.SessionEndedByPeerDisconnect(_logger, ex, ClientAddress); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionLogMessages.SessionEndedWithError(_logger, ex, ClientAddress); + } + finally + { + try + { + _closeCts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + if (_idleWatchTask is not null) + { + try + { + await _idleWatchTask.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + SessionCensus?.Unregister(this); + try + { + await FinalizeAdmissionAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed( + _logger, + ex, + "session", + ClientAddress.ToString()); + } + if (TakeThisWindow is not null) + { + try + { + await TakeThisWindow.ShutdownAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; peeks use the session token. + } + } + + if (Pipeline is not null) + { + try + { + await Pipeline.ShutdownAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; lookups use the session token. + } + } + + if (_response is not null) + { + try + { + await _response.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + } + + try + { + await StreamArticleTx.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + try + { + if (Connection is NntpConnection inputOwner) + { + await inputOwner.CompleteInputReaderAsync().ConfigureAwait(false); + } + else + { + await Connection.Input.CompleteAsync().ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort; reader may already be completed. + } + + try + { + await Connection.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Best-effort. + } + + _closeCts.Dispose(); + } + } + + private ValueTask SendGreetingAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + if (_authorization.PostingPermitted) + { + return response.WriteLineAsync(NntpResponses.GreetingPostingPermitted, cancellationToken); + } + + return response.WriteLineAsync(NntpResponses.GreetingPostingProhibited, cancellationToken); + } + + /// + /// Logs and dispatches one already-parsed command (shared by both RX strategies). + /// + internal async ValueTask DispatchCommandAsync( + NntpCommandDispatcher dispatcher, + NntpResponseWriter response, + ILogger logger, + NntpCommand command, + ReadOnlyMemory line, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(logger); + + if (command.Verb != NntpVerb.BenchIt + && !NntpCommandLogFormat.SuppressHotPathCommand(command.Verb) + && logger.IsEnabled(LogLevel.Debug)) + { + var client = NntpCommandLogFormat.Client(this); + var commandText = NntpCommandLogFormat.RedactRxCommand(command, line.Span); + CommandLogMessages.CommandRx(logger, client, commandText); + } + + await dispatcher + .DispatchAsync(this, command, line, response, cancellationToken, preReadArticle) + .ConfigureAwait(false); + } + + /// + /// Dispatches one parsed unit: authorized CHECK enters the CHECK pipeline; every other + /// command drains outstanding CHECK and TAKETHIS windows first and then runs serially. + /// + internal async ValueTask ProcessParsedCommandAsync( + NntpCommandDispatcher dispatcher, + NntpResponseWriter response, + ILogger logger, + NntpCommand command, + ReadOnlyMemory line, + NntpMultilineReadResult? preReadArticle, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(Pipeline); + ArgumentNullException.ThrowIfNull(TakeThisWindow); + + BeginCommandWork(); + try + { + if (IsByteQuotaExhausted) + { + await response + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + RequestClose(); + return; + } + + // Transit CHECK fast-path: only on a Transit host with transit authority. + // Reader hosts must fall through to the dispatcher role gate (502). + if (command.IsValid + && command.Verb == NntpVerb.Check + && ServerRole == NntpServerRole.Transit + && Authorization.AuthorizedTransit) + { + await TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + if (command.Verb != NntpVerb.BenchIt + && !NntpCommandLogFormat.SuppressHotPathCommand(command.Verb) + && logger.IsEnabled(LogLevel.Debug)) + { + var client = NntpCommandLogFormat.Client(this); + var commandText = NntpCommandLogFormat.RedactRxCommand(command, line.Span); + CommandLogMessages.CommandRx(logger, client, commandText); + } + + await Pipeline.SubmitAsync(command, line, cancellationToken).ConfigureAwait(false); + return; + } + + await TakeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + await Pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await DispatchCommandAsync(dispatcher, response, logger, command, line, preReadArticle, cancellationToken) + .ConfigureAwait(false); + } + finally + { + EndCommandWork(); + } + } + + /// + /// Waits until lastCommandActivity + IdleTime with no accepted command work, + /// then closes the session. One Delay per wait; no per-command timer or lock. + /// + private async Task WatchIdleAsync(CancellationToken cancellationToken) + { + Volatile.Write(ref _lastCommandTimestamp, _timeProvider.GetTimestamp()); + _idleWatchArmed.TrySetResult(); + try + { + while (!cancellationToken.IsCancellationRequested && Volatile.Read(ref _closeRequested) == 0) + { + if (Volatile.Read(ref _commandWork) > 0) + { + await Task.Delay(_commandIdleTimeout, _timeProvider, cancellationToken).ConfigureAwait(false); + continue; + } + + var last = Volatile.Read(ref _lastCommandTimestamp); + var epochAtIdle = Interlocked.Read(ref _idleState); + if ((epochAtIdle & IdleClosedFlag) != 0) + { + return; + } + + var elapsed = _timeProvider.GetElapsedTime(last, _timeProvider.GetTimestamp()); + var remaining = _commandIdleTimeout - elapsed; + if (remaining > TimeSpan.Zero) + { + await Task.Delay(remaining, _timeProvider, cancellationToken).ConfigureAwait(false); + continue; + } + + // Ownership snapshot: CAS this epoch later. BeginCommandWork increments + // the epoch before publishing work; if that increment lands first, + // CompareExchange(epochAtIdle) fails and this stale idle decision cannot close. + // Re-reading _idleState after the hold would observe the new epoch and + // incorrectly close after activity had already committed. + var aboutToCommit = IdleAboutToCommit; + aboutToCommit?.TrySetResult(); + if (IdleCommitHold is { } hold) + { + await hold.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Volatile.Read(ref _commandWork) > 0 + || Volatile.Read(ref _lastCommandTimestamp) != last) + { + continue; + } + + if (Interlocked.CompareExchange(ref _idleState, epochAtIdle | IdleClosedFlag, epochAtIdle) + != epochAtIdle) + { + continue; + } + + RequestCloseWithReason(TcpDisconnectReason.IdleTimeout); + return; + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// Waits until the CHECK window can accept another command (RX backpressure). + internal ValueTask WaitForCheckCapacityAsync(CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(Pipeline); + return Pipeline.IsFull + ? Pipeline.WaitForCapacityAsync(cancellationToken) + : ValueTask.CompletedTask; + } + + /// Waits until the TAKETHIS window can accept another command (RX backpressure). + internal ValueTask WaitForTakeThisCapacityAsync(CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(TakeThisWindow); + return TakeThisWindow.IsFull + ? TakeThisWindow.WaitForCapacityAsync(cancellationToken) + : ValueTask.CompletedTask; + } + + /// + /// Emits an already-decided news event. Failures are reported through diagnostics + /// and do not change protocol state. + /// + internal void TryWriteNews(in NewsLogEvent evt) + { + try + { + NewsLog.Write(in evt); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + var id = evt.MessageId.IsEmpty + ? "-" + : System.Text.Encoding.ASCII.GetString(evt.MessageId.Span); + SpoolLogMessages.NewsLogFailed(_logger, ex, id); + } + } + + /// Logs a parser rejection without converting the full command line to a string. + internal void LogCommandRejected(NntpCommand command, string detail) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + var client = NntpCommandLogFormat.Client(this); + CommandLogMessages.CommandRejected( + _logger, + client, + command.Verb, + command.Qualifier, + command.Status, + detail); + } + + /// + /// Peeks the application input pipe without consuming. Used to decide whether a partial + /// coalesced TX batch must flush before the session waits for more inbound octets. + /// + private static bool HasUnconsumedInput(PipeReader input) + { + try + { + if (!input.TryRead(out var result)) + { + return false; + } + + var has = !result.Buffer.IsEmpty; + // Peek only. Examined must stay at Start: AdvanceTo(Start, End) tells the Pipe + // the reader needs more data, so the next ReadAsync waits even though leftover + // TAKETHIS bytes are already buffered (same rule as NntpContinuousRxReader + // after a complete unit, and NntpConnection.EnsureApplicationInputDrainedForUpgrade). + input.AdvanceTo(result.Buffer.Start, result.Buffer.Start); + return has; + } + catch (InvalidOperationException) + { + return false; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpSessionCensus.cs b/src/VectorNNTPD.TransitService/Session/NntpSessionCensus.cs new file mode 100644 index 00000000..5a29f6d0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpSessionCensus.cs @@ -0,0 +1,145 @@ +using System.Collections.Concurrent; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Process-wide established-session registry. Independent of FeedDiagnostics. + /// + public sealed class NntpSessionCensus : INntpSessionCensus + { + private readonly ConcurrentDictionary _sessions = new(); + private readonly ITransitPeerMetrics? _peerMetrics; + + /// Initializes a census that optionally updates on register/unregister. + public NntpSessionCensus(ITransitPeerMetrics? peerMetrics = null) + { + _peerMetrics = peerMetrics; + } + + /// + public void Register(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + var peerId = session.Authorization.TransitPeerName; + _sessions[session] = peerId; + if (peerId is not null) + { + _peerMetrics?.RecordSessionRegistered(peerId); + } + } + + /// + public void Unregister(NntpSession session) + { + if (session is null) + { + return; + } + + if (_sessions.TryRemove(session, out var peerId) && peerId is not null) + { + _peerMetrics?.RecordSessionUnregistered(peerId); + } + } + + /// + public void ReleasePeerAttribution(NntpSession session) + { + ArgumentNullException.ThrowIfNull(session); + if (!_sessions.TryGetValue(session, out var peerId) || peerId is null) + { + return; + } + + if (_sessions.TryUpdate(session, null, peerId)) + { + _peerMetrics?.RecordSessionUnregistered(peerId); + } + } + + /// + public NntpSessionCensusSnapshot Capture() + { + var idle = 0; + var receiving = 0; + var waitingHistory = 0; + var waitingQueue = 0; + var waitingWindow = 0; + var completing = 0; + var active = 0; + var takeThisOccupied = 0; + var takeThisOccupiedMax = 0; + var takeThisFullSessions = 0; + var takeThisProcessing = 0; + var takeThisProcessingMax = 0; + foreach (var session in _sessions.Keys) + { + active++; + switch (session.ActivityState) + { + case FeedSessionState.Receiving: + receiving++; + break; + case FeedSessionState.WaitingHistory: + waitingHistory++; + break; + case FeedSessionState.WaitingQueue: + waitingQueue++; + break; + case FeedSessionState.WaitingWindow: + waitingWindow++; + break; + case FeedSessionState.Completing: + completing++; + break; + default: + idle++; + break; + } + + var window = session.TakeThisWindow; + if (window is null) + { + continue; + } + + var occupied = window.Occupied; + takeThisOccupied += occupied; + if (occupied > takeThisOccupiedMax) + { + takeThisOccupiedMax = occupied; + } + + if (occupied >= TakeThisPipeline.Depth) + { + takeThisFullSessions++; + } + + var processing = window.ActiveArticleProcessing; + takeThisProcessing += processing; + var processingPeak = window.MaxActiveArticleProcessing; + if (processingPeak > takeThisProcessingMax) + { + takeThisProcessingMax = processingPeak; + } + } + + return new NntpSessionCensusSnapshot( + active, + active, + idle, + receiving, + waitingHistory, + waitingQueue, + waitingWindow, + completing, + takeThisOccupied, + takeThisOccupiedMax, + takeThisFullSessions, + takeThisProcessing, + takeThisProcessingMax); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpSessionLifetime.cs b/src/VectorNNTPD.TransitService/Session/NntpSessionLifetime.cs new file mode 100644 index 00000000..02f3a584 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpSessionLifetime.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// Observable lifetime of an connection. + /// + /// → is the one-shot transition that may + /// release distributed admission. Later teardown signals observe + /// or and do nothing. + /// + public enum NntpSessionLifetime + { + /// Constructed; has not started. + Created = 0, + + /// Greeting/command loop is running. + Running = 1, + + /// The first teardown path has claimed cleanup. + Finalizing = 2, + + /// Admission release and connection cleanup have finished. + Finalized = 3, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpSessionMode.cs b/src/VectorNNTPD.TransitService/Session/NntpSessionMode.cs new file mode 100644 index 00000000..5eeb32e0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpSessionMode.cs @@ -0,0 +1,19 @@ +namespace VectorNNTPD.TransitService.Session +{ + /// NNTP operating mode (distinct from authentication/authorization). + public enum NntpSessionMode + { + /// No MODE command has successfully selected an operating mode yet. + Unspecified = 0, + + /// Reader mode after a successful MODE READER. + Reader = 1, + + /// + /// Transit/streaming operating mode. MODE STREAM does not set this (RFC 4644 §2.3); + /// AUTHINFO Transit authority is selected separately via + /// . + /// + Stream = 2, + } +} diff --git a/src/VectorNNTPD.TransitService/Session/NntpStreamDataPlaneRx.cs b/src/VectorNNTPD.TransitService/Session/NntpStreamDataPlaneRx.cs new file mode 100644 index 00000000..50c8f152 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/NntpStreamDataPlaneRx.cs @@ -0,0 +1,131 @@ +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Default / transit STREAM RX: continuous command/article units from the connection input pipe. + /// + /// + /// Authorized TAKETHIS is admitted into : HistoryDB peek starts + /// at the command line, then this RX task frames one stuffed-wire copy with + /// , attaches the owned buffer to a slot, and returns so + /// the next command can be parsed. Pipeline workers never read Connection.Input. + /// Unauthorized TAKETHIS is returned as a command line so 480/502 gates do not consume a + /// following QUIT. + /// + internal sealed class NntpStreamDataPlaneRx + { + private readonly NntpSession _session; + private readonly NntpCommandDispatcher _dispatcher; + private readonly ILogger _logger; + private readonly NntpContinuousRxParser _parser = new(); + + /// Initializes a new instance of the class. + public NntpStreamDataPlaneRx(NntpSession session, NntpCommandDispatcher dispatcher, ILogger logger) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(dispatcher); + ArgumentNullException.ThrowIfNull(logger); + _session = session; + _dispatcher = dispatcher; + _logger = logger; + } + + /// + /// Reads and dispatches one protocol unit. Returns on EOF or an + /// incomplete TAKETHIS article. + /// + public async ValueTask ProcessOneAsync(NntpResponseWriter response, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(response); + if (_session.Pipeline?.IsFull == true || _session.TakeThisWindow?.IsFull == true) + { + _session.SetActivityState(FeedSessionState.WaitingWindow); + } + + await _session.WaitForCheckCapacityAsync(cancellationToken).ConfigureAwait(false); + await _session.WaitForTakeThisCapacityAsync(cancellationToken).ConfigureAwait(false); + _session.SetActivityState(FeedSessionState.Idle); + + // Command line only. Authorized TAKETHIS starts HistoryDB peek, frames the + // article on this RX task, hands the owned buffer to the pipeline, and returns. + // Unauthorized TAKETHIS stays a command so 480/502 gates do not consume a + // following QUIT. + var unit = await NntpContinuousRxReader + .ReadUnitAsync( + _session.Connection.Input, + _parser, + consumeTakeThisArticle: false, + _session.ArticleIngestion.MaxArticleBytes, + cancellationToken) + .ConfigureAwait(false); + + if (unit.Kind == NntpContinuousRxKind.NeedMore) + { + return false; + } + + if (_session.HasSaslExchange) + { + _session.BeginCommandWork(); + try + { + var takeThisWindow = _session.TakeThisWindow + ?? throw new InvalidOperationException( + "SASL exchange requires an initialized TakeThis admission window."); + var pipeline = _session.Pipeline + ?? throw new InvalidOperationException( + "SASL exchange requires an initialized CHECK pipeline."); + await takeThisWindow.DrainAsync(cancellationToken).ConfigureAwait(false); + await pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await Commands.AuthInfo + .ContinueSaslAsync(_session, response, _parser.CurrentCommandLine, cancellationToken) + .ConfigureAwait(false); + } + finally + { + _session.EndCommandWork(); + } + + return true; + } + + // Transit TAKETHIS admit: only on a Transit host with transit authority. + // Reader hosts leave TAKETHIS as a command line so the dispatcher returns 502. + if (unit.Command.IsValid + && unit.Command.Verb == NntpVerb.TakeThis + && _session.ServerRole == Hosting.NntpServerRole.Transit + && _session.Authorization.AuthorizedTransit) + { + // Capture once: nullable session properties must not be re-read between check and use + // (concurrent teardown can clear them after a prior null check). + var takeThisWindow = _session.TakeThisWindow + ?? throw new InvalidOperationException( + "Transit TAKETHIS admit requires an initialized TakeThis admission window."); + var pipeline = _session.Pipeline + ?? throw new InvalidOperationException( + "Transit TAKETHIS admit requires an initialized CHECK pipeline."); + await pipeline.DrainAsync(cancellationToken).ConfigureAwait(false); + await takeThisWindow + .AdmitAuthorizedAsync(unit.Command, _parser.CurrentCommandLine, cancellationToken) + .ConfigureAwait(false); + return true; + } + + var preRead = unit.Kind == NntpContinuousRxKind.TakeThis ? unit.Article : (NntpMultilineReadResult?)null; + await _session + .ProcessParsedCommandAsync( + _dispatcher, + response, + _logger, + unit.Command, + _parser.CurrentCommandLine, + preRead, + cancellationToken) + .ConfigureAwait(false); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/SessionLogMessages.cs b/src/VectorNNTPD.TransitService/Session/SessionLogMessages.cs new file mode 100644 index 00000000..f7894869 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/SessionLogMessages.cs @@ -0,0 +1,32 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Session +{ + /// Source-generated session and transit-identification log messages. + internal static partial class SessionLogMessages + { + [LoggerMessage( + EventId = 1611, + Level = LogLevel.Debug, + Message = "NNTP session ended with an error for {Client}")] + public static partial void SessionEndedWithError(ILogger logger, Exception exception, IPAddress Client); + + [LoggerMessage( + EventId = 1612, + Level = LogLevel.Debug, + Message = "NNTP session ended; peer or transport disconnected for {Client}")] + public static partial void SessionEndedByPeerDisconnect(ILogger logger, Exception exception, IPAddress Client); + + [LoggerMessage( + EventId = 1700, + Level = LogLevel.Information, + Message = "Trusted Transit peers configured ({Count}): {Peers}")] + public static partial void TransitPeersConfigured(ILogger logger, int Count, string Peers); + + [LoggerMessage( + EventId = 1701, + Level = LogLevel.Warning, + Message = "Transit peer identification is ambiguous for {ClientAddress}; matching peers: {PeerNames}")] + public static partial void TransitPeerAmbiguous(ILogger logger, IPAddress ClientAddress, string PeerNames); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/SpeedTest/ISpeedTestCoordinator.cs b/src/VectorNNTPD.TransitService/Session/SpeedTest/ISpeedTestCoordinator.cs new file mode 100644 index 00000000..61d8cfb1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/SpeedTest/ISpeedTestCoordinator.cs @@ -0,0 +1,66 @@ +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.SpeedTest +{ + /// + /// Read-only Transit lookup and concurrency gate for SPEEDTEST. + /// + /// + /// Resolves Transit identifiers against the current immutable snapshot. Does not open + /// outbound sockets and does not retain a mutable options object across a reload. + /// + public interface ISpeedTestCoordinator + { + /// + /// Resolves against the current Transit snapshot + /// using exact ordinal identifier match (the configuration dictionary key). + /// + /// + /// when a configured Transit peer exists with that exact identifier. + /// + bool TryResolvePeer(ReadOnlySpan identifier, out TransitPeerPolicy? peer); + + /// + /// Attempts to acquire a SPEEDTEST slot for using + /// the current configured limits. + /// + bool TryAcquire(string identifier, out SpeedTestLease? lease); + } + + /// + /// Held SPEEDTEST concurrency slot. Dispose releases host and per-peer counters. + /// + public sealed class SpeedTestLease : IDisposable + { + private readonly Action _release; + private readonly string _peerName; + private int _disposed; + + internal SpeedTestLease(string peerName, SpeedTestLimits limits, Action release) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + ArgumentNullException.ThrowIfNull(release); + _peerName = peerName; + Limits = limits; + _release = release; + } + + /// Gets the limits captured when this slot was acquired. + public SpeedTestLimits Limits { get; } + + /// Gets the Transit identifier this slot is counted against. + public string PeerName => _peerName; + + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _release(_peerName); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestCoordinator.cs b/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestCoordinator.cs new file mode 100644 index 00000000..04fb43a5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestCoordinator.cs @@ -0,0 +1,205 @@ +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session.SpeedTest +{ + /// + /// Snapshot-backed peer lookup and bounded SPEEDTEST concurrency. + /// + /// + /// Each resolve reads . Limits are read from + /// at acquire time. In-flight tests keep the limits + /// captured on the lease. This type does not initiate outbound Transit connections. + /// + public sealed class SpeedTestCoordinator : ISpeedTestCoordinator + { + private readonly TransitConfigurationStore _store; + private readonly IOptionsMonitor _options; + private readonly object _gate = new(); + private readonly ConcurrentDictionary _perPeer = new(StringComparer.Ordinal); + private int _global; + + /// Creates a coordinator bound to the live Transit store and NNTPD options. + public SpeedTestCoordinator(TransitConfigurationStore store, IOptionsMonitor options) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(options); + _store = store; + _options = options; + } + + /// Creates a coordinator for tests with a fixed options snapshot. + public static SpeedTestCoordinator Create( + TransitConfigurationStore store, + NntpdOptions? options = null) => + new(store, new StaticOptions(options ?? new NntpdOptions())); + + /// + public bool TryResolvePeer(ReadOnlySpan peerName, out TransitPeerPolicy? peer) + { + peer = null; + if (peerName.IsEmpty) + { + return false; + } + + var snapshot = _store.Current; + foreach (var candidate in snapshot.Peers.Values) + { + if (NameEquals(candidate.Identifier, peerName)) + { + peer = candidate; + return true; + } + } + + return false; + } + + /// + public bool TryAcquire(string peerName, out SpeedTestLease? lease) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + lease = null; + var speed = _options.CurrentValue.SpeedTest ?? new SpeedTestOptions(); + var limits = speed.Snapshot(); + + lock (_gate) + { + if (_global >= limits.MaxConcurrent) + { + return false; + } + + var peerCount = _perPeer.GetValueOrDefault(peerName); + if (peerCount >= limits.MaxConcurrentPerPeer) + { + return false; + } + + _global++; + _perPeer[peerName] = peerCount + 1; + } + + lease = new SpeedTestLease(peerName, limits, Release); + return true; + } + + /// Gets the current host-wide in-flight count (tests). + internal int GlobalInFlight + { + get + { + lock (_gate) + { + return _global; + } + } + } + + /// Gets the current in-flight count for (tests). + internal int InFlightFor(string peerName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + lock (_gate) + { + return _perPeer.GetValueOrDefault(peerName); + } + } + + private void Release(string peerName) + { + lock (_gate) + { + _global = Math.Max(0, _global - 1); + if (_perPeer.TryGetValue(peerName, out var count)) + { + if (count <= 1) + { + _perPeer.TryRemove(peerName, out _); + } + else + { + _perPeer[peerName] = count - 1; + } + } + } + } + + /// + /// Exact Transit name match. ASCII names compare as ordinal bytes; non-ASCII names + /// compare as UTF-8 of the configured string (the dictionary key representation). + /// + internal static bool NameEquals(string name, ReadOnlySpan token) + { + ArgumentNullException.ThrowIfNull(name); + if (IsAscii(name)) + { + if (name.Length != token.Length) + { + return false; + } + + for (var i = 0; i < name.Length; i++) + { + if ((byte)name[i] != token[i]) + { + return false; + } + } + + return true; + } + + var max = Encoding.UTF8.GetMaxByteCount(name.Length); + if (max > 1024) + { + return false; + } + + Span utf8 = stackalloc byte[max]; + if (!Encoding.UTF8.TryGetBytes(name, utf8, out var written)) + { + return false; + } + + return utf8[..written].SequenceEqual(token); + } + + private static bool IsAscii(string name) + { + foreach (var ch in name) + { + if (ch > 0x7F) + { + return false; + } + } + + return true; + } + + private sealed class StaticOptions : IOptionsMonitor + { + public StaticOptions(NntpdOptions value) => CurrentValue = value; + + public NntpdOptions CurrentValue { get; } + + public NntpdOptions Get(string? name) => CurrentValue; + + public IDisposable OnChange(Action listener) => EmptyDisposable.Instance; + } + + private sealed class EmptyDisposable : IDisposable + { + public static readonly EmptyDisposable Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestPayload.cs b/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestPayload.cs new file mode 100644 index 00000000..d70c9f86 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/SpeedTest/SpeedTestPayload.cs @@ -0,0 +1,71 @@ +namespace VectorNNTPD.TransitService.Session.SpeedTest +{ + /// + /// Deterministic synthetic SPEEDTEST payload. One immortal chunk is reused for every write. + /// + /// + /// + /// The payload is not an article, is not read from disk, and is not customer or feed data. + /// It is a fixed ASCII pattern chosen so NNTP dot-stuffing never applies: no line begins + /// with .. Content characteristics are therefore irrelevant to compression or + /// article-pipeline behaviour. + /// + /// + /// Each line is octets: one #, then repeating digits + /// 0–9, then CRLF. is concatenated + /// lines (). Callers write a CRLF-aligned slice of that buffer + /// and must not retain a reference after the awaited flush when they later reuse the + /// same immortal memory for the next write (writes are sequential and flush-awaited). + /// + /// + internal static class SpeedTestPayload + { + /// One payload line including CRLF. + public const int LineBytes = 1024; + + /// Lines packed into the reusable chunk. + public const int LinesPerChunk = 64; + + /// Reusable chunk size (64 KiB). + public const int ChunkBytes = LineBytes * LinesPerChunk; + + /// Immortal reusable chunk. The TX pump only reads it. + public static readonly ReadOnlyMemory Chunk = BuildChunk(); + + /// + /// Returns a CRLF-aligned prefix of no larger than + /// . Empty when fewer than remain. + /// + public static ReadOnlyMemory Take(long remainingBytes) + { + if (remainingBytes < LineBytes) + { + return ReadOnlyMemory.Empty; + } + + var take = remainingBytes >= ChunkBytes + ? ChunkBytes + : (int)(remainingBytes - (remainingBytes % LineBytes)); + return Chunk[..take]; + } + + private static byte[] BuildChunk() + { + var chunk = new byte[ChunkBytes]; + for (var line = 0; line < LinesPerChunk; line++) + { + var offset = line * LineBytes; + chunk[offset] = (byte)'#'; + for (var i = 1; i < LineBytes - 2; i++) + { + chunk[offset + i] = (byte)('0' + (i % 10)); + } + + chunk[offset + LineBytes - 2] = (byte)'\r'; + chunk[offset + LineBytes - 1] = (byte)'\n'; + } + + return chunk; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/TakeThisPipeline.cs b/src/VectorNNTPD.TransitService/Session/TakeThisPipeline.cs new file mode 100644 index 00000000..d04ce7fc --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/TakeThisPipeline.cs @@ -0,0 +1,1141 @@ +using Microsoft.Extensions.Logging; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.Session +{ + using VectorNNTPD.Common.Core; + /// + /// Per-session bounded TAKETHIS window: one RX owner frames articles, then detached + /// slots complete Peek / enqueue / ordered 239 off the RX stack. + /// + /// + /// + /// The session RX task is the only consumer of Connection.Input. It parses the + /// TAKETHIS command, starts HistoryDB peek, frames the article with + /// (one owned stuffed-wire buffer, terminator omitted), + /// attaches that buffer to a slot, and returns. Each occupied slot destuffs and + /// builds on its own completion + /// task. They never call ReadAsync on the session pipe. Depth matches the + /// CHECK window and bounds how many owned article buffers and process results + /// may be retained. + /// + /// + /// After detach, waits for Peek if needed, then + /// destuffs and builds ArticleRecord on that slot's task (concurrent across + /// occupied slots). Enqueue / Remember / news - / ordered 239/439/400 stay + /// behind _emitGate so publication remains command-ordered. Emit never runs + /// on the RX task. A slot is occupied from Message-ID copy / peek start until the + /// response writer accepts the ordered line (or cancel). + /// + /// + internal sealed class TakeThisPipeline + { + /// + /// Outstanding TAKETHIS executions per session. Architectural constant; not configurable. + /// + public const int Depth = CheckPipeline.Depth; + + private readonly NntpSession _session; + private readonly NntpResponseWriter _response; + private readonly ILogger _logger; + private readonly Slot?[] _slots; + private readonly Lock _gate = new(); + // codeql[cs/missed-using-statement] Owned by TakeThisPipeline until ShutdownAsync; a using at this declaration would release it when the constructor returns. + private readonly SemaphoreSlim _emitGate = new(1, 1); + // codeql[cs/missed-using-statement] Owned by TakeThisPipeline until ShutdownAsync; a using at this declaration would cancel in-flight work when the constructor returns. + private readonly CancellationTokenSource _lifetimeCts = new(); + private readonly TakeThisStageSessionLog? _stageLog; + private int _head; + private int _count; + private int _peakOccupied; + private int _shutDown; + private int _inFlightCompletions; + private int _activeArticleReads; + private int _maxActiveArticleReads; + private int _activeArticleProcessing; + private int _maxActiveArticleProcessing; + private TaskCompletionSource _progress = NewProgress(); + + /// Initializes a new instance of the class. + public TakeThisPipeline(NntpSession session, NntpResponseWriter response) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(response); + _session = session; + _response = response; + _logger = NntpCommandLoggers.For(typeof(TakeThis)); + _slots = new Slot?[Depth]; + _stageLog = TakeThisStageProbe.CreateSessionLog(); + } + + /// Gets the number of occupied slots. + public int Occupied + { + get + { + lock (_gate) + { + return _count; + } + } + } + + /// Gets whether the window cannot accept another TAKETHIS without waiting. + public bool IsFull => Occupied >= Depth; + + /// Gets the peak occupied slot count observed on this session. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Gets the number of completion tasks still running (at most ). + internal int InFlightCompletions => Volatile.Read(ref _inFlightCompletions); + + /// + /// Test-only: set after HistoryDB peek is started and before article receive begins. + /// + internal Action? AfterPeekStarted { get; set; } + + /// + /// Test-only: set immediately before is awaited. + /// + internal Action? AfterReceiveStarted { get; set; } + + /// + /// Test-only: set after returns (including cancel). + /// + internal Action? AfterReceiveCompleted { get; set; } + + /// + /// Test-only: when set, awaits this task after + /// and before . + /// Production never sets this. + /// + internal Task? HoldReceive { get; set; } + + /// + /// Test-only: invoked after the owned article is stored on the slot and before + /// admission returns to the RX loop. Production never sets this. + /// + internal Action>? AfterArticleDetached { get; set; } + + /// + /// Test-only: awaited after response composition and before + /// . + /// The slot remains occupied. Production never sets this. + /// + internal Func? BeforeEnqueueProbe { get; set; } + + /// Gets how many calls are on this session's RX stack. + internal int ActiveArticleReads => Volatile.Read(ref _activeArticleReads); + + /// Gets the peak concurrent article-receive count observed on this session (must stay 1). + internal int MaxActiveArticleReads => Volatile.Read(ref _maxActiveArticleReads); + + /// + /// Gets how many destuff/parse/CRC tasks are running on this session (at most ). + /// + internal int ActiveArticleProcessing => Volatile.Read(ref _activeArticleProcessing); + + /// Gets the peak concurrent destuff/parse/CRC count observed on this session. + internal int MaxActiveArticleProcessing => Volatile.Read(ref _maxActiveArticleProcessing); + + /// + /// Test-only: awaited on a slot after Peek is Unseen and before destuff/parse/CRC. + /// Production never sets this. + /// + internal Func, Task>? HoldProcess { get; set; } + + /// + /// Test-only: invoked when destuff/parse/CRC starts for a slot. Production never sets this. + /// + internal Action>? AfterProcessStarted { get; set; } + + /// + /// Test-only: invoked when destuff/parse/CRC finishes for a slot. Production never sets this. + /// + internal Action>? AfterProcessCompleted { get; set; } + + /// Waits until a slot is free. The caller must not read RX while this is pending. + public async ValueTask WaitForCapacityAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + ThrowIfShutDown(); + if (_count < Depth) + { + return; + } + + wait = _progress.Task; + } + + var waitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordOccupiedFullWait(waitStart); + } + } + + /// + /// Starts HistoryDB peek, frames the STREAM article on the RX task, attaches the + /// owned buffer to a slot, and returns. Does not wait for Peek, destuff, enqueue, + /// Remember, or 239 after the article is detached from the PipeReader. + /// + public async ValueTask AdmitAuthorizedAsync( + NntpCommand command, + ReadOnlyMemory line, + CancellationToken cancellationToken) + { + ThrowIfShutDown(); + var probe = _session.FeedProbe; + probe?.RecordCommand(); + _session.SetActivityState(FeedSessionState.Receiving); + var ownedId = command.ArgumentMemory(line).ToArray(); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var marks = _stageLog is null ? null : new TakeThisStageMarks + { + CommandParsedTs = started, + PeekStartTs = started, + }; + var lookup = marks is not null + ? marks.ObservePeek(TakeThis.PeekAsync(_session, ownedId, cancellationToken)) + : TakeThis.PeekAsync(_session, ownedId, cancellationToken); + + AfterPeekStarted?.Invoke(); + + Slot slot; + lock (_gate) + { + ThrowIfShutDown(); + if (_count >= Depth) + { + throw new InvalidOperationException("TAKETHIS pipeline accepted a command without a free slot."); + } + + NntpArticlePolicyCapture.Capture( + _session, + _session.ArticleIngestion.MaxArticleBytes, + out var capturedMaxArticleBytes, + out var capturedSiteName); + slot = new Slot + { + MessageId = ownedId, + StartedTimestamp = started, + Index = (_head + _count) % Depth, + Marks = marks, + MaxArticleBytes = capturedMaxArticleBytes, + SiteNameUtf8 = capturedSiteName, + }; + _slots[slot.Index] = slot; + _count++; + _session.BeginCommandWork(); + if (_count > _peakOccupied) + { + _peakOccupied = _count; + } + + _stageLog?.NoteOccupied(_count); + IngestionPipelineMetrics.Shared.ObserveTakeThis( + _count, + Volatile.Read(ref _activeArticleProcessing)); + } + + marks?.ReceiveStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + + AfterReceiveStarted?.Invoke(); + var reads = Interlocked.Increment(ref _activeArticleReads); + ObservePeak(ref _maxActiveArticleReads, reads); + + try + { + IHaveArticleReadResult read; + try + { + if (HoldReceive is { } holdReceive) + { + await holdReceive.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + read = await IHaveArticleReader + .ReadAsync(_session.Connection.Input, slot.MaxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + Interlocked.Decrement(ref _activeArticleReads); + AfterReceiveCompleted?.Invoke(); + CancelAndRelease(slot); + return; + } + + Interlocked.Decrement(ref _activeArticleReads); + AfterReceiveCompleted?.Invoke(); + + if (marks is not null) + { + marks.ReceiveCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + marks.ArticleBytes = read.Payload.Length; + } + + if (probe is not null) + { + var receiveStart = marks?.ReceiveStartTs ?? started; + probe.RecordArticleReceived( + read.Payload.Length, + System.Diagnostics.Stopwatch.GetTimestamp() - receiveStart); + } + + _session.RecordPeerArticleReceived(read.Payload.Length); + _session.SetActivityState(FeedSessionState.WaitingHistory); + + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.Read = read; + slot.ArticleReady = true; + SignalProgressNoLock(); + } + + AfterArticleDetached?.Invoke(read.Payload); + _ = CompleteWhenAsync(slot, lookup, cancellationToken); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + CancelAndRelease(slot); + throw; + } + } + + /// Waits until every slot has been accepted by the TX writer (or cancelled). + public async ValueTask DrainAsync(CancellationToken cancellationToken) + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// Stops emission and waits until every slot is released. + public async ValueTask ShutdownAsync() + { + if (Interlocked.Exchange(ref _shutDown, 1) == 1) + { + return; + } + + try + { + await _lifetimeCts.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + lock (_gate) + { + for (var i = 0; i < Depth; i++) + { + if (_slots[i] is { } slot) + { + slot.Cancelled = true; + } + } + + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + try + { + while (true) + { + Task wait; + lock (_gate) + { + if (_count == 0) + { + return; + } + + wait = _progress.Task; + } + + await wait.ConfigureAwait(false); + } + } + finally + { + _lifetimeCts.Dispose(); + WriteStageLog(); + } + } + + private async Task CompleteWhenAsync( + Slot slot, + ValueTask lookup, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _inFlightCompletions); + try + { + // Leave the RX stack before Peek/enqueue/239, including when Peek already completed. + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + + HistoryLookupResult peek; + var peekWaitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + peek = await lookup.ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) + { + peek = HistoryLookupResult.Unavailable; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + peek = HistoryLookupResult.Unavailable; + } + + _session.FeedProbe?.RecordHistory( + peek, + System.Diagnostics.Stopwatch.GetTimestamp() - peekWaitStart); + + if (Volatile.Read(ref _shutDown) == 1 + || cancellationToken.IsCancellationRequested) + { + CancelAndRelease(slot); + return; + } + + ArticleProcessOutcome process = default; + try + { + using var processCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _lifetimeCts.Token); + process = await ProcessArticleAsync(slot, peek, processCts.Token).ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + CancelAndRelease(slot); + return; + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + return; + } + + var emit = false; + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + if (Volatile.Read(ref _shutDown) == 1) + { + slot.PeekCompleted = true; + slot.Processed = true; + slot.Cancelled = true; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot.PeekCompleted = true; + slot.Peek = peek; + slot.Processed = true; + slot.Process = process; + emit = HeadIsReadyNoLock(); + SignalProgressNoLock(); + } + + if (emit) + { + try + { + await EmitReadySerializedAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || Volatile.Read(ref _shutDown) == 1) + { + CancelAndRelease(slot); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + CancelAndRelease(slot); + } + finally + { + Interlocked.Decrement(ref _inFlightCompletions); + } + } + + private async ValueTask EmitReadySerializedAsync(CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetimeCts.Token); + var token = linked.Token; + var emitGateStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await _emitGate.WaitAsync(token).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordEmitGateWait(emitGateStart); + try + { + while (true) + { + Slot? slot; + lock (_gate) + { + if (Volatile.Read(ref _shutDown) == 1) + { + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + return; + } + + slot = TryBeginEmitHeadNoLock(); + } + + if (slot is null) + { + return; + } + + try + { + await EmitOccupiedAsync(slot, token).ConfigureAwait(false); + lock (_gate) + { + ReleaseHeadIfNoLock(slot); + SignalProgressNoLock(); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception) + { + lock (_gate) + { + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + + throw; + } + } + } + finally + { + _emitGate.Release(); + } + } + + private async ValueTask EmitOccupiedAsync(Slot slot, CancellationToken cancellationToken) + { + if (slot.Marks is { } emitMarks) + { + emitMarks.EmitStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + emitMarks.OccupiedAtEmit = Occupied; + } + + var read = slot.Read; + if (read.Status == NntpMultilineReadStatus.Incomplete) + { + return; + } + + if (read.Status == NntpMultilineReadStatus.TooLarge) + { + const string tooLarge = "rejected too large"; + IngressNewsEvents.TryWriteRejected( + _session, + SlotMessageId(slot), + 439, + IngressNewsReasons.ArticleTooLarge, + read.Payload.Length); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, tooLarge, TransferLogKind.Rejected); + return; + } + + if (slot.Peek == HistoryLookupResult.Unavailable) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + if (slot.Peek == HistoryLookupResult.Seen) + { + if (slot.Marks is { } seenMarks) + { + seenMarks.Outcome = "seen"; + } + + var probe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var responseStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.ArticleTransferredOkPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, "accepted duplicate", TransferLogKind.Accepted); + probe?.RecordArticleCompleted( + duplicate: true, + System.Diagnostics.Stopwatch.GetTimestamp() - responseStart); + _session.SetActivityState(FeedSessionState.Idle); + RecordAccepted(slot); + return; + } + + var queue = _session.ArticleIngestion; + if (!queue.IsAccepting) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + var messageIdText = System.Text.Encoding.ASCII.GetString(slot.MessageId); + var process = slot.Process; + if (process.RecordRejected) + { + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + IngressNewsReasons.ForExistingRejectDetail(process.RejectDetail), + process.Size); + var rejectProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var rejectStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, process.RejectDetail ?? string.Empty, TransferLogKind.Rejected); + rejectProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - rejectStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + if (process.UncarriedRejected) + { + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + process.UncarriedReason ?? IngressNewsEvents.NewsgroupNotCarried, + process.Inbound?.Payload.Length ?? read.Payload.Length); + var uncarriedProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var uncarriedStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, IngressNewsEvents.NewsgroupNotCarried, TransferLogKind.Rejected); + uncarriedProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - uncarriedStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + var inbound = process.Inbound; + if (inbound is null) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + ArticleEnqueueResult enqueue; + try + { + if (slot.Marks is { } enqueueMarks) + { + enqueueMarks.EnqueueStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + var probe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.WaitingQueue); + var queueStart = System.Diagnostics.Stopwatch.GetTimestamp(); + enqueue = await queue.EnqueueAsync(inbound, cancellationToken).ConfigureAwait(false); + IngestionPipelineMetrics.Shared.RecordTakeThisEnqueue(queueStart); + probe?.RecordQueue(enqueue, System.Diagnostics.Stopwatch.GetTimestamp() - queueStart, read.Payload.Length); + if (slot.Marks is { } acceptedMarks) + { + acceptedMarks.EnqueueAcceptedTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested + || _session.Connection.ConnectionClosed.IsCancellationRequested) + { + return; + } + + if (enqueue == ArticleEnqueueResult.Unavailable) + { + await TakeThis.FailTemporaryAsync(_session, _response, cancellationToken).ConfigureAwait(false); + WriteTxIfDebug(slot, "temporary failure", TransferLogKind.Temporary); + return; + } + + if (enqueue == ArticleEnqueueResult.Rejected) + { + const string budget = "rejected exceeds queue budget"; + IngressNewsEvents.TryWriteRejected( + _session, + messageIdText, + 439, + IngressNewsReasons.QueueCapacityExceeded, + inbound.Payload.Length); + var rejectProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var rejectStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.TransferRejectedPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, budget, TransferLogKind.Rejected); + rejectProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - rejectStart); + _session.SetActivityState(FeedSessionState.Idle); + return; + } + + if (slot.Marks is { } rememberMarks) + { + rememberMarks.Outcome = "unseen"; + rememberMarks.RememberStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + _session.HistoryDb?.Remember(slot.MessageId); + if (slot.Marks is { } rememberedMarks) + { + rememberedMarks.RememberCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + var completeProbe = _session.FeedProbe; + _session.SetActivityState(FeedSessionState.Completing); + var completeStart = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyAsync(slot, NntpResponses.ArticleTransferredOkPrefix, cancellationToken) + .ConfigureAwait(false); + WriteTxIfDebug(slot, "accepted", TransferLogKind.Accepted); + completeProbe?.RecordArticleCompleted( + duplicate: false, + System.Diagnostics.Stopwatch.GetTimestamp() - completeStart); + _session.SetActivityState(FeedSessionState.Idle); + RecordAccepted(slot); + } + + private enum TransferLogKind + { + Accepted, + Rejected, + Temporary, + } + + private void WriteTxIfDebug(Slot slot, string detail, TransferLogKind kind) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + var status = kind switch + { + TransferLogKind.Rejected => NntpCommandStatusText.FormatTakeThisRejected(slot.MessageId), + TransferLogKind.Temporary => NntpResponseStatus.ServiceTemporarilyUnavailable, + _ => NntpCommandStatusText.FormatTakeThisAccepted(slot.MessageId), + }; + TakeThis.WriteCompletion(_logger, _session, slot.StartedTimestamp, detail, status); + } + + private ValueTask EnqueueReplyAsync( + Slot slot, + ReadOnlyMemory prefix, + CancellationToken cancellationToken) + { + var owned = NntpResponseCompose.Concat(prefix.Span, slot.MessageId, NntpResponses.Crlf.Span); + if (slot.Marks is { } marks) + { + return EnqueueReplyTimedAsync(marks, owned, cancellationToken); + } + + return EnqueueReplyImmediateAsync(owned, cancellationToken); + } + + private async ValueTask EnqueueReplyImmediateAsync( + ReadOnlyMemory owned, + CancellationToken cancellationToken) + { + if (BeforeEnqueueProbe is { } probe) + { + await probe().ConfigureAwait(false); + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + } + + await _response.EnqueueLineImmediateAsync(owned, cancellationToken).ConfigureAwait(false); + } + + private async ValueTask EnqueueReplyTimedAsync( + TakeThisStageMarks marks, + ReadOnlyMemory owned, + CancellationToken cancellationToken) + { + marks.ReplyEnqueueStartTs = System.Diagnostics.Stopwatch.GetTimestamp(); + await EnqueueReplyImmediateAsync(owned, cancellationToken).ConfigureAwait(false); + marks.ReplyEnqueueCompleteTs = System.Diagnostics.Stopwatch.GetTimestamp(); + } + + private void RecordAccepted(Slot slot) + { + if (_stageLog is null || slot.Marks is not { } marks) + { + return; + } + + _stageLog.Record(marks.ToSample(_peakOccupied)); + } + + private void WriteStageLog() + { + if (_stageLog is null || !TakeThisStageProbe.IsEnabled) + { + return; + } + + var dir = TakeThisStageProbe.OutputDirectory; + if (dir is null) + { + return; + } + + try + { + var remote = _session.ClientIdentity.TcpPeer?.ToString() ?? "session"; + var path = _stageLog.Write(dir, remote); + if (_logger.IsEnabled(LogLevel.Information)) + { + var samples = _stageLog.Count; + _logger.LogInformation( + "TAKETHIS stage timing wrote {Path} samples={Samples} peak={Peak}", + path, + samples, + _stageLog.PeakOccupied); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Diagnostic dump must not affect shutdown. + } + } + + private bool HeadIsReadyNoLock() + { + if (_count == 0) + { + return false; + } + + var slot = _slots[_head]; + return slot is + { + ArticleReady: true, + PeekCompleted: true, + Processed: true, + Cancelled: false, + Emitting: false, + }; + } + + /// + /// Destuff/parse/CRC for an unseen completed article. Independent per slot. + /// History Seen/Unavailable and receive failures skip this work. + /// + /// Occupied slot that already owns stuffed wire. + /// Already-awaited HistoryDB result for this slot. + /// Session or pipeline lifetime cancellation. + /// The destuff/parse outcome for the ordered publisher. + private async ValueTask ProcessArticleAsync( + Slot slot, + HistoryLookupResult peek, + CancellationToken cancellationToken) + { + var read = slot.Read; + if (read.Status != NntpMultilineReadStatus.Completed + || peek != HistoryLookupResult.Unseen) + { + return default; + } + + var processing = Interlocked.Increment(ref _activeArticleProcessing); + ObservePeak(ref _maxActiveArticleProcessing, processing); + + AfterProcessStarted?.Invoke(slot.MessageId); + try + { + if (HoldProcess is { } hold) + { + await hold(slot.MessageId).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Volatile.Read(ref _shutDown) == 1 || cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + var messageIdText = SlotMessageId(slot); + if (!TakeThis.TryCreateQueuedRecord( + _session, + read.Payload, + stuffed: true, + messageIdText, + slot.MaxArticleBytes, + slot.SiteNameUtf8, + out var inbound, + out var recordReject, + out var canonicalSize)) + { + return new ArticleProcessOutcome + { + RecordRejected = true, + RejectDetail = recordReject, + Size = canonicalSize != 0 ? canonicalSize : read.Payload.Length, + }; + } + + if (IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + _session.Transit, + _session.NewsgroupCatalogue, + out var uncarriedReason)) + { + return new ArticleProcessOutcome + { + UncarriedRejected = true, + UncarriedReason = uncarriedReason, + Inbound = inbound, + }; + } + + return new ArticleProcessOutcome { Inbound = inbound }; + } + finally + { + Interlocked.Decrement(ref _activeArticleProcessing); + AfterProcessCompleted?.Invoke(slot.MessageId); + } + } + + private Slot? TryBeginEmitHeadNoLock() + { + if (!HeadIsReadyNoLock()) + { + return null; + } + + var slot = _slots[_head]!; + slot.Emitting = true; + return slot; + } + + private void ReleaseHeadIfNoLock(Slot slot) + { + if (_count == 0 || !ReferenceEquals(_slots[_head], slot)) + { + return; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + slot.Emitting = false; + } + + private void CancelAndRelease(Slot slot) + { + lock (_gate) + { + if (!IsPresentNoLock(slot)) + { + return; + } + + slot.ArticleReady = true; + slot.PeekCompleted = true; + slot.Processed = true; + slot.Cancelled = true; + slot.Emitting = false; + ReleaseCancelledPrefixNoLock(); + SignalProgressNoLock(); + } + } + + private void ReleaseCancelledPrefixNoLock() + { + while (_count > 0) + { + var slot = _slots[_head]; + if (slot is null || slot.Emitting || !slot.Cancelled) + { + break; + } + + _slots[_head] = null; + _head = (_head + 1) % Depth; + _count--; + _session.EndCommandWork(); + } + } + + private bool IsPresentNoLock(Slot slot) => + (uint)slot.Index < Depth && ReferenceEquals(_slots[slot.Index], slot); + + private void ThrowIfShutDown() + { + if (Volatile.Read(ref _shutDown) == 1) + { + throw new OperationCanceledException("TAKETHIS pipeline has shut down."); + } + } + + private void SignalProgressNoLock() + { + var previous = _progress; + _progress = NewProgress(); + previous.TrySetResult(); + } + + /// + /// Records when it exceeds . + /// Concurrent observers cannot publish a lower peak over a higher one. + /// + private static void ObservePeak(ref int peak, int observed) + { + while (true) + { + var current = Volatile.Read(ref peak); + if (observed <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref peak, observed, current) == current) + { + return; + } + } + } + + private static string SlotMessageId(Slot slot) => + System.Text.Encoding.ASCII.GetString(slot.MessageId); + + private static TaskCompletionSource NewProgress() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed class Slot + { + public required byte[] MessageId { get; init; } + + public required int Index { get; init; } + + public required long StartedTimestamp { get; init; } + + public TakeThisStageMarks? Marks { get; init; } + + public IHaveArticleReadResult Read { get; set; } + + public int MaxArticleBytes { get; init; } + + public byte[]? SiteNameUtf8 { get; init; } + + public HistoryLookupResult Peek { get; set; } + + public bool ArticleReady { get; set; } + + public bool PeekCompleted { get; set; } + + public bool Processed { get; set; } + + public ArticleProcessOutcome Process { get; set; } + + public bool Cancelled { get; set; } + + public bool Emitting { get; set; } + } + + /// + /// Per-slot destuff/parse/CRC outcome consumed by the ordered publisher. + /// + private readonly struct ArticleProcessOutcome + { + public bool RecordRejected { get; init; } + + public bool UncarriedRejected { get; init; } + + public string? RejectDetail { get; init; } + + public string? UncarriedReason { get; init; } + + public InboundArticle? Inbound { get; init; } + + public int Size { get; init; } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Session/TakeThisStageProbe.cs b/src/VectorNNTPD.TransitService/Session/TakeThisStageProbe.cs new file mode 100644 index 00000000..fb8bd800 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/TakeThisStageProbe.cs @@ -0,0 +1,468 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.History; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Off-by-default TAKETHIS pipeline stage sampler. Disabled unless + /// VECTORNNTP_TAKETHIS_TIMING is set or is called. + /// When disabled, callers must not allocate per-article state. + /// + internal static class TakeThisStageProbe + { + /// Environment variable that enables server-side TAKETHIS stage timing. + public const string EnvironmentVariableName = "VECTORNNTP_TAKETHIS_TIMING"; + + private static ProbeState? _state; + + static TakeThisStageProbe() + { + var value = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(value) || + value.Equals("0", StringComparison.OrdinalIgnoreCase) || + value.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var dir = value.Equals("1", StringComparison.OrdinalIgnoreCase) || + value.Equals("true", StringComparison.OrdinalIgnoreCase) + ? DefaultDirectory() + : value; + Enable(dir); + } + + /// Gets whether stage timestamps should be recorded. + public static bool IsEnabled => Volatile.Read(ref _state) is not null; + + /// Gets the directory that receives session dumps, or . + public static string? OutputDirectory => Volatile.Read(ref _state)?.Directory; + + /// Enables sampling and writes dumps under . + public static void Enable(string outputDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(outputDirectory); + Directory.CreateDirectory(outputDirectory); + Volatile.Write(ref _state, new ProbeState(outputDirectory)); + } + + /// Disables sampling (tests). + public static void Disable() => Volatile.Write(ref _state, null); + + /// Creates a per-session log when enabled; otherwise . + public static TakeThisStageSessionLog? CreateSessionLog() => + IsEnabled ? new TakeThisStageSessionLog() : null; + + /// + /// Repo-local diagnostic dump directory. + /// + /// + /// Deliberately walks from the process working directory toward a folder that + /// contains VectorNNTPD.TransitService.sln or .artifacts/. This is + /// development-only output, off unless is set. + /// It is not application-local runtime state and must not be used for certs/logs. + /// + internal static string DefaultDirectory() + { + var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.TransitService.sln")) || + Directory.Exists(Path.Join(dir.FullName, ".artifacts"))) + { + return Path.Join(dir.FullName, ".artifacts", "takethis-timing"); + } + + dir = dir.Parent; + } + + return Path.Join(Directory.GetCurrentDirectory(), ".artifacts", "takethis-timing"); + } + + private sealed class ProbeState(string directory) + { + public string Directory { get; } = directory; + } + } + + /// Per-session collection of TAKETHIS stage samples. + internal sealed class TakeThisStageSessionLog + { + private readonly ConcurrentBag _samples = []; + private int _peakOccupied; + private int _accepted; + + /// Gets the number of recorded accepted samples. + public int Count => _samples.Count; + + /// Gets the peak occupied slot count observed while sampling. + public int PeakOccupied => Volatile.Read(ref _peakOccupied); + + /// Records occupied-window observations (all admits, not only accepted samples). + public void NoteOccupied(int occupied) + { + var current = Volatile.Read(ref _peakOccupied); + while (occupied > current) + { + var original = Interlocked.CompareExchange(ref _peakOccupied, occupied, current); + if (original == current) + { + break; + } + + current = original; + } + } + + /// Adds one accepted TAKETHIS stage sample. + public void Record(in TakeThisStageSample sample) + { + Interlocked.Increment(ref _accepted); + _samples.Add(sample); + } + + /// Writes CSV and a percentile report for this session. + public string Write(string directory, string sessionLabel) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionLabel); + Directory.CreateDirectory(directory); + var samples = _samples.ToArray(); + Array.Sort(samples, static (a, b) => a.CommandParsedTs.CompareTo(b.CommandParsedTs)); + var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture); + var safe = Sanitize(sessionLabel); + var csvPath = Path.Join(directory, $"{stamp}-{safe}.csv"); + var txtPath = Path.Join(directory, $"{stamp}-{safe}.txt"); + File.WriteAllText(csvPath, TakeThisStageReport.FormatCsv(samples), Encoding.UTF8); + var report = TakeThisStageReport.Format( + samples, + PeakOccupied, + sessionLabel, + csvPath); + File.WriteAllText(txtPath, report, Encoding.UTF8); + return txtPath; + } + + private static string Sanitize(string value) + { + var chars = value.ToCharArray(); + for (var i = 0; i < chars.Length; i++) + { + if (!char.IsAsciiLetterOrDigit(chars[i]) && chars[i] is not '-' and not '_') + { + chars[i] = '_'; + } + } + + return new string(chars); + } + } + + /// Monotonic timestamps for one TAKETHIS pipeline slot. + internal sealed class TakeThisStageMarks + { + public long CommandParsedTs; + public long PeekStartTs; + public long PeekCompleteTs; + public long ReceiveStartTs; + public long ReceiveCompleteTs; + public long EmitStartTs; + public long EnqueueStartTs; + public long EnqueueAcceptedTs; + public long RememberStartTs; + public long RememberCompleteTs; + public long ReplyEnqueueStartTs; + public long ReplyEnqueueCompleteTs; + public int OccupiedAtEmit; + public int ArticleBytes; + public string Outcome = "accepted"; + + /// Wraps so peek completion is stamped when Redis returns. + public ValueTask ObservePeek(ValueTask lookup) => + ObservePeekAsync(this, lookup); + + public TakeThisStageSample ToSample(int peakOccupied) => + new( + CommandParsedTs, + PeekStartTs, + PeekCompleteTs, + ReceiveStartTs, + ReceiveCompleteTs, + EmitStartTs, + EnqueueStartTs, + EnqueueAcceptedTs, + RememberStartTs, + RememberCompleteTs, + ReplyEnqueueStartTs, + ReplyEnqueueCompleteTs, + OccupiedAtEmit, + peakOccupied, + ArticleBytes, + Outcome); + + private static async ValueTask ObservePeekAsync( + TakeThisStageMarks marks, + ValueTask lookup) + { + try + { + var result = await lookup.ConfigureAwait(false); + marks.PeekCompleteTs = Stopwatch.GetTimestamp(); + return result; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + marks.PeekCompleteTs = Stopwatch.GetTimestamp(); + throw; + } + } + } + + /// One accepted TAKETHIS pipeline transaction (monotonic timestamps). + internal readonly record struct TakeThisStageSample( + long CommandParsedTs, + long PeekStartTs, + long PeekCompleteTs, + long ReceiveStartTs, + long ReceiveCompleteTs, + long EmitStartTs, + long EnqueueStartTs, + long EnqueueAcceptedTs, + long RememberStartTs, + long RememberCompleteTs, + long ReplyEnqueueStartTs, + long ReplyEnqueueCompleteTs, + int OccupiedAtEmit, + int PeakOccupied, + int ArticleBytes, + string Outcome); + + /// Percentile report for TAKETHIS stage samples. + internal static class TakeThisStageReport + { + public static string FormatCsv(IReadOnlyList samples) + { + var sb = new StringBuilder(); + sb.AppendLine( + "command_parsed_ts,peek_us,receive_us,ready_to_emit_us,enqueue_us,remember_us,reply_enqueue_us,server_txn_us,occupied_at_emit,peak_occupied,article_bytes,outcome,dominant"); + foreach (var sample in samples) + { + var i = Intervals(sample); + sb.AppendLine(string.Create( + CultureInfo.InvariantCulture, + $"{sample.CommandParsedTs},{i.PeekUs},{i.ReceiveUs},{i.ReadyToEmitUs},{i.EnqueueUs},{i.RememberUs},{i.ReplyEnqueueUs},{i.ServerTxnUs},{sample.OccupiedAtEmit},{sample.PeakOccupied},{sample.ArticleBytes},{sample.Outcome},{i.Dominant}")); + } + + return sb.ToString(); + } + + public static string Format( + IReadOnlyList samples, + int peakOccupied, + string sessionLabel, + string? csvPath) + { + var sb = new StringBuilder(); + sb.AppendLine("TAKETHIS PIPELINE STAGE TIMING — DIAGNOSTIC"); + sb.AppendLine(); + sb.AppendLine("Off-by-default server probe. Production path is unchanged when disabled."); + sb.AppendLine("Clock: Stopwatch.GetTimestamp (monotonic). 239 enqueue is Channel accept, not socket flush."); + sb.AppendLine(); + sb.AppendLine($"Session: {sessionLabel}"); + sb.AppendLine($"Samples: {samples.Count:N0}"); + var seen = samples.Count(static s => s.Outcome == "seen"); + var unseen = samples.Count(static s => s.Outcome == "unseen"); + sb.AppendLine($"Seen / unseen: {seen:N0} / {unseen:N0}"); + sb.AppendLine($"Peak occupied: {peakOccupied} (window {TakeThisPipeline.Depth})"); + if (samples.Count == 0) + { + sb.AppendLine(); + sb.AppendLine("No accepted samples."); + return sb.ToString(); + } + + var maxAtEmit = samples.Max(static s => s.OccupiedAtEmit); + sb.AppendLine($"Max occupied@emit: {maxAtEmit}"); + sb.AppendLine($"Average article: {samples.Average(static s => (double)s.ArticleBytes):N0} bytes"); + sb.AppendLine(); + + WriteWindow(sb, "ALL SAMPLES", samples); + if (samples.Count >= 20) + { + var mid = samples.Count / 2; + WriteWindow(sb, "FIRST HALF", samples.Take(mid).ToArray()); + WriteWindow(sb, "SECOND HALF", samples.Skip(mid).ToArray()); + } + + if (csvPath is not null) + { + sb.AppendLine($"CSV: {csvPath}"); + } + + return sb.ToString(); + } + + private static void WriteWindow(StringBuilder sb, string title, IReadOnlyList samples) + { + sb.AppendLine(title); + sb.AppendLine($"{"Interval",-28} {"Mean",10} {"P50",10} {"P95",10} {"P99",10} {"Max",10}"); + foreach (var row in Rows(samples)) + { + sb.AppendLine(row); + } + + sb.AppendLine(); + var votes = new Dictionary(StringComparer.Ordinal); + foreach (var name in samples.Select(static sample => Intervals(sample).Dominant)) + { + votes[name] = votes.TryGetValue(name, out var n) ? n + 1 : 1; + } + + sb.AppendLine("Dominant interval (largest of peek / receive / ready→emit / enqueue / remember / 239 enqueue):"); + foreach (var pair in votes.OrderByDescending(static p => p.Value)) + { + var pct = 100.0 * pair.Value / samples.Count; + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, $" {pair.Key,-22} {pair.Value,8:N0} {pct,6:F1}%")); + } + + sb.AppendLine(); + } + + private static IEnumerable Rows(IReadOnlyList samples) + { + yield return Stats("msgid → peek start", samples, static i => i.MsgidToPeekUs); + yield return Stats("HistoryDB Peek", samples, static i => i.PeekUs); + yield return Stats("article receive", samples, static i => i.ReceiveUs); + yield return Stats("max(peek, receive)", samples, static i => i.OverlapUs); + yield return Stats("peek leftover after RX", samples, static i => i.PeekAfterReceiveUs); + yield return Stats("ready → emit (order)", samples, static i => i.ReadyToEmitUs); + yield return Stats("queue EnqueueAsync", samples, static i => i.EnqueueUs); + yield return Stats("HistoryDB Remember", samples, static i => i.RememberUs); + yield return Stats("239 Channel enqueue", samples, static i => i.ReplyEnqueueUs); + yield return Stats("server txn (parse→239)", samples, static i => i.ServerTxnUs); + } + + private static string Stats( + string name, + IReadOnlyList samples, + Func selector) + { + var values = new double[samples.Count]; + double sum = 0; + for (var i = 0; i < samples.Count; i++) + { + var value = selector(Intervals(samples[i])); + values[i] = value; + sum += value; + } + + Array.Sort(values); + var mean = sum / values.Length; + return string.Create( + CultureInfo.InvariantCulture, + $"{name,-28} {mean,10:F1} {Percentile(values, 0.50),10:F1} {Percentile(values, 0.95),10:F1} {Percentile(values, 0.99),10:F1} {values[^1],10:F1}"); + } + + internal static double Percentile(double[] sorted, double p) + { + if (sorted.Length == 1) + { + return sorted[0]; + } + + var position = p * (sorted.Length - 1); + var lo = (int)position; + var hi = Math.Min(lo + 1, sorted.Length - 1); + var frac = position - lo; + return sorted[lo] + ((sorted[hi] - sorted[lo]) * frac); + } + + internal static StageIntervals Intervals(in TakeThisStageSample sample) + { + var peek = Us(sample.PeekStartTs, sample.PeekCompleteTs); + var receive = Us(sample.ReceiveStartTs, sample.ReceiveCompleteTs); + var ready = Math.Max(sample.PeekCompleteTs, sample.ReceiveCompleteTs); + var readyToEmit = Us(ready, sample.EmitStartTs); + var enqueue = Us(sample.EnqueueStartTs, sample.EnqueueAcceptedTs); + var remember = Us(sample.RememberStartTs, sample.RememberCompleteTs); + var reply = Us(sample.ReplyEnqueueStartTs, sample.ReplyEnqueueCompleteTs); + var txn = Us(sample.CommandParsedTs, sample.ReplyEnqueueCompleteTs); + var msgidToPeek = Us(sample.CommandParsedTs, sample.PeekStartTs); + var leftover = sample.PeekCompleteTs > sample.ReceiveCompleteTs + ? Us(sample.ReceiveCompleteTs, sample.PeekCompleteTs) + : 0; + var overlap = Math.Max(peek, receive); + + var dominant = "article receive"; + var best = receive; + if (peek > best) + { + best = peek; + dominant = "HistoryDB Peek"; + } + + if (readyToEmit > best) + { + best = readyToEmit; + dominant = "ordered emission"; + } + + if (enqueue > best) + { + best = enqueue; + dominant = "queue admission"; + } + + if (remember > best) + { + best = remember; + dominant = "HistoryDB Remember"; + } + + if (reply > best) + { + dominant = "TX/flush (239 enqueue)"; + } + + return new StageIntervals( + msgidToPeek, + peek, + receive, + overlap, + leftover, + readyToEmit, + enqueue, + remember, + reply, + txn, + dominant); + } + + private static double Us(long start, long end) + { + if (start <= 0 || end <= 0 || end < start) + { + return 0; + } + + return Stopwatch.GetElapsedTime(start, end).TotalMicroseconds; + } + + internal readonly record struct StageIntervals( + double MsgidToPeekUs, + double PeekUs, + double ReceiveUs, + double OverlapUs, + double PeekAfterReceiveUs, + double ReadyToEmitUs, + double EnqueueUs, + double RememberUs, + double ReplyEnqueueUs, + double ServerTxnUs, + string Dominant); + } +} diff --git a/src/VectorNNTPD.TransitService/Session/TransitPeerAuthorization.cs b/src/VectorNNTPD.TransitService/Session/TransitPeerAuthorization.cs new file mode 100644 index 00000000..0330452c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Session/TransitPeerAuthorization.cs @@ -0,0 +1,180 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Session +{ + /// + /// Resolves connection-time transit/streaming peer identity from the active Transit snapshot. + /// + public interface ITransitPeerAuthorization + { + /// Gets whether any Transit peers are configured in the current snapshot. + bool IsEnabled { get; } + + /// + /// Returns the initial for . + /// + /// + /// Matching uses the precomputed IP-only ACL (literal prefixes plus already-resolved + /// AllowFrom addresses). This method never performs DNS, reverse DNS, or network I/O. + /// A unique match retains the named peer policy and grants streaming. + /// Feed commands are granted immediately only when that peer has no receive credentials. + /// When both username and password are set, feed commands wait for AUTHINFO. + /// Ambiguous multi-peer matches grant nothing. + /// Other addresses receive . + /// + NntpAuthorization Resolve(IPAddress effectiveClientAddress); + } + + /// + /// Snapshot-backed implementation of . + /// + public sealed class TransitPeerAuthorization : ITransitPeerAuthorization + { + private readonly TransitConfigurationStore _store; + private readonly ITransitDnsAddressCache _dns; + private readonly ILogger? _logger; + + /// Disabled shared instance (empty snapshot). + public static TransitPeerAuthorization Disabled { get; } = CreateStatic(TransitConfigurationSnapshot.Empty); + + /// Creates an instance from the live snapshot store (DI). + [ActivatorUtilitiesConstructor] + public TransitPeerAuthorization( + TransitConfigurationStore store, + ITransitDnsAddressCache dns, + TransitConfigurationHotReload hotReload, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(dns); + ArgumentNullException.ThrowIfNull(hotReload); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _dns = dns; + _logger = logger; + if (!store.Current.IsEmpty) + { + SessionLogMessages.TransitPeersConfigured( + logger, + store.Current.Peers.Count, + string.Join(", ", store.Current.Peers.Values.Select(static p => p.Identifier + " (" + p.PeerName + ")"))); + } + } + + private TransitPeerAuthorization( + TransitConfigurationStore store, + ITransitDnsAddressCache dns, + ILogger? logger = null) + { + _store = store; + _dns = dns; + _logger = logger; + } + + /// Creates an instance bound to an existing snapshot store (tests). + public static TransitPeerAuthorization CreateForStore( + TransitConfigurationStore store, + ITransitDnsAddressCache? dns = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(store); + return new TransitPeerAuthorization(store, dns ?? new EmptyDnsCache(), logger); + } + + /// Creates a static instance from a complete snapshot (tests). + public static TransitPeerAuthorization CreateStatic( + TransitConfigurationSnapshot snapshot, + ITransitDnsAddressCache? dns = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(snapshot); + var store = new TransitConfigurationStore(); + store.Replace(snapshot); + return new TransitPeerAuthorization(store, dns ?? new EmptyDnsCache(), logger); + } + + /// + public bool IsEnabled => !_store.Current.IsEmpty; + + /// + public NntpAuthorization Resolve(IPAddress effectiveClientAddress) + { + ArgumentNullException.ThrowIfNull(effectiveClientAddress); + var address = TrustedProxyHosts.Canonicalize(effectiveClientAddress); + var snapshot = _store.Current; + if (snapshot.IsEmpty) + { + return NntpAuthorization.Unauthenticated; + } + + List? matches = null; + TransitPeerPolicy? matched = null; + foreach (var peer in snapshot.Peers.Values) + { + if (!MatchesIpAcl(peer, address)) + { + continue; + } + + matches ??= []; + matches.Add(peer.Identifier + " (" + peer.PeerName + ")"); + matched = peer; + } + + if (matches is { Count: > 1 }) + { + if (_logger is not null) + { + SessionLogMessages.TransitPeerAmbiguous(_logger, address, string.Join(", ", matches)); + } + return NntpAuthorization.Unauthenticated; + } + + return matched is null + ? NntpAuthorization.Unauthenticated + : NntpAuthorization.ForTransitPeer(matched); + } + + /// + /// Matches against the peer's already-materialized IP ACL. + /// Does not call . + /// + private bool MatchesIpAcl(TransitPeerPolicy peer, IPAddress address) + { + foreach (var prefix in peer.LiteralPrefixes) + { + if (prefix.Contains(address)) + { + return true; + } + } + + foreach (var hostname in peer.DnsHostnames) + { + if (_dns.GetResolved(hostname).Contains(address)) + { + return true; + } + } + + return false; + } + + private sealed class EmptyDnsCache : ITransitDnsAddressCache + { + public IReadOnlySet GetResolved(string hostname) => new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteBatchId.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteBatchId.cs new file mode 100644 index 00000000..2ac5f52a --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteBatchId.cs @@ -0,0 +1,37 @@ +using System.Diagnostics.CodeAnalysis; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Identity of one MySQL-committed consume batch used for Redis APPLY idempotency. + /// + /// + /// Generated after MySQL commits and reused for every Redis retry of that batch in this + /// process. It is not persisted. A process crash drops it and the batch is not replayed. + /// + internal static class AccountByteBatchId + { + /// HASH field prefix for an applied batch on the remaining-quota key. + public const string FieldPrefix = "b:"; + + /// + /// Maximum applied-batch marks retained per remaining key. Floor-only APPLY + /// stays correct if an older mark is evicted. + /// + public const int MaxRetainedMarks = 256; + + /// Creates a new unique batch identity. + public static string Create() => Guid.NewGuid().ToString("N"); + + /// Returns whether is a usable identity. + public static bool IsValid([NotNullWhen(true)] string? batchId) => + !string.IsNullOrWhiteSpace(batchId); + + /// HASH field name for . + public static string FieldName(string batchId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + return FieldPrefix + batchId; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteDefaults.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteDefaults.cs new file mode 100644 index 00000000..7beb621c --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteDefaults.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Operational timings for account byte-quota reconciliation. + /// + /// There is no AccountBytes timer. owns the + /// single ~10-second cycle. Batching is not byte-exact. Redis is never used on + /// the NNTP write path. + /// + internal static class AccountByteDefaults + { + /// Documented batch cadence; the live scheduler is . + public static readonly TimeSpan ReconciliationPeriod = SessionStateDefaults.RenewalPeriod; + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteEngine.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteEngine.cs new file mode 100644 index 00000000..443571d6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteEngine.cs @@ -0,0 +1,164 @@ +using System.Globalization; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// In-process remaining-quota algorithm. Production Redis Lua is a faithful port. + /// Remaining never increases and never goes negative. + /// + /// + /// APPLY is idempotent per batch id. The remaining update is + /// min(current, mysqlRemainingAfter) (or initialize from MySQL when missing). + /// consumed is not subtracted: MySQL already subtracted that batch, and + /// another node's earlier APPLY may already have floored to a mysql_after that + /// includes this consume. Decrementing would double-count and is not retry-safe. + /// + internal sealed class AccountByteEngine + { + private readonly object _gate = new(); + private readonly Dictionary _remaining = new(StringComparer.Ordinal); + private readonly Dictionary> _applied = new(StringComparer.Ordinal); + + /// + /// Applies one consumed batch identified by . + /// Repeating the same id does not change remaining. + /// + public long Apply(string key, string batchId, long consumed, long mysqlRemainingAfter) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + if (!AccountByteBatchId.IsValid(batchId)) + { + throw new ArgumentException("Batch id is required for idempotent APPLY.", nameof(batchId)); + } + + _ = ClampNonNegative(consumed); + mysqlRemainingAfter = ClampNonNegative(mysqlRemainingAfter); + lock (_gate) + { + if (IsApplied(key, batchId)) + { + return _remaining.TryGetValue(key, out var existing) + ? ClampNonNegative(existing) + : 0; + } + + if (!_remaining.TryGetValue(key, out var current)) + { + _remaining[key] = mysqlRemainingAfter; + MarkApplied(key, batchId); + TrimApplied(key, batchId); + return mysqlRemainingAfter; + } + + current = ClampNonNegative(current); + var next = current < mysqlRemainingAfter ? current : mysqlRemainingAfter; + _remaining[key] = next; + MarkApplied(key, batchId); + TrimApplied(key, batchId); + return next; + } + } + + /// Reads remaining, or when absent. + public long Observe(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + return _remaining.TryGetValue(key, out var current) ? ClampNonNegative(current) : AccountByteKeys.Missing; + } + } + + /// Overwrites remaining without consume semantics (tests). + internal void Write(string key, long remaining) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + _remaining[key] = ClampNonNegative(remaining); + } + } + + /// Removes remaining and applied-batch marks. Returns 1 if the key existed. + internal long Delete(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + var existed = _remaining.Remove(key); + existed = _applied.Remove(key) || existed; + return existed ? 1 : 0; + } + } + + /// Returns whether the remaining key exists (tests). + internal bool Contains(string key) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + lock (_gate) + { + return _remaining.ContainsKey(key); + } + } + + /// Returns whether was already applied (tests). + internal bool WasApplied(string key, string batchId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + lock (_gate) + { + return IsApplied(key, batchId); + } + } + + internal static long ClampNonNegative(long value) => value < 0 ? 0 : value; + + internal static string Format(long remaining) => + remaining.ToString(CultureInfo.InvariantCulture); + + private bool IsApplied(string key, string batchId) => + _applied.TryGetValue(key, out var batches) && batches.Contains(batchId); + + private void MarkApplied(string key, string batchId) + { + if (!_applied.TryGetValue(key, out var batches)) + { + batches = new HashSet(StringComparer.Ordinal); + _applied[key] = batches; + } + + batches.Add(batchId); + } + + private void TrimApplied(string key, string keepBatchId) + { + if (!_applied.TryGetValue(key, out var batches) + || batches.Count <= AccountByteBatchId.MaxRetainedMarks) + { + return; + } + + var extra = batches.Count - AccountByteBatchId.MaxRetainedMarks; + var doomed = new List(extra); + foreach (var id in batches) + { + if (id == keepBatchId) + { + continue; + } + + doomed.Add(id); + if (doomed.Count == extra) + { + break; + } + } + + foreach (var id in doomed) + { + batches.Remove(id); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteKeys.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteKeys.cs new file mode 100644 index 00000000..5caf79a9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteKeys.cs @@ -0,0 +1,48 @@ +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Redis key material for cluster-wide remaining byte quota. + /// + /// + /// Key: nntpd:bytes:{sha256hex(accountName)} (HASH). The plaintext AUTHINFO + /// username is hashed so Redis listings do not expose account names. Field + /// remaining is the remaining byte count. Field b:{batchId} marks an + /// applied consume batch for idempotent APPLY. This is not a lease and has no key TTL. + /// + internal static class AccountByteKeys + { + /// ASCII namespace for remaining-quota keys. + public static ReadOnlySpan NamespacePrefix => "nntpd:bytes:"u8; + + /// Sentinel returned by OBSERVE when the key is missing. + public const long Missing = -1; + + /// Builds the remaining-quota Redis key for . + public static byte[] Create(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteLogMessages.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteLogMessages.cs new file mode 100644 index 00000000..9fcc4216 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteLogMessages.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Source-generated AccountBytes logs. EventIds 2700–2711. + internal static partial class AccountByteLogMessages + { + [LoggerMessage(EventId = 2700, Level = LogLevel.Information, Message = "Account-byte reconciliation started interval={Interval}")] + public static partial void ReconciliationStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 2701, Level = LogLevel.Information, Message = "Account-byte reconciliation stopped")] + public static partial void ReconciliationStopped(ILogger logger); + + [LoggerMessage(EventId = 2702, Level = LogLevel.Warning, Message = "Account-byte reconciliation pass failed")] + public static partial void ReconciliationPassFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2703, Level = LogLevel.Warning, Message = "Account-byte durable consume failed user={Username} bytes={Bytes}")] + public static partial void DurableConsumeFailed(ILogger logger, Exception exception, string username, long bytes); + + [LoggerMessage(EventId = 2704, Level = LogLevel.Warning, Message = "Account-byte Redis apply unavailable user={Username} consumed={Consumed} mysqlRemaining={MysqlRemaining}")] + public static partial void RedisApplyUnavailable(ILogger logger, string username, long consumed, long mysqlRemaining); + + [LoggerMessage(EventId = 2705, Level = LogLevel.Information, Message = "Account-byte quota exhausted user={Username} remaining={Remaining}")] + public static partial void QuotaExhausted(ILogger logger, string username, long remaining); + + [LoggerMessage(EventId = 2706, Level = LogLevel.Debug, Message = "Account-byte batch reconciled user={Username} consumed={Consumed} remaining={Remaining}")] + public static partial void BatchReconciled(ILogger logger, string username, long consumed, long remaining); + + [LoggerMessage(EventId = 2707, Level = LogLevel.Warning, Message = "Account-byte observe failed user={Username}")] + public static partial void ObserveFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2708, Level = LogLevel.Warning, Message = "Account-byte durable remaining query failed user={Username}")] + public static partial void DurableQueryFailed(ILogger logger, Exception exception, string username); + + [LoggerMessage(EventId = 2709, Level = LogLevel.Information, Message = "Account-byte final reconciliation started")] + public static partial void FinalReconciliationStarted(ILogger logger); + + [LoggerMessage(EventId = 2710, Level = LogLevel.Warning, Message = "Account-byte final reconciliation failed")] + public static partial void FinalReconciliationFailed(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 2711, Level = LogLevel.Information, Message = "Account-byte Redis state deleted user={Username} existed={Existed}")] + public static partial void ClusterStateDeleted(ILogger logger, string username, long existed); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteScripts.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteScripts.cs new file mode 100644 index 00000000..c81c4baa --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteScripts.cs @@ -0,0 +1,110 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Redis Lua for cluster remaining-quota. Semantics match . + /// The remaining HASH has no key TTL. Batch fields are marks only; they do not expire + /// the remaining value. + /// + internal static class AccountByteScripts + { + /// + /// Applies one consumed batch idempotently. Remaining is floored to + /// mysqlRemainingAfter and never increased. The same batch id is a no-op. + /// + /// + /// KEYS[1] remaining HASH. + /// ARGV: batchId, consumed, mysqlRemainingAfter. + /// consumed is accepted for the call contract but is not subtracted. + /// MySQL already subtracted the batch; decrementing Redis is not retry-safe + /// when another node has already floored to a mysql_after that includes it. + /// Returns remaining >= 0. + /// + public const string Apply = + """ + -- nntpd-bytes-apply + -- Retain at most 256 batch marks (AccountByteBatchId.MaxRetainedMarks). + local function trim_marks(key, batchField) + if redis.call('HLEN', key) <= 257 then + return + end + local fields = redis.call('HKEYS', key) + for i = 1, #fields do + local f = fields[i] + if f ~= 'remaining' and f ~= batchField then + redis.call('HDEL', key, f) + if redis.call('HLEN', key) <= 257 then + return + end + end + end + end + local key = KEYS[1] + local batchId = ARGV[1] + local mysql = tonumber(ARGV[3]) + if mysql == nil or mysql < 0 then + mysql = 0 + end + if batchId == nil or batchId == '' then + batchId = '' + end + local batchField = 'b:' .. batchId + if redis.call('EXISTS', key) == 0 then + if batchId ~= '' then + redis.call('HSET', key, 'remaining', tostring(mysql), batchField, '1') + trim_marks(key, batchField) + else + redis.call('HSET', key, 'remaining', tostring(mysql)) + end + return mysql + end + if batchId ~= '' and redis.call('HEXISTS', key, batchField) == 1 then + local cur = tonumber(redis.call('HGET', key, 'remaining')) + if cur == nil or cur < 0 then + return 0 + end + return cur + end + local current = tonumber(redis.call('HGET', key, 'remaining')) + if current == nil or current < 0 then + current = 0 + end + local next = current + if next > mysql then + next = mysql + end + if batchId ~= '' then + redis.call('HSET', key, 'remaining', tostring(next), batchField, '1') + trim_marks(key, batchField) + else + redis.call('HSET', key, 'remaining', tostring(next)) + end + return next + """; + + /// Reads remaining. Missing key returns -1. + /// KEYS[1] remaining HASH. No ARGV. Never creates or writes the key. + public const string Observe = + """ + -- nntpd-bytes-observe + if redis.call('EXISTS', KEYS[1]) == 0 then + return -1 + end + local n = tonumber(redis.call('HGET', KEYS[1], 'remaining')) + if n == nil or n < 0 then + return 0 + end + return n + """; + + /// + /// Deletes the remaining-quota HASH. Used only for explicit operator top-up + /// invalidation. Never creates or writes the key. + /// + /// KEYS[1] remaining HASH. No ARGV. Returns 1 if the key existed. + public const string Delete = + """ + -- nntpd-bytes-delete + return redis.call('DEL', KEYS[1]) + """; + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteTracker.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteTracker.cs new file mode 100644 index 00000000..40f144e5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/AccountByteTracker.cs @@ -0,0 +1,458 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Process-local account byte accumulator and crash-safe batch reconciler. + /// + /// + /// + /// Hot path: is an . + /// Pending and in-flight totals live only in this process. A crash loses unreconciled bytes + /// (at-most-once durable charge). That is the accepted approximation; durable quota must + /// never be charged twice for the same batch. + /// + /// + /// Reconciliation is MySQL first, then Redis APPLY with a per-batch id. After a + /// successful MySQL consume in this process, the same batch id is reused for every + /// Redis retry. Redis treats that id as exactly-once. The id is not durable. + /// Restart never replays a batch. + /// + /// + internal sealed class AccountByteTracker : IAccountByteAccountant + { + private readonly ConcurrentDictionary _ledgers = new(StringComparer.Ordinal); + private readonly IAccountByteDurableStore _durable; + private readonly IAccountByteStore _cluster; + private readonly ILogger _logger; + private readonly SemaphoreSlim _reconcileGate = new(1, 1); + + /// Initializes a new instance of the class. + public AccountByteTracker( + IAccountByteDurableStore durable, + IAccountByteStore cluster, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(durable); + ArgumentNullException.ThrowIfNull(cluster); + ArgumentNullException.ThrowIfNull(logger); + _durable = durable; + _cluster = cluster; + _logger = logger; + } + + /// + public IAccountByteSink CreateSink(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return NullAccountByteSink.Instance; + } + + return new AccountLedgerSink(GetOrAdd(accountName)); + } + + /// + public bool IsExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return false; + } + + return _ledgers.TryGetValue(accountName, out var ledger) + && Volatile.Read(ref ledger.Exhausted) == 1; + } + + /// + public void MarkExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + var ledger = GetOrAdd(accountName); + if (Interlocked.Exchange(ref ledger.Exhausted, 1) == 0) + { + AccountByteLogMessages.QuotaExhausted(_logger, accountName, remaining: 0); + } + } + + /// + public void ClearExhausted(string accountName) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + if (_ledgers.TryGetValue(accountName, out var ledger)) + { + Volatile.Write(ref ledger.Exhausted, 0); + } + } + + /// + public async ValueTask DeleteAccountByteStateAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + long? deleted; + try + { + deleted = await _cluster.DeleteAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + return false; + } + + if (deleted is null) + { + AccountByteLogMessages.RedisApplyUnavailable(_logger, accountName, 0, 0); + return false; + } + + ClearExhausted(accountName); + AccountByteLogMessages.ClusterStateDeleted(_logger, accountName, deleted.Value); + return true; + } + + /// + public async ValueTask ObserveRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + long? redis = null; + try + { + redis = await _cluster.ObserveAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + } + + AccountByteConsumeResult? durable = null; + try + { + durable = await _durable.QueryRemainingAsync(accountName, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException ex) + { + AccountByteLogMessages.DurableQueryFailed(_logger, ex, accountName); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.DurableQueryFailed(_logger, ex, accountName); + } + + if (durable is { Succeeded: false }) + { + return null; + } + + var mysqlRemaining = durable is { Succeeded: true } row + ? AccountByteEngine.ClampNonNegative(row.Remaining) + : (long?)null; + + if (redis is { } live && live != AccountByteKeys.Missing) + { + var clusterRemaining = AccountByteEngine.ClampNonNegative(live); + if (mysqlRemaining is { } durableRemaining) + { + var effective = clusterRemaining < durableRemaining ? clusterRemaining : durableRemaining; + if (clusterRemaining > durableRemaining) + { + await FloorRedisToMysqlAsync(accountName, durableRemaining, cancellationToken) + .ConfigureAwait(false); + } + + return effective; + } + + return clusterRemaining; + } + + return mysqlRemaining; + } + + /// + public async ValueTask CommitDurableAsync(CancellationToken cancellationToken = default) + { + await _reconcileGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + foreach (var pair in _ledgers) + { + cancellationToken.ThrowIfCancellationRequested(); + await CommitDurableAccountAsync(pair.Key, pair.Value, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _reconcileGate.Release(); + } + } + + /// + public async ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default) + { + await _reconcileGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + foreach (var pair in _ledgers) + { + cancellationToken.ThrowIfCancellationRequested(); + await ApplyCommittedAccountAsync(pair.Key, pair.Value, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _reconcileGate.Release(); + } + } + + /// + public async ValueTask ReconcileAsync(CancellationToken cancellationToken = default) + { + await CommitDurableAsync(cancellationToken).ConfigureAwait(false); + await ApplyCommittedAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch) + { + batch = default; + if (string.IsNullOrWhiteSpace(accountName) + || !_ledgers.TryGetValue(accountName, out var ledger) + || Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return false; + } + + var batchId = ledger.BatchId; + if (!AccountByteBatchId.IsValid(batchId)) + { + return false; + } + + batch = new AccountByteCommittedBatch( + accountName, + batchId, + ledger.LastConsumed, + ledger.LastMysqlRemaining); + return true; + } + + /// + public void CompleteApply(string accountName, long remaining) + { + if (string.IsNullOrWhiteSpace(accountName) + || !_ledgers.TryGetValue(accountName, out var ledger) + || Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return; + } + + var consumed = ledger.LastConsumed; + var mysqlRemaining = ledger.LastMysqlRemaining; + Interlocked.Exchange(ref ledger.InFlight, 0); + ledger.BatchId = null; + Volatile.Write(ref ledger.MysqlCommitted, 0); + AccountByteLogMessages.BatchReconciled(_logger, accountName, consumed, remaining); + if (remaining == 0 || mysqlRemaining == 0) + { + MarkExhausted(accountName); + } + } + + /// Returns the process-local pending total (tests). + internal long PendingBytes(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + ? Volatile.Read(ref ledger.Pending) + : 0; + + /// Returns the process-local in-flight total (tests). + internal long InFlightBytes(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + ? Volatile.Read(ref ledger.InFlight) + : 0; + + /// Returns whether MySQL already consumed the current in-flight batch (tests). + internal bool MysqlCommitted(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) + && Volatile.Read(ref ledger.MysqlCommitted) == 1; + + /// Returns the in-process Redis batch id awaiting APPLY acknowledgment (tests). + internal string? CurrentBatchId(string accountName) => + _ledgers.TryGetValue(accountName, out var ledger) ? ledger.BatchId : null; + + private AccountLedger GetOrAdd(string accountName) => + _ledgers.GetOrAdd(accountName, static _ => new AccountLedger()); + + private async ValueTask FloorRedisToMysqlAsync( + string accountName, + long mysqlRemaining, + CancellationToken cancellationToken) + { + try + { + _ = await _cluster + .ApplyAsync( + accountName, + AccountByteBatchId.Create(), + consumed: 0, + mysqlRemaining, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.ObserveFailed(_logger, ex, accountName); + } + } + + private async ValueTask CommitDurableAccountAsync( + string accountName, + AccountLedger ledger, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref ledger.MysqlCommitted) == 1) + { + return; + } + + var snapshot = Interlocked.Exchange(ref ledger.Pending, 0); + if (snapshot > 0) + { + Interlocked.Add(ref ledger.InFlight, snapshot); + } + + var inFlight = Volatile.Read(ref ledger.InFlight); + if (inFlight == 0) + { + return; + } + + AccountByteConsumeResult result; + try + { + result = await _durable.ConsumeAsync(accountName, inFlight, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException ex) + { + AccountByteLogMessages.DurableConsumeFailed(_logger, ex, accountName, inFlight); + return; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.DurableConsumeFailed(_logger, ex, accountName, inFlight); + return; + } + + if (!result.Succeeded) + { + Interlocked.Exchange(ref ledger.InFlight, 0); + ledger.BatchId = null; + return; + } + + ledger.LastConsumed = result.Consumed; + ledger.LastMysqlRemaining = result.Remaining; + ledger.BatchId = AccountByteBatchId.Create(); + Volatile.Write(ref ledger.MysqlCommitted, 1); + if (result.Remaining == 0) + { + MarkExhausted(accountName); + } + } + + private async ValueTask ApplyCommittedAccountAsync( + string accountName, + AccountLedger ledger, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref ledger.MysqlCommitted) != 1) + { + return; + } + + var consumed = ledger.LastConsumed; + var mysqlRemaining = ledger.LastMysqlRemaining; + var batchId = ledger.BatchId; + if (!AccountByteBatchId.IsValid(batchId)) + { + batchId = AccountByteBatchId.Create(); + ledger.BatchId = batchId; + } + + var applied = await _cluster + .ApplyAsync(accountName, batchId, consumed, mysqlRemaining, cancellationToken) + .ConfigureAwait(false); + if (applied is null) + { + AccountByteLogMessages.RedisApplyUnavailable(_logger, accountName, consumed, mysqlRemaining); + return; + } + + CompleteApply(accountName, applied.Value); + } + + private sealed class AccountLedger + { + public long Pending; + public long InFlight; + public int Exhausted; + public int MysqlCommitted; + public long LastConsumed; + public long LastMysqlRemaining; + public string? BatchId; + } + + private sealed class AccountLedgerSink(AccountLedger ledger) : IAccountByteSink + { + public void ObserveCopied(int bytes) + { + if (bytes <= 0) + { + return; + } + + Interlocked.Add(ref ledger.Pending, bytes); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteAccountant.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteAccountant.cs new file mode 100644 index 00000000..829c9cf1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteAccountant.cs @@ -0,0 +1,170 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Process-local download-quota bookkeeping. Redis and MySQL are used by + /// AUTHINFO observe and the periodic reconciler, never on the NNTP write path. + /// + public interface IAccountByteAccountant + { + /// + /// Returns a sink that s into this + /// account's pending total. No-op when is empty. + /// + IAccountByteSink CreateSink(string accountName); + + /// Gets whether this process has observed remaining == 0 for the account. + bool IsExhausted(string accountName); + + /// + /// Reads cluster remaining (Redis if present, otherwise durable MySQL). + /// Does not create or raise a Redis key. Returns when + /// remaining cannot be determined. + /// + /// + /// Live remaining is min(Redis, MySQL) when Redis exists. A stale-high Redis + /// key is floored down to MySQL; observe never raises Redis and never uses the + /// AUTHINFO user-record cache. + /// + ValueTask ObserveRemainingAsync(string accountName, CancellationToken cancellationToken = default); + + /// Marks the account exhausted for all local sessions of this process. + void MarkExhausted(string accountName); + + /// + /// Clears the process-local exhausted flag. Used after an explicit top-up + /// invalidation or when live remaining is observed greater than zero. + /// + void ClearExhausted(string accountName); + + /// + /// Deletes cluster Redis remaining-quota state after an operator MySQL top-up. + /// The next observe uses current durable remaining. Does not increase Redis. + /// + /// + /// when the key was removed or already absent; + /// when Redis was unavailable. + /// + ValueTask DeleteAccountByteStateAsync(string accountName, CancellationToken cancellationToken = default); + + /// + /// Snapshots pending bytes and persists one account-level MySQL consume per + /// account with work. Does not APPLY Redis. Used by + /// before the combined Redis cycle. + /// + ValueTask CommitDurableAsync(CancellationToken cancellationToken = default); + + /// + /// APPLY any MySQL-committed batches that are still awaiting Redis. Combined + /// RENEW+APPLY that already called is skipped. + /// + ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default); + + /// + /// Snapshots pending, consumes MySQL, and APPLYs leftover batches. Used by + /// tests and shutdown. + /// + ValueTask ReconcileAsync(CancellationToken cancellationToken = default); + + /// Returns the MySQL-committed batch awaiting Redis APPLY, if any. + bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch); + + /// + /// Clears the in-process committed batch after a successful Redis APPLY + /// (combined EVAL or APPLY-only). + /// + void CompleteApply(string accountName, long remaining); + } + + /// One account-wide MySQL-committed consume awaiting Redis APPLY. + public readonly struct AccountByteCommittedBatch + { + /// Initializes a committed batch snapshot. + public AccountByteCommittedBatch(string accountName, string batchId, long consumed, long mysqlRemaining) + { + AccountName = accountName; + BatchId = batchId; + Consumed = consumed; + MysqlRemaining = mysqlRemaining; + } + + /// Gets the AUTHINFO account name. + public string AccountName { get; } + + /// Gets the in-process batch identity. + public string BatchId { get; } + + /// Gets bytes MySQL already subtracted for this batch. + public long Consumed { get; } + + /// Gets durable remaining after this consume. + public long MysqlRemaining { get; } + } + + /// No-op accountant for tests and unauthenticated identities. + public sealed class NullAccountByteAccountant : IAccountByteAccountant + { + /// Shared no-op instance. + public static NullAccountByteAccountant Instance { get; } = new(); + + private NullAccountByteAccountant() + { + } + + /// + public IAccountByteSink CreateSink(string accountName) => NullAccountByteSink.Instance; + + /// + public bool IsExhausted(string accountName) => false; + + /// + public ValueTask ObserveRemainingAsync(string accountName, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + /// + public void MarkExhausted(string accountName) + { + } + + /// + public void ClearExhausted(string accountName) + { + } + + /// + public ValueTask DeleteAccountByteStateAsync( + string accountName, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(true); + + /// + public ValueTask CommitDurableAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public ValueTask ApplyCommittedAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public ValueTask ReconcileAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + + /// + public bool TryGetCommittedBatch(string accountName, out AccountByteCommittedBatch batch) + { + batch = default; + return false; + } + + /// + public void CompleteApply(string accountName, long remaining) + { + } + } + + /// Sink that ignores copied bytes. + internal sealed class NullAccountByteSink : IAccountByteSink + { + public static NullAccountByteSink Instance { get; } = new(); + + public void ObserveCopied(int bytes) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteDurableStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteDurableStore.cs new file mode 100644 index 00000000..46a89ac4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteDurableStore.cs @@ -0,0 +1,21 @@ +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Durable remaining-quota store (MySQL nntpusers.account_byte_limit). + internal interface IAccountByteDurableStore + { + /// + /// Atomically subtracts from remaining quota, clamping at zero. + /// + ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default); + + /// Reads remaining without mutating. Does not use the AUTHINFO user-record cache. + ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteSink.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteSink.cs new file mode 100644 index 00000000..474a67cc --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteSink.cs @@ -0,0 +1,14 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// + /// Hot-path observer for NNTP application bytes copied into the outbound PipeWriter. + /// + /// + /// Implementations must be allocation-free and must not perform Redis or MySQL I/O. + /// + public interface IAccountByteSink + { + /// Records copied into the output pipe for this account. + void ObserveCopied(int bytes); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteStore.cs new file mode 100644 index 00000000..610f03d5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/IAccountByteStore.cs @@ -0,0 +1,36 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Cluster remaining-quota store. Not used on the NNTP write path. + internal interface IAccountByteStore + { + /// + /// Atomically applies one MySQL-committed batch. The same + /// is a no-op. Remaining is floored to + /// and never increased. + /// + /// Remaining after the apply, or when the store is unavailable. + ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default); + + /// + /// Reads remaining. means unavailable; + /// means the key is absent. + /// + ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default); + + /// + /// Deletes cluster remaining-quota state for an explicit MySQL top-up. + /// The next observe/apply treats the key as missing and may initialize from + /// current durable remaining. Does not raise remaining. + /// + /// + /// 1 if a key was removed, 0 if it was already absent, + /// or when the store is unavailable. + /// + ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs new file mode 100644 index 00000000..3b741360 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteDurableStore.cs @@ -0,0 +1,103 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// In-memory durable remaining quota for tests. Shared across trackers as one MySQL. + internal sealed class InMemoryAccountByteDurableStore : IAccountByteDurableStore + { + private readonly Lock _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + + /// When set, consume/query fail as if MySQL were down. + public bool Unavailable { get; set; } + + /// Gets how many consume operations mutated or inspected a row. + public int ConsumeCalls { get; private set; } + + /// Gets how many remaining-quota reads ran. + public int QueryCalls { get; private set; } + + /// Adds or replaces remaining quota for an account. + public void SeedByteAccount(string accountName, long remaining) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(remaining); + lock (_gate) + { + _accounts[accountName] = new AccountRow(remaining); + } + } + + /// Returns stored remaining, or when missing. + public long? Remaining(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var row) ? row.Remaining : null; + } + } + + /// + public ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + throw new NntpDbUnavailableException("MySQL byte-quota consume unavailable."); + } + + lock (_gate) + { + ConsumeCalls++; + if (!_accounts.TryGetValue(accountName, out var row)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var consumed = bytes > row.Remaining ? row.Remaining : bytes; + row.Remaining -= consumed; + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, row.Remaining, consumed)); + } + } + + /// + public ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + throw new NntpDbUnavailableException("MySQL byte-quota query unavailable."); + } + + lock (_gate) + { + QueryCalls++; + if (!_accounts.TryGetValue(accountName, out var row)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, row.Remaining, 0)); + } + } + + private sealed class AccountRow(long remaining) + { + public long Remaining { get; set; } = remaining; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs new file mode 100644 index 00000000..baa57dd1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/InMemoryAccountByteStore.cs @@ -0,0 +1,83 @@ +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Process-local cluster fake. Two trackers sharing one instance behave as two nodes. + internal sealed class InMemoryAccountByteStore : IAccountByteStore + { + private readonly AccountByteEngine _engine = new(); + + /// When set, apply/observe fail as if Redis were down. + public bool Unavailable { get; set; } + + /// Gets how many apply operations reached the engine. + public int ApplyCalls { get; private set; } + + /// Gets how many observe operations reached the engine. + public int ObserveCalls { get; private set; } + + /// + /// When greater than zero, the next apply runs the engine then returns + /// (Redis executed, response lost). + /// + public int HideNextApplyResults { get; set; } + + /// Gets the shared engine (tests). + internal AccountByteEngine Engine => _engine; + + /// + public ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + ApplyCalls++; + var remaining = _engine.Apply(EncodingKey(accountName), batchId, consumed, mysqlRemainingAfter); + if (HideNextApplyResults > 0) + { + HideNextApplyResults--; + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(remaining); + } + + /// + public ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + ObserveCalls++; + return ValueTask.FromResult(_engine.Observe(EncodingKey(accountName))); + } + + /// + public ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(_engine.Delete(EncodingKey(accountName))); + } + + internal static string EncodingKey(string accountName) => + System.Text.Encoding.UTF8.GetString(AccountByteKeys.Create(accountName)); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs new file mode 100644 index 00000000..dbc80277 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/MysqlAccountByteDurableStore.cs @@ -0,0 +1,74 @@ +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Durable remaining-quota store over the existing NntpDB pool. + internal sealed class MysqlAccountByteDurableStore : IAccountByteDurableStore + { + private readonly NntpDbService _nntpDb; + + /// Initializes a new instance of the class. + public MysqlAccountByteDurableStore(NntpDbService nntpDb) + { + ArgumentNullException.ThrowIfNull(nntpDb); + _nntpDb = nntpDb; + } + + /// + public async ValueTask ConsumeAsync( + string accountName, + long bytes, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.ConsumeAccountBytesAsync(accountName, bytes, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + throw new NntpDbUnavailableException("MySQL account byte-quota consume failed.", ex); + } + } + + /// + public async ValueTask QueryRemainingAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + try + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + return await connection.QueryAccountByteRemainingAsync(accountName, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (NntpDbUnavailableException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + throw new NntpDbUnavailableException("MySQL account byte-quota query failed.", ex); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/RedisAccountByteStore.cs b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/RedisAccountByteStore.cs new file mode 100644 index 00000000..4b5e3ed4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/BytesAccounting/RedisAccountByteStore.cs @@ -0,0 +1,112 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.SessionState.BytesAccounting +{ + /// Redis remaining-quota store. One EVAL per observe/apply. No key TTL. + internal sealed class RedisAccountByteStore : IAccountByteStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisAccountByteStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask ApplyAsync( + string accountName, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + return ExecuteAsync( + AccountByteScripts.Apply, + accountName, + [ + Utf8(batchId), + Utf8(AccountByteEngine.ClampNonNegative(consumed).ToString(CultureInfo.InvariantCulture)), + Utf8(AccountByteEngine.ClampNonNegative(mysqlRemainingAfter).ToString(CultureInfo.InvariantCulture)), + ], + static result => result < 0 ? 0L : result, + cancellationToken); + } + + /// + public ValueTask ObserveAsync(string accountName, CancellationToken cancellationToken = default) => + ExecuteAsync( + AccountByteScripts.Observe, + accountName, + [], + static result => result, + cancellationToken); + + /// + public ValueTask DeleteAsync(string accountName, CancellationToken cancellationToken = default) => + ExecuteAsync( + AccountByteScripts.Delete, + accountName, + [], + static result => result < 0 ? 0L : result, + cancellationToken); + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] values, + Func map, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return null; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return null; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + [AccountByteKeys.Create(accountName)], + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return null; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return null; + } + } + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/DistributedSessionStateTracker.cs b/src/VectorNNTPD.TransitService/SessionState/DistributedSessionStateTracker.cs new file mode 100644 index 00000000..ec513214 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/DistributedSessionStateTracker.cs @@ -0,0 +1,726 @@ +using System.Collections.Concurrent; +using System.Net; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState.RateLimiting; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Hybrid authenticated-session admission: Redis is the cluster authority for + /// both account_session_limit and account_srcip_limit. + /// + /// + /// Both limits are cluster-wide and evaluated in one atomic membership operation. + /// Limit 0 is unlimited. The source-IP local hot path is used only while + /// account_session_limit is unlimited and this node already holds a live + /// source-IP lease. A finite session limit requires every new session to take + /// the distributed path so the global session count stays exact. New admissions + /// that require Redis fail closed when membership is unavailable. + /// Owner identity is {nodeId}:{incarnation}; incarnation is a process-lifetime + /// identifier so a restart cannot mutate a previous incarnation's unexpired fields. + /// Redis TTL is crash recovery only: while this process is alive and renewal + /// succeeds, idle authenticated sessions keep consuming cluster capacity. + /// + public sealed class DistributedSessionStateTracker : ISessionStateTracker, ISessionStateLeaseManager + { + private readonly ISessionStateStore _membership; + private readonly IAccountByteAccountant? _bytes; + private readonly IAccountRateAllocator _rates; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly string _ownerId; + private readonly TimeSpan _leaseTtl; + private readonly TimeSpan _hotPathSkew; + private readonly ConcurrentDictionary _accountGates = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + private long _generations; + + /// Initializes a production tracker with default lease timings. + public DistributedSessionStateTracker( + ISessionStateStore membership, + ILogger logger, + string nodeId) + : this(membership, logger, nodeId, TimeProvider.System, SessionStateDefaults.LeaseTtl, SessionStateDefaults.HotPathSkew, incarnation: null, bytes: null, rates: null) + { + } + + /// Initializes a tracker with explicit timing and incarnation (tests). + internal DistributedSessionStateTracker( + ISessionStateStore membership, + ILogger logger, + string nodeId, + TimeProvider timeProvider, + TimeSpan? leaseTtl = null, + TimeSpan? hotPathSkew = null, + string? incarnation = null, + IAccountByteAccountant? bytes = null, + IAccountRateAllocator? rates = null) + { + ArgumentNullException.ThrowIfNull(membership); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentNullException.ThrowIfNull(timeProvider); + _membership = membership; + _bytes = bytes; + _rates = rates ?? NullAccountRateAllocator.Instance; + _logger = logger; + _nodeId = nodeId; + _ownerId = string.Concat(nodeId, ":", string.IsNullOrWhiteSpace(incarnation) ? Guid.NewGuid().ToString("N") : incarnation); + _timeProvider = timeProvider; + _leaseTtl = leaseTtl ?? SessionStateDefaults.LeaseTtl; + _hotPathSkew = hotPathSkew ?? SessionStateDefaults.HotPathSkew; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(_leaseTtl, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThan(_hotPathSkew, TimeSpan.Zero); + } + + /// Gets the configured node identity. + internal string NodeId => _nodeId; + + /// Gets {nodeId}:{incarnation} written into Redis ownership fields. + internal string OwnerId => _ownerId; + + /// Gets how many admits were satisfied from a live local source-IP lease. + internal int LocalHotPathAdmits { get; private set; } + + /// Gets how many admits required a distributed membership call. + internal int DistributedAdmits { get; private set; } + + /// Gets how many admits were rejected by the distinct-IP limit. + internal int DistributedSourceRejects { get; private set; } + + /// Gets how many admits were rejected by the cluster session limit. + internal int DistributedSessionRejects { get; private set; } + + /// Gets how many admits were rejected by the distinct-IP limit. + internal int DistributedRejects => DistributedSourceRejects; + + /// Gets how many admits failed closed because membership was unavailable. + internal int UnavailableRejects { get; private set; } + + /// Gets how many admitted sessions this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default) => + TryAdmitAsync(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit, rateLimitBps: 0, cancellationToken); + + /// + public async ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(sourceAddress); + + var ip = SourceAddressIdentity.Format(sourceAddress); + var accountGate = _accountGates.GetOrAdd(accountName, static _ => new SemaphoreSlim(1, 1)); + await accountGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (TryAdmitLocalHotPath(accountName, sessionId, ip, sessionLimit, srcIpLimit, rateLimitBps, out var local)) + { + return local; + } + + if (sessionLimit <= 0 && srcIpLimit <= 0 && rateLimitBps <= 0) + { + return SessionAdmissionResult.Success; + } + + var trackSessions = sessionLimit > 0 || rateLimitBps > 0; + long sessionGeneration; + long sourceGeneration; + lock (_gate) + { + _accounts.TryGetValue(accountName, out var account); + sessionGeneration = trackSessions + ? (account is { SessionGeneration: not 0 } ? account.SessionGeneration : NextGeneration()) + : 0; + sourceGeneration = srcIpLimit > 0 + && account is not null + && account.Ips.TryGetValue(ip, out var owner) + && owner is { Distributed: true, Generation: not 0 } + ? owner.Generation + : (srcIpLimit > 0 ? NextGeneration() : 0); + } + + var now = _timeProvider.GetUtcNow(); + SessionStateAdmitResult membership; + try + { + membership = await _membership.TryAdmitAsync( + accountName, + ip, + _ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + _leaseTtl, + cancellationToken, + trackSessions).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // TRY_ADMIT is atomic. Cancellation after a successful EVAL (or a + // store that throws OCE after writing) must still decrement, or + // this process has no local session that can later release. + await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!membership.Accepted) + { + if (membership.Status == SessionStateAdmitStatus.Unavailable) + { + UnavailableRejects++; + SessionStateLogMessages.SessionStateUnavailable(_logger, accountName, ip, srcIpLimit, _nodeId); + return SessionAdmissionResult.Unavailable; + } + + if (membership.Status == SessionStateAdmitStatus.RejectedSessionLimit) + { + DistributedSessionRejects++; + SessionStateLogMessages.SessionAdmissionRejected(_logger, accountName, ip, sessionLimit, _nodeId); + return SessionAdmissionResult.SessionLimitExceeded; + } + + DistributedSourceRejects++; + SessionStateLogMessages.SourceAddressRejected(_logger, accountName, ip, srcIpLimit, _nodeId); + return SessionAdmissionResult.SourceAddressLimitExceeded; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + CommitDistributedAdmit( + accountName, + sessionId, + ip, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + trackSessions); + } + catch (OperationCanceledException) + { + await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + DistributedAdmits++; + if (rateLimitBps > 0) + { + _rates.ObserveClusterSessionCount(accountName, membership.SessionTotal); + } + + SessionStateLogMessages.SessionAdmittedDistributed( + _logger, + accountName, + ip, + srcIpLimit, + _nodeId, + membership.Status.ToString()); + return SessionAdmissionResult.Success; + } + finally + { + accountGate.Release(); + } + } + + /// + public async ValueTask ReleaseAsync( + string accountName, + string sessionId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + + var accountGate = _accountGates.GetOrAdd(accountName, static _ => new SemaphoreSlim(1, 1)); + await accountGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + string? ip = null; + long sessionGeneration = 0; + long sourceGeneration = 0; + var releaseMembership = false; + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account) + || !account.Sessions.Remove(sessionId, out var session)) + { + return; + } + + ReleaseCalls++; + + ip = session.Ip; + sessionGeneration = account.SessionGeneration; + var distributedSession = account.DistributedSession; + if (account.Ips.TryGetValue(ip, out var owner)) + { + owner.Count--; + sourceGeneration = owner.Generation; + if (owner.Count <= 0) + { + account.Ips.Remove(ip); + releaseMembership = owner.Distributed || distributedSession; + } + else + { + releaseMembership = distributedSession; + } + } + else + { + releaseMembership = distributedSession; + } + + if (account.Sessions.Count == 0) + { + _accounts.Remove(accountName); + } + } + + if (releaseMembership && ip is not null) + { + var remaining = await _membership.ReleaseAsync( + accountName, + ip, + _ownerId, + sessionGeneration, + sourceGeneration, + cancellationToken) + .ConfigureAwait(false); + // Redis unavailable maps to 0, the same integer as "last cluster session + // released". Observing 0 while local R sessions remain treats remotes as + // gone and can raise joiners from blocked to the full account rate. + if (remaining > 0) + { + _rates.ObserveClusterSessionCount(accountName, remaining); + } + + SessionStateLogMessages.SessionStateReleased(_logger, accountName, ip, _nodeId); + } + } + finally + { + accountGate.Release(); + } + } + + /// + public async ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + List<(string Account, long SessionGeneration, List<(string Ip, long Generation)> Sources)> owners; + lock (_gate) + { + owners = []; + foreach (var (accountName, account) in _accounts) + { + List<(string Ip, long Generation)> sources = []; + foreach (var (ip, owner) in account.Ips) + { + if (owner is { Distributed: true, Count: > 0 }) + { + sources.Add((ip, owner.Generation)); + } + } + + if (!account.DistributedSession && sources.Count == 0) + { + continue; + } + + owners.Add(( + accountName, + account.DistributedSession ? account.SessionGeneration : 0, + sources)); + } + } + + var now = _timeProvider.GetUtcNow(); + foreach (var (accountName, sessionGeneration, sources) in owners) + { + cancellationToken.ThrowIfCancellationRequested(); + SessionStateRenewResult status; + try + { + status = await RenewAccountAsync( + accountName, + sessionGeneration, + sources, + now, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateLeaseRenewFailed( + _logger, + ex, + accountName, + sources.Count > 0 ? sources[0].Ip : "-", + _nodeId); + continue; + } + + if (status.Status == SessionStateRenewStatus.Renewed) + { + if (status.SessionTotal > 0) + { + _rates.ObserveClusterSessionCount(accountName, status.SessionTotal); + } + + foreach (var (ip, _) in sources) + { + MarkLeaseRenewed(accountName, ip, now); + SessionStateLogMessages.SessionStateLeaseRenewed(_logger, accountName, ip, _nodeId); + } + + continue; + } + + if (status.Status == SessionStateRenewStatus.Unavailable) + { + SessionStateLogMessages.SessionStateLeaseRenewUnavailable( + _logger, + accountName, + sources.Count > 0 ? sources[0].Ip : "-", + _nodeId); + continue; + } + + foreach (var (ip, _) in sources) + { + InvalidateHotPath(accountName, ip); + SessionStateLogMessages.SessionStateLeaseLost(_logger, accountName, ip, _nodeId); + } + } + } + + private async ValueTask RenewAccountAsync( + string accountName, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + CancellationToken cancellationToken) + { + if (_bytes is not null + && _bytes.TryGetCommittedBatch(accountName, out var batch)) + { + var combined = await _membership.RenewAndApplyAsync( + accountName, + _ownerId, + sessionGeneration, + sources, + now, + _leaseTtl, + batch.BatchId, + batch.Consumed, + batch.MysqlRemaining, + cancellationToken).ConfigureAwait(false); + if (combined.Remaining is { } remaining) + { + _bytes.CompleteApply(accountName, remaining); + } + + return new SessionStateRenewResult(combined.Renew); + } + + return await _membership.RenewAsync( + accountName, + _ownerId, + sessionGeneration, + sources, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + + /// + public async ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) + { + List accounts; + lock (_gate) + { + accounts = []; + foreach (var (accountName, account) in _accounts) + { + if (account.DistributedSession || account.Ips.Values.Any(static owner => owner.Distributed)) + { + accounts.Add(accountName); + } + } + } + + foreach (var accountName in accounts) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await _membership.ReleaseOwnerAsync(accountName, _ownerId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateReleaseFailed(_logger, ex, accountName, _ownerId); + } + } + } + + /// Returns the local authenticated session count for . + internal int GetLocalSessionCount(string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + } + } + + /// Returns the local reference count for an account/source pair. + internal int GetLocalIpCount(string accountName, string normalizedSourceIp) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(normalizedSourceIp, out var owner) + ? owner.Count + : 0; + } + } + + private bool TryAdmitLocalHotPath( + string accountName, + string sessionId, + string ip, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + out SessionAdmissionResult result) + { + lock (_gate) + { + _accounts.TryGetValue(accountName, out var account); + if (account is not null && account.Sessions.ContainsKey(sessionId)) + { + result = SessionAdmissionResult.Success; + return true; + } + + if (sessionLimit <= 0 && srcIpLimit <= 0 && rateLimitBps <= 0) + { + account ??= GetOrCreateAccount(accountName); + account.Sessions[sessionId] = new SessionAdmission(ip); + if (!account.Ips.TryGetValue(ip, out var unlimited)) + { + unlimited = new IpOwner(); + account.Ips[ip] = unlimited; + } + + unlimited.Count++; + result = SessionAdmissionResult.Success; + return true; + } + + // A finite session limit or positive rate share consumes a cluster-wide + // slot; the source-IP hot path must not skip Redis. + if (sessionLimit > 0 || rateLimitBps > 0) + { + result = SessionAdmissionResult.Success; + return false; + } + + var now = _timeProvider.GetUtcNow(); + if (account is not null + && account.Ips.TryGetValue(ip, out var owner) + && owner.Distributed + && owner.HotPathValid + && owner.LeaseExpiresAt - _hotPathSkew > now) + { + owner.Count++; + owner.SrcIpLimit = srcIpLimit; + account.Sessions[sessionId] = new SessionAdmission(ip); + LocalHotPathAdmits++; + SessionStateLogMessages.SessionAdmittedLocal(_logger, accountName, ip, srcIpLimit, _nodeId); + result = SessionAdmissionResult.Success; + return true; + } + + if (account is not null && account.Sessions.Count == 0 && account.Ips.Count == 0) + { + _accounts.Remove(accountName); + } + + result = SessionAdmissionResult.Success; + return false; + } + } + + private void CommitDistributedAdmit( + string accountName, + string sessionId, + string ip, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + bool trackSessions) + { + lock (_gate) + { + var account = GetOrCreateAccount(accountName); + if (account.Sessions.ContainsKey(sessionId)) + { + return; + } + + if (sessionLimit > 0 || trackSessions) + { + account.DistributedSession = true; + account.SessionGeneration = sessionGeneration; + } + + if (!account.Ips.TryGetValue(ip, out var owner)) + { + owner = new IpOwner(); + account.Ips[ip] = owner; + } + + owner.Count++; + if (srcIpLimit > 0) + { + owner.Generation = sourceGeneration; + owner.SrcIpLimit = srcIpLimit; + owner.Distributed = true; + owner.HotPathValid = sessionLimit <= 0; + owner.LeaseExpiresAt = now + _leaseTtl; + } + + account.Sessions[sessionId] = new SessionAdmission(ip); + } + } + + private AccountAdmission GetOrCreateAccount(string accountName) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + account = new AccountAdmission(); + _accounts[accountName] = account; + } + + return account; + } + + private void MarkLeaseRenewed(string accountName, string ip, DateTimeOffset now) + { + lock (_gate) + { + if (_accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(ip, out var owner) + && owner.Distributed) + { + owner.LeaseExpiresAt = now + _leaseTtl; + owner.HotPathValid = !account.DistributedSession; + } + } + } + + private void InvalidateHotPath(string accountName, string ip) + { + lock (_gate) + { + if (_accounts.TryGetValue(accountName, out var account) + && account.Ips.TryGetValue(ip, out var owner)) + { + owner.HotPathValid = false; + owner.LeaseExpiresAt = DateTimeOffset.MinValue; + } + } + } + + private long NextGeneration() => Interlocked.Increment(ref _generations); + + private sealed class AccountAdmission + { + public Dictionary Sessions { get; } = new(StringComparer.Ordinal); + + public Dictionary Ips { get; } = new(StringComparer.Ordinal); + + public bool DistributedSession { get; set; } + + public long SessionGeneration { get; set; } + } + + private sealed class SessionAdmission + { + public SessionAdmission(string ip) + { + Ip = ip; + } + + public string Ip { get; } + } + + private sealed class IpOwner + { + public int Count { get; set; } + + public long Generation { get; set; } + + public int SrcIpLimit { get; set; } + + public bool Distributed { get; set; } + + public bool HotPathValid { get; set; } + + public DateTimeOffset LeaseExpiresAt { get; set; } + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/ISessionStateLeaseManager.cs b/src/VectorNNTPD.TransitService/SessionState/ISessionStateLeaseManager.cs new file mode 100644 index 00000000..f84293f4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/ISessionStateLeaseManager.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Process-local lease renewal and shutdown release for cluster session-state ownership. + /// + /// + /// Renewal proves this node is still responsible for its authenticated sessions. + /// It is independent of NNTP traffic. One renew operation per account extends + /// session and source-IP ownership together, or extends none of them. + /// The background is the sole scheduler; this + /// abstraction owns the actual ownership snapshot and Redis operations. When a + /// MySQL-committed byte batch exists for an owned account, renewal and APPLY + /// share one Redis EVAL. A successful renew also returns the cluster session + /// total so local rate caps can be updated without a second scheduler. + /// + public interface ISessionStateLeaseManager + { + /// + /// Refreshes this incarnation's session and source-IP leases for accounts that + /// still have local authenticated sessions. Does not recreate missing ownership. + /// + ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default); + + /// Releases leftover node/incarnation ownership without touching the database. + ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/ISessionStateStore.cs b/src/VectorNNTPD.TransitService/SessionState/ISessionStateStore.cs new file mode 100644 index 00000000..fbfc338e --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/ISessionStateStore.cs @@ -0,0 +1,77 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Cluster-wide account session and source-IP membership. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Admit, release, and renew each use one atomic operation. + /// + public interface ISessionStateStore + { + /// + /// Atomically admits one session and this owner's source-IP membership. + /// + ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false); + + /// + /// Atomically decrements this owner's session count and source-IP count. + /// Returns the remaining cluster session total. Unavailable membership + /// returns 0; callers must not treat that as a live total. + /// + ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's session and source-IP leases. + /// All requested fields are extended or none are. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's session/source leases and APPLYs one + /// MySQL-committed byte batch for the same account. APPLY runs even when + /// renewal is lost. + /// is when this store cannot apply byte state. + /// + ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default); + + /// Deletes leftover ownership for this process incarnation. + ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/ISessionStateTracker.cs b/src/VectorNNTPD.TransitService/SessionState/ISessionStateTracker.cs new file mode 100644 index 00000000..5923cc08 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/ISessionStateTracker.cs @@ -0,0 +1,47 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Atomically admits and releases concurrent authenticated reader sessions per account. + /// + /// + /// session_limit 0 and srcip_limit 0 are unlimited. + /// session_limit is the cluster-wide cap on authenticated sessions. + /// srcip_limit is a cluster-wide cap on distinct source IPs that currently + /// have at least one admitted session, not a per-IP session cap. Production uses + /// : both limits are enforced + /// atomically through Redis ownership leases. + /// + public interface ISessionStateTracker + { + /// + /// Attempts to admit for . + /// Re-admitting the same session id is idempotent. + /// + ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default); + + /// + /// Admits a session and, when (bits per second) + /// is greater than zero, tracks the cluster session count for rate allocation. + /// + ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) => + TryAdmitAsync(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit, cancellationToken); + + /// Releases a previously admitted session. Idempotent when the session is unknown. + ValueTask ReleaseAsync(string accountName, string sessionId, CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateStore.cs b/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateStore.cs new file mode 100644 index 00000000..330c4ab4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateStore.cs @@ -0,0 +1,256 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Process-local cluster fake for session + source-IP membership. Two trackers + /// sharing one instance behave as two NNTP nodes. Ordinary unit tests must not + /// require live Redis. + /// + internal sealed class InMemorySessionStateStore : ISessionStateStore + { + private readonly SessionStateEngine _engine = new(); + + /// When set, admit/release/renew fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Optional gate that blocks the next admit until completed or cancelled. + public TaskCompletionSource? BlockAdmit { get; set; } + + /// Optional gate that blocks the next release until completed or cancelled. + public TaskCompletionSource? BlockRelease { get; set; } + + /// Optional gate that blocks the next renew until completed or cancelled. + public TaskCompletionSource? BlockRenew { get; set; } + + /// Set when is entered, before waits. + public TaskCompletionSource? NotifyRenewStarted { get; set; } + + /// When set, throws for this account so other accounts can still renew. + public string? FaultRenewAccount { get; set; } + + /// Gets how many membership renew operations were attempted. + public int RenewCalls { get; private set; } + + /// Gets the shared admission engine (tests). + internal SessionStateEngine Engine => _engine; + + /// In-memory source hash identity for . + internal static string SourceKey(string accountName) => "src:" + accountName; + + /// In-memory session hash identity for . + internal static string SessionKey(string accountName) => "sess:" + accountName; + + /// Returns whether currently holds . + internal bool HasSourceOwner(string accountName, string normalizedSourceIp, string ownerId, long nowUnixMs) => + _engine.HasSourceOwner(SourceKey(accountName), normalizedSourceIp, ownerId, nowUnixMs); + + /// Returns the cluster-wide authenticated session count for . + internal int ActiveSessionCount(string accountName, long nowUnixMs) => + _engine.ActiveSessionCount(SessionKey(accountName), nowUnixMs); + + /// Returns this owner's unexpired session count. + internal int OwnerSessionCount(string accountName, string ownerId, long nowUnixMs) => + _engine.OwnerSessionCount(SessionKey(accountName), ownerId, nowUnixMs); + + /// Returns distinct unexpired source IPs for . + internal IReadOnlyCollection ActiveSourceIps(string accountName, long nowUnixMs) => + _engine.ActiveSourceIps(SourceKey(accountName), nowUnixMs); + + /// Returns the stored session-ownership expiry, or when absent. + internal long? SessionExpiry(string accountName, string ownerId) => + _engine.TryGetOwnership(SessionKey(accountName), ownerId, out var expiry, out _, out _) + ? expiry + : null; + + /// Returns the stored source-ownership expiry, or when absent. + internal long? SourceExpiry(string accountName, string normalizedSourceIp, string ownerId) => + _engine.TryGetOwnership( + SourceKey(accountName), + SessionStateKeys.SourceField(normalizedSourceIp, ownerId), + out var expiry, + out _, + out _) + ? expiry + : null; + + /// Changes a source field's generation so a later renew is all-or-nothing rejected. + internal void BreakSourceGeneration(string accountName, string normalizedSourceIp, string ownerId) + { + var field = SessionStateKeys.SourceField(normalizedSourceIp, ownerId); + if (!_engine.TryGetOwnership(SourceKey(accountName), field, out var expiry, out var generation, out var count)) + { + throw new InvalidOperationException("Source ownership is missing."); + } + + _engine.WriteOwnership(SourceKey(accountName), field, expiry, generation + 1, count); + } + + /// + public async ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (BlockAdmit is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return new SessionStateAdmitResult(SessionStateAdmitStatus.Unavailable); + } + + var code = _engine.TryAdmit( + SourceKey(accountName), + SessionKey(accountName), + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + sessionGeneration, + sourceGeneration, + trackSessions); + var status = SessionStateEngine.UnpackAdmitStatus(code); + var total = SessionStateEngine.UnpackAdmitSessionTotal(code); + return status switch + { + SessionStateEngine.AcceptedExisting => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedExisting, total), + SessionStateEngine.AcceptedNew => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedNew, total), + SessionStateEngine.RejectedSessionLimit => + new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSessionLimit), + _ => new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSourceLimit), + }; + } + + /// + public async ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (BlockRelease is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return 0; + } + + return (int)_engine.Release( + SourceKey(accountName), + SessionKey(accountName), + normalizedSourceIp, + ownerId, + sessionGeneration, + sourceGeneration); + } + + /// + public async ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentNullException.ThrowIfNull(sources); + if (FaultRenewAccount is { } fault + && string.Equals(fault, accountName, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Injected renewal fault."); + } + + NotifyRenewStarted?.TrySetResult(); + if (BlockRenew is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + RenewCalls++; + if (Unavailable) + { + return new SessionStateRenewResult(SessionStateRenewStatus.Unavailable); + } + + var result = _engine.Renew( + SourceKey(accountName), + SessionKey(accountName), + ownerId, + sessionGeneration, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + sources); + return result == 0 + ? new SessionStateRenewResult(SessionStateRenewStatus.Lost) + : new SessionStateRenewResult(SessionStateRenewStatus.Renewed, (int)result); + } + + /// + public async ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + _ = batchId; + _ = consumed; + _ = mysqlRemainingAfter; + var status = await RenewAsync( + accountName, + ownerId, + sessionGeneration, + sources, + now, + leaseTtl, + cancellationToken).ConfigureAwait(false); + return new SessionStateRenewAndApplyResult(status.Status, remaining: null); + } + + /// + public ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + if (!Unavailable) + { + _ = _engine.ReleaseOwner(SourceKey(accountName), SessionKey(accountName), ownerId); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateTracker.cs b/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateTracker.cs new file mode 100644 index 00000000..3277e7a8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/InMemorySessionStateTracker.cs @@ -0,0 +1,175 @@ +using System.Net; +using VectorNNTPD.TransitService.SessionState.RateLimiting; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Process-local atomic admission tracker used by unit tests. Production cluster + /// session and source-IP admission uses . + /// + public sealed class InMemorySessionStateTracker : ISessionStateTracker + { + private readonly object _gate = new(); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + private readonly IAccountRateAllocator _rates; + + /// Initializes a process-local tracker. + public InMemorySessionStateTracker(IAccountRateAllocator? rates = null) + { + _rates = rates ?? NullAccountRateAllocator.Instance; + } + + /// Gets how many admitted sessions this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask( + TryAdmit(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit)); + } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit, + int rateLimitBps, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = TryAdmit(accountName, sessionId, sourceAddress, sessionLimit, srcIpLimit); + if (result == SessionAdmissionResult.Success && rateLimitBps > 0) + { + _rates.ObserveClusterSessionCount(accountName, GetLocalSessionCount(accountName)); + } + + return new ValueTask(result); + } + + /// + public ValueTask ReleaseAsync( + string accountName, + string sessionId, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Release(accountName, sessionId); + return ValueTask.CompletedTask; + } + + /// Synchronous admit used by existing in-process tests. + public SessionAdmissionResult TryAdmit( + string accountName, + string sessionId, + IPAddress sourceAddress, + int sessionLimit, + int srcIpLimit) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(sourceAddress); + + var ip = SourceAddressIdentity.Format(sourceAddress); + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + account = new AccountAdmission(); + _accounts[accountName] = account; + } + + if (account.Sessions.TryGetValue(sessionId, out var existingIp)) + { + _ = existingIp; + return SessionAdmissionResult.Success; + } + + if (sessionLimit > 0 && account.Sessions.Count >= sessionLimit) + { + return SessionAdmissionResult.SessionLimitExceeded; + } + + if (srcIpLimit > 0 + && !account.IpCounts.ContainsKey(ip) + && account.IpCounts.Count >= srcIpLimit) + { + return SessionAdmissionResult.SourceAddressLimitExceeded; + } + + account.Sessions[sessionId] = ip; + account.IpCounts[ip] = account.IpCounts.GetValueOrDefault(ip) + 1; + return SessionAdmissionResult.Success; + } + } + + /// + public void Release(string accountName, string sessionId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + lock (_gate) + { + if (!_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + if (!account.Sessions.Remove(sessionId, out var ip)) + { + return; + } + + ReleaseCalls++; + + if (account.IpCounts.TryGetValue(ip, out var count)) + { + if (count <= 1) + { + account.IpCounts.Remove(ip); + } + else + { + account.IpCounts[ip] = count - 1; + } + } + + if (account.Sessions.Count == 0) + { + _accounts.Remove(accountName); + _rates.ObserveClusterSessionCount(accountName, 0); + } + else + { + _rates.ObserveClusterSessionCount(accountName, account.Sessions.Count); + } + } + } + + private int GetLocalSessionCount(string accountName) + { + lock (_gate) + { + return _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + } + } + + /// Formats the admitted source identity. IPv4-mapped IPv6 becomes IPv4 text. + internal static string FormatSourceAddress(IPAddress address) => SourceAddressIdentity.Format(address); + + private sealed class AccountAdmission + { + public Dictionary Sessions { get; } = new(StringComparer.Ordinal); + public Dictionary IpCounts { get; } = new(StringComparer.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateAllocator.cs b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateAllocator.cs new file mode 100644 index 00000000..b82032e8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateAllocator.cs @@ -0,0 +1,239 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.TransitService.SessionState.RateLimiting +{ + /// + /// Process-local map of account sessions to outbound caps. Cluster session + /// counts come from SessionState admit/release/renew; the write path never + /// consults this type. + /// + /// + /// A cross-node join would overshoot if the new session took an equal share + /// while remotes still held floor(rate / previousCount). The unused + /// remainder accountBytes % remotes is safe only when every remote is + /// that previous equal-share group. A second joining node sees remotes that + /// include the first joiner and would take another remainder while the + /// established node still holds nearly the whole aggregate. While remotes + /// exist, new sessions are blocked. Established local sessions may only drop + /// to a budget that assumes previously split remotes still hold this node's + /// last per-session cap and newly observed remotes hold 0. Equal shares + /// resume only when this node owns every remaining session. + /// + internal sealed class AccountRateAllocator : IAccountRateAllocator + { + private readonly ConcurrentDictionary _clusterCounts = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _accounts = new(StringComparer.Ordinal); + + /// Initializes a process-local allocator. + public AccountRateAllocator(TimeProvider? timeProvider = null, TimeSpan? equalSplitHold = null) + { + _ = timeProvider; + _ = equalSplitHold; + } + + /// + public void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(cap); + if (rateBps <= 0) + { + cap.UpdateMaxSendBytesPerSecond(0); + return; + } + + var account = _accounts.GetOrAdd(accountName, static _ => new AccountRates()); + lock (account.Gate) + { + account.RateBps = rateBps; + account.Sessions[sessionId] = cap; + if (_clusterCounts.TryGetValue(accountName, out var observed) && observed > 0) + { + account.ClusterCount = observed; + } + + Apply(account); + } + } + + /// + public void Unregister(string accountName, string sessionId) + { + if (string.IsNullOrWhiteSpace(accountName) + || string.IsNullOrWhiteSpace(sessionId) + || !_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + lock (account.Gate) + { + account.Sessions.TryRemove(sessionId, out _); + if (account.Sessions.IsEmpty) + { + _accounts.TryRemove(accountName, out _); + return; + } + + Apply(account); + } + } + + /// + public void ObserveClusterSessionCount(string accountName, int clusterSessionCount) + { + if (string.IsNullOrWhiteSpace(accountName)) + { + return; + } + + var count = clusterSessionCount < 0 ? 0 : clusterSessionCount; + _clusterCounts[accountName] = count; + if (!_accounts.TryGetValue(accountName, out var account)) + { + return; + } + + lock (account.Gate) + { + account.ClusterCount = count; + Apply(account); + } + } + + /// + public long? GetSessionCap(string accountName, string sessionId) + { + if (string.IsNullOrWhiteSpace(accountName) + || string.IsNullOrWhiteSpace(sessionId) + || !_accounts.TryGetValue(accountName, out var account)) + { + return null; + } + + lock (account.Gate) + { + return account.Sessions.TryGetValue(sessionId, out var cap) + ? cap.MaxSendBytesPerSecond + : null; + } + } + + /// Returns the last observed cluster count (tests). + internal int ClusterCount(string accountName) => + _clusterCounts.TryGetValue(accountName, out var count) ? count : 0; + + /// Returns how many local sessions are registered (tests). + internal int LocalSessionCount(string accountName) => + _accounts.TryGetValue(accountName, out var account) ? account.Sessions.Count : 0; + + private void Apply(AccountRates account) + { + var rate = account.RateBps; + if (rate <= 0) + { + return; + } + + var count = account.ClusterCount; + if (count <= 0) + { + count = account.Sessions.Count; + } + + var local = account.Sessions.Count; + if (local == 0) + { + return; + } + + var remotes = count - local; + var target = AccountRateFormula.EqualShareCap(rate, count); + if (remotes <= 0) + { + foreach (var pair in account.Sessions) + { + pair.Value.UpdateMaxSendBytesPerSecond(target); + } + + account.LastCount = count; + account.LastLocal = local; + return; + } + + var accountBytes = AccountRateFormula.AccountBytesPerSecond(rate); + long remoteHold; + if (account.LastCount <= 0) + { + var remoteShare = AccountRateFormula.PerSessionBytesPerSecond(rate, remotes); + remoteHold = checked(remoteShare * remotes); + } + else + { + var oldRemotes = account.LastCount - account.LastLocal; + if (oldRemotes < 0) + { + oldRemotes = 0; + } + + var oldShare = 0L; + // codeql[cs/linq/missed-select] Allocation-free max scan; Select would materialize caps. + foreach (var pair in account.Sessions) + { + var cap = pair.Value.MaxSendBytesPerSecond; + if (cap > oldShare) + { + oldShare = cap; + } + } + + if (oldShare <= 0) + { + oldShare = AccountRateFormula.PerSessionBytesPerSecond(rate, account.LastCount); + } + + remoteHold = checked(oldShare * oldRemotes); + } + + var localBudget = accountBytes - remoteHold; + var perLocal = localBudget > 0 ? localBudget / local : 0; + foreach (var pair in account.Sessions) + { + var current = pair.Value.MaxSendBytesPerSecond; + if (current <= 0) + { + pair.Value.UpdateMaxSendBytesPerSecond(AccountRateFormula.BlockedBytesPerSecond); + continue; + } + + var safe = target; + if (perLocal < safe) + { + safe = perLocal; + } + + if (current < safe) + { + safe = current; + } + + pair.Value.UpdateMaxSendBytesPerSecond( + safe > 0 ? safe : AccountRateFormula.BlockedBytesPerSecond); + } + + account.LastCount = count; + account.LastLocal = local; + } + + private sealed class AccountRates + { + public object Gate { get; } = new(); + public int RateBps; + public int ClusterCount; + public int LastCount; + public int LastLocal; + public ConcurrentDictionary Sessions { get; } = new(StringComparer.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateFormula.cs b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateFormula.cs new file mode 100644 index 00000000..31ab4d52 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/AccountRateFormula.cs @@ -0,0 +1,107 @@ +namespace VectorNNTPD.TransitService.SessionState.RateLimiting +{ + /// + /// Converts account_rate_limit (bits per second) to integer bytes/sec and + /// divides that aggregate by the cluster-wide active session count. + /// + /// + /// account_rate_limit = 0 is unlimited (existing policy). Conversion is + /// floor(bps / 8). Division uses integer floor so + /// sum(per-session caps) <= account bytes/sec. + /// The denominator is the live active session count, never + /// account_session_limit. A positive configured rate that floors to + /// 0 bytes/sec is , not unlimited. + /// + public static class AccountRateFormula + { + /// Bits in one byte. Database account_rate_limit is bits/sec. + public const int BitsPerByte = 8; + + /// + /// Account-wide cap in bytes/sec: floor(rateBps / 8). + /// <= 0 is unlimited and returns 0. + /// A positive rate below 8 bps returns 0 here; callers that apply a + /// limiter cap must use so that is blocked. + /// + public static long AccountBytesPerSecond(int rateBps) + { + if (rateBps <= 0) + { + return 0; + } + + // int.MaxValue / 8 fits in int and long; no Mbps multiplier remains. + return (long)rateBps / BitsPerByte; + } + + /// + /// Per-session cap in bytes/sec: floor(accountBytesPerSec / activeSessions). + /// Returns 0 (unlimited / no allocation) when the account rate is unlimited + /// or there are no active sessions. A positive rate that floors to 0 is + /// not unlimited; use on the write path. + /// + public static long PerSessionBytesPerSecond(int rateBps, int activeSessions) + { + if (rateBps <= 0 || activeSessions <= 0) + { + return 0; + } + + return AccountBytesPerSecond(rateBps) / activeSessions; + } + + /// + /// Equal-share cap for a positive rate. A floor of 0 is + /// so the limiter cannot treat it as unlimited. + /// + public static long EqualShareCap(int rateBps, int activeSessions) + { + if (rateBps <= 0) + { + return 0; + } + + var perSession = PerSessionBytesPerSecond(rateBps, activeSessions); + return perSession > 0 ? perSession : BlockedBytesPerSecond; + } + + /// + /// Returns whether perSession * activeSessions stays at or under the account cap. + /// + public static bool AggregateDoesNotExceed(int rateBps, int activeSessions) + { + if (rateBps <= 0 || activeSessions <= 0) + { + return true; + } + + var perSession = PerSessionBytesPerSecond(rateBps, activeSessions); + return perSession * activeSessions <= AccountBytesPerSecond(rateBps); + } + + /// + /// Bytes/sec a newly admitted session may take while remotes may still hold + /// floor(account / previousSessions). This is the unused remainder + /// after that previous equal split. 0 means no residual; callers must + /// not treat it as unlimited. + /// + public static long ConservativeJoinBytesPerSecond(int rateBps, int previousActiveSessions) + { + if (rateBps <= 0 || previousActiveSessions <= 0) + { + return 0; + } + + return AccountBytesPerSecond(rateBps) % previousActiveSessions; + } + + /// + /// Negative sentinel: block writes until the cap is replaced. Distinct from + /// 0, which remains unlimited. + /// + public const long BlockedBytesPerSecond = -1; + + /// Counts a stored cap toward a cluster aggregate. Blocked contributes 0; unlimited is 0. + public static long AllocatedBytesPerSecond(long cap) => cap > 0 ? cap : 0; + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IAccountRateAllocator.cs b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IAccountRateAllocator.cs new file mode 100644 index 00000000..5c24a6d4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IAccountRateAllocator.cs @@ -0,0 +1,60 @@ +namespace VectorNNTPD.TransitService.SessionState.RateLimiting +{ + /// + /// Process-local account rate allocation. SessionState observes the cluster + /// session count; this type updates every registered local session cap. + /// + public interface IAccountRateAllocator + { + /// + /// Registers a connected session. is + /// account_rate_limit in bits per second, not Mbps. Applies the last + /// observed cluster count, or one session when none has been observed yet. + /// + void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps); + + /// Drops a session. Remaining local sessions keep the last observed count. + void Unregister(string accountName, string sessionId); + + /// + /// Applies the cluster session total to local caps. Existing local sessions + /// that already hold a positive equal-share cap may only drop to a share + /// that fits accountBytes − remotes × floor(accountBytes / remotes). + /// A newly admitted session on a node that does not own every session is + /// blocked. Equal shares resume when this node owns every remaining session. + /// + void ObserveClusterSessionCount(string accountName, int clusterSessionCount); + + /// Returns the current cap for a registered session (tests). + long? GetSessionCap(string accountName, string sessionId); + } + + /// No-op allocator for hosts and tests that do not attach rate limiters. + public sealed class NullAccountRateAllocator : IAccountRateAllocator + { + /// Shared no-op instance. + public static NullAccountRateAllocator Instance { get; } = new(); + + private NullAccountRateAllocator() + { + } + + /// + public void Register(string accountName, string sessionId, IOutboundRateCap cap, int rateBps) + { + } + + /// + public void Unregister(string accountName, string sessionId) + { + } + + /// + public void ObserveClusterSessionCount(string accountName, int clusterSessionCount) + { + } + + /// + public long? GetSessionCap(string accountName, string sessionId) => null; + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IOutboundRateCap.cs b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IOutboundRateCap.cs new file mode 100644 index 00000000..dcb8ad25 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/IOutboundRateCap.cs @@ -0,0 +1,15 @@ +namespace VectorNNTPD.TransitService.SessionState.RateLimiting +{ + /// Dynamically adjustable outbound bytes-per-second cap. + public interface IOutboundRateCap + { + /// + /// Gets the current cap. 0 is unlimited. < 0 blocks writes + /// until a later update (not unlimited). + /// + long MaxSendBytesPerSecond { get; } + + /// Replaces the cap. Existing and future writes observe the new value. + void UpdateMaxSendBytesPerSecond(long bytesPerSecond); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RateLimiting/OutboundRateLimiter.cs b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/OutboundRateLimiter.cs new file mode 100644 index 00000000..1898d375 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RateLimiting/OutboundRateLimiter.cs @@ -0,0 +1,199 @@ +namespace VectorNNTPD.TransitService.SessionState.RateLimiting +{ + /// + /// Stream wrapper that throttles writes to a dynamically adjustable bytes/sec cap. + /// Reads, flushes, and cap == 0 are passthrough. cap < 0 blocks + /// until the cap is updated. + /// + /// + /// Adapted from Vector.NNTP DynamicSendRateLimitedStream: one-second write window, + /// 25 ms backoff, live during writes. + /// Writes are serialized. Cancellation aborts the backoff. Dispose does not wait + /// for an in-flight write; the write token must be cancelled by the connection. + /// + public sealed class OutboundRateLimiter : Stream, IOutboundRateCap + { + private readonly Stream _inner; + private readonly bool _leaveInnerOpen; + private readonly TimeProvider _time; + private readonly SemaphoreSlim _writeGate = new(1, 1); + private long _maxSendBytesPerSecond; + private long _windowStartMs; + private long _bytesInWindow; + private int _disposed; + + /// Initializes a limiter over . + public OutboundRateLimiter( + Stream inner, + long initialMaxSendBytesPerSecond, + bool leaveInnerOpen, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + _leaveInnerOpen = leaveInnerOpen; + _time = timeProvider ?? TimeProvider.System; + _maxSendBytesPerSecond = initialMaxSendBytesPerSecond; + _windowStartMs = NowMs(); + } + + /// + public override bool CanRead => _inner.CanRead; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => _inner.CanWrite; + + /// + public override long Length => throw new NotSupportedException(); + + /// + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// + public long MaxSendBytesPerSecond => Interlocked.Read(ref _maxSendBytesPerSecond); + + /// + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => + Interlocked.Exchange(ref _maxSendBytesPerSecond, bytesPerSecond); + + /// + public override void Flush() => _inner.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _inner.FlushAsync(cancellationToken); + + /// + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + _inner.ReadAsync(buffer, cancellationToken); + + /// + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + /// + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var cap = Interlocked.Read(ref _maxSendBytesPerSecond); + if (buffer.IsEmpty || cap == 0) + { + await _inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + return; + } + + await _writeGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var offset = 0; + while (offset < buffer.Length) + { + cap = Interlocked.Read(ref _maxSendBytesPerSecond); + if (cap == 0) + { + await _inner.WriteAsync(buffer[offset..], cancellationToken).ConfigureAwait(false); + return; + } + + if (cap < 0) + { + await Task.Delay(TimeSpan.FromMilliseconds(25), _time, cancellationToken) + .ConfigureAwait(false); + continue; + } + + var sliceBytes = buffer.Length - offset; + if (sliceBytes > cap) + { + sliceBytes = cap > int.MaxValue ? int.MaxValue : (int)cap; + } + + await ThrottleAsync(sliceBytes, cap, cancellationToken).ConfigureAwait(false); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + await _inner.WriteAsync(buffer.Slice(offset, sliceBytes), cancellationToken) + .ConfigureAwait(false); + _bytesInWindow += sliceBytes; + offset += sliceBytes; + } + } + finally + { + _writeGate.Release(); + } + } + + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// + public override void SetLength(long value) => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writeGate.Dispose(); + if (!_leaveInnerOpen) + { + _inner.Dispose(); + } + } + + base.Dispose(disposing); + } + + /// + public override async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _writeGate.Dispose(); + if (!_leaveInnerOpen) + { + await _inner.DisposeAsync().ConfigureAwait(false); + } + } + + await base.DisposeAsync().ConfigureAwait(false); + } + + private async Task ThrottleAsync(int nextBytes, long cap, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + var now = NowMs(); + if (now - _windowStartMs >= 1000) + { + _windowStartMs = now; + _bytesInWindow = 0; + } + + if (_bytesInWindow + nextBytes <= cap) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(25), _time, cancellationToken).ConfigureAwait(false); + } + } + + private long NowMs() => _time.GetUtcNow().ToUnixTimeMilliseconds(); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/RedisSessionStateStore.cs b/src/VectorNNTPD.TransitService/SessionState/RedisSessionStateStore.cs new file mode 100644 index 00000000..6c06ec99 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/RedisSessionStateStore.cs @@ -0,0 +1,256 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Redis-backed cluster session + source-IP membership using one atomic Lua EVAL + /// on the shared connection. + /// + internal sealed class RedisSessionStateStore : ISessionStateStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisSessionStateStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) => + ExecuteAsync( + SessionStateScripts.TryAdmit, + accountName, + MembershipKeys(accountName), + [ + Utf8(normalizedSourceIp), + Utf8(ownerId), + Utf8(sessionLimit.ToString(CultureInfo.InvariantCulture)), + Utf8(srcIpLimit.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(sourceGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(trackSessions ? "1" : "0"), + ], + result => + { + var status = SessionStateEngine.UnpackAdmitStatus(result); + var total = SessionStateEngine.UnpackAdmitSessionTotal(result); + return status switch + { + SessionStateEngine.AcceptedExisting => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedExisting, total), + SessionStateEngine.AcceptedNew => + new SessionStateAdmitResult(SessionStateAdmitStatus.AcceptedNew, total), + SessionStateEngine.RejectedSessionLimit => + new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSessionLimit), + _ => new SessionStateAdmitResult(SessionStateAdmitStatus.RejectedSourceLimit), + }; + }, + new SessionStateAdmitResult(SessionStateAdmitStatus.Unavailable), + cancellationToken); + + /// + public ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) => + ExecuteAsync( + SessionStateScripts.Release, + accountName, + MembershipKeys(accountName), + [ + Utf8(normalizedSourceIp), + Utf8(ownerId), + Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)), + Utf8(sourceGeneration.ToString(CultureInfo.InvariantCulture)), + ], + static result => result < 0 ? 0 : (int)result, + fallback: 0, + cancellationToken); + + /// + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + var values = new ReadOnlyMemory[5 + (sources.Count * 2)]; + values[0] = Utf8(ownerId); + values[1] = Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)); + values[2] = Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)); + values[3] = Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)); + values[4] = Utf8(sources.Count.ToString(CultureInfo.InvariantCulture)); + for (var i = 0; i < sources.Count; i++) + { + values[5 + (i * 2)] = Utf8(sources[i].Ip); + values[6 + (i * 2)] = Utf8(sources[i].Generation.ToString(CultureInfo.InvariantCulture)); + } + + return ExecuteAsync( + SessionStateScripts.Renew, + accountName, + MembershipKeys(accountName), + values, + result => result == 0 + ? new SessionStateRenewResult(SessionStateRenewStatus.Lost) + : new SessionStateRenewResult(SessionStateRenewStatus.Renewed, (int)result), + new SessionStateRenewResult(SessionStateRenewStatus.Unavailable), + cancellationToken); + } + + /// + public ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + ArgumentException.ThrowIfNullOrWhiteSpace(batchId); + var values = new ReadOnlyMemory[8 + (sources.Count * 2)]; + values[0] = Utf8(ownerId); + values[1] = Utf8(sessionGeneration.ToString(CultureInfo.InvariantCulture)); + values[2] = Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)); + values[3] = Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)); + values[4] = Utf8(sources.Count.ToString(CultureInfo.InvariantCulture)); + for (var i = 0; i < sources.Count; i++) + { + values[5 + (i * 2)] = Utf8(sources[i].Ip); + values[6 + (i * 2)] = Utf8(sources[i].Generation.ToString(CultureInfo.InvariantCulture)); + } + + values[5 + (sources.Count * 2)] = Utf8(batchId); + values[6 + (sources.Count * 2)] = Utf8( + AccountByteEngine.ClampNonNegative(consumed).ToString(CultureInfo.InvariantCulture)); + values[7 + (sources.Count * 2)] = Utf8( + AccountByteEngine.ClampNonNegative(mysqlRemainingAfter) + .ToString(CultureInfo.InvariantCulture)); + + return ExecuteAsync( + SessionStateScripts.RenewAndApply, + accountName, + CombinedKeys(accountName), + values, + packed => + { + var (status, remaining) = SessionStateBytePack.Decode(packed); + return new SessionStateRenewAndApplyResult(status, remaining); + }, + new SessionStateRenewAndApplyResult(SessionStateRenewStatus.Unavailable, remaining: null), + cancellationToken); + } + + /// + public async ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + SessionStateScripts.ReleaseOwner, + accountName, + MembershipKeys(accountName), + [Utf8(ownerId)], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + private async ValueTask ExecuteAsync( + string script, + string accountName, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + keys, + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + } + + private static ReadOnlyMemory[] MembershipKeys(string accountName) => + [SessionStateKeys.CreateSource(accountName), SessionStateKeys.CreateSession(accountName)]; + + private static ReadOnlyMemory[] CombinedKeys(string accountName) => + [ + SessionStateKeys.CreateSource(accountName), + SessionStateKeys.CreateSession(accountName), + AccountByteKeys.Create(accountName), + ]; + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionAdmissionResult.cs b/src/VectorNNTPD.TransitService/SessionState/SessionAdmissionResult.cs new file mode 100644 index 00000000..29b165f6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionAdmissionResult.cs @@ -0,0 +1,24 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// Outcome of authenticated-session admission. + public enum SessionAdmissionResult + { + /// The session was admitted, or admission was skipped because both limits are 0. + Success, + + /// account_session_limit would be exceeded. Wire: 481 Too many sessions. + SessionLimitExceeded, + + /// + /// A new distinct source address would exceed account_srcip_limit. + /// Wire: 481 Too many source addresses. + /// + SourceAddressLimitExceeded, + + /// + /// Cluster membership could not be established. New admission that requires Redis + /// fails closed. Wire: 503 Temporary authentication failure. + /// + Unavailable, + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateAdmitResult.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateAdmitResult.cs new file mode 100644 index 00000000..f2a3cc40 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateAdmitResult.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// Cluster membership outcome for one combined session/source-IP admit attempt. + public enum SessionStateAdmitStatus + { + /// Admitted; the source IP was already active. + AcceptedExisting, + + /// Admitted; a new distinct source IP was established. + AcceptedNew, + + /// Rejected: cluster account_session_limit would be exceeded. + RejectedSessionLimit, + + /// Rejected: cluster account_srcip_limit would be exceeded. + RejectedSourceLimit, + + /// Redis (or the membership store) was unavailable. Fail closed. + Unavailable, + } + + /// Result of a cluster session/source-IP admit attempt. + public readonly struct SessionStateAdmitResult + { + /// Initializes a membership admit result. + public SessionStateAdmitResult(SessionStateAdmitStatus status, int sessionTotal = 0) + { + Status = status; + SessionTotal = sessionTotal < 0 ? 0 : sessionTotal; + } + + /// Gets the membership outcome. + public SessionStateAdmitStatus Status { get; } + + /// + /// Cluster-wide unexpired session count after this attempt (0 when rejected + /// or unavailable). + /// + public int SessionTotal { get; } + + /// Gets whether membership was granted. + public bool Accepted => + Status is SessionStateAdmitStatus.AcceptedExisting or SessionStateAdmitStatus.AcceptedNew; + } + + /// Outcome of a node ownership lease renewal. + public enum SessionStateRenewStatus + { + /// This owner's requested leases were refreshed. + Renewed, + + /// At least one requested ownership field was gone or mismatched. + Lost, + + /// The membership store could not be reached. + Unavailable, + } + + /// Renewal outcome plus the cluster session total used for rate allocation. + public readonly struct SessionStateRenewResult + { + /// Initializes a renewal result. + public SessionStateRenewResult(SessionStateRenewStatus status, int sessionTotal = 0) + { + Status = status; + SessionTotal = sessionTotal < 0 ? 0 : sessionTotal; + } + + /// Gets the renewal outcome. + public SessionStateRenewStatus Status { get; } + + /// Unexpired cluster session count when is Renewed. + public int SessionTotal { get; } + + /// Converts a status-only result (session total 0). + public static implicit operator SessionStateRenewResult(SessionStateRenewStatus status) => + new(status); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateBytePack.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateBytePack.cs new file mode 100644 index 00000000..bd0c4b3b --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateBytePack.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Encodes combined RENEW+APPLY into one Redis integer. Remaining is always + /// non-negative; a lost renewal is -remaining - 1. + /// + internal static class SessionStateBytePack + { + /// Packs renew status and remaining for a combined EVAL return. + public static long Encode(bool renewed, long remaining) + { + if (remaining < 0) + { + remaining = 0; + } + + return renewed ? remaining : -remaining - 1; + } + + /// Unpacks a combined EVAL integer into renew status and remaining. + public static (SessionStateRenewStatus Status, long Remaining) Decode(long packed) + { + if (packed >= 0) + { + return (SessionStateRenewStatus.Renewed, packed); + } + + var remaining = -packed - 1; + return (SessionStateRenewStatus.Lost, remaining < 0 ? 0 : remaining); + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateDefaults.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateDefaults.cs new file mode 100644 index 00000000..d60cc9fa --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateDefaults.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// Lease and hot-path constants for cluster-wide session and source-IP admission. + /// + /// The Redis TTL is crash/failure recovery only. It is not an idle-session + /// timeout. A healthy node renews every 10 seconds while authenticated sessions + /// exist, so ownership stays alive indefinitely even when clients issue no + /// commands. A 30-second lease recovers crashed-node ownership quickly without + /// treating a brief Redis blip as membership loss. Two missed heartbeats expire + /// the lease. The 2-second hot-path skew refuses a local fast-path decision that + /// would outlive the distributed lease. + /// + internal static class SessionStateDefaults + { + /// Distributed ownership TTL written into Redis on admit and renew. + public static readonly TimeSpan LeaseTtl = TimeSpan.FromSeconds(30); + + /// + /// Interval between process-wide lease renewal passes. + /// uses the same 10-second period. + /// + public static readonly TimeSpan RenewalPeriod = TimeSpan.FromSeconds(10); + + /// + /// Local hot-path requires the known lease to remain valid for at least this long. + /// + public static readonly TimeSpan HotPathSkew = TimeSpan.FromSeconds(2); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateEngine.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateEngine.cs new file mode 100644 index 00000000..f8eb4f89 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateEngine.cs @@ -0,0 +1,521 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Atomic cluster session + source-IP membership algorithm. The Redis Lua scripts + /// are a faithful port of these operations. + /// + /// + /// Two hashes per account, mutated in one locked/EVAL step: + /// source fields {ip}\x1f{ownerId} and session fields {ownerId}, + /// values {expiryUnixMs}|{generation}|{count}. Session limit is checked + /// before source-IP limit. Rejection makes no state changes. The session count is + /// the sum of unexpired per-owner counts; the source count is the number of + /// distinct unexpired IPs. + /// + internal sealed class SessionStateEngine + { + /// Accepted; the source IP was already active. + public const long AcceptedExisting = 3; + + /// Accepted; a new distinct source IP was established. + public const long AcceptedNew = 2; + + /// Rejected: cluster session limit would be exceeded. + public const long RejectedSessionLimit = 1; + + /// Rejected: cluster source-address limit would be exceeded. + public const long RejectedSourceLimit = 0; + + /// Packs an accepted admit status with the cluster session total. + public static long PackAdmit(long status, int sessionTotal) => + status <= RejectedSessionLimit + ? status + : status | ((long)(sessionTotal < 0 ? 0 : sessionTotal) << 2); + + /// Reads the raw admit status from a packed EVAL integer. + public static long UnpackAdmitStatus(long packed) => packed <= RejectedSessionLimit ? packed : packed & 3; + + /// Reads the cluster session total from a packed accepted EVAL integer. + public static int UnpackAdmitSessionTotal(long packed) => + packed <= RejectedSessionLimit ? 0 : (int)(packed >> 2); + + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// Returns the unpacked admit status (tests that ignore the packed total). + internal long TryAdmitStatus( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long nowUnixMs, + long leaseMs, + long sessionGeneration, + long sourceGeneration, + bool trackSessions = false) => + UnpackAdmitStatus( + TryAdmit( + sourceKey, + sessionKey, + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + nowUnixMs, + leaseMs, + sessionGeneration, + sourceGeneration, + trackSessions)); + + /// + /// Atomically admits one session and this owner's source-IP membership. + /// + public long TryAdmit( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long nowUnixMs, + long leaseMs, + long sessionGeneration, + long sourceGeneration, + bool trackSessions = false) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + var sources = GetOrCreateHash(sourceKey); + var sessions = GetOrCreateHash(sessionKey); + PruneExpired(sources, nowUnixMs); + PruneExpired(sessions, nowUnixMs); + + var sessionTotal = SumCounts(sessions); + if (sessionLimit > 0 && sessionTotal >= sessionLimit) + { + RemoveIfEmpty(sourceKey, sources); + RemoveIfEmpty(sessionKey, sessions); + return RejectedSessionLimit; + } + + var activeIps = DistinctActiveIps(sources); + if (srcIpLimit > 0 && !activeIps.Contains(normalizedSourceIp) && activeIps.Count >= srcIpLimit) + { + RemoveIfEmpty(sourceKey, sources); + RemoveIfEmpty(sessionKey, sessions); + return RejectedSourceLimit; + } + + var expiry = nowUnixMs + leaseMs; + if (sessionLimit > 0 || trackSessions) + { + IncrementOwned(sessions, ownerId, expiry, sessionGeneration); + } + + if (srcIpLimit > 0) + { + IncrementOwned( + sources, + SessionStateKeys.SourceField(normalizedSourceIp, ownerId), + expiry, + sourceGeneration); + } + + var status = activeIps.Contains(normalizedSourceIp) ? AcceptedExisting : AcceptedNew; + return PackAdmit(status, SumCounts(sessions)); + } + } + + /// + /// Decrements this owner's session count and source-IP count when generations match. + /// + public long Release( + string sourceKey, + string sessionKey, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + DecrementOwned(sessions, ownerId, sessionGeneration); + RemoveIfEmpty(sessionKey, sessions); + } + + if (_hashes.TryGetValue(sourceKey, out var sources)) + { + DecrementOwned(sources, SessionStateKeys.SourceField(normalizedSourceIp, ownerId), sourceGeneration); + RemoveIfEmpty(sourceKey, sources); + } + + return _hashes.TryGetValue(sessionKey, out var remaining) + ? SumCounts(remaining) + : 0; + } + } + + /// Deletes every field owned by (process shutdown). + public long ReleaseOwner(string sourceKey, string sessionKey, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + sessions.Remove(ownerId); + RemoveIfEmpty(sessionKey, sessions); + } + + if (_hashes.TryGetValue(sourceKey, out var sources)) + { + List? owned = null; + // codeql[cs/linq/missed-where] Lock-held key collection without LINQ allocation. + foreach (var field in sources.Keys) + { + if (SessionStateKeys.TrySplitField(field, out _, out var fieldOwner) + && string.Equals(fieldOwner, ownerId, StringComparison.Ordinal)) + { + owned ??= []; + owned.Add(field); + } + } + + if (owned is not null) + { + foreach (var field in owned) + { + sources.Remove(field); + } + } + + RemoveIfEmpty(sourceKey, sources); + } + + return 1; + } + } + + /// + /// Refreshes this owner's unexpired matching session and source leases. + /// Returns 1 when every requested field was renewed; 0 when any was lost. + /// + public long Renew( + string sourceKey, + string sessionKey, + string ownerId, + long sessionGeneration, + long nowUnixMs, + long leaseMs, + IReadOnlyList<(string Ip, long Generation)> sources) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + ArgumentNullException.ThrowIfNull(sources); + + lock (_gate) + { + if (sessionGeneration != 0 + && !CanRenewField(sessionKey, ownerId, sessionGeneration, nowUnixMs, out _)) + { + return 0; + } + + foreach (var (ip, generation) in sources) + { + if (!CanRenewField( + sourceKey, + SessionStateKeys.SourceField(ip, ownerId), + generation, + nowUnixMs, + out _)) + { + return 0; + } + } + + if (sessionGeneration != 0) + { + RenewField(sessionKey, ownerId, sessionGeneration, nowUnixMs, leaseMs); + } + + foreach (var (ip, generation) in sources) + { + RenewField( + sourceKey, + SessionStateKeys.SourceField(ip, ownerId), + generation, + nowUnixMs, + leaseMs); + } + + if (_hashes.TryGetValue(sessionKey, out var sessions)) + { + PruneExpired(sessions, nowUnixMs); + RemoveIfEmpty(sessionKey, sessions); + var total = SumCounts(sessions); + return total < 1 ? 1 : total; + } + + return 1; + } + } + + /// Returns the cluster-wide authenticated session count. + internal int ActiveSessionCount(string sessionKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + lock (_gate) + { + if (!_hashes.TryGetValue(sessionKey, out var sessions)) + { + return 0; + } + + PruneExpired(sessions, nowUnixMs); + RemoveIfEmpty(sessionKey, sessions); + return SumCounts(sessions); + } + } + + /// Returns distinct unexpired source IPs. + internal IReadOnlyCollection ActiveSourceIps(string sourceKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + lock (_gate) + { + if (!_hashes.TryGetValue(sourceKey, out var sources)) + { + return []; + } + + PruneExpired(sources, nowUnixMs); + RemoveIfEmpty(sourceKey, sources); + return DistinctActiveIps(sources); + } + } + + /// Returns this owner's unexpired session count. + internal int OwnerSessionCount(string sessionKey, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sessionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(sessionKey, out var sessions) + || !sessions.TryGetValue(ownerId, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out _, out var count) + || expiry <= nowUnixMs) + { + return 0; + } + + return count; + } + } + + /// Reads a HASH field without mutating it (tests). + internal bool TryGetOwnership( + string key, + string field, + out long expiryUnixMs, + out long generation, + out int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + if (_hashes.TryGetValue(key, out var hash) + && hash.TryGetValue(field, out var current) + && SessionStateKeys.TrySplitValue(current, out expiryUnixMs, out generation, out count)) + { + return true; + } + } + + expiryUnixMs = 0; + generation = 0; + count = 0; + return false; + } + + /// Overwrites a HASH field without admit/release semantics (tests). + internal void WriteOwnership(string key, string field, long expiryUnixMs, long generation, int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + GetOrCreateHash(key)[field] = SessionStateKeys.Value(expiryUnixMs, generation, count); + } + } + + /// Returns whether this owner currently holds . + internal bool HasSourceOwner(string sourceKey, string normalizedSourceIp, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey); + var field = SessionStateKeys.SourceField(normalizedSourceIp, ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(sourceKey, out var sources) + || !sources.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out _, out var count)) + { + return false; + } + + return expiry > nowUnixMs && count > 0; + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private bool CanRenewField(string key, string field, long generation, long nowUnixMs, out int count) + { + count = 0; + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out count) + || storedGeneration != generation + || expiry <= nowUnixMs) + { + return false; + } + + return true; + } + + private void RenewField(string key, string field, long generation, long nowUnixMs, long leaseMs) + { + if (!CanRenewField(key, field, generation, nowUnixMs, out var count) + || !_hashes.TryGetValue(key, out var hash)) + { + return; + } + + hash[field] = SessionStateKeys.Value(nowUnixMs + leaseMs, generation, count); + } + + private static void IncrementOwned(Dictionary hash, string field, long expiry, long generation) + { + // Same generation: this owner still holds the field. A different generation + // is a new local epoch (local state was wiped) and must replace stale count. + if (hash.TryGetValue(field, out var current) + && SessionStateKeys.TrySplitValue(current, out _, out var storedGeneration, out var count) + && storedGeneration == generation) + { + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count + 1); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, generation, 1); + } + + private static void DecrementOwned(Dictionary hash, string field, long generation) + { + if (!hash.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out var count) + || storedGeneration != generation) + { + return; + } + + if (count <= 1) + { + hash.Remove(field); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count - 1); + } + + private static void PruneExpired(Dictionary hash, long nowUnixMs) + { + List? expired = null; + foreach (var (field, value) in hash) + { + if (!SessionStateKeys.TrySplitValue(value, out var expiry, out _, out _) || expiry <= nowUnixMs) + { + expired ??= []; + expired.Add(field); + } + } + + if (expired is null) + { + return; + } + + foreach (var field in expired) + { + hash.Remove(field); + } + } + + private static int SumCounts(Dictionary hash) + { + var total = 0; + // codeql[cs/linq/missed-where] TrySplitValue out-params; lock-held sum without LINQ allocation. + foreach (var value in hash.Values) + { + if (SessionStateKeys.TrySplitValue(value, out _, out _, out var count)) + { + total += count; + } + } + + return total; + } + + private static HashSet DistinctActiveIps(Dictionary hash) + { + var active = new HashSet(StringComparer.Ordinal); + // codeql[cs/linq/missed-where] TrySplitField out-params; lock-held scan without LINQ allocation. + foreach (var field in hash.Keys) + { + if (SessionStateKeys.TrySplitField(field, out var ip, out _)) + { + active.Add(ip); + } + } + + return active; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateKeys.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateKeys.cs new file mode 100644 index 00000000..b8315bb0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateKeys.cs @@ -0,0 +1,120 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Redis key material for cluster-wide account session and source-IP membership. + /// + /// + /// Keys: nntpd:srcip:{sha256hex(account)} and nntpd:sess:{sha256hex(account)}. + /// Account names are hashed so Redis key listings do not expose usernames. + /// Source field: {normalizedIp}\x1f{ownerId}. + /// Session field: {ownerId}. + /// Value: {expiryUnixMs}|{generation}|{count}. + /// ownerId is {nodeId}:{incarnation} so a process restart cannot + /// overwrite a previous incarnation's unexpired ownership. + /// + internal static class SessionStateKeys + { + /// ASCII namespace for source-IP ownership hashes. + public static ReadOnlySpan SourceNamespacePrefix => "nntpd:srcip:"u8; + + /// ASCII namespace for session ownership hashes. + public static ReadOnlySpan SessionNamespacePrefix => "nntpd:sess:"u8; + + /// Separates the normalized IP from the owner id in a source HASH field. + public const char FieldSeparator = '\u001f'; + + /// Separates expiry, generation, and count in a HASH value. + public const char ValueSeparator = '|'; + + /// Builds the source-IP Redis HASH key for . + public static byte[] CreateSource(string accountName) => Create(SourceNamespacePrefix, accountName); + + /// Builds the session Redis HASH key for . + public static byte[] CreateSession(string accountName) => Create(SessionNamespacePrefix, accountName); + + /// Builds the HASH field for a node-owned source IP. + public static string SourceField(string normalizedSourceIp, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(normalizedSourceIp); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + return string.Concat(normalizedSourceIp, FieldSeparator, ownerId); + } + + /// Formats a lease value as expiryMs|generation|count. + public static string Value(long expiryUnixMs, long generation, int count) => + string.Concat( + expiryUnixMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + count.ToString(CultureInfo.InvariantCulture)); + + /// Parses a source HASH field into source IP and owner id. + public static bool TrySplitField(string field, out string ip, out string ownerId) + { + ArgumentNullException.ThrowIfNull(field); + var separator = field.IndexOf(FieldSeparator, StringComparison.Ordinal); + if (separator <= 0 || separator >= field.Length - 1) + { + ip = string.Empty; + ownerId = string.Empty; + return false; + } + + ip = field[..separator]; + ownerId = field[(separator + 1)..]; + return true; + } + + /// Parses a HASH value into expiry, generation, and count. + public static bool TrySplitValue(string value, out long expiryUnixMs, out long generation, out int count) + { + ArgumentNullException.ThrowIfNull(value); + expiryUnixMs = 0; + generation = 0; + count = 0; + var first = value.IndexOf(ValueSeparator, StringComparison.Ordinal); + if (first <= 0 || first >= value.Length - 1) + { + return false; + } + + var second = value.IndexOf(ValueSeparator, first + 1); + if (second <= first + 1 || second >= value.Length - 1) + { + return false; + } + + return long.TryParse(value.AsSpan(0, first), out expiryUnixMs) + && long.TryParse(value.AsSpan(first + 1, second - first - 1), out generation) + && int.TryParse(value.AsSpan(second + 1), out count); + } + + private static byte[] Create(ReadOnlySpan prefix, string accountName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(accountName); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(accountName, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(accountName), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateLogMessages.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateLogMessages.cs new file mode 100644 index 00000000..5117ef14 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// Source-generated session-state diagnostics. Never includes secrets. + internal static partial class SessionStateLogMessages + { + [LoggerMessage(EventId = 2310, Level = LogLevel.Debug, Message = "Session-state admission accepted locally user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SessionAdmittedLocal(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2311, Level = LogLevel.Debug, Message = "Session-state admission accepted via cluster membership user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId} path={Path}")] + public static partial void SessionAdmittedDistributed(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId, string path); + + [LoggerMessage(EventId = 2312, Level = LogLevel.Information, Message = "Session-state source-address admission rejected user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SourceAddressRejected(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2323, Level = LogLevel.Information, Message = "Session admission rejected user={Username} ip={SourceIp} limit={SessionLimit} node={NodeId}")] + public static partial void SessionAdmissionRejected(ILogger logger, string username, string sourceIp, int sessionLimit, string nodeId); + + [LoggerMessage(EventId = 2313, Level = LogLevel.Warning, Message = "Session-state admission failed closed; cluster membership unavailable user={Username} ip={SourceIp} limit={SrcIpLimit} node={NodeId}")] + public static partial void SessionStateUnavailable(ILogger logger, string username, string sourceIp, int srcIpLimit, string nodeId); + + [LoggerMessage(EventId = 2314, Level = LogLevel.Debug, Message = "Session-state node ownership released user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateReleased(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2315, Level = LogLevel.Debug, Message = "Session-state lease renewed user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewed(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2316, Level = LogLevel.Warning, Message = "Session-state lease renewal failed user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewFailed(ILogger logger, Exception exception, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2317, Level = LogLevel.Warning, Message = "Session-state lease renewal skipped; membership unavailable user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseRenewUnavailable(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2318, Level = LogLevel.Warning, Message = "Session-state lease lost; local hot path invalidated user={Username} ip={SourceIp} node={NodeId}")] + public static partial void SessionStateLeaseLost(ILogger logger, string username, string sourceIp, string nodeId); + + [LoggerMessage(EventId = 2319, Level = LogLevel.Warning, Message = "Session-state release failed user={Username} ip={SourceIp}")] + public static partial void SessionStateReleaseFailed(ILogger logger, Exception exception, string username, string sourceIp); + + [LoggerMessage(EventId = 2320, Level = LogLevel.Information, Message = "Session-state lease renewal started interval={Interval}")] + public static partial void SessionStateRenewalStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 2321, Level = LogLevel.Information, Message = "Session-state lease renewal stopped")] + public static partial void SessionStateRenewalStopped(ILogger logger); + + [LoggerMessage(EventId = 2322, Level = LogLevel.Warning, Message = "Session-state lease renewal pass failed")] + public static partial void SessionStateRenewalPassFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateRenewAndApplyResult.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateRenewAndApplyResult.cs new file mode 100644 index 00000000..4b166d95 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateRenewAndApplyResult.cs @@ -0,0 +1,22 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// Outcome of one atomic session-renewal plus byte-quota APPLY. + public readonly struct SessionStateRenewAndApplyResult + { + /// Initializes a combined renew/apply result. + public SessionStateRenewAndApplyResult(SessionStateRenewStatus renew, long? remaining) + { + Renew = renew; + Remaining = remaining; + } + + /// Gets the session/source lease renewal outcome. + public SessionStateRenewStatus Renew { get; } + + /// + /// Remaining after APPLY, or when this store did not + /// apply byte state (in-memory renew-only path). + /// + public long? Remaining { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateScripts.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateScripts.cs new file mode 100644 index 00000000..7feba162 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateScripts.cs @@ -0,0 +1,481 @@ +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Redis Lua scripts for atomic cluster session + source-IP admission. + /// Semantics match . + /// + internal static class SessionStateScripts + { + /// + /// Atomically evaluates both limits and, on accept, increments session and + /// source-IP ownership. Rejection writes no fields. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: ip, owner, sessionLimit, srcIpLimit, nowMs, leaseMs, sessionGen, sourceGen, + /// trackSessions (1 to increment the session hash when sessionLimit is 0). + /// Accept returns status + (sessionTotal * 4) (3 existing source, 2 new source). + /// Reject remains 1 session-limit, 0 source-limit. Session limit is checked first. + /// + public const string TryAdmit = + """ + -- nntpd-admit-try + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local ip = ARGV[1] + local owner = ARGV[2] + local sessionLimit = tonumber(ARGV[3]) + local srcLimit = tonumber(ARGV[4]) + local now = tonumber(ARGV[5]) + local lease = tonumber(ARGV[6]) + local sessionGen = ARGV[7] + local sourceGen = ARGV[8] + local srcField = ip .. '\31' .. owner + local expiry = now + lease + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function prune(key) + local entries = redis.call('HGETALL', key) + local expired = {} + for i = 1, #entries, 2 do + local exp = parse_value(entries[i + 1]) + if exp == nil or exp <= now then + expired[#expired + 1] = entries[i] + end + end + if #expired > 0 then + redis.call('HDEL', key, unpack(expired)) + end + end + + local function increment(key, field, gen) + local cur = redis.call('HGET', key, field) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen and storedGen == gen then + redis.call('HSET', key, field, tostring(expiry) .. '|' .. storedGen .. '|' .. tostring((count or 0) + 1)) + return + end + end + redis.call('HSET', key, field, tostring(expiry) .. '|' .. gen .. '|1') + end + + prune(srcKey) + prune(sessKey) + if redis.call('HLEN', srcKey) == 0 then + redis.call('DEL', srcKey) + end + if redis.call('HLEN', sessKey) == 0 then + redis.call('DEL', sessKey) + end + + local sessionTotal = 0 + local sessEntries = redis.call('HGETALL', sessKey) + for i = 1, #sessEntries, 2 do + local _, _, count = parse_value(sessEntries[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + if sessionLimit ~= nil and sessionLimit > 0 and sessionTotal >= sessionLimit then + return 1 + end + + local active = {} + local srcEntries = redis.call('HGETALL', srcKey) + for i = 1, #srcEntries, 2 do + local f = srcEntries[i] + local sep = string.find(f, '\31', 1, true) + local fip = sep and string.sub(f, 1, sep - 1) or f + active[fip] = true + end + + if srcLimit ~= nil and srcLimit > 0 and not active[ip] then + local distinct = 0 + for _ in pairs(active) do + distinct = distinct + 1 + end + if distinct >= srcLimit then + return 0 + end + end + + local trackSessions = tonumber(ARGV[9]) == 1 + if (sessionLimit ~= nil and sessionLimit > 0) or trackSessions then + increment(sessKey, owner, sessionGen) + end + if srcLimit ~= nil and srcLimit > 0 then + increment(srcKey, srcField, sourceGen) + end + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local _, _, count = parse_value(counted[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + if active[ip] then + return 3 + (sessionTotal * 4) + end + return 2 + (sessionTotal * 4) + """; + + /// + /// Decrements this owner's session and source-IP counts when generations match. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: ip, owner, sessionGen, sourceGen. Returns the remaining cluster session total. + /// + public const string Release = + """ + -- nntpd-admit-release + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local ip = ARGV[1] + local owner = ARGV[2] + local sessionGen = ARGV[3] + local sourceGen = ARGV[4] + local srcField = ip .. '\31' .. owner + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function decrement(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen then + return + end + if count == nil or count <= 1 then + redis.call('HDEL', key, field) + else + redis.call('HSET', key, field, tostring(exp) .. '|' .. storedGen .. '|' .. tostring(count - 1)) + end + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + end + + decrement(sessKey, owner, sessionGen) + decrement(srcKey, srcField, sourceGen) + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local _, _, count = parse_value(counted[i + 1]) + sessionTotal = sessionTotal + (count or 0) + end + return sessionTotal + """; + + /// + /// Refreshes this owner's unexpired matching session and source leases. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash. + /// ARGV: owner, sessionGen, nowMs, leaseMs, ipCount, ip1, gen1, ... + /// Returns 0 when any requested field was lost. On success returns the + /// unexpired cluster session total (minimum 1 so a source-only renew is not Lost). + /// A sessionGen of 0 skips the session field. + /// + public const string Renew = + """ + -- nntpd-admit-renew + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local owner = ARGV[1] + local sessionGen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local ipCount = tonumber(ARGV[5]) or 0 + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function can_renew(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return false + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return false + end + return true, count + end + + local function renew_field(key, field, gen, count) + redis.call('HSET', key, field, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + end + + if sessionGen ~= '0' then + local ok = can_renew(sessKey, owner, sessionGen) + if not ok then + return 0 + end + end + + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok = can_renew(srcKey, ip .. '\31' .. owner, gen) + if not ok then + return 0 + end + i = i + 1 + end + + if sessionGen ~= '0' then + local ok, count = can_renew(sessKey, owner, sessionGen) + if ok then + renew_field(sessKey, owner, sessionGen, count) + end + end + + i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok, count = can_renew(srcKey, ip .. '\31' .. owner, gen) + if ok then + renew_field(srcKey, ip .. '\31' .. owner, gen, count) + end + i = i + 1 + end + local sessionTotal = 0 + local counted = redis.call('HGETALL', sessKey) + for i = 1, #counted, 2 do + local exp, _, count = parse_value(counted[i + 1]) + if exp ~= nil and exp > now then + sessionTotal = sessionTotal + (count or 0) + end + end + if sessionTotal < 1 then + sessionTotal = 1 + end + return sessionTotal + """; + + /// + /// Renews this owner's unexpired matching leases, then APPLYs one byte batch. + /// APPLY always runs; a lost renewal does not skip quota reconciliation. + /// + /// + /// KEYS[1] source hash, KEYS[2] session hash, KEYS[3] remaining-quota HASH. + /// ARGV: owner, sessionGen, nowMs, leaseMs, ipCount, ip1, gen1, ..., + /// batchId, consumed, mysqlRemainingAfter. + /// Returns packed : remaining when renewed, + /// -remaining - 1 when lost. APPLY is floor-only and idempotent. + /// + public const string RenewAndApply = + """ + -- nntpd-admit-renew-and-apply + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local bytesKey = KEYS[3] + local owner = ARGV[1] + local sessionGen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local ipCount = tonumber(ARGV[5]) or 0 + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function can_renew(key, field, gen) + local cur = redis.call('HGET', key, field) + if not cur then + return false + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return false + end + return true, count + end + + local function renew_field(key, field, gen, count) + redis.call('HSET', key, field, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + end + + local function trim_marks(key, batchField) + if redis.call('HLEN', key) <= 257 then + return + end + local fields = redis.call('HKEYS', key) + for i = 1, #fields do + local f = fields[i] + if f ~= 'remaining' and f ~= batchField then + redis.call('HDEL', key, f) + if redis.call('HLEN', key) <= 257 then + return + end + end + end + end + + local function apply_bytes() + local batchId = ARGV[6 + (ipCount * 2)] + local mysql = tonumber(ARGV[8 + (ipCount * 2)]) + if mysql == nil or mysql < 0 then + mysql = 0 + end + if batchId == nil or batchId == '' then + batchId = '' + end + local batchField = 'b:' .. batchId + if redis.call('EXISTS', bytesKey) == 0 then + if batchId ~= '' then + redis.call('HSET', bytesKey, 'remaining', tostring(mysql), batchField, '1') + trim_marks(bytesKey, batchField) + else + redis.call('HSET', bytesKey, 'remaining', tostring(mysql)) + end + return mysql + end + if batchId ~= '' and redis.call('HEXISTS', bytesKey, batchField) == 1 then + local cur = tonumber(redis.call('HGET', bytesKey, 'remaining')) + if cur == nil or cur < 0 then + return 0 + end + return cur + end + local current = tonumber(redis.call('HGET', bytesKey, 'remaining')) + if current == nil or current < 0 then + current = 0 + end + local next = current + if next > mysql then + next = mysql + end + if batchId ~= '' then + redis.call('HSET', bytesKey, 'remaining', tostring(next), batchField, '1') + trim_marks(bytesKey, batchField) + else + redis.call('HSET', bytesKey, 'remaining', tostring(next)) + end + return next + end + + local renewed = 1 + if sessionGen ~= '0' then + local ok = can_renew(sessKey, owner, sessionGen) + if not ok then + renewed = 0 + end + end + + if renewed == 1 then + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok = can_renew(srcKey, ip .. '\31' .. owner, gen) + if not ok then + renewed = 0 + break + end + i = i + 1 + end + end + + if renewed == 1 then + if sessionGen ~= '0' then + local ok, count = can_renew(sessKey, owner, sessionGen) + if ok then + renew_field(sessKey, owner, sessionGen, count) + end + end + local i = 0 + while i < ipCount do + local ip = ARGV[6 + (i * 2)] + local gen = ARGV[7 + (i * 2)] + local ok, count = can_renew(srcKey, ip .. '\31' .. owner, gen) + if ok then + renew_field(srcKey, ip .. '\31' .. owner, gen, count) + end + i = i + 1 + end + end + + local remaining = apply_bytes() + if remaining == nil or remaining < 0 then + remaining = 0 + end + if renewed == 1 then + return remaining + end + return -remaining - 1 + """; + + /// Deletes every field owned by this process incarnation. + /// KEYS[1] source hash, KEYS[2] session hash. ARGV: owner. Always returns 1. + public const string ReleaseOwner = + """ + -- nntpd-admit-release-owner + local srcKey = KEYS[1] + local sessKey = KEYS[2] + local owner = ARGV[1] + redis.call('HDEL', sessKey, owner) + if redis.call('HLEN', sessKey) == 0 then + redis.call('DEL', sessKey) + end + local entries = redis.call('HGETALL', srcKey) + local owned = {} + for i = 1, #entries, 2 do + local f = entries[i] + local sep = string.find(f, '\31', 1, true) + local fieldOwner = sep and string.sub(f, sep + 1) or '' + if fieldOwner == owner then + owned[#owned + 1] = f + end + end + if #owned > 0 then + redis.call('HDEL', srcKey, unpack(owned)) + end + if redis.call('HLEN', srcKey) == 0 then + redis.call('DEL', srcKey) + end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SessionStateService.cs b/src/VectorNNTPD.TransitService/SessionState/SessionStateService.cs new file mode 100644 index 00000000..6739792f --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SessionStateService.cs @@ -0,0 +1,177 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Sole periodic scheduler for SessionState: lease renewal, source-address + /// ownership, byte-quota reconciliation, and rate observation. + /// + /// + /// This service is a scheduler. It does not accept connections, authenticate clients, + /// or keep its own session registry. Every renewal interval it commits account-wide + /// MySQL remaining-quota consumes, then asks + /// to renew currently active local ownership. When an account has both distributed + /// ownership and a committed byte batch, renewal and Redis APPLY share one EVAL. + /// Accounts with pending bytes but no SessionState ownership receive APPLY-only. + /// Each account with distributed ownership is one Redis operation covering session + /// count, every locally owned source IP, and byte state when present. Renewal is + /// node liveness, not client activity: idle authenticated sessions keep consuming + /// cluster capacity while this process can renew. stops the + /// loop, performs a bounded two-pass final byte reconcile, then releases leftover incarnation + /// ownership. Listeners are registered after this service so they stop first and + /// sessions can finalize before renewal ends. Process crash is detected by lease + /// expiry after renewals stop. Unreconciled process-memory pending bytes are lost. + /// + public sealed class SessionStateService : IApplicationService + { + private readonly ISessionStateLeaseManager _leases; + private readonly IAccountByteAccountant _bytes; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public SessionStateService( + ISessionStateLeaseManager leases, + IAccountByteAccountant bytes, + ILogger logger) + : this(leases, bytes, logger, TimeProvider.System, SessionStateDefaults.RenewalPeriod) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal SessionStateService( + ISessionStateLeaseManager leases, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + : this(leases, NullAccountByteAccountant.Instance, logger, timeProvider, interval) + { + } + + /// Initializes a new instance with byte accounting and an explicit interval (tests). + internal SessionStateService( + ISessionStateLeaseManager leases, + IAccountByteAccountant bytes, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(leases); + ArgumentNullException.ThrowIfNull(bytes); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _leases = leases; + _bytes = bytes; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "SessionState"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + AccountByteLogMessages.FinalReconciliationStarted(_logger); + try + { + await _bytes.ReconcileAsync(cancellationToken).ConfigureAwait(false); + // Pending observed while a MySQL-committed batch was still awaiting APPLY + // stays in Pending. A second pass consumes it after the first APPLY completes. + await _bytes.ReconcileAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + AccountByteLogMessages.FinalReconciliationFailed(_logger, ex); + } + + try + { + await _leases.ReleaseAllOwnershipAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + SessionStateLogMessages.SessionStateRenewalStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + try + { + await ReconcileOnceAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + SessionStateLogMessages.SessionStateRenewalPassFailed(_logger, ex); + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + SessionStateLogMessages.SessionStateRenewalStopped(_logger); + } + + private async Task ReconcileOnceAsync(CancellationToken cancellationToken) + { + await _bytes.CommitDurableAsync(cancellationToken).ConfigureAwait(false); + await _leases.RenewLeasesAsync(cancellationToken).ConfigureAwait(false); + await _bytes.ApplyCommittedAsync(cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/SessionState/SourceAddressIdentity.cs b/src/VectorNNTPD.TransitService/SessionState/SourceAddressIdentity.cs new file mode 100644 index 00000000..30f54cc6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/SessionState/SourceAddressIdentity.cs @@ -0,0 +1,36 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.SessionState +{ + /// + /// Canonical source-IP identity used by session and cluster source-IP admission. + /// + /// + /// IPv4-mapped IPv6 addresses are converted to IPv4 so ::ffff:192.0.2.10 + /// and 192.0.2.10 are one identity. Other IPv4 and IPv6 addresses are + /// independent; equivalent textual forms of the same address normalize through + /// . + /// + internal static class SourceAddressIdentity + { + /// Returns the canonical textual identity for . + public static string Format(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return Normalize(address).ToString(); + } + + /// + /// Network-order address bytes after the same IPv4-mapped normalization as + /// . IPv4 is 4 bytes; IPv6 is 16 bytes. + /// + public static byte[] ToNetworkBytes(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return Normalize(address).GetAddressBytes(); + } + + private static IPAddress Normalize(IPAddress address) => + address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/ArticlePlacement.cs b/src/VectorNNTPD.TransitService/Storage/ArticlePlacement.cs new file mode 100644 index 00000000..21bd38c9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/ArticlePlacement.cs @@ -0,0 +1,145 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.TransitService.Storage +{ + /// Observable result of one single-copy placement attempt. + public enum ArticlePlacementKind : byte + { + /// Journal accept completed. Matches ArticleAcceptOutcome.Accepted (1). + Accepted = 1, + + /// Identical article already represented. Matches ArticleAcceptOutcome.Duplicate (2). + Duplicate = 2, + + /// Conflicting ArtHash or ArtSize. Matches ArticleAcceptOutcome.Conflict (3). + Conflict = 3, + + /// Journal pressure rejection. Matches ArticleAcceptOutcome.RejectedPressure (4). + RejectedPressure = 4, + + /// Canonical or integrity rejection. Matches ArticleAcceptOutcome.RejectedInvalid (5). + RejectedInvalid = 5, + + /// Capacity rejection. Matches ArticleAcceptOutcome.RejectedCapacity (6). + RejectedCapacity = 6, + + /// No eligible registry entry. No dial. + NoActiveServer = 7, + + /// Connect, TLS, protocol FAIL, or I/O failure before END was fully written. + TransportFailure = 8, + + /// Caller cancelled before the server could accept. + Cancelled = 9, + + /// END was written and RESULT was not observed. The journal may already hold the article. + AcknowledgementNotObserved = 10, + } + + /// One finished placement attempt. + /// Classification. + /// Transport or protocol failure name. Never article bytes. + public readonly record struct ArticlePlacementResult(ArticlePlacementKind Kind, string? Failure = null) + { + /// Gets whether the StorageServer accepted responsibility or already held the same article. + public bool Succeeded => Kind is ArticlePlacementKind.Accepted or ArticlePlacementKind.Duplicate; + + /// Maps a RESULT outcome byte. Unknown bytes are transport failures. + public static ArticlePlacementResult FromOutcomeByte(byte outcome) => outcome switch + { + (byte)ArticlePlacementKind.Accepted => new(ArticlePlacementKind.Accepted), + (byte)ArticlePlacementKind.Duplicate => new(ArticlePlacementKind.Duplicate), + (byte)ArticlePlacementKind.Conflict => new(ArticlePlacementKind.Conflict), + (byte)ArticlePlacementKind.RejectedPressure => new(ArticlePlacementKind.RejectedPressure), + (byte)ArticlePlacementKind.RejectedInvalid => new(ArticlePlacementKind.RejectedInvalid), + (byte)ArticlePlacementKind.RejectedCapacity => new(ArticlePlacementKind.RejectedCapacity), + _ => new(ArticlePlacementKind.TransportFailure, "result-outcome"), + }; + } + + /// Selects one placement target from an already-active registry snapshot. + public static class StorageServerPlacementSelector + { + /// + /// Selects the eligible entry with the greatest , + /// then the lowest , then ordinal FQDN. + /// + public static bool TrySelect(IReadOnlyList active, out StorageServerFleetEntry selected) + { + ArgumentNullException.ThrowIfNull(active); + selected = default; + var found = false; + foreach (var entry in active) + { + if (entry.VatpPort is not (>= 1 and <= 65535)) + { + continue; + } + + if (!found || Prefer(entry, selected)) + { + selected = entry; + found = true; + } + } + + return found; + } + + private static bool Prefer(in StorageServerFleetEntry candidate, in StorageServerFleetEntry current) + { + if (candidate.AvailableBytes != current.AvailableBytes) + { + return candidate.AvailableBytes > current.AvailableBytes; + } + + if (candidate.ServerId != current.ServerId) + { + return candidate.ServerId < current.ServerId; + } + + return string.CompareOrdinal(candidate.Fqdn, current.Fqdn) < 0; + } + } + + /// Builds VATP META from a CanonicalV1 record. + public static class ArticlePlacementMeta + { + private static readonly NntpArticleHeaderName[] DateHeaders = + [ + NntpArticleHeaderName.Date, + NntpArticleHeaderName.InjectionDate, + NntpArticleHeaderName.NntpPostingDate, + NntpArticleHeaderName.Posted, + NntpArticleHeaderName.XDate, + NntpArticleHeaderName.DeliveryDate, + ]; + + /// + /// Recovers the winning Date-family header by locating fields that match the record, + /// then builds META with . + /// + public static bool TryFromRecord(in ArticleRecord record, out ArticleCanonicalTransferMeta meta) + { + meta = default; + if (record.ParseStatus != ArticleParseStatus.CanonicalV1 || record.ArtSize <= 0) + { + return false; + } + + var artData = record.ArtData.Span; + foreach (var name in DateHeaders) + { + if (ArticleFieldTable.Locate(artData, name).Equals(record.Fields)) + { + meta = ArticleCanonicalTransferMeta.FromRecord(in record, name); + return true; + } + } + + return false; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/ArticlePlacementClient.cs b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementClient.cs new file mode 100644 index 00000000..f3745ae6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementClient.cs @@ -0,0 +1,673 @@ +using System.Buffers; +using System.Globalization; +using System.Net.Security; +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Storage +{ + using VectorNNTPD.Common.Core; + /// + /// Long-lived STORE client. One TLS connection per placement target. Does not use . + /// + /// + /// Connections stay open while they are healthy. Each connection has one frame writer and at most + /// outstanding STORE streams. A lost RESULT after END is + /// and is not retried. Placement itself + /// stays disabled in the spool writer. + /// + public sealed class ArticlePlacementClient : IArticlePlacementClient, IAsyncDisposable + { + private static readonly TimeSpan InitialBackoff = TimeSpan.FromMilliseconds(200); + private static readonly TimeSpan MaximumBackoff = TimeSpan.FromSeconds(5); + + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ArticleTransferLimits _limits; + private readonly TimeProvider _time; + private readonly object _gate = new(); + private readonly Dictionary _connections = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary> _dialing = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _nextDial = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _backoff = new(StringComparer.OrdinalIgnoreCase); + private int _disposed; + private int _connects; + private int _dials; + + /// Optional certificate callback for tests only. + internal RemoteCertificateValidationCallback? ServerCertificateValidationCallback { get; set; } + + /// Optional TCP connect host override for tests. TLS TargetHost remains the advertised FQDN. + internal string? TestTcpConnectHost { get; set; } + + /// Gets how many TCP connections this client has established. + internal int ConnectCount => Volatile.Read(ref _connects); + + /// Gets how many dial attempts have started, including failures. + internal int DialAttempts => Volatile.Read(ref _dials); + + /// Gets the outstanding STORE limit applied to each connection. + internal int MaxOutstandingStores { get; } + + /// Sums wait and RESULT counters across connections still owned by this client. + internal PlacementTransportCounters Snapshot() + { + lock (_gate) + { + var counters = default(PlacementTransportCounters); + foreach (var connection in _connections.Values) + { + counters = counters.Add(connection); + } + + return counters; + } + } + + /// Initializes a placement client with two outstanding STOREs and the production VATP limits. + public ArticlePlacementClient(ILogger logger) + : this(logger, ArticleIngestionOptions.DefaultMaxOutstandingStoresPerConnection) + { + } + + /// Initializes a placement client with an explicit outstanding-STORE limit. + public ArticlePlacementClient(ILogger logger, int maxOutstandingStores) + : this(logger, new VatpClientOptions(), ArticleTransferLimits.Default, maxOutstandingStores, TimeProvider.System) + { + } + + internal ArticlePlacementClient( + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + int maxOutstandingStores = ArticleIngestionOptions.DefaultMaxOutstandingStoresPerConnection, + TimeProvider? time = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(limits); + if (maxOutstandingStores is not (1 or 2)) + { + throw new ArgumentOutOfRangeException( + nameof(maxOutstandingStores), + maxOutstandingStores, + "Outstanding STORE operations per connection must be 1 or 2."); + } + + _logger = logger; + _options = options; + _limits = limits; + _time = time ?? TimeProvider.System; + MaxOutstandingStores = maxOutstandingStores; + } + + /// + public async ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + if (Volatile.Read(ref _disposed) != 0) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "shutdown"); + } + + if (target.VatpPort is not int port || port is < 1 or > 65535) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "vatp-port"); + } + + if (!ArticlePlacementMeta.TryFromRecord(in record, out var meta)) + { + return new ArticlePlacementResult(ArticlePlacementKind.RejectedInvalid, "meta"); + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + var leased = await LeaseAsync(target.Fqdn, port, cancellationToken).ConfigureAwait(false); + if (leased.Failure is { } failure) + { + return failure; + } + + var connection = leased.Connection + ?? throw new InvalidOperationException("Placement lease returned neither a connection nor a failure."); + var result = await connection.PlaceAsync(record, meta, cancellationToken).ConfigureAwait(false); + if (connection.IsDead) + { + Forget(target.Fqdn, port, connection); + try + { + await connection.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or TimeoutException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + return result; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "timeout"); + } + catch (OperationCanceledException) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, DescribeTransportFailure(ex)); + } + } + + /// Closes every placement connection and rejects later calls. + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + ArticlePlacementConnection[] connections; + lock (_gate) + { + connections = _connections.Values.ToArray(); + _connections.Clear(); + } + + var outstanding = 0; + foreach (var connection in connections) + { + outstanding += connection.ActiveStores; + await connection.DisposeAsync().ConfigureAwait(false); + } + + ArticlePlacementTransportLogMessages.Shutdown(_logger, outstanding); + } + + private async Task<(ArticlePlacementConnection? Connection, ArticlePlacementResult? Failure)> LeaseAsync( + string fqdn, + int port, + CancellationToken cancellationToken) + { + var key = fqdn + ":" + port.ToString(CultureInfo.InvariantCulture); + Task? pending = null; + var owner = false; + ArticlePlacementConnection? dead = null; + ArticlePlacementResult? deferred = null; + lock (_gate) + { + if (_disposed != 0) + { + return (null, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "shutdown")); + } + + if (_connections.TryGetValue(key, out var live) && !live.IsDead) + { + ArticlePlacementTransportLogMessages.Reused(_logger, fqdn, port, live.ActiveStores); + return (live, null); + } + + if (_connections.TryGetValue(key, out dead)) + { + _connections.Remove(key); + } + + if (_dialing.TryGetValue(key, out pending)) + { + // Dial already in flight; await the captured pending task below. + } + else if (_nextDial.TryGetValue(key, out var next) && _time.GetUtcNow() < next) + { + var wait = next - _time.GetUtcNow(); + ArticlePlacementTransportLogMessages.ReconnectDeferred( + _logger, fqdn, port, (long)wait.TotalMilliseconds, "backoff"); + deferred = new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "backoff"); + } + else + { + pending = DialAsync(fqdn, port, key); + _dialing[key] = pending; + owner = true; + } + } + + if (dead is not null) + { + try + { + await dead.DisposeAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or TimeoutException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + if (deferred is not null) + { + return (null, deferred); + } + + try + { + var connection = await pending!.WaitAsync(cancellationToken).ConfigureAwait(false); + return (connection, null); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return (null, new ArticlePlacementResult(ArticlePlacementKind.Cancelled)); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is SocketException or IOException or TimeoutException or OperationCanceledException) + { + if (owner) + { + NoteFailure(key); + } + + var failure = DescribeTransportFailure(ex); + ArticlePlacementTransportLogMessages.ConnectionFailed(_logger, fqdn, port, failure); + return (null, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, failure)); + } + finally + { + if (owner) + { + _ = ReleaseDialAsync(key, pending!); + } + } + } + + private async Task ReleaseDialAsync(string key, Task pending) + { + try + { + await pending.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + lock (_gate) + { + if (_dialing.TryGetValue(key, out var current) && ReferenceEquals(current, pending)) + { + _dialing.Remove(key); + } + } + } + + private async Task DialAsync(string fqdn, int port, string key) + { + Interlocked.Increment(ref _dials); + var connection = await ArticlePlacementConnection.EstablishAsync( + fqdn, + port, + TestTcpConnectHost ?? fqdn, + _logger, + _options, + _limits, + MaxOutstandingStores, + ServerCertificateValidationCallback, + CancellationToken.None) + .ConfigureAwait(false); + Interlocked.Increment(ref _connects); + lock (_gate) + { + _backoff[key] = InitialBackoff; + _nextDial.Remove(key); + if (_disposed != 0) + { + _ = connection.DisposeAsync(); + throw new IOException("shutdown"); + } + + _connections[key] = connection; + } + + return connection; + } + + private void NoteFailure(string key) + { + lock (_gate) + { + var current = _backoff.TryGetValue(key, out var backoff) ? backoff : InitialBackoff; + _nextDial[key] = _time.GetUtcNow().Add(current); + var doubled = TimeSpan.FromTicks(Math.Min(current.Ticks * 2, MaximumBackoff.Ticks)); + _backoff[key] = doubled; + } + } + + private void Forget(string fqdn, int port, ArticlePlacementConnection connection) + { + var key = fqdn + ":" + port.ToString(CultureInfo.InvariantCulture); + lock (_gate) + { + if (_connections.TryGetValue(key, out var current) && ReferenceEquals(current, connection)) + { + _connections.Remove(key); + } + } + } + + private static string DescribeTransportFailure(Exception ex) + { + if (ex is TimeoutException) + { + return "timeout"; + } + + if (ex is SocketException socket) + { + return string.Create( + CultureInfo.InvariantCulture, + $"SocketException: SocketErrorCode={socket.SocketErrorCode} HResult={socket.HResult} Message={socket.Message}"); + } + + return ex.GetType().Name; + } + } + + /// Wait and RESULT totals for connections still held by one placement client. + internal readonly record struct PlacementTransportCounters( + long SlotWaitTicks, + long CreditWaitTicks, + long ResultCount, + long ResultTicks, + long ResultMaxTicks, + int PeakActive, + long DataFramesWritten) + { + /// Adds one connection's counters. + internal PlacementTransportCounters Add(ArticlePlacementConnection connection) => + new( + SlotWaitTicks + connection.SlotWaitTicks, + CreditWaitTicks + connection.CreditWaitTicks, + ResultCount + connection.ResultCount, + ResultTicks + connection.ResultTicks, + Math.Max(ResultMaxTicks, connection.ResultMaxTicks), + Math.Max(PeakActive, connection.PeakActive), + DataFramesWritten + connection.DataFramesWritten); + } + + /// STORE/META/DATA/END sender over an already connected stream. + internal static class VatpPlacementTransfer + { + public static async Task PlaceAsync( + Stream stream, + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + ArticleTransferLimits limits, + TimeSpan ioTimeout, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + var reader = new FrameReader(stream, ioTimeout); + var endWritten = false; + try + { + cancellationToken.ThrowIfCancellationRequested(); + var maxFrame = limits.DefaultMaxFramePayload; + await WriteAsync(stream, VatpFrameEncoder.EncodeHello(maxFrame), cancellationToken).ConfigureAwait(false); + var hello = await reader.ReadAsync(maxFrame, cancellationToken).ConfigureAwait(false); + if (hello is null) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "hello-closed"); + } + + if (hello.Value.Header.Type != VatpFrameType.Hello + || !VatpHello.TryDecode(hello.Value.Payload, out var decodedHello, out _)) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "hello"); + } + + maxFrame = Math.Min(decodedHello.MaxFramePayload, VatpProtocol.DefaultMaxFramePayload); + const uint streamId = 1; + var id = new byte[ArticleId.Length]; + record.ArtId.CopyTo(id); + await WriteAsync(stream, VatpFrameEncoder.EncodeStore(streamId, id), cancellationToken).ConfigureAwait(false); + var metaBytes = VatpMetaCodec.Encode(in meta); + await WriteAsync(stream, VatpFrameEncoder.EncodeMeta(streamId, metaBytes), cancellationToken).ConfigureAwait(false); + + var window = new ArticleTransferWindow(0, limits.MaxStreamCreditBytes); + var offset = 0; + var artData = record.ArtData; + while (offset < record.ArtSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var length = ArticleTransferReadyRing.ComputeDataPayloadLength( + record.ArtSize - offset, + window.Credit, + maxFrame); + if (length <= 0) + { + var paced = await ReadControlAsync(reader, maxFrame, endWritten: false, cancellationToken) + .ConfigureAwait(false); + if (paced.Terminal is { } terminal) + { + return terminal; + } + + if (paced.AddedCredit > 0) + { + _ = window.Add(paced.AddedCredit); + } + + continue; + } + + if (!window.TryConsume(length)) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window"); + } + + var fin = offset + length == record.ArtSize; + await WriteAsync( + stream, + VatpFrameEncoder.EncodeData(streamId, artData.Slice(offset, length), fin), + cancellationToken) + .ConfigureAwait(false); + offset += length; + } + + await WriteAsync(stream, VatpFrameEncoder.EncodeEnd(streamId), cancellationToken).ConfigureAwait(false); + endWritten = true; + while (true) + { + var final = await ReadControlAsync(reader, maxFrame, endWritten: true, cancellationToken) + .ConfigureAwait(false); + if (final.Terminal is { } terminal) + { + return terminal; + } + + if (final.AddedCredit > 0) + { + _ = window.Add(final.AddedCredit); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (!endWritten) + { + try + { + await WriteAsync(stream, VatpFrameEncoder.EncodeCancel(1), CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + + return new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or TimeoutException or ObjectDisposedException) + { + return endWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, ex.GetType().Name); + } + } + + private readonly record struct ControlRead(ArticlePlacementResult? Terminal, uint AddedCredit); + + private static async Task ReadControlAsync( + FrameReader reader, + uint maxFrame, + bool endWritten, + CancellationToken cancellationToken) + { + var frame = await reader.ReadAsync(maxFrame, cancellationToken).ConfigureAwait(false); + if (frame is null) + { + return new ControlRead( + endWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"), + 0); + } + + switch (frame.Value.Header.Type) + { + case VatpFrameType.Window: + return VatpControlPayload.TryDecodeWindow(Copy(frame.Value.Payload), out var add) + ? new ControlRead(null, add) + : new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window"), 0); + case VatpFrameType.Result: + if (!VatpResultPayload.TryDecode(frame.Value.Payload, out var outcome, out _)) + { + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "result"), 0); + } + + return new ControlRead(ArticlePlacementResult.FromOutcomeByte(outcome), 0); + case VatpFrameType.Fail: + if (!VatpControlPayload.TryDecodeFail(Copy(frame.Value.Payload), out var error, out _)) + { + return new ControlRead( + new ArticlePlacementResult( + endWritten + ? ArticlePlacementKind.AcknowledgementNotObserved + : ArticlePlacementKind.TransportFailure, + "fail"), + 0); + } + + if (error == VatpErrorCode.Cancelled) + { + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.Cancelled), 0); + } + + return new ControlRead(new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, error.ToString()), 0); + default: + return new ControlRead( + new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, frame.Value.Header.Type.ToString()), + 0); + } + } + + private static byte[] Copy(ReadOnlySequence payload) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return bytes; + } + + private static async Task WriteAsync( + Stream stream, + VatpFrameEncoder.EncodedFrame frame, + CancellationToken cancellationToken) + { + if (!frame.Header.IsEmpty) + { + await stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (!frame.Payload.IsEmpty) + { + await stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + + private sealed class FrameReader + { + private readonly Stream _stream; + private readonly TimeSpan _ioTimeout; + private readonly byte[] _buffer = new byte[64 * 1024]; + private readonly List _pending = []; + + public FrameReader(Stream stream, TimeSpan ioTimeout) + { + _stream = stream; + _ioTimeout = ioTimeout; + } + + public async Task ReadAsync(uint maxFramePayload, CancellationToken cancellationToken) + { + while (true) + { + var parsed = VatpFrameParser.ParseOneFrame(new ReadOnlySequence(_pending.ToArray()), maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + throw new IOException(parsed.Error.ToString()); + } + + if (parsed.Status == VatpFrameParseStatus.Success && parsed.Frame is { } frame) + { + _pending.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + return CopyFrame(frame); + } + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(_ioTimeout); + int read; + try + { + read = await _stream.ReadAsync(_buffer, timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("placement-io"); + } + + if (read == 0) + { + return null; + } + + for (var i = 0; i < read; i++) + { + _pending.Add(_buffer[i]); + } + } + } + + private static VatpParsedFrame CopyFrame(VatpParsedFrame frame) + { + var payload = new byte[frame.Payload.Length]; + frame.Payload.CopyTo(payload); + return new VatpParsedFrame(frame.Header, new ReadOnlySequence(payload)); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/ArticlePlacementConnection.cs b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementConnection.cs new file mode 100644 index 00000000..7fec3f53 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementConnection.cs @@ -0,0 +1,918 @@ +using System.Buffers; +using System.Net.Security; +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Storage +{ + /// + /// One TLS connection to one placement target. One writer. At most + /// STORE streams. + /// + internal sealed class ArticlePlacementConnection : IAsyncDisposable + { + private readonly string _fqdn; + private readonly int _port; + private readonly TcpClient _tcp; + private readonly SslStream _ssl; + private readonly ILogger _logger; + private readonly ArticleTransferLimits _limits; + private readonly TimeSpan _ioTimeout; + private readonly SemaphoreSlim _slots; + private readonly SemaphoreSlim _write = new(1, 1); + private readonly object _tableGate = new(); + private readonly Dictionary _operations = []; + private readonly CancellationTokenSource _lifetime = new(); + private readonly FrameBuffer _frames = new(); + private Task? _reader; + private int _dead; + private int _disposed; + private int _active; + private int _peakActive; + private uint _nextStreamId = 1; + private uint _maxFramePayload; + private long _slotWaitTicks; + private long _creditWaitTicks; + private long _resultTicks; + private long _resultCount; + private long _resultMaxTicks; + private long _dataFramesWritten; + private int _storesAwaitingInitialCredit; + private readonly object _readTimeoutGate = new(); + private CancellationTokenSource? _activeReadTimeout; + + private ArticlePlacementConnection( + string fqdn, + int port, + TcpClient tcp, + SslStream ssl, + ILogger logger, + ArticleTransferLimits limits, + TimeSpan ioTimeout, + int maxOutstanding, + uint maxFramePayload) + { + _fqdn = fqdn; + _port = port; + _tcp = tcp; + _ssl = ssl; + _logger = logger; + _limits = limits; + _ioTimeout = ioTimeout; + MaxOutstanding = maxOutstanding; + _maxFramePayload = maxFramePayload; + _slots = new SemaphoreSlim(maxOutstanding, maxOutstanding); + } + + public int MaxOutstanding { get; } + + /// Gets the highest number of STORE operations that were in flight together. + internal int PeakActive => Volatile.Read(ref _peakActive); + + /// Gets ticks spent waiting for a free STORE slot. + internal long SlotWaitTicks => Volatile.Read(ref _slotWaitTicks); + + /// Gets ticks spent waiting for WINDOW credit. + internal long CreditWaitTicks => Volatile.Read(ref _creditWaitTicks); + + /// Gets how many RESULT waits were recorded. + internal long ResultCount => Volatile.Read(ref _resultCount); + + /// Gets the total ticks spent waiting for RESULT after END. + internal long ResultTicks => Volatile.Read(ref _resultTicks); + + /// Gets the longest RESULT wait in ticks. + internal long ResultMaxTicks => Volatile.Read(ref _resultMaxTicks); + + /// Gets DATA frames this connection has written. Credit is zero until the server sends WINDOW. + internal long DataFramesWritten => Volatile.Read(ref _dataFramesWritten); + + public bool IsDead => Volatile.Read(ref _dead) != 0; + + public int ActiveStores => Volatile.Read(ref _active); + + public static async Task EstablishAsync( + string fqdn, + int port, + string connectHost, + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + int maxOutstanding, + RemoteCertificateValidationCallback? certificateValidation, + CancellationToken cancellationToken) + { + var tcp = new TcpClient { NoDelay = true }; + try + { + tcp.SendBufferSize = 1 << 20; + tcp.ReceiveBufferSize = 1 << 20; + } + catch (SocketException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(options.ConnectTimeout); + try + { + await tcp.ConnectAsync(connectHost, port, connectCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + if (ex.InnerException is SocketException) + { + throw ex.InnerException; + } + + throw new TimeoutException("timeout"); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + SslStream ssl; + try + { + ssl = await VatpTlsClient.AuthenticateAsClientAsync( + tcp.GetStream(), + fqdn, + options.TlsHandshakeTimeout, + certificateValidation, + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("timeout"); + } + var connection = new ArticlePlacementConnection( + fqdn, + port, + tcp, + ssl, + logger, + limits, + options.IoTimeout, + maxOutstanding, + limits.DefaultMaxFramePayload); + try + { + await connection.ExchangeHelloAsync(cancellationToken).ConfigureAwait(false); + connection._reader = Task.Run(connection.ReadLoopAsync); + ArticlePlacementTransportLogMessages.Established(logger, fqdn, port, maxOutstanding); + return connection; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public async Task PlaceAsync( + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + CancellationToken cancellationToken) + { + if (IsDead || Volatile.Read(ref _disposed) != 0) + { + return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + } + + var slotWait = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + await _slots.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Add(ref _slotWaitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - slotWait); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + catch (ObjectDisposedException) + { + return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + + try + { + using (var operation = Begin(record)) + { + try + { + var now = Interlocked.Increment(ref _active); + int previousPeak; + do + { + previousPeak = Volatile.Read(ref _peakActive); + if (now <= previousPeak) + { + break; + } + } + while (Interlocked.CompareExchange(ref _peakActive, now, previousPeak) != previousPeak); + + return await SendAsync(operation, record, meta, cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _active); + lock (_tableGate) + { + _operations.Remove(operation.StreamId); + } + } + } + } + finally + { + try + { + _slots.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + var outstanding = ActiveStores; + MarkDead(); + try + { + await _lifetime.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + try + { + if (_reader is not null) + { + await _reader.WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + } + } + catch (TimeoutException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + FaultOutstanding(shutdown: true); + ArticlePlacementTransportLogMessages.Closed(_logger, _fqdn, _port, outstanding); + await _ssl.DisposeAsync().ConfigureAwait(false); + _tcp.Dispose(); + _slots.Dispose(); + _write.Dispose(); + _lifetime.Dispose(); + } + + private StoreOperation Begin(ArticleRecord record) + { + lock (_tableGate) + { + if (_nextStreamId == 0) + { + MarkDead(); + throw new IOException("stream-id-space"); + } + + var streamId = _nextStreamId++; + var operation = new StoreOperation(streamId, record.ArtId, _limits); + _operations.Add(streamId, operation); + return operation; + } + } + + private async Task SendAsync( + StoreOperation operation, + ArticleRecord record, + ArticleCanonicalTransferMeta meta, + CancellationToken cancellationToken) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetime.Token); + try + { + var idBytes = new byte[ArticleId.Length]; + record.ArtId.CopyTo(idBytes); + await WriteFrameAsync(VatpFrameEncoder.EncodeStore(operation.StreamId, idBytes), linked.Token) + .ConfigureAwait(false); + operation.Started = true; + var metaBytes = VatpMetaCodec.Encode(in meta); + operation.AwaitingInitialCredit = 1; + Interlocked.Increment(ref _storesAwaitingInitialCredit); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(operation.StreamId, metaBytes), linked.Token) + .ConfigureAwait(false); + DisarmReadTimeout(); + + var artData = record.ArtData; + var offset = 0; + while (offset < record.ArtSize) + { + linked.Token.ThrowIfCancellationRequested(); + int length; + lock (operation.Gate) + { + length = ArticleTransferReadyRing.ComputeDataPayloadLength( + record.ArtSize - offset, + operation.Window.Credit, + _maxFramePayload); + if (length > 0 && !operation.Window.TryConsume(length)) + { + length = 0; + } + } + + if (length <= 0) + { + var creditWait = System.Diagnostics.Stopwatch.GetTimestamp(); + await operation.Credit.WaitAsync(linked.Token).ConfigureAwait(false); + Interlocked.Add(ref _creditWaitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - creditWait); + if (operation.IsCompleted) + { + return operation.Result; + } + + continue; + } + + var fin = offset + length == record.ArtSize; + await WriteFrameAsync( + VatpFrameEncoder.EncodeData(operation.StreamId, artData.Slice(offset, length), fin), + linked.Token) + .ConfigureAwait(false); + Interlocked.Increment(ref _dataFramesWritten); + offset += length; + } + + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(operation.StreamId), linked.Token).ConfigureAwait(false); + operation.EndWritten = true; + var resultWait = System.Diagnostics.Stopwatch.GetTimestamp(); + var result = await operation.Completion.Task.WaitAsync(_ioTimeout, linked.Token).ConfigureAwait(false); + NoteResultWait(System.Diagnostics.Stopwatch.GetTimestamp() - resultWait); + LogOutcome(operation, result); + return result; + } + catch (TimeoutException) + { + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "timeout"); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + catch (OperationCanceledException) when (operation.IsCompleted) + { + LogOutcome(operation, operation.Result); + return operation.Result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (operation.EndWritten) + { + var ambiguous = new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved); + Finish(operation, ambiguous); + LogOutcome(operation, ambiguous); + return ambiguous; + } + + await TryWriteCancelAsync(operation).ConfigureAwait(false); + var cancelled = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + Finish(operation, cancelled); + LogOutcome(operation, cancelled); + return cancelled; + } + catch (OperationCanceledException) + { + var result = _disposed != 0 + ? operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException) + { + MarkDead(); + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, ex.GetType().Name); + Finish(operation, result); + LogOutcome(operation, result); + return result; + } + } + + private async Task ExchangeHelloAsync(CancellationToken cancellationToken) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeHello(_maxFramePayload), cancellationToken).ConfigureAwait(false); + var frame = await ReadFrameAsync(cancellationToken).ConfigureAwait(false); + if (frame is null + || frame.Value.Header.Type != VatpFrameType.Hello + || !VatpHello.TryDecode(frame.Value.Payload, out var hello, out _)) + { + throw new IOException("hello"); + } + + _maxFramePayload = Math.Min(hello.MaxFramePayload, VatpProtocol.DefaultMaxFramePayload); + } + + private async Task ReadLoopAsync() + { + try + { + while (!IsDead && Volatile.Read(ref _disposed) == 0) + { + var frame = await ReadFrameAsync(_lifetime.Token).ConfigureAwait(false); + if (frame is null) + { + MarkDead(); + FaultOutstanding(shutdown: false); + return; + } + + Dispatch(frame.Value); + } + } + catch (OperationCanceledException) + { + FaultOutstanding(shutdown: Volatile.Read(ref _disposed) != 0); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException or TimeoutException) + { + MarkDead(); + ArticlePlacementTransportLogMessages.ConnectionFailed(_logger, _fqdn, _port, ex.GetType().Name); + FaultOutstanding(shutdown: false); + } + } + + private void Dispatch(VatpParsedFrame frame) + { + if (frame.Header.Type == VatpFrameType.Hello) + { + return; + } + + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + MarkDead(); + FaultOutstanding(shutdown: false); + return; + } + + StoreOperation? operation; + lock (_tableGate) + { + _operations.TryGetValue(frame.Header.StreamId, out operation); + } + + if (operation is null) + { + return; + } + + switch (frame.Header.Type) + { + case VatpFrameType.Window: + if (!VatpControlPayload.TryDecodeWindow(Copy(frame.Payload), out var add)) + { + Finish(operation, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "window")); + break; + } + + var gainedInitialCredit = false; + lock (operation.Gate) + { + gainedInitialCredit = operation.AwaitingInitialCredit == 1 && add > 0; + _ = operation.Window.Add(add); + } + + if (gainedInitialCredit && Interlocked.Exchange(ref operation.AwaitingInitialCredit, 0) == 1) + { + Interlocked.Decrement(ref _storesAwaitingInitialCredit); + } + + try + { + operation.Credit.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + break; + case VatpFrameType.Result: + if (!VatpResultPayload.TryDecode(frame.Payload, out var outcome, out _)) + { + Finish(operation, new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "result")); + break; + } + + Finish(operation, ArticlePlacementResult.FromOutcomeByte(outcome)); + break; + case VatpFrameType.Fail: + if (!VatpControlPayload.TryDecodeFail(Copy(frame.Payload), out var error, out _)) + { + Finish(operation, new ArticlePlacementResult( + operation.EndWritten + ? ArticlePlacementKind.AcknowledgementNotObserved + : ArticlePlacementKind.TransportFailure, + "fail")); + break; + } + + Finish(operation, error == VatpErrorCode.Cancelled + ? new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, error.ToString())); + break; + default: + Finish(operation, new ArticlePlacementResult( + ArticlePlacementKind.TransportFailure, + frame.Header.Type.ToString())); + break; + } + } + + private void FaultOutstanding(bool shutdown) + { + StoreOperation[] operations; + lock (_tableGate) + { + operations = _operations.Values.ToArray(); + } + + foreach (var operation in operations) + { + var result = operation.EndWritten + ? new ArticlePlacementResult(ArticlePlacementKind.AcknowledgementNotObserved) + : shutdown + ? new ArticlePlacementResult(ArticlePlacementKind.Cancelled) + : new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "closed"); + Finish(operation, result); + } + } + + private async Task TryWriteCancelAsync(StoreOperation operation) + { + if (IsDead) + { + return; + } + + try + { + await WriteFrameAsync(VatpFrameEncoder.EncodeCancel(operation.StreamId), CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is IOException or ObjectDisposedException or SocketException or OperationCanceledException) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task WriteFrameAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + await _write.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + try + { + if (!frame.Header.IsEmpty) + { + await _ssl.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (!frame.Payload.IsEmpty) + { + await _ssl.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + MarkDead(); + throw; + } + } + finally + { + try + { + _write.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private void NoteResultWait(long ticks) + { + Interlocked.Increment(ref _resultCount); + Interlocked.Add(ref _resultTicks, ticks); + long previous; + do + { + previous = Volatile.Read(ref _resultMaxTicks); + if (ticks <= previous) + { + return; + } + } + while (Interlocked.CompareExchange(ref _resultMaxTicks, ticks, previous) != previous); + } + + private async Task ReadFrameAsync(CancellationToken cancellationToken) + { + while (true) + { + if (_frames.TryTake(_maxFramePayload, out var frame)) + { + return frame; + } + + using (var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + { + lock (_readTimeoutGate) + { + _activeReadTimeout = timeout; + if (Volatile.Read(ref _storesAwaitingInitialCredit) == 0) + { + timeout.CancelAfter(_ioTimeout); + } + } + + int read; + try + { + read = await _frames.ReadFromAsync(_ssl, timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException("placement-io"); + } + finally + { + lock (_readTimeoutGate) + { + if (ReferenceEquals(_activeReadTimeout, timeout)) + { + _activeReadTimeout = null; + } + } + } + + if (read == 0) + { + return null; + } + } + } + } + + /// Completes one STORE and drops its initial-credit wait from the read-timeout count. + private void Finish(StoreOperation operation, ArticlePlacementResult result) + { + if (operation.TryFinish(result)) + { + Interlocked.Decrement(ref _storesAwaitingInitialCredit); + } + } + + /// + /// Disarms the outstanding read while at least one STORE is waiting for its first WINDOW. + /// The next read arms the timeout again once that count returns to zero. + /// + private void DisarmReadTimeout() + { + lock (_readTimeoutGate) + { + if (Volatile.Read(ref _storesAwaitingInitialCredit) == 0) + { + return; + } + + var timeout = _activeReadTimeout; + if (timeout is null) + { + return; + } + + try + { + timeout.CancelAfter(Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + private void LogOutcome(StoreOperation operation, ArticlePlacementResult result) + { + if (result.Kind == ArticlePlacementKind.AcknowledgementNotObserved) + { + ArticlePlacementTransportLogMessages.StoreAmbiguous(_logger, _fqdn, _port, operation.StreamId); + return; + } + + if (result.Succeeded) + { + ArticlePlacementTransportLogMessages.StoreCompleted( + _logger, _fqdn, _port, operation.StreamId, result.Kind.ToString()); + return; + } + + ArticlePlacementTransportLogMessages.StoreRejected( + _logger, _fqdn, _port, operation.StreamId, result.Kind.ToString(), result.Failure); + } + + private void MarkDead() => Volatile.Write(ref _dead, 1); + + private static byte[] Copy(ReadOnlySequence payload) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return bytes; + } + + private sealed class StoreOperation : IDisposable + { + private int _completed; + + public StoreOperation(uint streamId, ArticleId articleId, ArticleTransferLimits limits) + { + StreamId = streamId; + ArticleId = articleId; + Window = new ArticleTransferWindow(0, limits.MaxStreamCreditBytes); + Credit = new SemaphoreSlim(0); + Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public uint StreamId { get; } + + public ArticleId ArticleId { get; } + + public object Gate { get; } = new(); + + // codeql[cs/missed-readonly-modifier] Mutable struct; readonly would copy on Add/TryConsume and break window credit. + public ArticleTransferWindow Window; + + public SemaphoreSlim Credit { get; } + + public TaskCompletionSource Completion { get; } + + public bool Started { get; set; } + + private volatile bool _endWritten; + + public bool EndWritten + { + get => _endWritten; + set => _endWritten = value; + } + + public bool IsCompleted => Volatile.Read(ref _completed) != 0; + + public ArticlePlacementResult Result { get; private set; } + + /// 1 after META until the first WINDOW or a terminal result, then 0. + public int AwaitingInitialCredit; + + /// + /// Completes the operation once. + /// Returns when this call clears a pending initial-credit wait. + /// + public bool TryFinish(ArticlePlacementResult result) + { + var releasedCreditWait = Interlocked.Exchange(ref AwaitingInitialCredit, 0) == 1; + if (Interlocked.Exchange(ref _completed, 1) != 0) + { + return releasedCreditWait; + } + + Result = result; + Completion.TrySetResult(result); + try + { + Credit.Release(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + return releasedCreditWait; + } + + public void Dispose() => Credit.Dispose(); + } + + /// Accumulates socket bytes and yields one parsed frame at a time. + private sealed class FrameBuffer + { + private byte[] _buffer = new byte[64 * 1024]; + private int _start; + private int _end; + + public bool TryTake(uint maxFramePayload, out VatpParsedFrame frame) + { + frame = default; + if (_end <= _start) + { + return false; + } + + var parsed = VatpFrameParser.ParseOneFrame( + new ReadOnlySequence(_buffer, _start, _end - _start), + maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + throw new IOException(parsed.Error.ToString()); + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } parsedFrame) + { + return false; + } + + var payload = new byte[parsedFrame.Payload.Length]; + parsedFrame.Payload.CopyTo(payload); + frame = new VatpParsedFrame(parsedFrame.Header, new ReadOnlySequence(payload)); + _start += checked((int)parsed.ConsumedBytes); + if (_start == _end) + { + _start = 0; + _end = 0; + } + + return true; + } + + public async ValueTask ReadFromAsync(Stream stream, CancellationToken cancellationToken) + { + Compact(); + if (_end == _buffer.Length) + { + Array.Resize(ref _buffer, _buffer.Length * 2); + } + + var read = await stream.ReadAsync(_buffer.AsMemory(_end), cancellationToken).ConfigureAwait(false); + _end += read; + return read; + } + + private void Compact() + { + if (_start == 0) + { + return; + } + + if (_start == _end) + { + _start = 0; + _end = 0; + return; + } + + Buffer.BlockCopy(_buffer, _start, _buffer, 0, _end - _start); + _end -= _start; + _start = 0; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/ArticlePlacementTransportLogMessages.cs b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementTransportLogMessages.cs new file mode 100644 index 00000000..8d33062b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/ArticlePlacementTransportLogMessages.cs @@ -0,0 +1,62 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.TransitService.Storage +{ + /// Source-generated logs for the long-lived placement connection. Not per DATA frame. + internal static partial class ArticlePlacementTransportLogMessages + { + [LoggerMessage( + EventId = 2640, + Level = LogLevel.Information, + Message = "Placement connection established fqdn={Fqdn} vatpPort={VatpPort} outstandingLimit={OutstandingLimit}")] + public static partial void Established(ILogger logger, string Fqdn, int VatpPort, int OutstandingLimit); + + [LoggerMessage( + EventId = 2641, + Level = LogLevel.Debug, + Message = "Placement connection reused fqdn={Fqdn} vatpPort={VatpPort} activeStores={ActiveStores}")] + public static partial void Reused(ILogger logger, string Fqdn, int VatpPort, int ActiveStores); + + [LoggerMessage( + EventId = 2642, + Level = LogLevel.Warning, + Message = "Placement reconnect deferred fqdn={Fqdn} vatpPort={VatpPort} backoffMs={BackoffMs} failure={Failure}")] + public static partial void ReconnectDeferred(ILogger logger, string Fqdn, int VatpPort, long BackoffMs, string Failure); + + [LoggerMessage( + EventId = 2643, + Level = LogLevel.Warning, + Message = "Placement connection failed fqdn={Fqdn} vatpPort={VatpPort} failure={Failure}")] + public static partial void ConnectionFailed(ILogger logger, string Fqdn, int VatpPort, string Failure); + + [LoggerMessage( + EventId = 2644, + Level = LogLevel.Information, + Message = "Placement connection closed fqdn={Fqdn} vatpPort={VatpPort} outstanding={Outstanding}")] + public static partial void Closed(ILogger logger, string Fqdn, int VatpPort, int Outstanding); + + [LoggerMessage( + EventId = 2645, + Level = LogLevel.Debug, + Message = "Placement STORE completed fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId} outcome={Outcome}")] + public static partial void StoreCompleted(ILogger logger, string Fqdn, int VatpPort, uint StreamId, string Outcome); + + [LoggerMessage( + EventId = 2646, + Level = LogLevel.Warning, + Message = "Placement STORE rejected fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId} outcome={Outcome} failure={Failure}")] + public static partial void StoreRejected(ILogger logger, string Fqdn, int VatpPort, uint StreamId, string Outcome, string? Failure); + + [LoggerMessage( + EventId = 2647, + Level = LogLevel.Warning, + Message = "Placement STORE acknowledgement not observed fqdn={Fqdn} vatpPort={VatpPort} streamId={StreamId}")] + public static partial void StoreAmbiguous(ILogger logger, string Fqdn, int VatpPort, uint StreamId); + + [LoggerMessage( + EventId = 2648, + Level = LogLevel.Information, + Message = "Placement transport shutdown outstanding={Outstanding}")] + public static partial void Shutdown(ILogger logger, int Outstanding); + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/IArticlePlacementClient.cs b/src/VectorNNTPD.TransitService/Storage/IArticlePlacementClient.cs new file mode 100644 index 00000000..af2f6194 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/IArticlePlacementClient.cs @@ -0,0 +1,17 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Storage +{ + /// + /// Sends one CanonicalV1 article to one StorageServer. + /// The client keeps one TLS connection per target and does not dial for every article. + /// + public interface IArticlePlacementClient + { + /// Places on . One attempt. + ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupContracts.cs b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupContracts.cs new file mode 100644 index 00000000..40f139a9 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupContracts.cs @@ -0,0 +1,348 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Storage +{ + /// One validated StorageServer presence response retained for a single lookup. + /// Responding StorageServer id. + /// Responding StorageServer FQDN. + /// TLS VATP listen port for retrieval. + internal readonly record struct StorageArticleCandidate(int ServerId, string Fqdn, int VatpPort); + + /// + /// Later positives for one in-flight lookup. Completes when an alternate arrives or the + /// original lookup window closes. Does not start another lookup. + /// + internal interface IStorageArticleAlternateSource + { + /// Completes when the lookup window closes. + Task WhenClosed { get; } + + /// + /// Returns the second retained candidate, or when the original + /// lookup window closes without one. + /// + ValueTask WaitForAlternateAsync(CancellationToken cancellationToken); + } + + /// Outcome of a StorageServer fleet article-presence lookup. + public enum StorageArticleLookupOutcome + { + /// At least one StorageServer responded positively. + Found = 0, + + /// No positive response arrived before the lookup timeout. + NotFound = 1, + } + + /// Result of a StorageServer fleet article-presence lookup. + /// Classified outcome. + /// Logical lookup identity. + /// Queried article identity. + /// Winning StorageServer id when found. + /// Winning StorageServer FQDN when found. + /// TLS VATP listen port when found. + /// Detail when not found. + public sealed record StorageArticleLookupResult( + StorageArticleLookupOutcome Outcome, + Guid RequestId, + ArticleId ArticleId, + int? ServerId, + string? Fqdn, + int? VatpPort, + string? Error) + { + /// + /// Later candidates for this lookup, present only while the original lookup window can + /// still accept them. Null when the lookup did not find a first candidate. + /// + internal IStorageArticleAlternateSource? Alternates { get; init; } + + /// Builds a not-found result. + public static StorageArticleLookupResult NotFound(Guid requestId, ArticleId articleId, string error) => + new(StorageArticleLookupOutcome.NotFound, requestId, articleId, null, null, null, error); + } + + /// NNTPD client for StorageServer fleet article-presence lookups. + public interface IStorageArticleLookupClient + { + /// + /// Asks all StorageServers "who has this article?" once. The task completes when the + /// first valid positive arrives or the lookup timeout elapses. The registration stays + /// until that same timeout so one later positive can still be retained. + /// + Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken); + } + + /// + /// Correlates StorageServer lookup responses by AMQP CorrelationId. + /// Matching positives are retained until the lookup window closes. Duplicates and + /// unknowns are ignored. The registration is not removed on the first positive. + /// + internal sealed class StorageArticleLookupResponseRouter + { + private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); + + /// Gets outstanding correlation count (tests). + internal int OutstandingCount => _pending.Count; + + /// Registers a lookup awaiting responses. + internal bool TryRegister(string correlationId, StorageArticleLookupOperation operation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + ArgumentNullException.ThrowIfNull(operation); + if (operation.IsCompleted) + { + return false; + } + + if (!_pending.TryAdd(correlationId, new PendingLookup(operation))) + { + throw new InvalidOperationException($"Duplicate storage lookup correlation '{correlationId}'."); + } + + if (operation.IsCompleted) + { + _pending.TryRemove(correlationId, out _); + return false; + } + + return true; + } + + /// Removes a registration. + internal void Unregister(string correlationId) + { + if (!string.IsNullOrWhiteSpace(correlationId)) + { + _pending.TryRemove(correlationId, out _); + } + } + + /// Cancels all outstanding lookups. + internal void CancelAll() + { + // codeql[cs/linq/missed-where] ConcurrentDictionary iteration with TryRemove side effects. + foreach (var pair in _pending) + { + if (_pending.TryRemove(pair.Key, out var pending)) + { + pending.Operation.TrySetCanceled(); + } + } + } + + /// + /// Dispatches a parsed positive response. Returns when the + /// response was retained as a candidate. Does not remove the registration. + /// + internal bool TryComplete(string? correlationId, StorageArticleLookupResponse response) + { + ArgumentNullException.ThrowIfNull(response); + if (string.IsNullOrWhiteSpace(correlationId)) + { + return false; + } + + if (!_pending.TryGetValue(correlationId, out var pending)) + { + return false; + } + + if (pending.Operation.RequestId != response.RequestId) + { + return false; + } + + if (pending.Operation.ArticleId != response.ArticleId) + { + return false; + } + + return pending.Operation.TryAccept(response); + } + + private sealed record PendingLookup(StorageArticleLookupOperation Operation); + } + + /// + /// One in-flight StorageServer fleet lookup. Retains at most two distinct candidates in + /// arrival order and wakes waiters on the first without closing the lookup window. + /// + internal sealed class StorageArticleLookupOperation : IStorageArticleAlternateSource + { + private const int MaxRetainedCandidates = 2; + private readonly object _gate = new(); + private readonly List _candidates = new(MaxRetainedCandidates); + private readonly HashSet _serverIds = []; + private readonly HashSet _endpoints = new(StringComparer.OrdinalIgnoreCase); + private readonly TaskCompletionSource _firstReady = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _alternateOrClosed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _closed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private bool _windowClosed; + + /// Initializes a lookup operation. + internal StorageArticleLookupOperation(Guid requestId, ArticleId articleId) + { + RequestId = requestId; + ArticleId = articleId; + } + + /// Logical request identity. + internal Guid RequestId { get; } + + /// Queried article identity. + internal ArticleId ArticleId { get; } + + /// Completes when the first candidate is retained or the window closes without one. + internal Task FirstReady => _firstReady.Task; + + /// + public Task WhenClosed => _closed.Task; + + /// Whether the window has been closed. + internal bool IsCompleted + { + get + { + lock (_gate) + { + return _windowClosed; + } + } + } + + /// Reads the first retained candidate. + internal bool TryGetFirst(out StorageArticleCandidate candidate) + { + lock (_gate) + { + if (_candidates.Count > 0) + { + candidate = _candidates[0]; + return true; + } + } + + candidate = default; + return false; + } + + /// + /// Retains when it is a new valid candidate. + /// Invalid, duplicate, and over-cap responses are ignored. + /// + internal bool TryAccept(StorageArticleLookupResponse response) + { + if (!TryValidate(response, out var candidate, out var endpointKey)) + { + return false; + } + + lock (_gate) + { + if (_windowClosed || _candidates.Count >= MaxRetainedCandidates) + { + return false; + } + + if (_serverIds.Contains(candidate.ServerId) || _endpoints.Contains(endpointKey)) + { + return false; + } + + _serverIds.Add(candidate.ServerId); + _endpoints.Add(endpointKey); + _candidates.Add(candidate); + if (_candidates.Count == 1) + { + _firstReady.TrySetResult(true); + } + else + { + _alternateOrClosed.TrySetResult(true); + } + + return true; + } + } + + /// Closes the window so no further candidate can be retained. + internal void CloseWindow() + { + lock (_gate) + { + if (_windowClosed) + { + return; + } + + _windowClosed = true; + } + + _firstReady.TrySetResult(false); + _alternateOrClosed.TrySetResult(false); + _closed.TrySetResult(); + } + + /// Cancels waiters. Used when the lookup service is stopping. + internal bool TrySetCanceled() + { + CloseWindow(); + return true; + } + + /// + public async ValueTask WaitForAlternateAsync(CancellationToken cancellationToken) + { + lock (_gate) + { + if (_candidates.Count >= 2) + { + return _candidates[1]; + } + + if (_windowClosed) + { + return null; + } + } + + await _alternateOrClosed.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_gate) + { + return _candidates.Count >= 2 ? _candidates[1] : null; + } + } + + private bool TryValidate( + StorageArticleLookupResponse response, + out StorageArticleCandidate candidate, + out string endpointKey) + { + candidate = default; + endpointKey = string.Empty; + if (!ServerIdRules.IsInRange(response.ServerId)) + { + return false; + } + + if (response.ArticleId != ArticleId + || !VatpEndpointFields.IsCanonicalFqdn(response.Fqdn) + || !VatpEndpointFields.IsCanonicalPort(response.VatpPort)) + { + return false; + } + + candidate = new StorageArticleCandidate(response.ServerId, response.Fqdn, response.VatpPort); + endpointKey = response.Fqdn + ":" + response.VatpPort.ToString(System.Globalization.CultureInfo.InvariantCulture); + return true; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupLogMessages.cs b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupLogMessages.cs new file mode 100644 index 00000000..56389700 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupLogMessages.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.TransitService.Storage +{ + /// Source-generated StorageServer article-lookup log messages. + internal static partial class StorageArticleLookupLogMessages + { + [LoggerMessage( + EventId = 2870, + Level = LogLevel.Information, + Message = "Storage article lookup ready (replyTo={ReplyTo}, generation={Generation})")] + public static partial void Ready(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2871, + Level = LogLevel.Information, + Message = "Storage article lookup session replaced (replyTo={ReplyTo}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string ReplyTo, long Generation); + + [LoggerMessage( + EventId = 2872, + Level = LogLevel.Error, + Message = "Storage article lookup session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2873, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for storage article lookup")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2874, + Level = LogLevel.Debug, + Message = "Storage article lookup published (requestId={RequestId}, correlationId={CorrelationId}, articleId={ArticleId}, generation={Generation})")] + public static partial void Published( + ILogger logger, + Guid RequestId, + string CorrelationId, + string ArticleId, + long Generation); + + [LoggerMessage( + EventId = 2875, + Level = LogLevel.Debug, + Message = "Storage article lookup response ignored (correlationId={CorrelationId}, reason={Reason})")] + public static partial void ResponseIgnored(ILogger logger, string CorrelationId, string Reason); + + [LoggerMessage( + EventId = 2876, + Level = LogLevel.Error, + Message = "Storage article lookup delivery handling failed")] + public static partial void DeliveryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2877, + Level = LogLevel.Information, + Message = "Storage article lookup stopped")] + public static partial void Stopped(ILogger logger); + + [LoggerMessage( + EventId = 2878, + Level = LogLevel.Debug, + Message = "Storage article lookup skipped because the registry has no StorageServers (articleId={ArticleId})")] + public static partial void SkippedEmptyRegistry(ILogger logger, string ArticleId); + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupService.cs b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupService.cs new file mode 100644 index 00000000..70737708 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageArticleLookupService.cs @@ -0,0 +1,480 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.RabbitMq; + +namespace VectorNNTPD.TransitService.Storage +{ + /// + /// NNTPD-owned StorageServer fleet article-presence lookup: publish to cache.requests, + /// consume replies on an ephemeral reply queue, and return the first valid positive immediately. + /// + /// + /// The registration stays until the existing lookup timeout so one later distinct positive + /// can be retained. Independent of ArticleWork RPC. Negative responses are not expected. + /// When has no entries, no cache.requests + /// publication is made and the existing miss result is returned so ARTICLE/HEAD/BODY/STAT + /// continue to ArticleWork. An empty registry is not an article-not-found result. + /// A registry that still contains any entry, including a stale or draining one, uses the + /// existing publication path. + /// + internal sealed class StorageArticleLookupService : IApplicationService, IStorageArticleLookupClient + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly IStorageServerRegistry? _storageServers; + private readonly StorageArticleLookupResponseRouter _router = new(); + private readonly SemaphoreSlim _publishGate = new(1, 1); + private readonly object _sessionGate = new(); + private readonly Guid _instanceId = Guid.NewGuid(); + private CancellationTokenSource? _shutdownCts; + private LookupSession? _session; + private int _started; + + /// Initializes a new storage article lookup service. + public StorageArticleLookupService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + ILogger logger, + TimeProvider? timeProvider = null, + IStorageServerRegistry? storageServers = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _storageServers = storageServers; + } + + /// + public string Name => "StorageArticleLookup"; + + /// + public Task? Execution => null; + + /// Gets the current reply queue name when attached (tests). + internal string? CurrentReplyTo + { + get + { + lock (_sessionGate) + { + return _session?.ReplyTo; + } + } + } + + /// Gets outstanding correlations (tests). + internal int OutstandingCorrelations => _router.OutstandingCount; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + /// + public async Task LookupAsync( + ArticleId articleId, + CancellationToken cancellationToken) + { + if (_storageServers is not null && _storageServers.Snapshot().Count == 0) + { + StorageArticleLookupLogMessages.SkippedEmptyRegistry(_logger, articleId.ToLowerHexString()); + return StorageArticleLookupResult.NotFound( + Guid.NewGuid(), + articleId, + "Storage article lookup timed out with no positive response."); + } + + if (Volatile.Read(ref _started) == 0 || _shutdownCts is null) + { + throw new InvalidOperationException("Storage article lookup is not started."); + } + + var requestId = Guid.NewGuid(); + var correlationId = Guid.NewGuid().ToString("D"); + var operation = new StorageArticleLookupOperation(requestId, articleId); + // codeql[cs/missed-using-statement] Linked CTS ownership transfers to ObserveLookupWindowAsync when the lookup window starts; disposed there or in the !windowStarted finally below. A using around LookupAsync would cancel the background window early. + var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token); + bool registered; + try + { + // codeql[cs/dispose-not-called-on-throw] linked is disposed in catch below, on !registered, or by ObserveLookupWindowAsync after ownership transfer; method-scoped using would cancel the window early. + registered = _router.TryRegister(correlationId, operation); + } + catch + { + linked.Dispose(); + throw; + } + + if (!registered) + { + // codeql[cs/dispose-not-called-on-throw] Explicit dispose on registration failure; throw path disposed in catch. Ownership otherwise transfers to ObserveLookupWindowAsync. + linked.Dispose(); + return StorageArticleLookupResult.NotFound(requestId, articleId, "Lookup could not be registered."); + } + + var windowStarted = false; + try + { + await PublishRequestAsync(requestId, correlationId, articleId, linked.Token).ConfigureAwait(false); + windowStarted = true; + _ = ObserveLookupWindowAsync(operation, correlationId, linked); + await operation.FirstReady.WaitAsync(linked.Token).ConfigureAwait(false); + if (operation.TryGetFirst(out var first)) + { + return new StorageArticleLookupResult( + StorageArticleLookupOutcome.Found, + requestId, + articleId, + first.ServerId, + first.Fqdn, + first.VatpPort, + Error: null) + { + Alternates = operation, + }; + } + + return StorageArticleLookupResult.NotFound( + requestId, + articleId, + "Storage article lookup timed out with no positive response."); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + operation.CloseWindow(); + throw; + } + catch (OperationCanceledException) + { + operation.CloseWindow(); + return StorageArticleLookupResult.NotFound( + requestId, + articleId, + "Storage article lookup was canceled."); + } + finally + { + if (!windowStarted) + { + _router.Unregister(correlationId); + linked.Dispose(); + } + } + } + + private async Task ObserveLookupWindowAsync( + StorageArticleLookupOperation operation, + string correlationId, + CancellationTokenSource linked) + { + using (linked) + { + try + { + await Task.Delay(CacheFleetTopology.LookupTimeout, _timeProvider, linked.Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + finally + { + operation.CloseWindow(); + _router.Unregister(correlationId); + } + } + } + + /// Completes a lookup from a parsed response body (tests). + internal bool TryDispatchResponse(string correlationId, StorageArticleLookupResponse response) => + _router.TryComplete(correlationId, response); + + private async Task PublishRequestAsync( + Guid requestId, + string correlationId, + ArticleId articleId, + CancellationToken cancellationToken) + { + LookupSession session; + lock (_sessionGate) + { + session = _session ?? throw new InvalidOperationException("Storage article lookup session is not ready."); + } + + if (!_rabbitMq.TryGetCurrent(out var handle) + || !handle.IsCurrent + || handle.Generation != session.Generation) + { + throw new InvalidOperationException( + $"Storage article lookup publication failed: connection generation {session.Generation} is not current."); + } + + var body = StorageArticleLookupWireProtocol.SerializeRequestV1( + new StorageArticleLookupRequest( + StorageArticleLookupWireProtocol.CurrentVersion, + requestId, + articleId)); + + await _publishGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + lock (_sessionGate) + { + if (!ReferenceEquals(_session, session)) + { + throw new InvalidOperationException("Storage article lookup session was replaced."); + } + } + + await session.PublishChannel + .PublishAsync( + CacheRequestsTopology.ExchangeName, + routingKey: string.Empty, + correlationId, + requestId.ToString("D"), + session.ReplyTo, + StorageArticleLookupWireProtocol.JsonContentType, + CacheRequestsTopology.ExpirationMilliseconds, + body, + cancellationToken) + .ConfigureAwait(false); + + StorageArticleLookupLogMessages.Published( + _logger, + requestId, + correlationId, + articleId.ToLowerHexString(), + session.Generation); + } + finally + { + _publishGate.Release(); + } + } + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + StorageArticleLookupLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for storage article lookup."); + } + + var generation = handle.Generation; + IRabbitMqRpcChannel? publishChannel = null; + IRabbitMqRpcChannel? consumeChannel = null; + try + { + publishChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + consumeChannel = await handle.Connection + .CreateRpcChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + var replyTo = CreateReplyQueueName(); + await consumeChannel.QueueDeclareAsync( + replyTo, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + _ = await consumeChannel + .ConsumeAsync(replyTo, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new LookupSession(generation, replyTo, publishChannel, consumeChannel); + LookupSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + StorageArticleLookupLogMessages.SessionReplaced(_logger, replyTo, generation); + } + else + { + StorageArticleLookupLogMessages.Ready(_logger, replyTo, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + StorageArticleLookupLogMessages.SessionAttachFailed(_logger, ex, generation); + if (publishChannel is not null) + { + await publishChannel.DisposeAsync().ConfigureAwait(false); + } + + if (consumeChannel is not null) + { + await consumeChannel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqRpcDelivery delivery) + { + try + { + if (!StorageArticleLookupWireProtocol.TryParseResponseV1( + delivery.Body.Span, + out var response, + out var reason) + || response is null) + { + StorageArticleLookupLogMessages.ResponseIgnored(_logger, delivery.CorrelationId ?? string.Empty, reason); + return; + } + + if (!_router.TryComplete(delivery.CorrelationId, response)) + { + StorageArticleLookupLogMessages.ResponseIgnored( + _logger, + delivery.CorrelationId ?? string.Empty, + "Unknown, duplicate, or mismatched correlation."); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageArticleLookupLogMessages.DeliveryFailed(_logger, ex); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageArticleLookupLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + _router.CancelAll(); + + LookupSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + StorageArticleLookupLogMessages.Stopped(_logger); + } + + private string CreateReplyQueueName() + { + var serverId = _nntpdOptions.Value.ServerId ?? 0; + return RabbitMqTopologyNames.Normalize($"nntpd.{serverId:00}.{_instanceId:N}.cache.lookup"); + } + + private sealed class LookupSession : IAsyncDisposable + { + internal LookupSession( + long generation, + string replyTo, + IRabbitMqRpcChannel publishChannel, + IRabbitMqRpcChannel consumeChannel) + { + Generation = generation; + ReplyTo = replyTo; + PublishChannel = publishChannel; + ConsumeChannel = consumeChannel; + } + + internal long Generation { get; } + + internal string ReplyTo { get; } + + internal IRabbitMqRpcChannel PublishChannel { get; } + + internal IRabbitMqRpcChannel ConsumeChannel { get; } + + public async ValueTask DisposeAsync() + { + await PublishChannel.DisposeAsync().ConfigureAwait(false); + await ConsumeChannel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageServerFleetConsumerService.cs b/src/VectorNNTPD.TransitService/Storage/StorageServerFleetConsumerService.cs new file mode 100644 index 00000000..e29b69ea --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageServerFleetConsumerService.cs @@ -0,0 +1,365 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.RabbitMq; + +namespace VectorNNTPD.TransitService.Storage +{ + /// + /// Consumes StorageServer advertisements from cache.broadcast into + /// . + /// + /// + /// Declares an ephemeral per-instance queue cache.<nntpd-fqdn> (non-durable, + /// exclusive, auto-delete) bound to the fanout exchange so every NNTPD receives every + /// advertisement without competing consumers. + /// + internal sealed class StorageServerFleetConsumerService : IApplicationService + { + private readonly IRabbitMqService _rabbitMq; + private readonly IOptions _nntpdOptions; + private readonly IStorageServerRegistry _registry; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly object _sessionGate = new(); + private CancellationTokenSource? _shutdownCts; + private ConsumeSession? _session; + private int _started; + + /// Initializes a new fleet advertisement consumer. + public StorageServerFleetConsumerService( + IRabbitMqService rabbitMq, + IOptions nntpdOptions, + IStorageServerRegistry registry, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(registry); + ArgumentNullException.ThrowIfNull(logger); + _rabbitMq = rabbitMq; + _nntpdOptions = nntpdOptions; + _registry = registry; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public string Name => "StorageServerFleetConsumer"; + + /// + public Task? Execution => null; + + /// Gets the current instance queue name when attached (tests). + internal string? CurrentQueueName + { + get + { + lock (_sessionGate) + { + return _session?.QueueName; + } + } + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + _shutdownCts = new CancellationTokenSource(); + _rabbitMq.ConnectionReplaced += OnConnectionReplaced; + await AttachCurrentSessionAsync(cancellationToken).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await StopCoreAsync().ConfigureAwait(false); + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public Task StopAsync(CancellationToken cancellationToken) => StopCoreAsync(); + + private async Task AttachCurrentSessionAsync(CancellationToken cancellationToken) + { + if (!_rabbitMq.TryGetCurrent(out var handle) || !handle.IsCurrent) + { + StorageServerFleetLogMessages.ConnectionNotReady(_logger); + throw new InvalidOperationException("RabbitMQ connection is not ready for StorageServer fleet consume."); + } + + var fqdn = _nntpdOptions.Value.Fqdn; + if (string.IsNullOrWhiteSpace(fqdn)) + { + throw new InvalidOperationException("StorageServer fleet consume requires the generated NNTPD FQDN."); + } + + var queueName = CacheBroadcastTopology.BuildInstanceQueueName(fqdn); + var generation = handle.Generation; + IRabbitMqManualAckChannel? channel = null; + try + { + channel = await handle.Connection + .CreateManualAckChannelAsync(generation, cancellationToken) + .ConfigureAwait(false); + + await channel.ExchangeDeclareAsync( + CacheBroadcastTopology.ExchangeName, + CacheBroadcastTopology.ExchangeTypeName, + CacheBroadcastTopology.ExchangeDurable, + CacheBroadcastTopology.ExchangeAutoDelete, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + await channel.QueueDeclareAsync( + queueName, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + await channel.QueueBindAsync( + queueName, + CacheBroadcastTopology.ExchangeName, + routingKey: string.Empty, + arguments: null, + cancellationToken) + .ConfigureAwait(false); + + var consumerTag = await channel + .BasicConsumeAsync(queueName, prefetchCount: 64, OnDeliveryAsync, cancellationToken) + .ConfigureAwait(false); + + var session = new ConsumeSession(generation, queueName, consumerTag, channel); + ConsumeSession? previous; + lock (_sessionGate) + { + previous = _session; + _session = session; + } + + if (previous is not null) + { + await previous.DisposeAsync().ConfigureAwait(false); + StorageServerFleetLogMessages.SessionReplaced(_logger, queueName, generation); + } + else + { + StorageServerFleetLogMessages.Ready(_logger, queueName, generation); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + StorageServerFleetLogMessages.SessionAttachFailed(_logger, ex, generation); + if (channel is not null) + { + await channel.DisposeAsync().ConfigureAwait(false); + } + + throw; + } + } + + private async Task OnDeliveryAsync(RabbitMqManualAckDelivery delivery) + { + ConsumeSession? session; + lock (_sessionGate) + { + session = _session; + } + + if (session is null || delivery.Generation != session.Generation) + { + return; + } + + try + { + if (!string.Equals( + delivery.ContentType, + StorageServerAdvertisementWireProtocol.JsonContentType, + StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(delivery.ContentType)) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, "Unexpected content type."); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + var receivedAt = _timeProvider.GetUtcNow(); + if (StorageServerLifecycleWireProtocol.TryParseV1( + delivery.Body.Span, + out var announcement, + out var lifecycleReason, + out var lifecycleRecognized)) + { + if (announcement is null) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, lifecycleReason); + } + else + { + _registry.ApplyLifecycle(announcement, receivedAt); + } + + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + if (lifecycleRecognized) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, lifecycleReason); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + if (!StorageServerAdvertisementWireProtocol.TryParseV1( + delivery.Body.Span, + out var advertisement, + out var reason) + || advertisement is null) + { + StorageServerFleetLogMessages.PayloadRejected(_logger, reason); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + return; + } + + _registry.ApplyAdvertisement(advertisement, receivedAt); + await session.Channel.BasicAckAsync(delivery.DeliveryTag, CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageServerFleetLogMessages.DeliveryFailed(_logger, ex); + try + { + await session.Channel + .BasicNackAsync(delivery.DeliveryTag, requeue: false, CancellationToken.None) + .ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception nackEx) + { + StorageServerFleetLogMessages.DeliveryFailed(_logger, nackEx); + } + } + } + + private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEventArgs eventArgs) + { + if (_shutdownCts is null || _shutdownCts.IsCancellationRequested || !eventArgs.IsReplacement) + { + return; + } + + _ = AttachReplacementAsync(); + } + + private async Task AttachReplacementAsync() + { + try + { + await AttachCurrentSessionAsync(_shutdownCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + StorageServerFleetLogMessages.SessionAttachFailed(_logger, ex, _rabbitMq.ConnectionGeneration); + } + } + + private async Task StopCoreAsync() + { + _rabbitMq.ConnectionReplaced -= OnConnectionReplaced; + if (_shutdownCts is not null) + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + } + + ConsumeSession? session; + lock (_sessionGate) + { + session = _session; + _session = null; + } + + if (session is not null) + { + await session.DisposeAsync().ConfigureAwait(false); + } + + _shutdownCts?.Dispose(); + _shutdownCts = null; + Interlocked.Exchange(ref _started, 0); + StorageServerFleetLogMessages.Stopped(_logger); + } + + private sealed class ConsumeSession : IAsyncDisposable + { + internal ConsumeSession( + long generation, + string queueName, + string consumerTag, + IRabbitMqManualAckChannel channel) + { + Generation = generation; + QueueName = queueName; + ConsumerTag = consumerTag; + Channel = channel; + } + + internal long Generation { get; } + + internal string QueueName { get; } + + internal string ConsumerTag { get; } + + internal IRabbitMqManualAckChannel Channel { get; } + + public async ValueTask DisposeAsync() + { + try + { + if (Channel.IsOpen) + { + await Channel.BasicCancelAsync(ConsumerTag, CancellationToken.None).ConfigureAwait(false); + } + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + // Channel may already be closed by connection loss. + } + + await Channel.DisposeAsync().ConfigureAwait(false); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageServerFleetLogMessages.cs b/src/VectorNNTPD.TransitService/Storage/StorageServerFleetLogMessages.cs new file mode 100644 index 00000000..0e12acda --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageServerFleetLogMessages.cs @@ -0,0 +1,48 @@ +namespace VectorNNTPD.TransitService.Storage +{ + /// Source-generated StorageServer fleet consumer log messages. + internal static partial class StorageServerFleetLogMessages + { + [LoggerMessage( + EventId = 2860, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer ready (queue={Queue}, generation={Generation})")] + public static partial void Ready(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2861, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer session replaced (queue={Queue}, generation={Generation})")] + public static partial void SessionReplaced(ILogger logger, string Queue, long Generation); + + [LoggerMessage( + EventId = 2862, + Level = LogLevel.Error, + Message = "StorageServer fleet consumer session attach failed (generation={Generation})")] + public static partial void SessionAttachFailed(ILogger logger, Exception exception, long Generation); + + [LoggerMessage( + EventId = 2863, + Level = LogLevel.Error, + Message = "RabbitMQ connection is not ready for StorageServer fleet consume")] + public static partial void ConnectionNotReady(ILogger logger); + + [LoggerMessage( + EventId = 2864, + Level = LogLevel.Warning, + Message = "Rejected StorageServer advertisement payload ({Reason})")] + public static partial void PayloadRejected(ILogger logger, string Reason); + + [LoggerMessage( + EventId = 2865, + Level = LogLevel.Error, + Message = "StorageServer fleet delivery handling failed")] + public static partial void DeliveryFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2866, + Level = LogLevel.Information, + Message = "StorageServer fleet consumer stopped")] + public static partial void Stopped(ILogger logger); + } +} diff --git a/src/VectorNNTPD.TransitService/Storage/StorageServerRegistry.cs b/src/VectorNNTPD.TransitService/Storage/StorageServerRegistry.cs new file mode 100644 index 00000000..d3faf1fd --- /dev/null +++ b/src/VectorNNTPD.TransitService/Storage/StorageServerRegistry.cs @@ -0,0 +1,267 @@ +using System.Collections.Concurrent; +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.TransitService.Storage +{ + /// Liveness classification for a known StorageServer. + public enum StorageServerFleetState + { + /// LastSeen is within the cache-fleet liveness window. + Active = 0, + + /// LastSeen is older than the cache-fleet liveness window. + Stale = 1, + } + + /// + /// One in-memory StorageServer fleet entry maintained from cache broadcast advertisements. + /// + /// Numeric StorageServer identity. + /// StorageServer FQDN (registry key). + /// Latest advertised total storage bytes. + /// Latest advertised used storage bytes. + /// Latest advertised available storage bytes. + /// UTC time when the latest accepted advertisement or Draining announcement was applied. + /// + /// Advertised TLS VATP port. Null entries stay in the liveness view and are not placement targets. + /// + /// + /// When , the entry is excluded from + /// even while is inside the liveness window. Liveness expiry does not clear it. + /// + /// + /// Newest advertisement or Draining timestamp accepted for this FQDN. An older timestamp is not applied. + /// This orders one advertisement stream. It is not a process identity. + /// + public readonly record struct StorageServerFleetEntry( + int ServerId, + string Fqdn, + long TotalBytes, + long UsedBytes, + long AvailableBytes, + DateTimeOffset LastSeen, + int? VatpPort = null, + bool IsDraining = false, + DateTimeOffset NewestMessageTimestamp = default) + { + /// + /// Classifies this entry relative to using + /// . + /// + public StorageServerFleetState GetState(DateTimeOffset utcNow) => + utcNow - LastSeen <= CacheFleetTopology.LivenessWindow + ? StorageServerFleetState.Active + : StorageServerFleetState.Stale; + + /// Returns whether this entry is active at . + public bool IsActive(DateTimeOffset utcNow) => GetState(utcNow) == StorageServerFleetState.Active; + } + + /// + /// In-memory authoritative view of currently known StorageServers for future placement decisions. + /// + /// + /// Hot-path lookups must use this registry only. Do not call RabbitMQ, disk, or network from + /// selection logic merely to learn which StorageServers are alive. + /// A periodic advertisement is the readiness, liveness, capacity, and VATP endpoint signal. + /// Draining removes the FQDN from immediately and does not change + /// capacity bytes. There is no separate Ready announcement and no process epoch. + /// + public interface IStorageServerRegistry + { + /// + /// Applies a parsed advertisement, creating or updating the FQDN-keyed entry and refreshing + /// when the advertisement is accepted. + /// + /// Validated advertisement payload. + /// UTC receive time used as LastSeen when the advertisement is accepted. + /// + /// An advertisement older than is ignored. + /// While the entry is draining, only a strictly newer advertisement is accepted, and that advertisement + /// clears draining. A restarted process whose timestamps are not strictly newer than the accepted + /// Draining timestamp cannot be distinguished from a delayed advertisement and stays out of + /// . Receive time is not compared with the message timestamp. + /// + void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc); + + /// Attempts to read the entry for . + bool TryGet(string fqdn, out StorageServerFleetEntry entry); + + /// Returns a snapshot of all known entries (active and stale). + IReadOnlyList Snapshot(); + + /// + /// Returns entries that are inside the liveness window and not draining at . + /// + IReadOnlyList GetActive(DateTimeOffset utcNow); + + /// + /// Applies a Draining announcement for the announcement FQDN. + /// + /// Validated Draining announcement. + /// UTC receive time used as LastSeen when the announcement is accepted. + /// + /// Draining removes the FQDN from immediately. Capacity bytes already recorded + /// for the FQDN are kept. A Draining timestamp older than + /// is ignored, so a delayed Draining + /// announcement does not override a newer advertisement. Liveness expiry does not clear Draining. + /// + void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc); + } + + /// Thread-safe in-memory StorageServer fleet registry. + public sealed class StorageServerRegistry : IStorageServerRegistry + { + private readonly ConcurrentDictionary _entries = + new(StringComparer.OrdinalIgnoreCase); + + /// + public void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc) + { + ArgumentNullException.ThrowIfNull(advertisement); + ArgumentException.ThrowIfNullOrWhiteSpace(advertisement.Fqdn); + + var key = advertisement.Fqdn.Trim(); + var receivedAt = receivedAtUtc.ToUniversalTime(); + _entries.AddOrUpdate( + key, + _ => EntryFromAdvertisement(advertisement, key, receivedAt), + (_, existing) => MergeAdvertisement(existing, advertisement, key, receivedAt)); + } + + /// + public void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc) + { + ArgumentNullException.ThrowIfNull(announcement); + ArgumentException.ThrowIfNullOrWhiteSpace(announcement.Fqdn); + if (announcement.State != StorageServerLifecycleState.Draining) + { + return; + } + + var key = announcement.Fqdn.Trim(); + var receivedAt = receivedAtUtc.ToUniversalTime(); + _entries.AddOrUpdate( + key, + _ => EntryFromDraining(announcement, key, receivedAt), + (_, existing) => MergeDraining(existing, announcement, key, receivedAt)); + } + + private static StorageServerFleetEntry EntryFromAdvertisement( + StorageServerAdvertisement advertisement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = advertisement.Timestamp.ToUniversalTime(); + return new StorageServerFleetEntry( + advertisement.ServerId, + key, + advertisement.TotalBytes, + advertisement.UsedBytes, + advertisement.AvailableBytes, + receivedAt, + advertisement.VatpPort, + IsDraining: false, + timestamp); + } + + private static StorageServerFleetEntry EntryFromDraining( + StorageServerLifecycleAnnouncement announcement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = announcement.Timestamp.ToUniversalTime(); + return new StorageServerFleetEntry( + announcement.ServerId, + key, + TotalBytes: 0, + UsedBytes: 0, + AvailableBytes: 0, + receivedAt, + announcement.VatpPort, + IsDraining: true, + timestamp); + } + + private static StorageServerFleetEntry MergeAdvertisement( + StorageServerFleetEntry existing, + StorageServerAdvertisement advertisement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = advertisement.Timestamp.ToUniversalTime(); + if (timestamp < existing.NewestMessageTimestamp) + { + return existing; + } + + if (existing.IsDraining && timestamp <= existing.NewestMessageTimestamp) + { + return existing; + } + + return new StorageServerFleetEntry( + advertisement.ServerId, + key, + advertisement.TotalBytes, + advertisement.UsedBytes, + advertisement.AvailableBytes, + receivedAt, + advertisement.VatpPort, + IsDraining: false, + timestamp); + } + + private static StorageServerFleetEntry MergeDraining( + StorageServerFleetEntry existing, + StorageServerLifecycleAnnouncement announcement, + string key, + DateTimeOffset receivedAt) + { + var timestamp = announcement.Timestamp.ToUniversalTime(); + if (timestamp < existing.NewestMessageTimestamp) + { + return existing; + } + + return existing with + { + ServerId = announcement.ServerId, + Fqdn = key, + LastSeen = receivedAt, + VatpPort = announcement.VatpPort, + IsDraining = true, + NewestMessageTimestamp = timestamp, + }; + } + + /// + public bool TryGet(string fqdn, out StorageServerFleetEntry entry) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fqdn); + return _entries.TryGetValue(fqdn.Trim(), out entry); + } + + /// + public IReadOnlyList Snapshot() => [.. _entries.Values]; + + /// + public IReadOnlyList GetActive(DateTimeOffset utcNow) + { + var now = utcNow.ToUniversalTime(); + List? active = null; + foreach (var entry in _entries.Values) + { + if (!entry.IsActive(now) || entry.IsDraining) + { + continue; + } + + active ??= []; + active.Add(entry); + } + + return active is null ? Array.Empty() : active; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryLogMessages.cs b/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryLogMessages.cs new file mode 100644 index 00000000..eca83621 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryLogMessages.cs @@ -0,0 +1,151 @@ +namespace VectorNNTPD.TransitService.Telemetry +{ + /// Source-generated standard application telemetry messages. + internal static partial class ApplicationTelemetryLogMessages + { + [LoggerMessage( + EventId = 2400, + Level = LogLevel.Information, + Message = "HistoryDb lookups={Lookups} hits={Hits} misses={Misses} errors={Errors} wait_ms={WaitMs}")] + public static partial void HistoryDb( + ILogger logger, + long Lookups, + long Hits, + long Misses, + long Errors, + long WaitMs); + + [LoggerMessage( + EventId = 2401, + Level = LogLevel.Information, + Message = "TransitIngressQueue articles={Articles} bytes={Bytes} peak_articles={PeakArticles} peak_bytes={PeakBytes} waiting={Waiting} limit={Limit} admit_fail={AdmissionFailures} admission_wait_ms={AdmissionWaitMs}")] + public static partial void TransitIngressQueue( + ILogger logger, + int Articles, + long Bytes, + int PeakArticles, + long PeakBytes, + int Waiting, + long Limit, + long AdmissionFailures, + long AdmissionWaitMs); + + [LoggerMessage( + EventId = 2402, + Level = LogLevel.Information, + Message = "ActiveSessions active={Active} established={Established} idle={Idle} receiving={Receiving} waiting_history={WaitingHistory} waiting_queue={WaitingQueue} waiting_window={WaitingWindow} completing={Completing}")] + public static partial void ActiveSessions( + ILogger logger, + int Active, + int Established, + int Idle, + int Receiving, + int WaitingHistory, + int WaitingQueue, + int WaitingWindow, + int Completing); + + [LoggerMessage( + EventId = 2403, + Level = LogLevel.Information, + Message = "{PeerName} active={Active}/{MaxIncomingConnections} peak={Peak} accepted={Accepted} transmitted={Transmitted} rejected={Rejected} articles={Articles} bytes={Bytes} avg_article={AverageArticleBytes} avg_mbps={AverageMbps:F2} checks={Checks}")] + public static partial void TransitPeer( + ILogger logger, + string PeerName, + int Active, + int MaxIncomingConnections, + int Peak, + long Accepted, + long Transmitted, + long Rejected, + long Articles, + long Bytes, + long AverageArticleBytes, + double AverageMbps, + long Checks); + + [LoggerMessage( + EventId = 2404, + Level = LogLevel.Information, + Message = "IngestionWorker items={Items} arts_s={ArticlesPerSec:F2} busy_pct={BusyPct:F1} idle_ms={IdleMs} busy_ms={BusyMs} dq_p95_ms={DequeueP95Ms} to_pub_p95_ms={ToPublishP95Ms} encode_p95_ms={EncodeP95Ms} news_p95_ms={NewsP95Ms} item_avg_ms={ItemAvgMs} item_p95_ms={ItemP95Ms}")] + public static partial void IngestionWorker( + ILogger logger, + long Items, + double ArticlesPerSec, + double BusyPct, + long IdleMs, + long BusyMs, + long DequeueP95Ms, + long ToPublishP95Ms, + long EncodeP95Ms, + long NewsP95Ms, + double ItemAvgMs, + long ItemP95Ms); + + [LoggerMessage( + EventId = 2405, + Level = LogLevel.Information, + Message = "OverviewDbHandoff gate_p95_ms={GateP95Ms} pub_p95_ms={PublishP95Ms} confirm_avg_ms={ConfirmAvgMs} confirm_p95_ms={ConfirmP95Ms} handoff_avg_ms={HandoffAvgMs} handoff_p95_ms={HandoffP95Ms} confirm_busy_pct={ConfirmBusyPct:F1} fail={ConfirmFailures} timeout={ConfirmTimeouts} in_flight={InFlightPublishes} in_flight_max={MaxInFlightPublishes}")] + public static partial void OverviewDbHandoff( + ILogger logger, + long GateP95Ms, + long PublishP95Ms, + double ConfirmAvgMs, + long ConfirmP95Ms, + double HandoffAvgMs, + long HandoffP95Ms, + double ConfirmBusyPct, + long ConfirmFailures, + long ConfirmTimeouts, + int InFlightPublishes, + int MaxInFlightPublishes); + + [LoggerMessage( + EventId = 2406, + Level = LogLevel.Information, + Message = "TakeThisPipeline occupied={Occupied} occupied_max={OccupiedMax} occupied_full_sessions={OccupiedFullSessions} occupied_full_waits={OccupiedFullWaits} processing={Processing} processing_max={ProcessingMax} emit_gate_p95_ms={EmitGateP95Ms} enqueue_avg_ms={EnqueueAvgMs} enqueue_p95_ms={EnqueueP95Ms} budget_wait_p95_ms={BudgetWaitP95Ms} budget_waits={BudgetWaits} budget_exhausted={BudgetExhausted}")] + public static partial void TakeThisPipeline( + ILogger logger, + int Occupied, + int OccupiedMax, + int OccupiedFullSessions, + long OccupiedFullWaits, + int Processing, + int ProcessingMax, + long EmitGateP95Ms, + double EnqueueAvgMs, + long EnqueueP95Ms, + long BudgetWaitP95Ms, + long BudgetWaits, + int BudgetExhausted); + + [LoggerMessage( + EventId = 2407, + Level = LogLevel.Information, + Message = "IngestionWorkerPool workers={Workers} min={MinWorkers} max={MaxWorkers} scale_up={ScaleUps} scale_down={ScaleDowns} queue_util={QueueUtilisation:F3} waiting={WaitingProducers}")] + public static partial void IngestionWorkerPool( + ILogger logger, + int Workers, + int MinWorkers, + int MaxWorkers, + long ScaleUps, + long ScaleDowns, + double QueueUtilisation, + int WaitingProducers); + + [LoggerMessage( + EventId = 2408, + Level = LogLevel.Information, + Message = "OverviewDbWorkQueue articles={Articles} bytes={Bytes} waiting={Waiting} publishers={Publishers} pub_min={MinPublishers} pub_max={MaxPublishers} scale_up={ScaleUps} scale_down={ScaleDowns}")] + public static partial void OverviewDbWorkQueue( + ILogger logger, + int Articles, + long Bytes, + int Waiting, + int Publishers, + int MinPublishers, + int MaxPublishers, + long ScaleUps, + long ScaleDowns); + } +} diff --git a/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryService.cs b/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryService.cs new file mode 100644 index 00000000..28a4c853 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Telemetry/ApplicationTelemetryService.cs @@ -0,0 +1,410 @@ +using System.Diagnostics; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Telemetry +{ + /// + /// Always-on one-minute application telemetry. Independent of FeedDiagnostics. + /// + public sealed class ApplicationTelemetryService : IApplicationService, IAsyncDisposable + { + /// Production emit period. + public static readonly TimeSpan Period = TimeSpan.FromMinutes(1); + + private readonly IArticleIngestionQueue _queue; + private readonly INntpSessionCensus _census; + private readonly IHistoryLookupMetrics? _history; + private readonly TransitConfigurationStore _transit; + private readonly ITransitPeerMetrics? _peerMetrics; + private readonly IngestionPipelineMetrics? _pipeline; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly IAsyncInterval? _injectedInterval; + private readonly TimeSpan _period; + private readonly CancellationTokenSource _runCts = new(); + private readonly object _emitGate = new(); + private Task? _execution; + private int _disposed; + private int _emitBusy; + private long _priorLookups; + private long _priorHits; + private long _priorMisses; + private long _priorErrors; + private long _priorWaitTicks; + private long _priorAdmissionFailures; + private long _priorAdmissionWaitTicks; + private readonly Dictionary _peerPriors = new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + public ApplicationTelemetryService( + IArticleIngestionQueue queue, + INntpSessionCensus census, + ILogger logger, + IHistoryLookupMetrics? history = null, + TimeProvider? timeProvider = null, + TransitConfigurationStore? transit = null, + ITransitPeerMetrics? peerMetrics = null, + IngestionPipelineMetrics? pipelineMetrics = null) + : this(queue, census, logger, history, timeProvider, interval: null, period: null, transit, peerMetrics, pipelineMetrics) + { + } + + /// Initializes a new instance with an injectable interval (tests). + internal ApplicationTelemetryService( + IArticleIngestionQueue queue, + INntpSessionCensus census, + ILogger logger, + IHistoryLookupMetrics? history, + TimeProvider? timeProvider, + IAsyncInterval? interval, + TimeSpan? period, + TransitConfigurationStore? transit = null, + ITransitPeerMetrics? peerMetrics = null, + IngestionPipelineMetrics? pipelineMetrics = null) + { + ArgumentNullException.ThrowIfNull(queue); + ArgumentNullException.ThrowIfNull(census); + ArgumentNullException.ThrowIfNull(logger); + _queue = queue; + _census = census; + _history = history; + _transit = transit ?? new TransitConfigurationStore(); + _peerMetrics = peerMetrics; + _pipeline = pipelineMetrics; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + _injectedInterval = interval; + _period = period ?? Period; + } + + /// + public string Name => "ApplicationTelemetry"; + + /// + public Task? Execution => _execution; + + /// Gets how many snapshots have been emitted (tests). + internal int EmitCount { get; private set; } + + /// Gets whether an emit is currently running (tests). + internal bool IsEmitBusy => Volatile.Read(ref _emitBusy) != 0; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _execution = RunAsync(_runCts.Token); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is null) + { + return; + } + + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ignored) when (!ExceptionHelpers.IsFatal(ignored)) + { + _ = ignored; // observed; intentionally ignored at this best-effort boundary + } + + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var owned = _injectedInterval is null; + var interval = _injectedInterval ?? new PeriodicTimerInterval(_period, _timeProvider); + try + { + while (!cancellationToken.IsCancellationRequested) + { + var more = await interval.WaitNextAsync(cancellationToken).ConfigureAwait(false); + if (!more) + { + break; + } + + Emit(); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + finally + { + if (owned) + { + await interval.DisposeAsync().ConfigureAwait(false); + } + } + } + + /// Emits one snapshot (tests). Production uses the interval loop only. + internal void Emit() + { + lock (_emitGate) + { + Interlocked.Exchange(ref _emitBusy, 1); + try + { + EmitCore(); + EmitCount++; + } + finally + { + Interlocked.Exchange(ref _emitBusy, 0); + } + } + } + + private void EmitCore() + { + var history = _history?.Capture() ?? default; + var lookups = NonNegativeDelta(history.Lookups, ref _priorLookups); + var hits = NonNegativeDelta(history.Hits, ref _priorHits); + var misses = NonNegativeDelta(history.Misses, ref _priorMisses); + var errors = NonNegativeDelta(history.Errors, ref _priorErrors); + var waitMs = TicksToMilliseconds(NonNegativeDelta(history.WaitTicks, ref _priorWaitTicks)); + ApplicationTelemetryLogMessages.HistoryDb(_logger, lookups, hits, misses, errors, waitMs); + + var admitFail = NonNegativeDelta(_queue.AdmissionFailureCount, ref _priorAdmissionFailures); + var admitWaitMs = TicksToMilliseconds( + NonNegativeDelta(_queue.AdmissionWaitTicks, ref _priorAdmissionWaitTicks)); + ApplicationTelemetryLogMessages.TransitIngressQueue( + _logger, + _queue.Count, + _queue.QueuedBytes, + _queue.PeakCount, + _queue.PeakQueuedBytes, + _queue.WaitingProducerCount, + _queue.MemoryLimitBytes, + admitFail, + admitWaitMs); + + var sessions = _census.Capture(); + ApplicationTelemetryLogMessages.ActiveSessions( + _logger, + sessions.Active, + sessions.Established, + sessions.Idle, + sessions.Receiving, + sessions.WaitingHistory, + sessions.WaitingQueue, + sessions.WaitingWindow, + sessions.Completing); + + EmitPipeline(sessions); + EmitConfiguredPeers(); + } + + private void EmitPipeline(NntpSessionCensusSnapshot sessions) + { + if (_pipeline is null) + { + return; + } + + var snapshot = _pipeline.CaptureInterval(); + var busyMs = IngestionPipelineSnapshot.TicksToMilliseconds(snapshot.BusyTicks); + var idleMs = IngestionPipelineSnapshot.TicksToMilliseconds(snapshot.IdleTicks); + ApplicationTelemetryLogMessages.IngestionWorker( + _logger, + snapshot.WorkerItems, + snapshot.ArticlesPerSecond(_period), + snapshot.BusyPercent, + idleMs, + busyMs, + snapshot.DequeueWait.P95Ms, + snapshot.ToPublishStart.P95Ms, + snapshot.Encode.P95Ms, + snapshot.News.P95Ms, + snapshot.WorkerItem.AvgMs, + snapshot.WorkerItem.P95Ms); + ApplicationTelemetryLogMessages.OverviewDbHandoff( + _logger, + snapshot.GateWait.P95Ms, + snapshot.Publish.P95Ms, + snapshot.Confirm.AvgMs, + snapshot.Confirm.P95Ms, + snapshot.Handoff.AvgMs, + snapshot.Handoff.P95Ms, + snapshot.ConfirmBusyPercent, + snapshot.ConfirmFailures, + snapshot.ConfirmTimeouts, + snapshot.InFlightPublishes, + snapshot.MaxInFlightPublishes); + + var utilisation = _queue.MemoryLimitBytes <= 0 + ? 0d + : Math.Clamp(_queue.QueuedBytes / (double)_queue.MemoryLimitBytes, 0d, 1d); + ApplicationTelemetryLogMessages.IngestionWorkerPool( + _logger, + snapshot.CurrentWorkers, + snapshot.MinWorkers, + snapshot.MaxWorkers, + snapshot.ScaleUps, + snapshot.ScaleDowns, + utilisation, + _queue.WaitingProducerCount); + + var overviewStage = _pipeline.CaptureOverviewDbStage(); + ApplicationTelemetryLogMessages.OverviewDbWorkQueue( + _logger, + overviewStage.WorkQueueCount, + overviewStage.WorkQueueBytes, + overviewStage.WorkQueueWaiting, + overviewStage.PublisherWorkers, + overviewStage.MinPublisherWorkers, + overviewStage.MaxPublisherWorkers, + overviewStage.PublisherScaleUps, + overviewStage.PublisherScaleDowns); + + var budgetExhausted = _queue.WaitingProducerCount > 0 + || _queue.QueuedBytes >= _queue.MemoryLimitBytes + ? 1 + : 0; + var occupiedMax = Math.Max(sessions.TakeThisOccupiedMax, snapshot.MaxTakeThisOccupied); + var processingMax = Math.Max(sessions.TakeThisProcessingMax, snapshot.MaxTakeThisProcessing); + ApplicationTelemetryLogMessages.TakeThisPipeline( + _logger, + sessions.TakeThisOccupied, + occupiedMax, + sessions.TakeThisFullSessions, + snapshot.OccupiedFullWaits, + sessions.TakeThisProcessing, + processingMax, + snapshot.EmitGateWait.P95Ms, + snapshot.TakeThisEnqueue.AvgMs, + snapshot.TakeThisEnqueue.P95Ms, + snapshot.BudgetWait.P95Ms, + snapshot.BudgetWaits, + budgetExhausted); + } + + private void EmitConfiguredPeers() + { + if (_peerMetrics is null) + { + return; + } + + var peers = _transit.Current.Peers; + if (peers.Count == 0) + { + return; + } + + foreach (var identifier in peers.Keys.OrderBy(static id => id, StringComparer.Ordinal)) + { + var policy = peers[identifier]; + var snapshot = _peerMetrics.Capture(identifier); + if (!_peerPriors.TryGetValue(identifier, out var prior)) + { + prior = new PeerPrior(); + _peerPriors[identifier] = prior; + } + + var accepted = NonNegativeDelta(snapshot.Accepted, ref prior.Accepted); + var rejected = NonNegativeDelta(snapshot.Rejected, ref prior.Rejected); + var articles = NonNegativeDelta(snapshot.ArticlesReceived, ref prior.Articles); + var bytes = NonNegativeDelta(snapshot.ArticleBytes, ref prior.Bytes); + var checks = NonNegativeDelta(snapshot.Checks, ref prior.Checks); + var transmitted = NonNegativeDelta(snapshot.Transmitted, ref prior.Transmitted); + var average = articles > 0 ? bytes / articles : 0; + ApplicationTelemetryLogMessages.TransitPeer( + _logger, + identifier, + snapshot.Active, + policy.MaxIncomingConnections, + snapshot.Peak, + accepted, + transmitted, + rejected, + articles, + bytes, + average, + AverageMbps(bytes, _period), + checks); + } + } + + /// + /// Decimal megabits/sec from interval article bytes over the telemetry period. + /// + internal static double AverageMbps(long bytes, TimeSpan interval) + { + if (bytes <= 0) + { + return 0; + } + + var seconds = interval.TotalSeconds; + if (seconds <= 0) + { + return 0; + } + + return bytes * 8d / seconds / 1_000_000d; + } + + private static long NonNegativeDelta(long current, ref long prior) + { + var previous = prior; + prior = current; + return current < previous ? 0 : current - previous; + } + + private static long TicksToMilliseconds(long ticks) + { + if (ticks <= 0) + { + return 0; + } + + return (long)Stopwatch.GetElapsedTime(0, ticks).TotalMilliseconds; + } + + private sealed class PeerPrior + { + public long Accepted; + public long Rejected; + public long Articles; + public long Bytes; + public long Checks; + public long Transmitted; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Telemetry/IAsyncInterval.cs b/src/VectorNNTPD.TransitService/Telemetry/IAsyncInterval.cs new file mode 100644 index 00000000..17e2386d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Telemetry/IAsyncInterval.cs @@ -0,0 +1,35 @@ +namespace VectorNNTPD.TransitService.Telemetry +{ + /// Wait-for-next-period seam used by application telemetry. + internal interface IAsyncInterval : IAsyncDisposable + { + /// + /// Waits until the next period. Returns when the interval is completed. + /// + ValueTask WaitNextAsync(CancellationToken cancellationToken); + } + + /// implementation of . + internal sealed class PeriodicTimerInterval : IAsyncInterval + { + private readonly PeriodicTimer _timer; + + /// Initializes a period timer using . + public PeriodicTimerInterval(TimeSpan period, TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(timeProvider); + _timer = new PeriodicTimer(period, timeProvider); + } + + /// + public ValueTask WaitNextAsync(CancellationToken cancellationToken) => + _timer.WaitForNextTickAsync(cancellationToken); + + /// + public ValueTask DisposeAsync() + { + _timer.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/ITransitDnsAddressCache.cs b/src/VectorNNTPD.TransitService/Transit/ITransitDnsAddressCache.cs new file mode 100644 index 00000000..fbb18edf --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/ITransitDnsAddressCache.cs @@ -0,0 +1,36 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Thread-safe cache of DNS-resolved AllowFrom addresses. + /// + /// + /// A refresh atomically replaces the address set for one hostname. Transient failures + /// keep the previous set. Readers never observe a partial hostname update. + /// + public interface ITransitDnsAddressCache + { + /// Minimum interval between DNS refreshes for one hostname. + static readonly TimeSpan MinimumRefreshInterval = TimeSpan.FromSeconds(60); + + /// Gets the current resolved address set for . + /// + /// In-memory only. Must not perform DNS, reverse DNS, or network I/O. + /// Connection-time identification calls this after refresh has materialized addresses. + /// + IReadOnlySet GetResolved(string hostname); + + /// Resolves every hostname referenced by . + Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken); + + /// Refreshes hostnames whose TTL/minimum interval has elapsed. + Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken); + + /// + /// Returns the delay until the next hostname is due, or + /// when no hostnames are configured. + /// + TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/ITransitDnsResolver.cs b/src/VectorNNTPD.TransitService/Transit/ITransitDnsResolver.cs new file mode 100644 index 00000000..f68341f0 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/ITransitDnsResolver.cs @@ -0,0 +1,65 @@ +using System.Net; + +namespace VectorNNTPD.TransitService.Transit +{ + /// Resolves Transit AllowFrom hostnames to A/AAAA addresses with optional DNS TTL. + public interface ITransitDnsResolver + { + /// Resolves asynchronously. + Task ResolveAsync(string hostname, CancellationToken cancellationToken); + } + + /// Classifies a DNS lookup outcome for AllowFrom refresh. + public enum TransitDnsOutcome + { + /// One or more addresses were returned. + Success = 0, + + /// + /// Authoritative empty result (NXDOMAIN or NOERROR with no A/AAAA). + /// The cached address set may be cleared. + /// + Empty = 1, + + /// + /// Temporary failure (timeout, SERVFAIL, transport error). + /// The last valid address set must be retained. + /// + TransientFailure = 2, + } + + /// Result of one hostname lookup. + public sealed class TransitDnsResolveResult + { + /// Creates a successful result with addresses and optional TTL. + public static TransitDnsResolveResult Success(IReadOnlyList addresses, TimeSpan? timeToLive) => + new(TransitDnsOutcome.Success, addresses, timeToLive); + + /// Creates an authoritative empty result. + public static TransitDnsResolveResult Empty(TimeSpan? timeToLive = null) => + new(TransitDnsOutcome.Empty, Array.Empty(), timeToLive); + + /// Creates a transient failure that must preserve the last address set. + public static TransitDnsResolveResult TransientFailure() => + new(TransitDnsOutcome.TransientFailure, Array.Empty(), timeToLive: null); + + private TransitDnsResolveResult(TransitDnsOutcome outcome, IReadOnlyList addresses, TimeSpan? timeToLive) + { + Outcome = outcome; + Addresses = addresses; + TimeToLive = timeToLive; + } + + /// Gets the lookup outcome. + public TransitDnsOutcome Outcome { get; } + + /// Gets resolved addresses (empty unless ). + public IReadOnlyList Addresses { get; } + + /// + /// Gets the DNS TTL when the resolver provided one; otherwise . + /// A missing TTL uses the 60-second minimum refresh interval as the fallback. + /// + public TimeSpan? TimeToLive { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/ITransitInboundConnectionLimiter.cs b/src/VectorNNTPD.TransitService/Transit/ITransitInboundConnectionLimiter.cs new file mode 100644 index 00000000..386c9530 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/ITransitInboundConnectionLimiter.cs @@ -0,0 +1,100 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Admits inbound connections associated with a named Transit peer. + /// + /// + /// Listeners remain Transit-facing. Redis is the admission authority for named + /// peers. A process-local count is diagnostics only. + /// + public interface ITransitInboundConnectionLimiter + { + /// + /// Attempts to consume one inbound slot for using the + /// current snapshot's MaxIncomingConnections. + /// + /// + /// An outcome that is admitted when a cluster slot was taken (or no slot is + /// required). Rejected when the peer is unknown, closed, at its cluster + /// limit, or Redis is unavailable. + /// + ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default); + } + + /// Outcome of a Transit inbound slot acquire. + public readonly struct TransitInboundAdmitResult + { + /// Initializes an inbound admit outcome. + public TransitInboundAdmitResult(bool admitted, TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + Admitted = admitted; + Lease = lease; + } + + /// Gets whether the connection may proceed. + public bool Admitted { get; } + + /// Gets the lease to dispose when the connection ends. Never null. + public TransitInboundConnectionLease Lease { get; } + + /// Rejected without a held slot. + public static TransitInboundAdmitResult Rejected { get; } = + new(false, TransitInboundConnectionLease.None); + + /// Admitted without a held slot (non-transit or disabled limiter). + public static TransitInboundAdmitResult Uncounted { get; } = + new(true, TransitInboundConnectionLease.None); + + /// Admitted with a slot that must be released exactly once. + public static TransitInboundAdmitResult Accept(TransitInboundConnectionLease lease) + { + ArgumentNullException.ThrowIfNull(lease); + return new TransitInboundAdmitResult(true, lease); + } + } + + /// Releases one inbound Transit peer slot when disposed. One-shot. + public sealed class TransitInboundConnectionLease : IAsyncDisposable, IDisposable + { + private readonly Func? _release; + private int _released; + + internal TransitInboundConnectionLease(Func release) + { + ArgumentNullException.ThrowIfNull(release); + _release = release; + } + + private TransitInboundConnectionLease() + { + } + + /// Empty lease (no slot was taken). + public static TransitInboundConnectionLease None { get; } = new(); + + /// Gets whether this lease holds a slot that has not yet been released. + public bool IsHeld => _release is not null && Volatile.Read(ref _released) == 0; + + /// + public async ValueTask DisposeAsync() + { + if (_release is null) + { + return; + } + + if (Interlocked.Exchange(ref _released, 1) != 0) + { + return; + } + + await _release(CancellationToken.None).ConfigureAwait(false); + } + + /// + public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/ITransitPeerMetrics.cs b/src/VectorNNTPD.TransitService/Transit/ITransitPeerMetrics.cs new file mode 100644 index 00000000..12b683ce --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/ITransitPeerMetrics.cs @@ -0,0 +1,98 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// Lifetime counters for one configured Transit peer. + public readonly struct TransitPeerMetricsSnapshot + { + /// Initializes a peer snapshot. + public TransitPeerMetricsSnapshot( + string peerId, + int active, + int peak, + long accepted, + long rejected, + long articlesReceived, + long articleBytes, + long checks, + long transmitted) + { + PeerId = peerId; + Active = active; + Peak = peak; + Accepted = accepted; + Rejected = rejected; + ArticlesReceived = articlesReceived; + ArticleBytes = articleBytes; + Checks = checks; + Transmitted = transmitted; + } + + /// Gets the Transit identifier (configuration dictionary key). + public string PeerId { get; } + + /// Gets current established sessions belonging to this peer. + public int Active { get; } + + /// Gets the lifetime high-water of simultaneous established sessions. + public int Peak { get; } + + /// Gets lifetime accepted inbound connections. + public long Accepted { get; } + + /// Gets lifetime rejected inbound connections. + public long Rejected { get; } + + /// Gets lifetime framed articles received from this peer. + public long ArticlesReceived { get; } + + /// Gets lifetime framed article payload bytes received from this peer. + public long ArticleBytes { get; } + + /// Gets lifetime CHECK operations for this peer. + public long Checks { get; } + + /// + /// Gets lifetime articles successfully transmitted to this peer. + /// Remains 0 until outbound delivery accounting increments it. + /// + public long Transmitted { get; } + } + + /// + /// Authoritative process-wide Transit peer counters. Independent of FeedDiagnostics. + /// + public interface ITransitPeerMetrics + { + /// Records a successful inbound admission for a named peer. + void RecordAccepted(string peerId); + + /// + /// Retracts a prior when the session is reclassified + /// as a non-transit reader after AUTHINFO. + /// + void RetractAccepted(string peerId); + + /// Records an inbound admission rejection for a named peer. + void RecordRejected(string peerId); + + /// Records that an established session for entered RunAsync. + void RecordSessionRegistered(string peerId); + + /// Records that an established session for left RunAsync. + void RecordSessionUnregistered(string peerId); + + /// Records one framed article received from . + void RecordArticleReceived(string peerId, int bytes); + + /// Records one CHECK for . + void RecordCheck(string peerId); + + /// + /// Records articles successfully transmitted to . + /// Call only from the outbound delivery path after a successful send. + /// + void RecordTransmitted(string peerId, long articles = 1); + + /// Captures lifetime counters for (zeros when unseen). + TransitPeerMetricsSnapshot Capture(string peerId); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/NewsfeedsPattern.cs b/src/VectorNNTPD.TransitService/Transit/NewsfeedsPattern.cs new file mode 100644 index 00000000..5a3e7523 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/NewsfeedsPattern.cs @@ -0,0 +1,457 @@ +using System.Text; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Compiled newsfeeds(5) subscription expression using INN uwildmat_poison semantics. + /// + /// + /// + /// The configuration value is a single comma-separated string, not a JSON array and not a + /// .NET glob or regular expression. Grammar and matching follow INN + /// libinn-uwildmat / newsfeeds(5): + /// + /// + /// Comma separates patterns; \, is a literal comma. + /// Each pattern is matched against the complete newsgroup name (anchored). + /// The rightmost matching pattern decides the result. + /// Leading ! excludes; leading @ poisons. + /// * any sequence, ? one Unicode scalar, [...] / [^...] sets, \ escapes. + /// + /// + /// Empty segments, a dangling escape, or an unclosed character set fail validation. + /// + /// + public sealed class NewsfeedsPattern + { + private readonly IReadOnlyList _segments; + + private NewsfeedsPattern(string expression, IReadOnlyList segments) + { + Expression = expression; + _segments = segments; + } + + /// Gets the original configuration expression. + public string Expression { get; } + + /// Parses and validates a newsfeeds-style expression. + public static bool TryParse(string? expression, out NewsfeedsPattern? pattern, out string? error) + { + pattern = null; + error = null; + if (expression is null) + { + error = "Patterns must not be null."; + return false; + } + + if (!TrySplit(expression, out var rawSegments, out error)) + { + return false; + } + + if (rawSegments.Count == 0) + { + error = "Patterns must contain at least one wildmat expression."; + return false; + } + + var segments = new List(rawSegments.Count); + for (var i = 0; i < rawSegments.Count; i++) + { + var raw = rawSegments[i]; + if (raw.Length == 0) + { + error = "Patterns contains an empty expression (leading, trailing, or doubled comma)."; + return false; + } + + var kind = NewsfeedsPatternDecision.Match; + var start = 0; + if (raw[0] == '!') + { + kind = NewsfeedsPatternDecision.Exclude; + start = 1; + } + else if (raw[0] == '@') + { + kind = NewsfeedsPatternDecision.Poison; + start = 1; + } + + if (start >= raw.Length) + { + error = "Patterns contains an empty wildmat after '!' or '@'."; + return false; + } + + var wildmat = raw[start..]; + if (!TryValidateWildmat(wildmat, out error)) + { + return false; + } + + segments.Add(new Segment(kind, wildmat)); + } + + pattern = new NewsfeedsPattern(expression, segments); + return true; + } + + /// + /// Returns whether is subscribed (inclusion, not exclude/poison). + /// + public bool MatchesNewsgroup(string newsgroup) => + Evaluate(newsgroup) == NewsfeedsPatternDecision.Match; + + /// Evaluates the expression against a single newsgroup name. + public NewsfeedsPatternDecision Evaluate(string newsgroup) + { + ArgumentNullException.ThrowIfNull(newsgroup); + var decision = NewsfeedsPatternDecision.NoMatch; + // codeql[cs/linq/missed-where] Hot-path wildmat evaluation without LINQ allocation. + foreach (var segment in _segments) + { + if (Wildmat.IsMatch(newsgroup, segment.Wildmat)) + { + decision = segment.Kind; + } + } + + return decision; + } + + /// + /// Evaluates poison/subscription across an article's newsgroups. + /// + /// + /// Any poison group poisons the article even when another cross-posted group would match. + /// Otherwise the article is subscribed if any group matches. + /// + public NewsfeedsPatternDecision EvaluateArticle(IEnumerable newsgroups) + { + ArgumentNullException.ThrowIfNull(newsgroups); + var subscribed = false; + // codeql[cs/linq/missed-select] Early-exit poison check; Select enumerator allocation on article path. + foreach (var group in newsgroups) + { + var decision = Evaluate(group); + if (decision == NewsfeedsPatternDecision.Poison) + { + return NewsfeedsPatternDecision.Poison; + } + + if (decision == NewsfeedsPatternDecision.Match) + { + subscribed = true; + } + } + + return subscribed ? NewsfeedsPatternDecision.Match : NewsfeedsPatternDecision.NoMatch; + } + + private static bool TrySplit(string expression, out List segments, out string? error) + { + segments = []; + error = null; + var current = new StringBuilder(); + var escaped = false; + foreach (var ch in expression) + { + if (escaped) + { + current.Append(ch); + escaped = false; + continue; + } + + if (ch == '\\') + { + current.Append(ch); + escaped = true; + continue; + } + + if (ch == ',') + { + segments.Add(current.ToString()); + current.Clear(); + continue; + } + + current.Append(ch); + } + + if (escaped) + { + error = "Patterns ends with a dangling backslash escape."; + return false; + } + + segments.Add(current.ToString()); + return true; + } + + private static bool TryValidateWildmat(string wildmat, out string? error) + { + error = null; + var escaped = false; + for (var i = 0; i < wildmat.Length; i++) + { + var ch = wildmat[i]; + if (escaped) + { + escaped = false; + continue; + } + + if (ch == '\\') + { + escaped = true; + continue; + } + + if (ch != '[') + { + continue; + } + + if (!TryConsumeCharacterSet(wildmat, ref i, out error)) + { + return false; + } + } + + if (escaped) + { + error = "Patterns contains a dangling backslash escape."; + return false; + } + + return true; + } + + private static bool TryConsumeCharacterSet(string wildmat, ref int index, out string? error) + { + error = null; + var i = index + 1; + if (i >= wildmat.Length) + { + error = "Patterns contains an unclosed character set."; + return false; + } + + if (wildmat[i] == '^') + { + i++; + } + + if (i < wildmat.Length && wildmat[i] == ']') + { + i++; + } + + while (i < wildmat.Length && wildmat[i] != ']') + { + i++; + } + + if (i >= wildmat.Length) + { + error = "Patterns contains an unclosed character set."; + return false; + } + + index = i; + return true; + } + + private readonly struct Segment + { + public Segment(NewsfeedsPatternDecision kind, string wildmat) + { + Kind = kind; + Wildmat = wildmat; + } + + public NewsfeedsPatternDecision Kind { get; } + + public string Wildmat { get; } + } + + /// INN uwildmat single-pattern matcher (no comma / ! / @ handling). + internal static class Wildmat + { + public static bool IsMatch(string text, string pattern) => + Match(text.AsSpan(), pattern.AsSpan()); + + private static bool Match(ReadOnlySpan text, ReadOnlySpan pattern) + { + while (!pattern.IsEmpty) + { + if (pattern[0] == '*') + { + while (pattern.Length > 1 && pattern[1] == '*') + { + pattern = pattern[1..]; + } + + pattern = pattern[1..]; + if (pattern.IsEmpty) + { + return true; + } + + var remaining = text; + while (true) + { + if (Match(remaining, pattern)) + { + return true; + } + + if (remaining.IsEmpty) + { + return false; + } + + remaining = ConsumeRune(remaining); + } + } + + if (text.IsEmpty) + { + return false; + } + + if (pattern[0] == '\\') + { + if (pattern.Length < 2) + { + return false; + } + + if (!TryConsumeLiteral(ref text, pattern[1])) + { + return false; + } + + pattern = pattern[2..]; + continue; + } + + if (pattern[0] == '?') + { + text = ConsumeRune(text); + pattern = pattern[1..]; + continue; + } + + if (pattern[0] == '[') + { + if (!TryMatchCharacterSet(ref text, ref pattern)) + { + return false; + } + + continue; + } + + if (!TryConsumeLiteral(ref text, pattern[0])) + { + return false; + } + + pattern = pattern[1..]; + } + + return text.IsEmpty; + } + + private static bool TryMatchCharacterSet(ref ReadOnlySpan text, ref ReadOnlySpan pattern) + { + if (text.IsEmpty) + { + return false; + } + + var body = pattern[1..]; + var negated = false; + if (!body.IsEmpty && body[0] == '^') + { + negated = true; + body = body[1..]; + } + + if (body.IsEmpty) + { + return false; + } + + Rune.DecodeFromUtf16(text, out var candidate, out var candidateLen); + var matched = false; + var i = 0; + if (body[0] == ']') + { + matched = candidate.Value == ']'; + i = 1; + } + + while (i < body.Length && body[i] != ']') + { + if (i + 2 < body.Length && body[i + 1] == '-' && body[i + 2] != ']') + { + var start = body[i]; + var end = body[i + 2]; + if (candidate.Value >= start && candidate.Value <= end) + { + matched = true; + } + + i += 3; + continue; + } + + if (candidate.Value == body[i]) + { + matched = true; + } + + i++; + } + + if (i >= body.Length || body[i] != ']') + { + return false; + } + + if (matched == negated) + { + return false; + } + + text = text[candidateLen..]; + pattern = body[(i + 1)..]; + return true; + } + + private static bool TryConsumeLiteral(ref ReadOnlySpan text, char expected) + { + if (text.IsEmpty || text[0] != expected) + { + return false; + } + + text = text[1..]; + return true; + } + + private static ReadOnlySpan ConsumeRune(ReadOnlySpan text) + { + Rune.DecodeFromUtf16(text, out _, out var len); + return text[len..]; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/NewsfeedsPatternDecision.cs b/src/VectorNNTPD.TransitService/Transit/NewsfeedsPatternDecision.cs new file mode 100644 index 00000000..65234b9c --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/NewsfeedsPatternDecision.cs @@ -0,0 +1,20 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Result of evaluating a newsfeeds(5) / uwildmat_poison subscription expression. + /// + public enum NewsfeedsPatternDecision + { + /// No pattern matched the newsgroup. + NoMatch = 0, + + /// The rightmost matching pattern is an inclusion (subscribed). + Match = 1, + + /// The rightmost matching pattern is an exclusion (!). + Exclude = 2, + + /// The rightmost matching pattern is poison (@). + Poison = 3, + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitCatalogueCompiler.cs b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueCompiler.cs new file mode 100644 index 00000000..2c264c62 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueCompiler.cs @@ -0,0 +1,217 @@ +using System.Globalization; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Transit +{ + /// Compiles one Transit catalogue record into the immutable snapshot. + internal static class TransitCatalogueCompiler + { + /// Maximum article-type mask. Matches . + public const int UnrestrictedArticleTypes = 65535; + + /// Compiles or throws when a plane is not a valid snapshot. + public static TransitConfigurationSnapshot Compile(TransitCatalogueRecord record) + { + ArgumentNullException.ThrowIfNull(record); + if (record.PublicationId < 1) + { + throw new InvalidOperationException("Transit publication_id must be at least 1."); + } + + var peers = new Dictionary(record.Peers.Count, StringComparer.Ordinal); + foreach (var peer in record.Peers) + { + if (!peers.TryAdd(peer.Identifier, CompilePeer(peer))) + { + throw new InvalidOperationException( + "Transit publication contains a duplicate peer identifier '" + peer.Identifier + "'."); + } + } + + return new TransitConfigurationSnapshot(peers, record.WantTrash, record.LogTrash, record.PublicationId); + } + + private static TransitPeerPolicy CompilePeer(TransitCataloguePeerRecord peer) + { + ValidateLimit(peer.Identifier, "max_inbound", peer.MaxInbound); + ValidateLimit(peer.Identifier, "max_outbound", peer.MaxOutbound); + ValidateSize(peer.Identifier, "receive max_article_bytes", peer.ReceiveMaxArticleBytes); + ValidateSendSize(peer.Identifier, peer.SendMaxArticleBytes); + var receiveTypes = ParseArticleTypes(peer.Identifier, "receive", peer.ReceiveArticleTypes); + var sendTypes = ParseArticleTypes(peer.Identifier, "send", peer.SendArticleTypes); + var receivePatterns = ParsePatterns(peer.Identifier, "receive", peer.ReceivePatterns); + var sendPatterns = ParsePatterns(peer.Identifier, "send", peer.SendPatterns); + ValidateCredentials(peer.Identifier, "receive", peer.ReceiveUsername, peer.ReceivePassword); + ValidateCredentials(peer.Identifier, "send", peer.SendUsername, peer.SendPassword); + if (!Enum.TryParse(peer.SslMode, ignoreCase: false, out var ssl) || !Enum.IsDefined(ssl)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' ssl_mode is not None, Tls, or StartTls."); + } + + if (string.IsNullOrWhiteSpace(peer.PeerName)) + { + throw new InvalidOperationException("Transit peer '" + peer.Identifier + "' peer_name is empty."); + } + + var literals = new List(); + var hostnames = new List(); + for (var i = 0; i < peer.AllowFrom.Count; i++) + { + if (!TransitAllowFromEntry.TryParse(peer.AllowFrom[i], out var entry, out var error)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' AllowFrom[" + i.ToString(CultureInfo.InvariantCulture) + + "] is invalid: " + (error ?? "unparsed.")); + } + + switch (entry) + { + case TransitAllowFromEntry.Literal literal: + literals.Add(literal.Prefix); + break; + case TransitAllowFromEntry.Hostname hostname: + if (!hostnames.Contains(hostname.DnsName, StringComparer.OrdinalIgnoreCase)) + { + hostnames.Add(hostname.DnsName); + } + + break; + default: + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' AllowFrom entry was not parsed."); + } + } + + var connectTo = new List(peer.Endpoints.Count); + var seenEndpoints = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < peer.Endpoints.Count; i++) + { + var endpoint = peer.Endpoints[i]; + var key = endpoint.Host + "\n" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!seenEndpoints.Add(key)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' has a duplicate Send endpoint."); + } + + var text = endpoint.Host.Contains(':', StringComparison.Ordinal) + ? "[" + endpoint.Host + "]:" + endpoint.Port.ToString(CultureInfo.InvariantCulture) + : endpoint.Host + ":" + endpoint.Port.ToString(CultureInfo.InvariantCulture); + if (!TransitConnectEndpoint.TryParse(text, out var parsed, out var error) || parsed is null) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' Send endpoint is invalid: " + (error ?? "unparsed.")); + } + + connectTo.Add(parsed); + } + + var exclusions = new List(peer.PathExclusions.Count); + var seenExclusions = new HashSet(StringComparer.Ordinal); + foreach (var token in peer.PathExclusions) + { + if (string.IsNullOrWhiteSpace(token) || !seenExclusions.Add(token)) + { + throw new InvalidOperationException( + "Transit peer '" + peer.Identifier + "' has a duplicate or empty Path exclusion."); + } + + exclusions.Add(token); + } + + return new TransitPeerPolicy( + peer.Identifier, + peer.PeerName, + peer.MaxInbound, + peer.MaxOutbound, + literals, + hostnames, + connectTo, + peer.ReceiveUsername, + peer.ReceivePassword, + ssl, + sendPatterns, + peer.DeferOnDuplicate, + peer.PathToken, + peer.ReceiveMaxArticleBytes, + sendTypes, + receivePatterns, + receiveTypes, + peer.SendMaxArticleBytes, + peer.SendUsername, + peer.SendPassword, + exclusions); + } + + private static void ValidateLimit(string identifier, string column, int value) + { + if (value < 0 || value > TransitPeerPolicy.MaxConnectionLimit) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + column + " is outside 0–" + + TransitPeerPolicy.MaxConnectionLimit.ToString(CultureInfo.InvariantCulture) + "."); + } + } + + private static void ValidateSize(string identifier, string column, long value) + { + if (value < 1 || value > int.MaxValue) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + column + " is outside 1–2147483647."); + } + } + + /// + /// Accepts SQL NULL as unlimited. An explicit send ceiling uses the same positive range as receive. + /// + private static void ValidateSendSize(string identifier, long? value) + { + if (value is null) + { + return; + } + + ValidateSize(identifier, "send max_article_bytes", value.Value); + } + + private static TransitMessageTypes ParseArticleTypes(string identifier, string plane, int mask) + { + if (mask < 0 || mask > UnrestrictedArticleTypes) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " article_types is outside 0–65535."); + } + + return (TransitMessageTypes)mask; + } + + private static NewsfeedsPattern ParsePatterns(string identifier, string plane, string expression) + { + NewsfeedsPattern? pattern = null; + string? error = null; + if (string.IsNullOrWhiteSpace(expression) + || !NewsfeedsPattern.TryParse(expression, out pattern, out error) + || pattern is null) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " patterns are invalid: " + + (error ?? "empty.")); + } + + return pattern; + } + + private static void ValidateCredentials(string identifier, string plane, string username, string password) + { + var userSet = username.Length > 0; + var passwordSet = password.Length > 0; + if (userSet != passwordSet) + { + throw new InvalidOperationException( + "Transit peer '" + identifier + "' " + plane + " credentials must both be set or both be blank."); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitCatalogueRecord.cs b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueRecord.cs new file mode 100644 index 00000000..d477015b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueRecord.cs @@ -0,0 +1,195 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// One published Transit catalogue row set, before pattern and endpoint compilation. + /// + public sealed class TransitCatalogueRecord + { + /// Empty published catalogue used by offline hosts. No peers; junk logging stays on. + public static TransitCatalogueRecord Empty { get; } = new(1, 1, 1, 1, true, true, []); + + /// Initializes a catalogue row set. + public TransitCatalogueRecord( + long publicationId, + long receiveRevision, + long sendRevision, + long globalRevision, + bool wantTrash, + bool logTrash, + IReadOnlyList peers) + { + ArgumentNullException.ThrowIfNull(peers); + PublicationId = publicationId; + ReceiveRevision = receiveRevision; + SendRevision = sendRevision; + GlobalRevision = globalRevision; + WantTrash = wantTrash; + LogTrash = logTrash; + Peers = peers; + } + + /// Gets nntptransitpublication.publication_id. + public long PublicationId { get; } + + /// Gets the receive revision named by the publication. + public long ReceiveRevision { get; } + + /// Gets the send revision named by the publication. + public long SendRevision { get; } + + /// Gets the global revision named by the publication. + public long GlobalRevision { get; } + + /// Gets the site-wide want-trash flag. + public bool WantTrash { get; } + + /// Gets the site-wide log-trash flag. + public bool LogTrash { get; } + + /// Gets every peer in the publication. Receive and send identifiers are the same set. + public IReadOnlyList Peers { get; } + } + + /// One peer's identity plus both planes, still in catalogue column form. + public sealed class TransitCataloguePeerRecord + { + /// Initializes one peer row. + public TransitCataloguePeerRecord( + string identifier, + string peerName, + int maxInbound, + string receiveUsername, + string receivePassword, + bool deferOnDuplicate, + long receiveMaxArticleBytes, + int receiveArticleTypes, + string receivePatterns, + IReadOnlyList allowFrom, + int maxOutbound, + string sslMode, + string sendUsername, + string sendPassword, + long? sendMaxArticleBytes, + int sendArticleTypes, + string sendPatterns, + string pathToken, + IReadOnlyList endpoints, + IReadOnlyList pathExclusions) + { + ArgumentNullException.ThrowIfNull(identifier); + ArgumentNullException.ThrowIfNull(peerName); + ArgumentNullException.ThrowIfNull(receiveUsername); + ArgumentNullException.ThrowIfNull(receivePassword); + ArgumentNullException.ThrowIfNull(receivePatterns); + ArgumentNullException.ThrowIfNull(allowFrom); + ArgumentNullException.ThrowIfNull(sslMode); + ArgumentNullException.ThrowIfNull(sendUsername); + ArgumentNullException.ThrowIfNull(sendPassword); + ArgumentNullException.ThrowIfNull(sendPatterns); + ArgumentNullException.ThrowIfNull(pathToken); + ArgumentNullException.ThrowIfNull(endpoints); + ArgumentNullException.ThrowIfNull(pathExclusions); + Identifier = identifier; + PeerName = peerName; + MaxInbound = maxInbound; + ReceiveUsername = receiveUsername; + ReceivePassword = receivePassword; + DeferOnDuplicate = deferOnDuplicate; + ReceiveMaxArticleBytes = receiveMaxArticleBytes; + ReceiveArticleTypes = receiveArticleTypes; + ReceivePatterns = receivePatterns; + AllowFrom = allowFrom; + MaxOutbound = maxOutbound; + SslMode = sslMode; + SendUsername = sendUsername; + SendPassword = sendPassword; + SendMaxArticleBytes = sendMaxArticleBytes; + SendArticleTypes = sendArticleTypes; + SendPatterns = sendPatterns; + PathToken = pathToken; + Endpoints = endpoints; + PathExclusions = pathExclusions; + } + + /// Gets the peer identifier. + public string Identifier { get; } + + /// Gets the display name. + public string PeerName { get; } + + /// Gets max_inbound. 0 closes receive. + public int MaxInbound { get; } + + /// Gets the receive username. + public string ReceiveUsername { get; } + + /// Gets the receive password. Never log this value. + public string ReceivePassword { get; } + + /// Gets whether duplicate offers are deferred. + public bool DeferOnDuplicate { get; } + + /// Gets the receive article size limit. + public long ReceiveMaxArticleBytes { get; } + + /// Gets the receive article-type mask. + public int ReceiveArticleTypes { get; } + + /// Gets the receive newsfeeds expression. + public string ReceivePatterns { get; } + + /// Gets AllowFrom entries in ordinal order. + public IReadOnlyList AllowFrom { get; } + + /// Gets max_outbound. 0 closes send. + public int MaxOutbound { get; } + + /// Gets ssl_mode (None, Tls, or StartTls). + public string SslMode { get; } + + /// Gets the send username. + public string SendUsername { get; } + + /// Gets the send password. Never log this value. + public string SendPassword { get; } + + /// + /// Gets the send article size limit in bytes. + /// is unlimited. A value is an explicit ceiling and is not filled from the receive limit. + /// + public long? SendMaxArticleBytes { get; } + + /// Gets the send article-type mask. + public int SendArticleTypes { get; } + + /// Gets the send newsfeeds expression. + public string SendPatterns { get; } + + /// Gets the Path token. + public string PathToken { get; } + + /// Gets ConnectTo endpoints in ordinal order. + public IReadOnlyList Endpoints { get; } + + /// Gets extra Path exclusions in ordinal order. + public IReadOnlyList PathExclusions { get; } + } + + /// One send endpoint stored as an unbracketed host and an explicit port. + public sealed class TransitCatalogueEndpointRecord + { + /// Initializes an endpoint row. + public TransitCatalogueEndpointRecord(string host, int port) + { + ArgumentNullException.ThrowIfNull(host); + Host = host; + Port = port; + } + + /// Gets the unbracketed host. + public string Host { get; } + + /// Gets the TCP port. + public int Port { get; } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitCatalogueService.cs b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueService.cs new file mode 100644 index 00000000..75d9e646 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitCatalogueService.cs @@ -0,0 +1,219 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Loads the published Transit catalogue from NntpDB and replaces the process snapshot. + /// + /// + /// Initial load failure prevents RUNNING. A failed refresh keeps the last snapshot. + /// The same publication_id does not replace the snapshot. Send configuration is stored + /// for a later feeder and does not open outbound connections. Passwords are not logged. + /// + internal sealed class TransitCatalogueService : IApplicationService, IAsyncDisposable + { + /// Refresh interval after a successful initial load. + public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(60); + + private readonly NntpDbService _nntpDb; + private readonly TransitConfigurationStore _store; + private readonly IOptionsMonitor _nntpdOptions; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private IDisposable? _optionsSubscription; + private Task? _execution; + private int _started; + private int _refreshing; + private int _disposed; + + /// Initializes a production catalogue service. + public TransitCatalogueService( + NntpDbService nntpDb, + TransitConfigurationStore store, + IOptionsMonitor nntpdOptions, + ILogger logger) + : this(nntpDb, store, nntpdOptions, logger, TimeProvider.System, RefreshInterval) + { + } + + /// Initializes a service with an explicit clock and interval (tests). + internal TransitCatalogueService( + NntpDbService nntpDb, + TransitConfigurationStore store, + IOptionsMonitor nntpdOptions, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(nntpDb); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(nntpdOptions); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _nntpDb = nntpDb; + _store = store; + _nntpdOptions = nntpdOptions; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "TransitCatalogue"; + + /// + public Task? Execution => _execution; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + _optionsSubscription = _nntpdOptions.OnChange(OnOptionsChanged); + _execution = RunAsync(_runCts.Token); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + if (ex is not OperationCanceledException) + { + TransitLogMessages.CatalogueInitialLoadFailed(_logger, ex); + } + + Interlocked.Exchange(ref _started, 0); + throw; + } + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + if (_execution is not null) + { + try + { + await _execution.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _optionsSubscription?.Dispose(); + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await TryRefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task TryRefreshAsync(CancellationToken cancellationToken) + { + if (Interlocked.CompareExchange(ref _refreshing, 1, 0) != 0) + { + return; + } + + try + { + var snapshot = await LoadAsync(cancellationToken).ConfigureAwait(false); + Publish(snapshot); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitLogMessages.CatalogueRefreshFailed(_logger, ex); + } + finally + { + Interlocked.Exchange(ref _refreshing, 0); + } + } + + private async ValueTask LoadAsync(CancellationToken cancellationToken) + { + await using var connection = await NntpDbConnections.OpenAsync(_nntpDb, cancellationToken).ConfigureAwait(false); + var record = await connection.QueryTransitCatalogueAsync(cancellationToken).ConfigureAwait(false); + if (record is null) + { + throw new InvalidOperationException( + "nntptransitcurrent is missing policy_id = 1. NNTPD will not invent a local Transit catalogue."); + } + + return TransitCatalogueCompiler.Compile(record); + } + + private void Publish(TransitConfigurationSnapshot snapshot) + { + ApplyGlobalFlags(_nntpdOptions.CurrentValue, snapshot); + if (_store.Current.PublicationId == snapshot.PublicationId && _store.Current.PublicationId != 0) + { + return; + } + + _store.Replace(snapshot); + TransitLogMessages.CataloguePublished(_logger, snapshot.PublicationId, snapshot.Peers.Count); + } + + private void OnOptionsChanged(NntpdOptions options, string? name) => + ApplyGlobalFlags(options, _store.Current); + + private static void ApplyGlobalFlags(NntpdOptions options, TransitConfigurationSnapshot snapshot) + { + if (snapshot.PublicationId == 0) + { + return; + } + + options.Transit.WantTrash = snapshot.WantTrash; + options.Transit.LogTrash = snapshot.LogTrash; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitConfigurationHotReload.cs b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationHotReload.cs new file mode 100644 index 00000000..d5b71849 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationHotReload.cs @@ -0,0 +1,73 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Applies updates to . + /// + /// + /// An empty top-level Transit section does not replace the store. The published + /// MySQL catalogue is authoritative, and a JSON reload must not wipe it. Non-empty + /// in-memory sections still replace the snapshot so tests can exercise admission. + /// Invalid non-empty reloads are ignored so the last valid snapshot remains. + /// + public sealed class TransitConfigurationHotReload : IDisposable + { + private readonly IDisposable? _subscription; + + /// Initializes hot reload from the options monitor into the snapshot store. + public TransitConfigurationHotReload( + IOptionsMonitor monitor, + TransitConfigurationStore store, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(monitor); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(logger); + + Apply(monitor.CurrentValue, store, logger, isReload: false); + _subscription = monitor.OnChange(options => Apply(options, store, logger, isReload: true)); + } + + /// + public void Dispose() => _subscription?.Dispose(); + + private static void Apply( + TransitPeersOptions options, + TransitConfigurationStore store, + ILogger logger, + bool isReload) + { + if (options.Count == 0) + { + return; + } + + var result = new TransitPeersOptionsValidator().Validate(Options.DefaultName, options); + if (result.Failed) + { + var first = result.Failures?.FirstOrDefault() ?? result.FailureMessage ?? "invalid Transit configuration"; + if (isReload) + { + TransitLogMessages.InvalidReloadIgnored(logger, first); + } + else + { + throw new OptionsValidationException( + Options.DefaultName, + typeof(TransitPeersOptions), + result.Failures); + } + + return; + } + + store.Replace(TransitConfigurationSnapshot.Create(options)); + if (isReload) + { + TransitLogMessages.ConfigurationReloaded(logger, options.Count); + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitConfigurationSnapshot.cs b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationSnapshot.cs new file mode 100644 index 00000000..22825c1d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationSnapshot.cs @@ -0,0 +1,161 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Immutable, fully validated Transit peer configuration snapshot. + /// + /// + /// Readers observe either the previous complete snapshot or this complete snapshot. + /// DNS-resolved addresses live in so hostname + /// refreshes do not rebuild unrelated peer policy. + /// + public sealed class TransitConfigurationSnapshot + { + /// Empty snapshot (no peers). Global junk flags stay at their process defaults. + public static TransitConfigurationSnapshot Empty { get; } = new(new Dictionary(StringComparer.Ordinal)); + + private static readonly NewsfeedsPattern OptionsReceivePatterns = ParseOptionsReceivePatterns(); + + internal TransitConfigurationSnapshot( + IReadOnlyDictionary peers, + bool wantTrash = true, + bool logTrash = true, + long publicationId = 0) + { + Peers = peers; + WantTrash = wantTrash; + LogTrash = logTrash; + PublicationId = publicationId; + } + + /// Gets peers keyed by configured identifier (ordinal, case-sensitive). + public IReadOnlyDictionary Peers { get; } + + /// Gets the published nntptransitpublication.publication_id, or 0 for an in-memory snapshot. + public long PublicationId { get; } + + /// Gets the site-wide want-trash flag from the published global revision. + public bool WantTrash { get; } + + /// Gets the site-wide log-trash flag from the published global revision. + public bool LogTrash { get; } + + /// Gets whether any peers are configured. + public bool IsEmpty => Peers.Count == 0; + + /// + /// Builds a snapshot from already-validated in-memory options. + /// + /// + /// has one size, one pattern, and one message-type list. + /// Those map to Receive size, Send patterns, and Send article types. Receive patterns + /// are * and Receive article types are unrestricted. The MySQL catalogue compiler + /// sets the two planes independently. + /// + public static TransitConfigurationSnapshot Create(TransitPeersOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var peers = new Dictionary(options.Count, StringComparer.Ordinal); + foreach (var (identifier, peer) in options) + { + peers[identifier] = CreatePeer(identifier, peer); + } + + return new TransitConfigurationSnapshot(peers); + } + + internal static TransitPeerPolicy CreatePeer(string identifier, TransitPeerOptions options) + { + var literals = new List(); + var hostnames = new List(); + foreach (var raw in options.AllowFrom ?? []) + { + if (!TransitAllowFromEntry.TryParse(raw, out var entry, out _)) + { + continue; + } + + switch (entry) + { + case TransitAllowFromEntry.Literal literal: + literals.Add(literal.Prefix); + break; + case TransitAllowFromEntry.Hostname hostname: + if (!hostnames.Contains(hostname.DnsName, StringComparer.OrdinalIgnoreCase)) + { + hostnames.Add(hostname.DnsName); + } + + break; + } + } + + var connectTo = new List(); + // codeql[cs/linq/missed-where] TryParse out-params; foreach preserves parse-and-collect clarity. + foreach (var raw in options.ConnectTo ?? []) + { + if (TransitConnectEndpoint.TryParse(raw, out var endpoint, out _) && endpoint is not null) + { + connectTo.Add(endpoint); + } + } + + if (!NewsfeedsPattern.TryParse(options.Patterns, out var patterns, out _) || patterns is null) + { + throw new InvalidOperationException("Patterns must be validated before snapshot creation."); + } + + if (!TransitSslParser.TryParse(options.Ssl, out var ssl, out _)) + { + throw new InvalidOperationException("Ssl must be validated before snapshot creation."); + } + + var username = (options.Username ?? string.Empty).Trim(); + var password = options.Password ?? string.Empty; + if (!TransitMessageTypesParser.TryParse(options.MessageTypes, out var messageTypes, out _)) + { + throw new InvalidOperationException("MessageTypes must be validated before snapshot creation."); + } + + var peerName = options.PeerName ?? string.Empty; + if (string.IsNullOrWhiteSpace(peerName)) + { + throw new InvalidOperationException("PeerName must be validated before snapshot creation."); + } + + return new TransitPeerPolicy( + identifier, + peerName, + options.MaxIncomingConnections!.Value, + options.MaxOutgoingConnections!.Value, + literals, + hostnames, + connectTo, + username, + password, + ssl, + patterns, + options.DeferOnDuplicate, + options.PathToken ?? string.Empty, + options.MaxSize, + messageTypes, + OptionsReceivePatterns, + TransitMessageTypes.All, + options.MaxSize, + string.Empty, + string.Empty, + []); + } + + private static NewsfeedsPattern ParseOptionsReceivePatterns() + { + if (!NewsfeedsPattern.TryParse("*", out var pattern, out _) || pattern is null) + { + throw new InvalidOperationException("Receive pattern '*' must parse."); + } + + return pattern; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitConfigurationStore.cs b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationStore.cs new file mode 100644 index 00000000..3c8669d1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitConfigurationStore.cs @@ -0,0 +1,56 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Holds the active immutable Transit configuration snapshot. + /// + /// + /// Replacement is atomic. Readers use without taking a lock. + /// A listener registered via observes complete snapshots only. + /// + public sealed class TransitConfigurationStore + { + private TransitConfigurationSnapshot _current = TransitConfigurationSnapshot.Empty; + private event Action? Changed; + + /// Gets the current complete snapshot. + public TransitConfigurationSnapshot Current => Volatile.Read(ref _current!); + + /// Replaces the active snapshot with a fully built instance. + public void Replace(TransitConfigurationSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Volatile.Write(ref _current, snapshot); + Changed?.Invoke(snapshot); + } + + /// Subscribes to complete snapshot replacements. + public IDisposable Subscribe(Action listener) + { + ArgumentNullException.ThrowIfNull(listener); + Changed += listener; + return new Subscription(this, listener); + } + + private sealed class Subscription : IDisposable + { + private TransitConfigurationStore? _store; + private Action? _listener; + + public Subscription(TransitConfigurationStore store, Action listener) + { + _store = store; + _listener = listener; + } + + public void Dispose() + { + var store = Interlocked.Exchange(ref _store, null); + var listener = Interlocked.Exchange(ref _listener, null); + if (store is not null && listener is not null) + { + store.Changed -= listener; + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitConnectionAdmission.cs b/src/VectorNNTPD.TransitService/Transit/TransitConnectionAdmission.cs new file mode 100644 index 00000000..b72edde4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitConnectionAdmission.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Admits an inbound connection against the identified Transit peer's MaxIncomingConnections. + /// + internal static class TransitConnectionAdmission + { + /// + /// Consumes a cluster slot when the session is associated with a Transit peer. + /// Non-transit clients are admitted without a slot and never create peer state. + /// + public static async ValueTask TryAdmitAsync( + ITransitInboundConnectionLimiter limiter, + NntpSession session, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(limiter); + ArgumentNullException.ThrowIfNull(session); + var peerName = session.Authorization.TransitPeerName; + if (peerName is null) + { + return TransitInboundAdmitResult.Uncounted; + } + + return await limiter.TryAcquireAsync(peerName, cancellationToken).ConfigureAwait(false); + } + + /// + /// Writes RFC 3977/4644 400 Service temporarily unavailable and completes. + /// + public static async ValueTask WriteUnavailableAsync( + INntpConnection connection, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(connection); + await using var writer = new NntpResponseWriter(connection.Output); + await writer + .WriteLineAsync(NntpResponses.ServiceTemporarilyUnavailable, cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitDnsAddressCache.cs b/src/VectorNNTPD.TransitService/Transit/TransitDnsAddressCache.cs new file mode 100644 index 00000000..f0dff13d --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitDnsAddressCache.cs @@ -0,0 +1,242 @@ +using System.Collections.Concurrent; +using System.Net; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Per-hostname DNS cache with TTL-based refresh and a 60-second minimum interval. + /// + public sealed class TransitDnsAddressCache : ITransitDnsAddressCache + { + private static readonly IReadOnlySet EmptySet = new HashSet(); + + private readonly ITransitDnsResolver _resolver; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _minimumRefreshInterval; + private readonly ConcurrentDictionary _hosts = + new(StringComparer.OrdinalIgnoreCase); + + /// Initializes the cache with the production 60-second minimum refresh interval. + public TransitDnsAddressCache( + ITransitDnsResolver resolver, + ILogger logger, + TimeProvider? timeProvider = null) + : this(resolver, logger, ITransitDnsAddressCache.MinimumRefreshInterval, timeProvider) + { + } + + /// Test constructor with an injectable minimum refresh interval. + internal TransitDnsAddressCache( + ITransitDnsResolver resolver, + ILogger logger, + TimeSpan minimumRefreshInterval, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(resolver); + ArgumentNullException.ThrowIfNull(logger); + _resolver = resolver; + _logger = logger; + _minimumRefreshInterval = minimumRefreshInterval; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + /// + /// Dictionary lookup only. Does not call , + /// Dns.GetHostEntry, Dns.GetHostAddresses, or perform network I/O. + /// + public IReadOnlySet GetResolved(string hostname) + { + ArgumentException.ThrowIfNullOrWhiteSpace(hostname); + return _hosts.TryGetValue(hostname, out var state) ? state.Addresses : EmptySet; + } + + /// + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + RefreshCoreAsync(snapshot, dueOnly: false, cancellationToken); + + /// + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + RefreshCoreAsync(snapshot, dueOnly: true, cancellationToken); + + /// + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + var names = CollectHostnames(snapshot); + if (names.Count == 0) + { + return Timeout.InfiniteTimeSpan; + } + + var now = _timeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var name in names.Keys) + { + if (!_hosts.TryGetValue(name, out var state)) + { + return TimeSpan.Zero; + } + + if (state.NextRefreshUtc < next) + { + next = state.NextRefreshUtc; + } + } + + var delay = next - now; + return delay < TimeSpan.Zero ? TimeSpan.Zero : delay; + } + + private async Task RefreshCoreAsync( + TransitConfigurationSnapshot snapshot, + bool dueOnly, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(snapshot); + var hosts = CollectHostnames(snapshot); + PruneUnused(hosts.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase)); + var now = _timeProvider.GetUtcNow(); + foreach (var (name, peerNames) in hosts) + { + cancellationToken.ThrowIfCancellationRequested(); + if (dueOnly + && _hosts.TryGetValue(name, out var existing) + && existing.NextRefreshUtc > now) + { + continue; + } + + await RefreshHostnameAsync(snapshot, name, peerNames, cancellationToken).ConfigureAwait(false); + } + } + + private async Task RefreshHostnameAsync( + TransitConfigurationSnapshot snapshot, + string hostname, + IReadOnlyList peerNames, + CancellationToken cancellationToken) + { + TransitDnsResolveResult result; + string? failureReason = null; + try + { + result = await _resolver.ResolveAsync(hostname, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + failureReason = ex.Message; + result = TransitDnsResolveResult.TransientFailure(); + } + + var now = _timeProvider.GetUtcNow(); + var next = now + RefreshInterval(result.TimeToLive); + switch (result.Outcome) + { + case TransitDnsOutcome.Success: + var set = Deduplicate(result.Addresses); + _hosts[hostname] = new HostState(set, next); + TransitLogMessages.DnsResolved(_logger, hostname, set.Count, next); + break; + case TransitDnsOutcome.Empty: + _hosts[hostname] = new HostState(new HashSet(), next); + LogDnsFailure( + snapshot, + peerNames, + hostname, + failureReason ?? "NXDOMAIN or NOERROR with no A/AAAA records"); + break; + default: + _hosts[hostname] = _hosts.TryGetValue(hostname, out var previous) + ? new HostState(previous.Addresses, next) + : new HostState(new HashSet(), next); + + LogDnsFailure(snapshot, peerNames, hostname, failureReason ?? "transient DNS failure"); + break; + } + } + + private void LogDnsFailure( + TransitConfigurationSnapshot snapshot, + IReadOnlyList peerIds, + string hostname, + string reason) + { + foreach (var peerId in peerIds) + { + var display = snapshot.Peers.TryGetValue(peerId, out var peer) ? peer.PeerName : peerId; + TransitLogMessages.DnsResolutionFailed(_logger, peerId, display, hostname, reason); + } + } + + private TimeSpan RefreshInterval(TimeSpan? ttl) + { + if (ttl is not { } value || value <= TimeSpan.Zero) + { + return _minimumRefreshInterval; + } + + return value > _minimumRefreshInterval ? value : _minimumRefreshInterval; + } + + private static Dictionary> CollectHostnames(TransitConfigurationSnapshot snapshot) + { + var names = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var peer in snapshot.Peers.Values) + { + foreach (var hostname in peer.DnsHostnames) + { + if (!names.TryGetValue(hostname, out var peers)) + { + peers = []; + names[hostname] = peers; + } + + if (!peers.Contains(peer.Identifier, StringComparer.Ordinal)) + { + peers.Add(peer.Identifier); + } + } + } + + return names; + } + + private void PruneUnused(HashSet live) + { + // codeql[cs/linq/missed-where] ConcurrentDictionary Keys with TryRemove side effects. + foreach (var key in _hosts.Keys) + { + if (!live.Contains(key)) + { + _hosts.TryRemove(key, out _); + } + } + } + + private static IReadOnlySet Deduplicate(IReadOnlyList addresses) + { + var set = new HashSet(); + foreach (var address in addresses) + { + set.Add(TrustedProxyHosts.Canonicalize(address)); + } + + return set; + } + + private sealed class HostState(IReadOnlySet addresses, DateTimeOffset nextRefreshUtc) + { + public IReadOnlySet Addresses { get; } = addresses; + + public DateTimeOffset NextRefreshUtc { get; } = nextRefreshUtc; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitDnsClientResolver.cs b/src/VectorNNTPD.TransitService/Transit/TransitDnsClientResolver.cs new file mode 100644 index 00000000..b49cf997 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitDnsClientResolver.cs @@ -0,0 +1,136 @@ +using System.Net; +using DnsClient; +using DnsClient.Protocol; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Resolves AllowFrom hostnames with DnsClient so A/AAAA record TTLs are available. + /// + /// + /// + /// Each lookup queries A and AAAA. DnsClient exposes TimeToLive on answer records; + /// the refresh interval uses the minimum TTL among returned records, floored at 60 seconds + /// by . + /// + /// + /// If a successful answer somehow omitted TTL information, + /// is and the cache falls back to the 60-second minimum. This resolver + /// does not invent a fixed polling interval in place of record TTLs. + /// + /// + public sealed class TransitDnsClientResolver : ITransitDnsResolver + { + private readonly ILookupClient _client; + + /// Initializes a resolver with the system DNS lookup client. + public TransitDnsClientResolver() + : this(new LookupClient()) + { + } + + /// Initializes a resolver with an injected lookup client (tests). + public TransitDnsClientResolver(ILookupClient client) + { + ArgumentNullException.ThrowIfNull(client); + _client = client; + } + + /// + public async Task ResolveAsync(string hostname, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(hostname); + var name = hostname.Trim().TrimEnd('.'); + + IDnsQueryResponse a; + IDnsQueryResponse aaaa; + try + { + a = await _client.QueryAsync(name, QueryType.A, cancellationToken: cancellationToken) + .ConfigureAwait(false); + aaaa = await _client.QueryAsync(name, QueryType.AAAA, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + return TransitDnsResolveResult.TransientFailure(); + } + + var aTransient = IsTransient(a); + var aaaaTransient = IsTransient(aaaa); + if (aTransient && aaaaTransient) + { + return TransitDnsResolveResult.TransientFailure(); + } + + var addresses = new List(); + var ttlSeconds = int.MaxValue; + var sawTtl = false; + Collect(a, addresses, ref ttlSeconds, ref sawTtl); + Collect(aaaa, addresses, ref ttlSeconds, ref sawTtl); + + if (addresses.Count == 0) + { + if (aTransient || aaaaTransient) + { + return TransitDnsResolveResult.TransientFailure(); + } + + return TransitDnsResolveResult.Empty(sawTtl ? TimeSpan.FromSeconds(ttlSeconds) : null); + } + + return TransitDnsResolveResult.Success( + addresses, + sawTtl ? TimeSpan.FromSeconds(ttlSeconds) : null); + } + + private static bool IsTransient(IDnsQueryResponse response) + { + if (!response.HasError) + { + return false; + } + + return response.Header.ResponseCode is not DnsHeaderResponseCode.NotExistentDomain; + } + + private static void Collect( + IDnsQueryResponse response, + List addresses, + ref int ttlSeconds, + ref bool sawTtl) + { + if (response.HasError) + { + return; + } + + foreach (var record in response.Answers) + { + IPAddress? address = record switch + { + ARecord a => a.Address, + AaaaRecord aaaa => aaaa.Address, + _ => null, + }; + if (address is null) + { + continue; + } + + addresses.Add(TrustedProxyHosts.Canonicalize(address)); + if (record.TimeToLive > 0) + { + sawTtl = true; + ttlSeconds = Math.Min(ttlSeconds, record.TimeToLive); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitDnsRefreshService.cs b/src/VectorNNTPD.TransitService/Transit/TransitDnsRefreshService.cs new file mode 100644 index 00000000..b6c56ff4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitDnsRefreshService.cs @@ -0,0 +1,139 @@ +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Session; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Refreshes Transit AllowFrom DNS names according to each hostname's TTL (minimum 60 seconds). + /// + /// + /// Does not poll VectorNNTPD.TransitService.json. Configuration reloads arrive through + /// (IOptionsMonitor). The wait interval is the + /// next hostname due time, not a fixed poll. + /// + public sealed class TransitDnsRefreshService : IApplicationService, IAsyncDisposable + { + private readonly TransitConfigurationStore _store; + private readonly ITransitDnsAddressCache _cache; + private readonly ILogger _logger; + private readonly CancellationTokenSource _runCts = new(); + private IDisposable? _subscription; + private TaskCompletionSource _wakeup = NewWakeup(); + private Task? _execution; + private int _disposed; + + /// Initializes a new instance of the class. + public TransitDnsRefreshService( + TransitConfigurationStore store, + ITransitDnsAddressCache cache, + ITransitPeerAuthorization authorization, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(cache); + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _cache = cache; + _logger = logger; + } + + /// + public string Name => "TransitDnsRefresh"; + + /// + public Task? Execution => _execution; + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + await _cache.RefreshAllAsync(_store.Current, cancellationToken).ConfigureAwait(false); + _subscription = _store.Subscribe(_ => _wakeup.TrySetResult()); + _execution = RunAsync(_runCts.Token); + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + _subscription?.Dispose(); + _subscription = null; + await _runCts.CancelAsync().ConfigureAwait(false); + _wakeup.TrySetResult(); + if (_execution is not null) + { + try + { + await _execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + // Expected. + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + _subscription?.Dispose(); + await _runCts.CancelAsync().ConfigureAwait(false); + _runCts.Dispose(); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var delay = _cache.GetDelayUntilNextRefresh(_store.Current); + var wakeup = _wakeup; + try + { + if (delay == Timeout.InfiniteTimeSpan) + { + await wakeup.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + else if (delay > TimeSpan.Zero) + { + var delayTask = Task.Delay(delay, cancellationToken); + await Task.WhenAny(delayTask, wakeup.Task).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + Interlocked.Exchange(ref _wakeup, NewWakeup()); + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await _cache.RefreshDueAsync(_store.Current, cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + // Shutdown. + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitLogMessages.RefreshLoopTerminated(_logger, ex); + throw; + } + } + + private static TaskCompletionSource NewWakeup() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitInboundConnectionLimiter.cs b/src/VectorNNTPD.TransitService/Transit/TransitInboundConnectionLimiter.cs new file mode 100644 index 00000000..a104a255 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitInboundConnectionLimiter.cs @@ -0,0 +1,65 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Transit-facing inbound limiter. Redis is authoritative for named peers; + /// the process-local count is diagnostics and lifecycle bookkeeping only. + /// + /// + /// Lowering MaxIncomingConnections on reload does not disconnect existing holders. + /// Removing a peer rejects new acquires for that name; existing leases still release. + /// 0 admits no new inbound connections. Redis unavailability fails closed. + /// + public sealed class TransitInboundConnectionLimiter : ITransitInboundConnectionLimiter + { + private readonly TransitConfigurationStore _store; + private readonly ITransitPeerStateTracker _tracker; + + /// Shared no-op limiter for tests that do not exercise inbound limits. + public static ITransitInboundConnectionLimiter Disabled { get; } = new DisabledLimiter(); + + /// Initializes a limiter bound to the active Transit snapshot and cluster tracker. + public TransitInboundConnectionLimiter(TransitConfigurationStore store, ITransitPeerStateTracker tracker) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(tracker); + _store = store; + _tracker = tracker; + } + + /// Gets the current local owned count for (tests/diagnostics). + public int GetCount(string peerName) => _tracker.GetLocalCount(peerName); + + /// + public async ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerName); + if (!_store.Current.Peers.TryGetValue(peerName, out var peer)) + { + return TransitInboundAdmitResult.Rejected; + } + + var max = peer.MaxIncomingConnections; + var admitted = await _tracker + .TryAdmitAsync(peerName, max, cancellationToken) + .ConfigureAwait(false); + if (!admitted.Accepted) + { + return TransitInboundAdmitResult.Rejected; + } + + var generation = admitted.Generation; + return TransitInboundAdmitResult.Accept( + new TransitInboundConnectionLease(ct => _tracker.ReleaseAsync(peerName, generation, ct))); + } + + private sealed class DisabledLimiter : ITransitInboundConnectionLimiter + { + public ValueTask TryAcquireAsync( + string peerName, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(TransitInboundAdmitResult.Uncounted); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitLogMessages.cs b/src/VectorNNTPD.TransitService/Transit/TransitLogMessages.cs new file mode 100644 index 00000000..199807a1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitLogMessages.cs @@ -0,0 +1,59 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// Source-generated Transit configuration and DNS log messages. + internal static partial class TransitLogMessages + { + [LoggerMessage( + EventId = 1702, + Level = LogLevel.Error, + Message = "Ignoring invalid Transit configuration reload; last valid snapshot remains active. {Failure}")] + public static partial void InvalidReloadIgnored(ILogger logger, string Failure); + + [LoggerMessage( + EventId = 1703, + Level = LogLevel.Information, + Message = "Transit peer configuration reloaded ({Count} peer(s))")] + public static partial void ConfigurationReloaded(ILogger logger, int Count); + + [LoggerMessage( + EventId = 1704, + Level = LogLevel.Information, + Message = "Transit AllowFrom DNS {Hostname} resolved to {Count} address(es); next refresh at {NextRefreshUtc}")] + public static partial void DnsResolved(ILogger logger, string Hostname, int Count, DateTimeOffset NextRefreshUtc); + + [LoggerMessage( + EventId = 1705, + Level = LogLevel.Warning, + Message = "Transit peer {PeerId} ({PeerName}) AllowFrom DNS resolution failed for {Hostname}: {Reason}")] + public static partial void DnsResolutionFailed( + ILogger logger, + string PeerId, + string PeerName, + string Hostname, + string Reason); + + [LoggerMessage( + EventId = 1706, + Level = LogLevel.Error, + Message = "Transit AllowFrom DNS refresh loop terminated unexpectedly")] + public static partial void RefreshLoopTerminated(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2270, + Level = LogLevel.Error, + Message = "Transit catalogue initial load failed")] + public static partial void CatalogueInitialLoadFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2271, + Level = LogLevel.Warning, + Message = "Transit catalogue refresh failed; last published snapshot remains active")] + public static partial void CatalogueRefreshFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2272, + Level = LogLevel.Information, + Message = "Transit catalogue published {PublicationId} ({PeerCount} peer(s))")] + public static partial void CataloguePublished(ILogger logger, long PublicationId, int PeerCount); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerMetrics.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerMetrics.cs new file mode 100644 index 00000000..d04b4730 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerMetrics.cs @@ -0,0 +1,155 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Always-on Transit peer counters keyed by configured identifier. + /// + public sealed class TransitPeerMetrics : ITransitPeerMetrics + { + private readonly ConcurrentDictionary _peers = new(StringComparer.Ordinal); + + /// + public void RecordAccepted(string peerId) => Get(peerId).IncrementAccepted(); + + /// + public void RetractAccepted(string peerId) => Get(peerId).DecrementAccepted(); + + /// + public void RecordRejected(string peerId) => Get(peerId).IncrementRejected(); + + /// + public void RecordSessionRegistered(string peerId) => Get(peerId).RegisterSession(); + + /// + public void RecordSessionUnregistered(string peerId) => Get(peerId).UnregisterSession(); + + /// + public void RecordArticleReceived(string peerId, int bytes) => Get(peerId).AddArticle(bytes); + + /// + public void RecordCheck(string peerId) => Get(peerId).IncrementChecks(); + + /// + public void RecordTransmitted(string peerId, long articles = 1) => Get(peerId).AddTransmitted(articles); + + /// + public TransitPeerMetricsSnapshot Capture(string peerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerId); + return _peers.TryGetValue(peerId, out var counters) + ? counters.Capture(peerId) + : new TransitPeerMetricsSnapshot(peerId, 0, 0, 0, 0, 0, 0, 0, 0); + } + + private Counters Get(string peerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(peerId); + return _peers.GetOrAdd(peerId, static _ => new Counters()); + } + + private sealed class Counters + { + private int _active; + private int _peak; + private long _accepted; + private long _rejected; + private long _articlesReceived; + private long _articleBytes; + private long _checks; + private long _transmitted; + + public void IncrementAccepted() => Interlocked.Increment(ref _accepted); + + public void DecrementAccepted() + { + while (true) + { + var current = Volatile.Read(ref _accepted); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref _accepted, current - 1, current) == current) + { + return; + } + } + } + + public void IncrementRejected() => Interlocked.Increment(ref _rejected); + + public void IncrementChecks() => Interlocked.Increment(ref _checks); + + public void RegisterSession() + { + var next = Interlocked.Increment(ref _active); + UpdatePeak(next); + } + + public void UnregisterSession() + { + while (true) + { + var current = Volatile.Read(ref _active); + if (current <= 0) + { + return; + } + + if (Interlocked.CompareExchange(ref _active, current - 1, current) == current) + { + return; + } + } + } + + public void AddArticle(int bytes) + { + Interlocked.Increment(ref _articlesReceived); + if (bytes > 0) + { + Interlocked.Add(ref _articleBytes, bytes); + } + } + + public void AddTransmitted(long articles) + { + if (articles > 0) + { + Interlocked.Add(ref _transmitted, articles); + } + } + + public TransitPeerMetricsSnapshot Capture(string peerId) => + new( + peerId, + Math.Max(0, Volatile.Read(ref _active)), + Volatile.Read(ref _peak), + Volatile.Read(ref _accepted), + Volatile.Read(ref _rejected), + Volatile.Read(ref _articlesReceived), + Volatile.Read(ref _articleBytes), + Volatile.Read(ref _checks), + Volatile.Read(ref _transmitted)); + + private void UpdatePeak(int candidate) + { + while (true) + { + var current = Volatile.Read(ref _peak); + if (candidate <= current) + { + return; + } + + if (Interlocked.CompareExchange(ref _peak, candidate, current) == current) + { + return; + } + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerPolicy.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerPolicy.cs new file mode 100644 index 00000000..e3970cf6 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerPolicy.cs @@ -0,0 +1,183 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Immutable policy snapshot for one named Transit peer. + /// + /// + /// Do not log . Use in diagnostics. + /// + public sealed class TransitPeerPolicy + { + /// Maximum accepted inbound or outbound connection limit. + public const int MaxConnectionLimit = 4096; + + internal TransitPeerPolicy( + string identifier, + string peerName, + int maxIncomingConnections, + int maxOutgoingConnections, + IReadOnlyList literalPrefixes, + IReadOnlyList dnsHostnames, + IReadOnlyList connectTo, + string username, + string password, + TransitSslMode ssl, + NewsfeedsPattern patterns, + bool deferOnDuplicate, + string pathToken, + long maxSize, + TransitMessageTypes messageTypes, + NewsfeedsPattern receivePatterns, + TransitMessageTypes receiveArticleTypes, + long? sendMaxArticleBytes, + string sendUsername, + string sendPassword, + IReadOnlyList pathExclusions) + { + ArgumentNullException.ThrowIfNull(receivePatterns); + ArgumentNullException.ThrowIfNull(sendUsername); + ArgumentNullException.ThrowIfNull(sendPassword); + ArgumentNullException.ThrowIfNull(pathExclusions); + Identifier = identifier; + PeerName = peerName; + MaxIncomingConnections = maxIncomingConnections; + MaxOutgoingConnections = maxOutgoingConnections; + LiteralPrefixes = literalPrefixes; + DnsHostnames = dnsHostnames; + ConnectTo = connectTo; + Username = username; + Password = password; + Ssl = ssl; + Patterns = patterns; + DeferOnDuplicate = deferOnDuplicate; + PathToken = pathToken; + MaxSize = maxSize; + MessageTypes = messageTypes; + ReceivePatterns = receivePatterns; + ReceiveArticleTypes = receiveArticleTypes; + SendMaxArticleBytes = sendMaxArticleBytes; + SendUsername = sendUsername; + SendPassword = sendPassword; + PathExclusions = pathExclusions; + } + + /// + /// Gets the stable protocol/machine identifier (Transit dictionary key). + /// + /// Exact configured string. Not derived from . + public string Identifier { get; } + + /// + /// Gets the human-readable administrative display name. + /// + /// Exact configured PeerName. Not a protocol argument. + public string PeerName { get; } + + /// Gets the inbound connection limit for this peer. + public int MaxIncomingConnections { get; } + + /// + /// Gets the outbound connection limit for this peer. + /// 0 closes send. Outbound feeding is not executed from this value. + /// + public int MaxOutgoingConnections { get; } + + /// Gets literal AllowFrom prefixes (addresses and CIDRs). + public IReadOnlyList LiteralPrefixes { get; } + + /// Gets AllowFrom DNS hostnames (resolved separately). + public IReadOnlyList DnsHostnames { get; } + + /// Gets parsed ConnectTo endpoints (not connected). + public IReadOnlyList ConnectTo { get; } + + /// Gets the Receive AUTHINFO username, or empty when receive credentials are not configured. + public string Username { get; } + + /// + /// Gets the peer AUTHINFO password, or empty when credentials are not configured. + /// Never write this value to logs. + /// + public string Password { get; } + + /// Gets a value indicating whether both username and password are configured. + public bool HasPeerCredentials => Username.Length > 0 && Password.Length > 0; + + /// Gets the canonical TLS mode. + public TransitSslMode Ssl { get; } + + /// Gets the Send newsfeeds expression. Inbound acceptance does not apply it. + public NewsfeedsPattern Patterns { get; } + + /// + /// Gets whether a History hit on CHECK and IHAVE is deferred + /// (431/436) instead of rejected (438/435). + /// TAKETHIS does not read this value. + /// + public bool DeferOnDuplicate { get; } + + /// + /// Gets the exact configured Path-header token (not consumed by routing yet). + /// + public string PathToken { get; } + + /// + /// Gets the Receive article size limit in bytes. + /// Inbound acceptance uses the smaller of this value and the published global maxartsize. + /// A non-positive value, or a value above , adds no peer ceiling. + /// + public long MaxSize { get; } + + /// Gets the Send article-type mask. Inbound acceptance does not apply it. + public TransitMessageTypes MessageTypes { get; } + + /// + /// Gets the Receive newsfeeds expression. + /// TAKETHIS and IHAVE accept an article only when returns match. + /// + public NewsfeedsPattern ReceivePatterns { get; } + + /// + /// Gets the Receive article-type mask. 65535 is unrestricted. + /// TAKETHIS and IHAVE require every flag to be present, via . + /// + public TransitMessageTypes ReceiveArticleTypes { get; } + + /// + /// Gets the Send article size limit in bytes. Independent of . + /// is unlimited. A value is an explicit ceiling. Outbound feeding does not apply it. + /// + public long? SendMaxArticleBytes { get; } + + /// Gets the Send username, or empty when send credentials are not configured. + public string SendUsername { get; } + + /// + /// Gets the Send password, or empty when send credentials are not configured. + /// Never write this value to logs. + /// + public string SendPassword { get; } + + /// Gets a value indicating whether both send username and password are configured. + public bool HasSendCredentials => SendUsername.Length > 0 && SendPassword.Length > 0; + + /// Gets additional Path hops that suppress sending to this peer. + public IReadOnlyList PathExclusions { get; } + + /// + public override string ToString() => Identifier; + + /// Returns whether and match this peer. + public bool CredentialsMatch(string username, string password) + { + ArgumentNullException.ThrowIfNull(username); + ArgumentNullException.ThrowIfNull(password); + return HasPeerCredentials + && string.Equals(Username, username, StringComparison.Ordinal) + && string.Equals(Password, password, StringComparison.Ordinal); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs new file mode 100644 index 00000000..18f55b06 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/DistributedTransitPeerStateTracker.cs @@ -0,0 +1,373 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Hybrid Transit inbound admission: Redis is the cluster authority for + /// MaxIncomingConnections. Local counts are bookkeeping only. + /// + /// + /// Identity is . Source IP is never + /// used as a Redis key. MaxIncomingConnections == 0 is closed, not + /// unlimited. Every named-peer admit requires a successful distributed TRY_ADMIT. + /// Owner identity is {nodeId}:{incarnation}. Redis TTL is crash recovery + /// only: while this process is alive and renewal succeeds, idle Transit + /// connections keep consuming cluster capacity. + /// + public sealed class DistributedTransitPeerStateTracker : ITransitPeerStateTracker, ITransitPeerStateLeaseManager + { + private readonly ITransitPeerStateStore _membership; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly string _ownerId; + private readonly TimeSpan _leaseTtl; + private readonly ConcurrentDictionary _identifierGates = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private readonly Dictionary _peers = new(StringComparer.Ordinal); + private readonly HashSet _ownedIdentifiers = new(StringComparer.Ordinal); + private long _generations; + private bool _accepting = true; + + /// Initializes a production tracker with default lease timings. + public DistributedTransitPeerStateTracker( + ITransitPeerStateStore membership, + ILogger logger, + string nodeId) + : this(membership, logger, nodeId, TimeProvider.System, TransitPeerStateDefaults.LeaseTtl) + { + } + + /// Initializes a tracker with explicit timing and incarnation (tests). + internal DistributedTransitPeerStateTracker( + ITransitPeerStateStore membership, + ILogger logger, + string nodeId, + TimeProvider timeProvider, + TimeSpan? leaseTtl = null, + string? incarnation = null) + { + ArgumentNullException.ThrowIfNull(membership); + ArgumentNullException.ThrowIfNull(logger); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + ArgumentNullException.ThrowIfNull(timeProvider); + _membership = membership; + _logger = logger; + _nodeId = nodeId; + _ownerId = string.Concat(nodeId, ":", string.IsNullOrWhiteSpace(incarnation) ? Guid.NewGuid().ToString("N") : incarnation); + _timeProvider = timeProvider; + _leaseTtl = leaseTtl ?? TransitPeerStateDefaults.LeaseTtl; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(_leaseTtl, TimeSpan.Zero); + } + + /// Gets the configured node identity. + internal string NodeId => _nodeId; + + /// Gets {nodeId}:{incarnation} written into Redis ownership fields. + internal string OwnerId => _ownerId; + + /// Gets how many admits required a distributed membership call that succeeded. + internal int DistributedAdmits { get; private set; } + + /// Gets how many admits were rejected by the cluster inbound limit or a closed peer. + internal int DistributedRejects { get; private set; } + + /// Gets how many admits failed closed because membership was unavailable. + internal int UnavailableRejects { get; private set; } + + /// Gets how many admitted connections this tracker actually released. + internal int ReleaseCalls { get; private set; } + + /// + public async ValueTask TryAdmitAsync( + string identifier, + int maxIncoming, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + + var identifierGate = _identifierGates.GetOrAdd(identifier, static _ => new SemaphoreSlim(1, 1)); + await identifierGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + long generation; + lock (_gate) + { + if (!_accepting) + { + DistributedRejects++; + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected); + } + + generation = _peers.TryGetValue(identifier, out var peer) && peer.Generation != 0 + ? peer.Generation + : NextGeneration(); + } + + var now = _timeProvider.GetUtcNow(); + TransitPeerStateAdmitResult membership; + try + { + membership = await _membership.TryAdmitAsync( + identifier, + _ownerId, + maxIncoming, + generation, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // TRY_ADMIT is atomic. Cancellation after a successful EVAL (or a + // store that throws OCE after writing) must still decrement, or + // this process has no local connection that can later release. + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + CancellationToken.None) + .ConfigureAwait(false); + throw; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + if (!membership.Accepted) + { + if (membership.Status == TransitPeerStateAdmitStatus.Unavailable) + { + UnavailableRejects++; + TransitPeerStateLogMessages.TransitPeerUnavailable(_logger, identifier, maxIncoming, _nodeId); + return membership; + } + + DistributedRejects++; + TransitPeerStateLogMessages.TransitPeerRejected(_logger, identifier, maxIncoming, _nodeId); + return membership; + } + + CommitDistributedAdmit(identifier, generation, now); + } + catch (OperationCanceledException) + { + if (membership.Accepted) + { + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + CancellationToken.None) + .ConfigureAwait(false); + } + + throw; + } + + DistributedAdmits++; + TransitPeerStateLogMessages.TransitPeerAdmitted(_logger, identifier, maxIncoming, _nodeId); + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation); + } + finally + { + identifierGate.Release(); + } + } + + /// + public async ValueTask ReleaseAsync( + string identifier, + long generation, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + + var identifierGate = _identifierGates.GetOrAdd(identifier, static _ => new SemaphoreSlim(1, 1)); + await identifierGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var releaseMembership = false; + lock (_gate) + { + if (!_peers.TryGetValue(identifier, out var peer) || peer.LocalCount <= 0) + { + return; + } + + ReleaseCalls++; + peer.LocalCount--; + releaseMembership = true; + if (peer.LocalCount <= 0) + { + _peers.Remove(identifier); + } + } + + if (releaseMembership) + { + await _membership.ReleaseAsync( + identifier, + _ownerId, + generation, + cancellationToken) + .ConfigureAwait(false); + TransitPeerStateLogMessages.TransitPeerReleased(_logger, identifier, _nodeId); + } + } + finally + { + identifierGate.Release(); + } + } + + /// + public async ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + List<(string Identifier, long Generation)> owners; + lock (_gate) + { + owners = []; + foreach (var (identifier, peer) in _peers) + { + if (peer.LocalCount > 0 && peer.Generation != 0) + { + owners.Add((identifier, peer.Generation)); + } + } + } + + var now = _timeProvider.GetUtcNow(); + foreach (var (identifier, generation) in owners) + { + cancellationToken.ThrowIfCancellationRequested(); + TransitPeerStateRenewStatus status; + try + { + status = await _membership.RenewAsync( + identifier, + _ownerId, + generation, + now, + _leaseTtl, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerLeaseRenewFailed(_logger, ex, identifier, _nodeId); + continue; + } + + if (status == TransitPeerStateRenewStatus.Renewed) + { + MarkLeaseRenewed(identifier, now); + TransitPeerStateLogMessages.TransitPeerLeaseRenewed(_logger, identifier, _nodeId); + continue; + } + + if (status == TransitPeerStateRenewStatus.Unavailable) + { + TransitPeerStateLogMessages.TransitPeerLeaseRenewUnavailable(_logger, identifier, _nodeId); + continue; + } + + TransitPeerStateLogMessages.TransitPeerLeaseLost(_logger, identifier, _nodeId); + } + } + + /// + public async ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) + { + List identifiers; + lock (_gate) + { + _accepting = false; + identifiers = [.. _ownedIdentifiers]; + } + + foreach (var identifier in identifiers) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await _membership.ReleaseOwnerAsync(identifier, _ownerId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerReleaseOwnerFailed(_logger, ex, identifier, _ownerId); + } + } + } + + /// + public int GetLocalCount(string identifier) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + lock (_gate) + { + return _peers.TryGetValue(identifier, out var peer) ? peer.LocalCount : 0; + } + } + + /// Returns whether this tracker still accepts new admissions (tests). + internal bool IsAccepting + { + get + { + lock (_gate) + { + return _accepting; + } + } + } + + private void CommitDistributedAdmit(string identifier, long generation, DateTimeOffset now) + { + lock (_gate) + { + if (!_peers.TryGetValue(identifier, out var peer)) + { + peer = new PeerOwnership(); + _peers[identifier] = peer; + } + + peer.Generation = generation; + peer.LocalCount++; + peer.LeaseExpiresAt = now + _leaseTtl; + _ownedIdentifiers.Add(identifier); + } + } + + private void MarkLeaseRenewed(string identifier, DateTimeOffset now) + { + lock (_gate) + { + if (_peers.TryGetValue(identifier, out var peer)) + { + peer.LeaseExpiresAt = now + _leaseTtl; + } + } + } + + private long NextGeneration() => Interlocked.Increment(ref _generations); + + private sealed class PeerOwnership + { + public int LocalCount { get; set; } + + public long Generation { get; set; } + + public DateTimeOffset LeaseExpiresAt { get; set; } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs new file mode 100644 index 00000000..08c12c61 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateLeaseManager.cs @@ -0,0 +1,28 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Process-local lease renewal and shutdown release for Transit inbound ownership. + /// + /// + /// Renewal proves this node is still responsible for its admitted Transit + /// connections. It is independent of NNTP traffic, article transfers, CHECK, + /// TAKETHIS, commands, bytes transferred, and idle state. + /// is the scheduler; this abstraction + /// owns the ownership snapshot and Redis operations. + /// + public interface ITransitPeerStateLeaseManager + { + /// + /// Refreshes this incarnation's inbound leases for peers that still have + /// local admitted connections. Does not recreate missing ownership. + /// + ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default); + + /// + /// Releases leftover node/incarnation ownership. After this call, new + /// admissions on this tracker fail closed so shutdown cannot race ReleaseOwner + /// with a later TRY_ADMIT. + /// + ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateStore.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateStore.cs new file mode 100644 index 00000000..9f2251e2 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateStore.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Cluster-wide Transit inbound-connection membership. Redis is the production + /// authority; tests use an in-memory store that runs the same algorithm. + /// Admit, release, and renew each use one atomic operation. + /// + public interface ITransitPeerStateStore + { + /// + /// Atomically admits one inbound connection for this owner when the cluster + /// count is below . + /// + ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// + /// Atomically decrements this owner's inbound count when the generation matches. + /// + ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default); + + /// + /// Atomically refreshes this owner's inbound lease. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default); + + /// Deletes leftover ownership for this process incarnation on this peer. + ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateTracker.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateTracker.cs new file mode 100644 index 00000000..73d22527 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/ITransitPeerStateTracker.cs @@ -0,0 +1,34 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Admits and releases inbound Transit connections against cluster-wide + /// MaxIncomingConnections for a . + /// + /// + /// Redis is authoritative. A local owned-count is diagnostics and lifecycle + /// bookkeeping only; it never permits admission without distributed authorization. + /// Source IP is not an identity here. MaxIncomingConnections == 0 is closed. + /// + public interface ITransitPeerStateTracker + { + /// + /// Attempts to admit one inbound connection for . + /// + ValueTask TryAdmitAsync( + string identifier, + int maxIncoming, + CancellationToken cancellationToken = default); + + /// + /// Releases one previously admitted inbound connection for this identifier + /// and generation. Idempotent when this process has no local ownership. + /// + ValueTask ReleaseAsync( + string identifier, + long generation, + CancellationToken cancellationToken = default); + + /// Returns the process-local owned count for . + int GetLocalCount(string identifier); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs new file mode 100644 index 00000000..6cc71348 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/InMemoryTransitPeerStateStore.cs @@ -0,0 +1,187 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Process-local cluster fake for Transit inbound membership. Two trackers + /// sharing one instance behave as two NNTP nodes. Ordinary unit tests must not + /// require live Redis. + /// + internal sealed class InMemoryTransitPeerStateStore : ITransitPeerStateStore + { + private readonly TransitPeerStateEngine _engine = new(); + + /// When set, admit/release/renew fail closed as if Redis were down. + public bool Unavailable { get; set; } + + /// Optional gate that blocks the next admit until completed or cancelled. + public TaskCompletionSource? BlockAdmit { get; set; } + + /// Optional gate that blocks the next release until completed or cancelled. + public TaskCompletionSource? BlockRelease { get; set; } + + /// Optional gate that blocks the next renew until completed or cancelled. + public TaskCompletionSource? BlockRenew { get; set; } + + /// Set when is entered, before waits. + public TaskCompletionSource? NotifyRenewStarted { get; set; } + + /// When set, throws after a successful engine write. + public Exception? ThrowAfterAdmit { get; set; } + + /// When set, throws for this identifier so others can still renew. + public string? FaultRenewIdentifier { get; set; } + + /// Gets how many membership renew operations were attempted. + /// + /// After reaches the engine, this increments only once that + /// renew has returned, so a non-zero count means the renewed expiry is already visible. + /// The unavailable path increments without an engine write. + /// + public int RenewCalls { get; private set; } + + /// Gets how many admit operations reached the engine. + public int AdmitCalls { get; private set; } + + /// Gets how many release operations reached the engine. + public int ReleaseCalls { get; private set; } + + /// Gets the shared admission engine (tests). + internal TransitPeerStateEngine Engine => _engine; + + /// In-memory hash identity for . + internal static string ConnectionKey(string identifier) => "tconn:" + identifier; + + /// Returns the cluster-wide unexpired inbound count. + internal int ActiveCount(string identifier, long nowUnixMs) => + _engine.ActiveCount(ConnectionKey(identifier), nowUnixMs); + + /// Returns this owner's unexpired inbound count. + internal int OwnerCount(string identifier, string ownerId, long nowUnixMs) => + _engine.OwnerCount(ConnectionKey(identifier), ownerId, nowUnixMs); + + /// Returns the stored ownership expiry, or when absent. + internal long? Expiry(string identifier, string ownerId) => + _engine.TryGetOwnership(ConnectionKey(identifier), ownerId, out var expiry, out _, out _) + ? expiry + : null; + + /// + public async ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (BlockAdmit is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Unavailable); + } + + AdmitCalls++; + var code = _engine.TryAdmit( + ConnectionKey(identifier), + ownerId, + maxIncoming, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds, + generation); + if (ThrowAfterAdmit is { } fault) + { + throw fault; + } + + return code == TransitPeerStateEngine.Accepted + ? new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation) + : new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected); + } + + /// + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (BlockRelease is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + return; + } + + ReleaseCalls++; + _ = _engine.Release(ConnectionKey(identifier), ownerId, generation); + } + + /// + public async ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (FaultRenewIdentifier is { } fault + && string.Equals(fault, identifier, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Injected renewal fault."); + } + + NotifyRenewStarted?.TrySetResult(); + if (BlockRenew is { } block) + { + await block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Unavailable) + { + RenewCalls++; + return TransitPeerStateRenewStatus.Unavailable; + } + + var renewed = _engine.Renew( + ConnectionKey(identifier), + ownerId, + generation, + now.ToUnixTimeMilliseconds(), + (long)leaseTtl.TotalMilliseconds) == 1; + RenewCalls++; + return renewed + ? TransitPeerStateRenewStatus.Renewed + : TransitPeerStateRenewStatus.Lost; + } + + /// + public ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + cancellationToken.ThrowIfCancellationRequested(); + if (!Unavailable) + { + _ = _engine.ReleaseOwner(ConnectionKey(identifier), ownerId); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs new file mode 100644 index 00000000..a4df8eeb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/RedisTransitPeerStateStore.cs @@ -0,0 +1,159 @@ +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Redis-backed cluster Transit inbound membership using one atomic Lua EVAL + /// on the shared connection. + /// + internal sealed class RedisTransitPeerStateStore : ITransitPeerStateStore + { + private readonly IRedisService _redis; + + /// Initializes a new instance of the class. + public RedisTransitPeerStateStore(IRedisService redis) + { + ArgumentNullException.ThrowIfNull(redis); + _redis = redis; + } + + /// + public ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + ExecuteAsync( + TransitPeerStateScripts.TryAdmit, + identifier, + [ + Utf8(ownerId), + Utf8(maxIncoming.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + ], + result => result == TransitPeerStateEngine.Accepted + ? new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Accepted, generation) + : new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Rejected), + new TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus.Unavailable), + cancellationToken); + + /// + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + TransitPeerStateScripts.Release, + identifier, + [ + Utf8(ownerId), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + ], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + /// + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + ExecuteAsync( + TransitPeerStateScripts.Renew, + identifier, + [ + Utf8(ownerId), + Utf8(generation.ToString(CultureInfo.InvariantCulture)), + Utf8(now.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture)), + Utf8(((long)leaseTtl.TotalMilliseconds).ToString(CultureInfo.InvariantCulture)), + ], + result => result == 1 + ? TransitPeerStateRenewStatus.Renewed + : TransitPeerStateRenewStatus.Lost, + TransitPeerStateRenewStatus.Unavailable, + cancellationToken); + + /// + public async ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) + { + _ = await ExecuteAsync( + TransitPeerStateScripts.ReleaseOwner, + identifier, + [Utf8(ownerId)], + static _ => true, + fallback: true, + cancellationToken).ConfigureAwait(false); + } + + private async ValueTask ExecuteAsync( + string script, + string identifier, + ReadOnlyMemory[] values, + Func map, + T fallback, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + if (!_redis.TryBeginOperation(out var isRecoveryProbe)) + { + return fallback; + } + + IRedisDatabase database; + try + { + database = _redis.Database; + } + catch (InvalidOperationException) + { + _redis.AbandonOperation(isRecoveryProbe); + return fallback; + } + + try + { + var result = await database.ScriptEvaluateAsync( + script, + [TransitPeerStateKeys.Create(identifier)], + values, + cancellationToken).ConfigureAwait(false); + _redis.CompleteOperation(isRecoveryProbe, succeeded: true); + return map(result); + } + catch (OperationCanceledException) + { + _redis.AbandonOperation(isRecoveryProbe); + throw; + } + catch (RedisUnavailableException) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false); + return fallback; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + _redis.CompleteOperation(isRecoveryProbe, succeeded: false, ex); + return fallback; + } + } + + private static byte[] Utf8(string value) => Encoding.UTF8.GetBytes(value); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs new file mode 100644 index 00000000..d026e715 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateAdmitResult.cs @@ -0,0 +1,54 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// Cluster membership outcome for one Transit inbound admit attempt. + public enum TransitPeerStateAdmitStatus + { + /// Admitted; this owner now holds one additional inbound connection. + Accepted, + + /// + /// Rejected: MaxIncomingConnections is 0 (closed) or the cluster + /// already holds at least that many unexpired inbound connections. + /// + Rejected, + + /// Redis (or the membership store) was unavailable. Fail closed. + Unavailable, + } + + /// Result of a cluster Transit inbound admit attempt. + public readonly struct TransitPeerStateAdmitResult + { + /// Initializes a Transit membership admit result. + public TransitPeerStateAdmitResult(TransitPeerStateAdmitStatus status, long generation = 0) + { + Status = status; + Generation = generation; + } + + /// Gets the membership outcome. + public TransitPeerStateAdmitStatus Status { get; } + + /// + /// Gets the generation that must be supplied to . + /// Zero when admission was not granted. + /// + public long Generation { get; } + + /// Gets whether membership was granted. + public bool Accepted => Status == TransitPeerStateAdmitStatus.Accepted; + } + + /// Outcome of a Transit node ownership lease renewal. + public enum TransitPeerStateRenewStatus + { + /// This owner's requested lease was refreshed. + Renewed, + + /// The requested ownership field was gone, expired, or generation-mismatched. + Lost, + + /// The membership store could not be reached. + Unavailable, + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateDefaults.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateDefaults.cs new file mode 100644 index 00000000..2645b077 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateDefaults.cs @@ -0,0 +1,30 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// Lease timings for cluster-wide Transit inbound connection ownership. + /// + /// These match the established SessionState operational values (30s lease, 10s + /// renewal, 2s hot-path skew) without introducing Transit-specific configuration. + /// Redis expiry is crash/failure recovery only. It is not a Transit idle timeout. + /// A healthy node renews every 10 seconds while it still owns inbound connections, + /// independently of NNTP traffic. Transit has no local admit hot path: every named + /// peer requires distributed authorization. MaxIncomingConnections == 0 + /// means closed, not unlimited. + /// + internal static class TransitPeerStateDefaults + { + /// Distributed ownership TTL written into Redis on admit and renew. + public static readonly TimeSpan LeaseTtl = TimeSpan.FromSeconds(30); + + /// + /// Interval between process-wide lease renewal passes. + /// uses the same 10-second period. + /// + public static readonly TimeSpan RenewalPeriod = TimeSpan.FromSeconds(10); + + /// + /// SessionState hot-path skew. Transit does not admit without Redis; the + /// constant is retained so operational timing stays aligned. + /// + public static readonly TimeSpan HotPathSkew = TimeSpan.FromSeconds(2); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateEngine.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateEngine.cs new file mode 100644 index 00000000..f2022665 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateEngine.cs @@ -0,0 +1,326 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Atomic cluster Transit inbound-connection algorithm. The Redis Lua scripts + /// are a faithful port of these operations. + /// + /// + /// One HASH per Transit Identifier. Field {ownerId}, value + /// {expiryUnixMs}|{generation}|{count}. The cluster count is the sum of + /// unexpired per-owner counts. max <= 0 rejects without incrementing. + /// Rejection after prune writes no ownership. Source IP is not part of this state. + /// + internal sealed class TransitPeerStateEngine + { + /// Accepted; ownership was created or incremented. + public const long Accepted = 1; + + /// Rejected: closed (max <= 0) or cluster count already at the limit. + public const long Rejected = 0; + + private readonly object _gate = new(); + private readonly Dictionary> _hashes = new(StringComparer.Ordinal); + + /// + /// Atomically admits one inbound connection for this owner when the cluster + /// count is below . + /// + public long TryAdmit( + string connectionKey, + string ownerId, + int maxIncoming, + long nowUnixMs, + long leaseMs, + long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + var owners = GetOrCreateHash(connectionKey); + PruneExpired(owners, nowUnixMs); + if (maxIncoming <= 0) + { + RemoveIfEmpty(connectionKey, owners); + return Rejected; + } + + if (SumCounts(owners) >= maxIncoming) + { + RemoveIfEmpty(connectionKey, owners); + return Rejected; + } + + IncrementOwned(owners, ownerId, nowUnixMs + leaseMs, generation); + return Accepted; + } + } + + /// + /// Decrements this owner's count when the generation matches. Deletes the + /// field at zero and the hash when empty. + /// + public long Release(string connectionKey, string ownerId, long generation) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(connectionKey, out var owners)) + { + DecrementOwned(owners, ownerId, generation); + RemoveIfEmpty(connectionKey, owners); + } + + return 1; + } + } + + /// + /// Refreshes this owner's unexpired matching lease. Missing, expired, or + /// generation-mismatched ownership is not recreated. + /// + public long Renew( + string connectionKey, + string ownerId, + long generation, + long nowUnixMs, + long leaseMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (!CanRenewField(connectionKey, ownerId, generation, nowUnixMs, out _)) + { + return 0; + } + + RenewField(connectionKey, ownerId, generation, nowUnixMs, leaseMs); + return 1; + } + } + + /// Deletes only this owner's field. Other owners and peers are untouched. + public long ReleaseOwner(string connectionKey, string ownerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + + lock (_gate) + { + if (_hashes.TryGetValue(connectionKey, out var owners)) + { + owners.Remove(ownerId); + RemoveIfEmpty(connectionKey, owners); + } + + return 1; + } + } + + /// Returns the cluster-wide unexpired inbound count for this peer. + internal int ActiveCount(string connectionKey, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + lock (_gate) + { + if (!_hashes.TryGetValue(connectionKey, out var owners)) + { + return 0; + } + + PruneExpired(owners, nowUnixMs); + RemoveIfEmpty(connectionKey, owners); + return SumCounts(owners); + } + } + + /// Returns this owner's unexpired inbound count. + internal int OwnerCount(string connectionKey, string ownerId, long nowUnixMs) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + ArgumentException.ThrowIfNullOrWhiteSpace(ownerId); + lock (_gate) + { + if (!_hashes.TryGetValue(connectionKey, out var owners) + || !owners.TryGetValue(ownerId, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out _, out var count) + || expiry <= nowUnixMs) + { + return 0; + } + + return count; + } + } + + /// Reads a HASH field without mutating it (tests). + internal bool TryGetOwnership( + string key, + string field, + out long expiryUnixMs, + out long generation, + out int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + if (_hashes.TryGetValue(key, out var hash) + && hash.TryGetValue(field, out var current) + && TransitPeerStateKeys.TrySplitValue(current, out expiryUnixMs, out generation, out count)) + { + return true; + } + } + + expiryUnixMs = 0; + generation = 0; + count = 0; + return false; + } + + /// Overwrites a HASH field without admit/release semantics (tests). + internal void WriteOwnership(string key, string field, long expiryUnixMs, long generation, int count) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(field); + lock (_gate) + { + GetOrCreateHash(key)[field] = TransitPeerStateKeys.Value(expiryUnixMs, generation, count); + } + } + + /// Returns field names currently stored for (tests). + internal IReadOnlyCollection OwnerFields(string connectionKey) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionKey); + lock (_gate) + { + return _hashes.TryGetValue(connectionKey, out var owners) + ? [.. owners.Keys] + : []; + } + } + + private Dictionary GetOrCreateHash(string key) + { + if (!_hashes.TryGetValue(key, out var hash)) + { + hash = new Dictionary(StringComparer.Ordinal); + _hashes[key] = hash; + } + + return hash; + } + + private void RemoveIfEmpty(string key, Dictionary hash) + { + if (hash.Count == 0) + { + _hashes.Remove(key); + } + } + + private bool CanRenewField(string key, string field, long generation, long nowUnixMs, out int count) + { + count = 0; + if (!_hashes.TryGetValue(key, out var hash) + || !hash.TryGetValue(field, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out count) + || storedGeneration != generation + || expiry <= nowUnixMs) + { + return false; + } + + return true; + } + + private void RenewField(string key, string field, long generation, long nowUnixMs, long leaseMs) + { + if (!CanRenewField(key, field, generation, nowUnixMs, out var count) + || !_hashes.TryGetValue(key, out var hash)) + { + return; + } + + hash[field] = TransitPeerStateKeys.Value(nowUnixMs + leaseMs, generation, count); + } + + private static void IncrementOwned(Dictionary hash, string field, long expiry, long generation) + { + // Same generation: this owner still holds the field. A different generation + // is a new local epoch (local state was wiped) and must replace stale count. + if (hash.TryGetValue(field, out var current) + && TransitPeerStateKeys.TrySplitValue(current, out _, out var storedGeneration, out var count) + && storedGeneration == generation) + { + hash[field] = TransitPeerStateKeys.Value(expiry, storedGeneration, count + 1); + return; + } + + hash[field] = TransitPeerStateKeys.Value(expiry, generation, 1); + } + + private static void DecrementOwned(Dictionary hash, string field, long generation) + { + if (!hash.TryGetValue(field, out var current) + || !TransitPeerStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out var count) + || storedGeneration != generation) + { + return; + } + + if (count <= 1) + { + hash.Remove(field); + return; + } + + hash[field] = TransitPeerStateKeys.Value(expiry, storedGeneration, count - 1); + } + + private static void PruneExpired(Dictionary hash, long nowUnixMs) + { + List? expired = null; + foreach (var (field, value) in hash) + { + if (!TransitPeerStateKeys.TrySplitValue(value, out var expiry, out _, out _) || expiry <= nowUnixMs) + { + expired ??= []; + expired.Add(field); + } + } + + if (expired is null) + { + return; + } + + foreach (var field in expired) + { + hash.Remove(field); + } + } + + private static int SumCounts(Dictionary hash) + { + var total = 0; + // codeql[cs/linq/missed-where] TrySplitValue out-params; lock-held sum without LINQ allocation. + foreach (var value in hash.Values) + { + if (TransitPeerStateKeys.TrySplitValue(value, out _, out _, out var count)) + { + total += count; + } + } + + return total; + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateKeys.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateKeys.cs new file mode 100644 index 00000000..f0a39fd8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateKeys.cs @@ -0,0 +1,84 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Redis key material for cluster-wide Transit inbound connection ownership. + /// + /// + /// Key: nntpd:tconn:{sha256hex(identifier)}. + /// Field: {ownerId} where ownerId is {nodeId}:{incarnation}. + /// Value: {expiryUnixMs}|{generation}|{count}. + /// The Transit dictionary key () is hashed + /// so Redis key listings do not expose peer names. Source IP is never part of this key. + /// Ownership uses value expiry, not a Redis key TTL. + /// + internal static class TransitPeerStateKeys + { + /// ASCII namespace for Transit inbound-connection ownership hashes. + public static ReadOnlySpan NamespacePrefix => "nntpd:tconn:"u8; + + /// Separates expiry, generation, and count in a HASH value. + public const char ValueSeparator = '|'; + + /// Builds the Transit inbound-connection Redis HASH key for . + public static byte[] Create(string identifier) + { + ArgumentException.ThrowIfNullOrWhiteSpace(identifier); + Span hash = stackalloc byte[32]; + var byteCount = Encoding.UTF8.GetByteCount(identifier); + if (byteCount <= 256) + { + Span utf8 = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(identifier, utf8); + SHA256.HashData(utf8, hash); + } + else + { + SHA256.HashData(Encoding.UTF8.GetBytes(identifier), hash); + } + + var hex = Convert.ToHexStringLower(hash); + var prefix = NamespacePrefix; + var key = new byte[prefix.Length + hex.Length]; + prefix.CopyTo(key); + Encoding.ASCII.GetBytes(hex, key.AsSpan(prefix.Length)); + return key; + } + + /// Formats a lease value as expiryMs|generation|count. + public static string Value(long expiryUnixMs, long generation, int count) => + string.Concat( + expiryUnixMs.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + generation.ToString(CultureInfo.InvariantCulture), + ValueSeparator, + count.ToString(CultureInfo.InvariantCulture)); + + /// Parses a HASH value into expiry, generation, and count. + public static bool TrySplitValue(string value, out long expiryUnixMs, out long generation, out int count) + { + ArgumentNullException.ThrowIfNull(value); + expiryUnixMs = 0; + generation = 0; + count = 0; + var first = value.IndexOf(ValueSeparator, StringComparison.Ordinal); + if (first <= 0 || first >= value.Length - 1) + { + return false; + } + + var second = value.IndexOf(ValueSeparator, first + 1); + if (second <= first + 1 || second >= value.Length - 1) + { + return false; + } + + return long.TryParse(value.AsSpan(0, first), out expiryUnixMs) + && long.TryParse(value.AsSpan(first + 1, second - first - 1), out generation) + && int.TryParse(value.AsSpan(second + 1), out count); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs new file mode 100644 index 00000000..60d2dd55 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateLogMessages.cs @@ -0,0 +1,42 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// Source-generated Transit peer-state diagnostics. Never includes secrets. + internal static partial class TransitPeerStateLogMessages + { + [LoggerMessage(EventId = 1710, Level = LogLevel.Debug, Message = "Transit peer-state admission accepted identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerAdmitted(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1711, Level = LogLevel.Information, Message = "Transit peer-state admission rejected identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerRejected(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1712, Level = LogLevel.Warning, Message = "Transit peer-state admission failed closed; cluster membership unavailable identifier={Identifier} limit={MaxIncoming} node={NodeId}")] + public static partial void TransitPeerUnavailable(ILogger logger, string identifier, int maxIncoming, string nodeId); + + [LoggerMessage(EventId = 1713, Level = LogLevel.Debug, Message = "Transit peer-state ownership released identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerReleased(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1714, Level = LogLevel.Debug, Message = "Transit peer-state lease renewed identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewed(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1715, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal failed identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewFailed(ILogger logger, Exception exception, string identifier, string nodeId); + + [LoggerMessage(EventId = 1716, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal skipped; membership unavailable identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseRenewUnavailable(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1717, Level = LogLevel.Warning, Message = "Transit peer-state lease lost; ownership not extended identifier={Identifier} node={NodeId}")] + public static partial void TransitPeerLeaseLost(ILogger logger, string identifier, string nodeId); + + [LoggerMessage(EventId = 1718, Level = LogLevel.Warning, Message = "Transit peer-state release-owner failed identifier={Identifier} owner={OwnerId}")] + public static partial void TransitPeerReleaseOwnerFailed(ILogger logger, Exception exception, string identifier, string ownerId); + + [LoggerMessage(EventId = 1719, Level = LogLevel.Information, Message = "Transit peer-state lease renewal started interval={Interval}")] + public static partial void TransitPeerRenewalStarted(ILogger logger, TimeSpan interval); + + [LoggerMessage(EventId = 1720, Level = LogLevel.Information, Message = "Transit peer-state lease renewal stopped")] + public static partial void TransitPeerRenewalStopped(ILogger logger); + + [LoggerMessage(EventId = 1721, Level = LogLevel.Warning, Message = "Transit peer-state lease renewal pass failed")] + public static partial void TransitPeerRenewalPassFailed(ILogger logger, Exception exception); + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateScripts.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateScripts.cs new file mode 100644 index 00000000..aacbfbe5 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateScripts.cs @@ -0,0 +1,190 @@ +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Redis Lua scripts for atomic cluster Transit inbound-connection admission. + /// Semantics match . + /// + internal static class TransitPeerStateScripts + { + /// + /// Atomically evaluates the inbound limit and, on accept, increments this + /// owner's count. Rejection writes no ownership increment. + /// + /// + /// KEYS[1] tconn hash. + /// ARGV: owner, maxIncoming, nowMs, leaseMs, generation. + /// Returns 1 accepted, 0 rejected (closed or at limit). + /// Expired fields are pruned before the limit is summed. + /// max <= 0 rejects without incrementing. + /// + public const string TryAdmit = + """ + -- nntpd-tconn-try + local key = KEYS[1] + local owner = ARGV[1] + local maxIncoming = tonumber(ARGV[2]) + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + local gen = ARGV[5] + local expiry = now + lease + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local function prune(hashKey) + local entries = redis.call('HGETALL', hashKey) + local expired = {} + for i = 1, #entries, 2 do + local exp = parse_value(entries[i + 1]) + if exp == nil or exp <= now then + expired[#expired + 1] = entries[i] + end + end + if #expired > 0 then + redis.call('HDEL', hashKey, unpack(expired)) + end + end + + local function increment(hashKey, field, requestedGen) + local cur = redis.call('HGET', hashKey, field) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen and storedGen == requestedGen then + redis.call('HSET', hashKey, field, tostring(expiry) .. '|' .. storedGen .. '|' .. tostring((count or 0) + 1)) + return + end + end + redis.call('HSET', hashKey, field, tostring(expiry) .. '|' .. requestedGen .. '|1') + end + + prune(key) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + + if maxIncoming == nil or maxIncoming <= 0 then + return 0 + end + + local total = 0 + local entries = redis.call('HGETALL', key) + for i = 1, #entries, 2 do + local _, _, count = parse_value(entries[i + 1]) + total = total + (count or 0) + end + if total >= maxIncoming then + return 0 + end + + increment(key, owner, gen) + return 1 + """; + + /// + /// Decrements this owner's count when the generation matches. + /// + /// + /// KEYS[1] tconn hash. ARGV: owner, generation. Always returns 1. + /// A mismatched generation is a no-op. The field is deleted at count 0; + /// the hash is deleted when empty. + /// + public const string Release = + """ + -- nntpd-tconn-release + local key = KEYS[1] + local owner = ARGV[1] + local gen = ARGV[2] + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local cur = redis.call('HGET', key, owner) + if cur then + local exp, storedGen, count = parse_value(cur) + if storedGen == gen then + if count == nil or count <= 1 then + redis.call('HDEL', key, owner) + else + redis.call('HSET', key, owner, tostring(exp) .. '|' .. storedGen .. '|' .. tostring(count - 1)) + end + end + end + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + return 1 + """; + + /// + /// Refreshes this owner's unexpired matching lease. Missing ownership is not recreated. + /// + /// + /// KEYS[1] tconn hash. ARGV: owner, generation, nowMs, leaseMs. + /// Returns 1 when the field was renewed, 0 when it was missing, expired, or mismatched. + /// + public const string Renew = + """ + -- nntpd-tconn-renew + local key = KEYS[1] + local owner = ARGV[1] + local gen = ARGV[2] + local now = tonumber(ARGV[3]) + local lease = tonumber(ARGV[4]) + + local function parse_value(v) + local p1 = string.find(v, '|', 1, true) + if not p1 then + return nil, nil, nil + end + local p2 = string.find(v, '|', p1 + 1, true) + if not p2 then + return nil, nil, nil + end + return tonumber(string.sub(v, 1, p1 - 1)), string.sub(v, p1 + 1, p2 - 1), tonumber(string.sub(v, p2 + 1)) + end + + local cur = redis.call('HGET', key, owner) + if not cur then + return 0 + end + local exp, storedGen, count = parse_value(cur) + if storedGen ~= gen or exp == nil or exp <= now then + return 0 + end + redis.call('HSET', key, owner, tostring(now + lease) .. '|' .. gen .. '|' .. tostring(count or 1)) + return 1 + """; + + /// Deletes only this owner's field on this peer hash. + /// KEYS[1] tconn hash. ARGV: owner. Always returns 1. + public const string ReleaseOwner = + """ + -- nntpd-tconn-release-owner + local key = KEYS[1] + local owner = ARGV[1] + redis.call('HDEL', key, owner) + if redis.call('HLEN', key) == 0 then + redis.call('DEL', key) + end + return 1 + """; + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateService.cs b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateService.cs new file mode 100644 index 00000000..676754fb --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitPeerState/TransitPeerStateService.cs @@ -0,0 +1,130 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Periodic distributed Transit inbound-ownership maintenance. + /// + /// + /// This service is a scheduler. It does not accept connections or keep its own + /// connection registry. Every renewal interval it asks + /// to renew currently active local + /// ownership. Renewal is process/node liveness, not connection activity: idle + /// Transit connections continue to consume cluster capacity while this process + /// can renew. stops the loop and then releases leftover + /// incarnation ownership. Listeners are registered after this service so they + /// stop first and admitted connections can finalize before renewal ends. + /// Process crash is detected by lease expiry after renewals stop. + /// + public sealed class TransitPeerStateService : IApplicationService + { + private readonly ITransitPeerStateLeaseManager _leases; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _runCts = new(); + private Task? _execution; + private int _started; + + /// Initializes a new instance of the class. + public TransitPeerStateService( + ITransitPeerStateLeaseManager leases, + ILogger logger) + : this(leases, logger, TimeProvider.System, TransitPeerStateDefaults.RenewalPeriod) + { + } + + /// Initializes a new instance with an explicit interval (tests). + internal TransitPeerStateService( + ITransitPeerStateLeaseManager leases, + ILogger logger, + TimeProvider timeProvider, + TimeSpan interval) + { + ArgumentNullException.ThrowIfNull(leases); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(interval, TimeSpan.Zero); + _leases = leases; + _logger = logger; + _timeProvider = timeProvider; + _interval = interval; + } + + /// + public string Name => "TransitPeerState"; + + /// + public Task? Execution => _execution; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _started, 1) == 1) + { + return Task.CompletedTask; + } + + _execution = Task.Run(() => RunAsync(_runCts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + await _runCts.CancelAsync().ConfigureAwait(false); + var execution = _execution; + if (execution is not null) + { + try + { + await execution.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + try + { + await _leases.ReleaseAllOwnershipAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + TransitPeerStateLogMessages.TransitPeerRenewalStarted(_logger, _interval); + try + { + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(_interval, _timeProvider, cancellationToken).ConfigureAwait(false); + try + { + await _leases.RenewLeasesAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + TransitPeerStateLogMessages.TransitPeerRenewalPassFailed(_logger, ex); + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + TransitPeerStateLogMessages.TransitPeerRenewalStopped(_logger); + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transit/TransitReceivePolicy.cs b/src/VectorNNTPD.TransitService/Transit/TransitReceivePolicy.cs new file mode 100644 index 00000000..c2cdc3b4 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transit/TransitReceivePolicy.cs @@ -0,0 +1,138 @@ +using System.Text; +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Transit +{ + /// + /// Applies one connection's captured Receive peer policy to an already classified article. + /// + /// + /// The peer object is the session authorization captured at identification. + /// Send , , + /// and are not read. + /// CHECK has no article body and does not call this type. + /// + internal static class TransitReceivePolicy + { + /// TAKETHIS/IHAVE detail when is outside the receive mask. + internal const string ArticleTypeRejectDetail = "rejected article type"; + + /// TAKETHIS/IHAVE detail when does not subscribe the article. + internal const string NewsgroupPatternRejectDetail = "rejected newsgroup pattern"; + + /// + /// Returns the receive ceiling for one article operation. + /// + /// + /// Published nntpsharedconfig.maxartsize, or the caller fallback when no snapshot is on the session. + /// This value is always the hard ceiling. + /// + /// Peer captured for this connection, or when the session is not a named peer. + /// + /// when is absent or its + /// is non-positive or above + /// (no peer ceiling). Otherwise the smaller of the two. + /// + internal static int EffectiveMaxArticleBytes(int globalOrFallback, TransitPeerPolicy? peer) + { + if (peer is null) + { + return globalOrFallback; + } + + var peerMax = peer.MaxSize; + if (peerMax <= 0 || peerMax > int.MaxValue) + { + return globalOrFallback; + } + + var peerCeiling = (int)peerMax; + return peerCeiling < globalOrFallback ? peerCeiling : globalOrFallback; + } + + /// + /// Returns whether the captured receive mask or newsgroup expression rejects . + /// + /// Peer captured for this connection. does not reject. + /// Classified CanonicalV1 record. Classification is not repeated here. + /// + /// or when this method returns . + /// + /// when the article must be rejected. + /// + /// Article types use : 65535 allows every combination, + /// and every other mask requires each flag on to be present. + /// Newsgroups use on the article's Newsgroups header. + /// Only is accepted. Exclude, poison, and no-match reject. + /// + internal static bool TryReject(TransitPeerPolicy? peer, in ArticleRecord record, out string rejectDetail) + { + rejectDetail = string.Empty; + if (peer is null) + { + return false; + } + + var allowed = (ArticleType)peer.ReceiveArticleTypes; + if (!ArticleTypeCapabilities.Allows(allowed, record.ArtType)) + { + rejectDetail = ArticleTypeRejectDetail; + return true; + } + + if (peer.ReceivePatterns.EvaluateArticle(SplitNewsgroups(record.Newsgroups)) != NewsfeedsPatternDecision.Match) + { + rejectDetail = NewsgroupPatternRejectDetail; + return true; + } + + return false; + } + + /// + /// Splits a Newsgroups header into trimmed group tokens for . + /// + /// Header value bytes from the canonical record. + /// Group names. Empty and whitespace-only tokens are omitted. + /// + /// The string conversion exists only because takes strings. + /// Comma separation and surrounding SP/HTAB trimming follow the header, not a second wildmat implementation. + /// + private static List SplitNewsgroups(ReadOnlySpan newsgroups) + { + var groups = new List(4); + var start = 0; + for (var i = 0; i <= newsgroups.Length; i++) + { + if (i < newsgroups.Length && newsgroups[i] != (byte)',') + { + continue; + } + + var token = newsgroups[start..i]; + start = i + 1; + var tokenStart = 0; + var tokenEnd = token.Length; + while (tokenStart < tokenEnd && IsSpace(token[tokenStart])) + { + tokenStart++; + } + + while (tokenEnd > tokenStart && IsSpace(token[tokenEnd - 1])) + { + tokenEnd--; + } + + if (tokenEnd > tokenStart) + { + groups.Add(Encoding.UTF8.GetString(token[tokenStart..tokenEnd])); + } + } + + return groups; + } + + /// Returns whether is SP or HTAB. + private static bool IsSpace(byte value) => value is (byte)' ' or (byte)'\t'; + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/IVatpArticleClient.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/IVatpArticleClient.cs new file mode 100644 index 00000000..106aaae1 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/IVatpArticleClient.cs @@ -0,0 +1,23 @@ +using VectorNNTPD.Common.Articles; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Fetches canonical articles from BackFiller or StorageServer listeners over VATP. + public interface IVatpArticleClient + { + /// + /// Fetches one article from the explicit VATP endpoint. + /// + /// Lowercase dotted DNS name of the listener. + /// TLS listen port in the range 1–65535. + /// Correlation id from ArticleWork or the storage lookup. + /// Expected ArticleId for OPEN binding. + /// Cancellation; sends CANCEL when the connection is usable. + Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken); + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpArticleClient.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpArticleClient.cs new file mode 100644 index 00000000..e01d71ac --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpArticleClient.cs @@ -0,0 +1,70 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Pool-backed VATP article fetch orchestration. + internal sealed class VatpArticleClient : IVatpArticleClient + { + private readonly VatpConnectionPool _pool; + private readonly ILogger _logger; + + public VatpArticleClient(VatpConnectionPool pool, ILogger logger) + { + _pool = pool; + _logger = logger; + } + + /// + public async Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + if (requestId == Guid.Empty) + { + return VatpFetchResult.ProtocolFailure("RequestId must not be empty.", requestId, articleId); + } + + if (!VatpEndpointFields.IsCanonicalFqdn(fqdn)) + { + return VatpFetchResult.ProtocolFailure("VATP endpoint requires a lowercase dotted DNS fqdn.", requestId, articleId); + } + + if (!VatpEndpointFields.IsCanonicalPort(vatpPort)) + { + return VatpFetchResult.ProtocolFailure("VATP endpoint requires a port in the range 1–65535.", requestId, articleId); + } + + var (connection, acquireFailure) = await _pool.AcquireAsync(fqdn, vatpPort, cancellationToken) + .ConfigureAwait(false); + if (acquireFailure is { } failure) + { + return VatpFetchResult.ConnectionFailure(failure.Error, requestId, articleId); + } + + if (connection is null) + { + return VatpFetchResult.ConnectionFailure("Failed to acquire VATP connection.", requestId, articleId); + } + + try + { + return await connection.FetchArticleAsync(requestId, articleId, cancellationToken).ConfigureAwait(false); + } + finally + { + if (connection.IsDead) + { + _pool.RemoveDead(connection); + } + else + { + _pool.Release(connection); + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientLogMessages.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientLogMessages.cs new file mode 100644 index 00000000..4491163b --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientLogMessages.cs @@ -0,0 +1,89 @@ +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Source-generated VATP client log messages. + internal static partial class VatpClientLogMessages + { + [LoggerMessage( + EventId = 2800, + Level = LogLevel.Debug, + Message = "VATP connecting to {Host}:{Port}.")] + public static partial void Connecting(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2801, + Level = LogLevel.Debug, + Message = "VATP TLS established for {Host}:{Port}.")] + public static partial void TlsEstablished(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2802, + Level = LogLevel.Debug, + Message = "VATP HELLO complete on {Host}:{Port} maxFramePayload={MaxFramePayload}.")] + public static partial void HelloComplete(ILogger logger, string Host, int Port, uint MaxFramePayload); + + [LoggerMessage( + EventId = 2803, + Level = LogLevel.Debug, + Message = "VATP OPEN streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void OpenSent( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2804, + Level = LogLevel.Debug, + Message = "VATP META streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void MetaReceived( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2805, + Level = LogLevel.Debug, + Message = "VATP transfer complete streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void TransferComplete( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2806, + Level = LogLevel.Debug, + Message = "VATP transfer failed streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex} errorCode={ErrorCode}.")] + public static partial void TransferFailed( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex, + VatpErrorCode ErrorCode); + + [LoggerMessage( + EventId = 2807, + Level = LogLevel.Debug, + Message = "VATP CANCEL streamId={StreamId} requestId={RequestId} articleId={ArticleIdHex}.")] + public static partial void CancelSent( + ILogger logger, + uint StreamId, + Guid RequestId, + string ArticleIdHex); + + [LoggerMessage( + EventId = 2808, + Level = LogLevel.Debug, + Message = "VATP connection closed {Host}:{Port}.")] + public static partial void ConnectionClosed(ILogger logger, string Host, int Port); + + [LoggerMessage( + EventId = 2809, + Level = LogLevel.Debug, + Message = "VATP protocol violation on {Host}:{Port}: {Detail}.")] + public static partial void ProtocolViolation(ILogger logger, string Host, int Port, string Detail); + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientOptions.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientOptions.cs new file mode 100644 index 00000000..5ccf6c50 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpClientOptions.cs @@ -0,0 +1,21 @@ +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Internal VATP client resource and timeout defaults. + internal sealed class VatpClientOptions + { + /// Maximum TLS connections per cache endpoint. + public int MaxConnectionsPerEndpoint { get; init; } = 4; + + /// TCP connect timeout. + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(15); + + /// TLS handshake timeout. + public TimeSpan TlsHandshakeTimeout { get; init; } = TimeSpan.FromSeconds(15); + + /// + /// Per-read I/O timeout while a connection is active. + /// It is not armed while a STORE on the connection is still waiting for its first WINDOW. + /// + public TimeSpan IoTimeout { get; init; } = TimeSpan.FromMinutes(5); + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnection.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnection.cs new file mode 100644 index 00000000..a4df7d24 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnection.cs @@ -0,0 +1,842 @@ +using System.Buffers; +using System.Net.Security; +using System.Net.Sockets; +using System.Threading.Channels; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// One multiplexed TLS+VATP connection to a cache listener. + internal sealed class VatpConnection : IAsyncDisposable + { + private readonly string _host; + private readonly int _port; + private readonly Stream _stream; + private readonly TcpClient _tcp; + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ArticleTransferLimits _limits; + private readonly ArticleTransferStreamTable _streamTable; + private readonly Dictionary _pending = new(); + private readonly object _gate = new(); + private readonly SemaphoreSlim _writeLock = new(1, 1); + private readonly Channel _outbound; + private uint _nextStreamId = 1; + private uint _maxFramePayload = VatpProtocol.DefaultMaxFramePayload; + private Task? _runTask; + private CancellationTokenSource? _runCts; + private int _disposed; + private int _dead; + + private VatpConnection( + string host, + int port, + TcpClient tcp, + Stream stream, + ILogger logger, + VatpClientOptions options, + ArticleTransferLimits limits, + uint maxFramePayload) + { + _host = host; + _port = port; + _tcp = tcp; + _stream = stream; + _logger = logger; + _options = options; + _limits = limits; + _streamTable = new ArticleTransferStreamTable(limits); + _maxFramePayload = maxFramePayload; + _outbound = Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + } + + /// Gets whether the connection is dead and must be removed from the pool. + public bool IsDead => Volatile.Read(ref _dead) != 0; + + /// Gets whether another stream can be opened on this connection. + public bool HasStreamCapacity + { + get + { + lock (_gate) + { + return !IsDead && _streamTable.Count < _limits.MaxStreamsPerConnection; + } + } + } + + /// Endpoint host used for TLS and logging. + public string Host => _host; + + /// Endpoint port. + public int Port => _port; + + internal static async Task ConnectAsync( + string host, + int port, + ILogger logger, + VatpClientOptions options, + RemoteCertificateValidationCallback? serverCertificateValidationCallback, + CancellationToken cancellationToken, + string? tcpConnectHost = null) + { + VatpClientLogMessages.Connecting(logger, host, port); + var tcp = new TcpClient(); + try + { + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectCts.CancelAfter(options.ConnectTimeout); + await tcp.ConnectAsync(tcpConnectHost ?? host, port, connectCts.Token).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + tcp.Dispose(); + throw new VatpConnectionException("TCP connect failed.", ex); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + throw new VatpConnectionException("TCP connect timed out."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + + Stream transport; + try + { + var ssl = await VatpTlsClient.AuthenticateAsClientAsync( + tcp.GetStream(), + host, + options.TlsHandshakeTimeout, + serverCertificateValidationCallback, + cancellationToken).ConfigureAwait(false); + transport = ssl; + VatpClientLogMessages.TlsEstablished(logger, host, port); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) when (ex is not OperationCanceledException) + { + tcp.Dispose(); + throw new VatpConnectionException("TLS handshake failed.", ex); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + tcp.Dispose(); + throw new VatpConnectionException("TLS handshake timed out."); + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + tcp.Dispose(); + throw; + } + + var limits = ArticleTransferLimits.Default; + var connection = new VatpConnection( + host, + port, + tcp, + transport, + logger, + options, + limits, + VatpProtocol.DefaultMaxFramePayload); + + try + { + var maxPayload = await connection.PerformHelloExchangeAsync(cancellationToken).ConfigureAwait(false); + connection._maxFramePayload = maxPayload; + VatpClientLogMessages.HelloComplete(logger, host, port, maxPayload); + connection.StartRunLoop(); + return connection; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + internal async Task FetchArticleAsync( + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (IsDead) + { + return VatpFetchResult.ConnectionFailure("VATP connection is no longer usable.", requestId, articleId); + } + + uint streamId; + PendingStream pending; + lock (_gate) + { + if (_streamTable.Count >= _limits.MaxStreamsPerConnection) + { + return VatpFetchResult.ConnectionFailure("VATP connection stream table is full.", requestId, articleId); + } + + streamId = AllocateStreamIdLocked(); + var openResult = _streamTable.TryOpen(streamId, requestId, articleId, out var receiveStream); + if (!openResult.Success || receiveStream is null) + { + RecycleStreamIdLocked(streamId); + return VatpFetchResult.ProtocolFailure( + $"Failed to open local stream: {openResult.Error}", + requestId, + articleId); + } + + pending = new PendingStream(receiveStream, articleId); + _pending[streamId] = pending; + } + + var articleHex = articleId.ToLowerHexString(); + VatpClientLogMessages.OpenSent(_logger, streamId, requestId, articleHex); + + Span artBytes = stackalloc byte[ArticleId.Length]; + articleId.CopyTo(artBytes); + await EnqueueFrameAsync( + VatpFrameEncoder.EncodeOpen(streamId, requestId, artBytes), + cancellationToken).ConfigureAwait(false); + + using var cancelReg = cancellationToken.Register(static state => + { + var tuple = ((VatpConnection Connection, uint StreamId))state!; + _ = tuple.Connection.TrySendCancelAsync(tuple.StreamId); + }, (this, streamId)); + + try + { + return await pending.Completion.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return VatpFetchResult.Cancelled(requestId, articleId, pending.ReceiveStream.ReceivedBytes); + } + finally + { + lock (_gate) + { + _pending.Remove(streamId); + _streamTable.TryRemove(streamId); + RecycleStreamIdLocked(streamId); + } + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + MarkDead("Connection disposed."); + try + { + _runCts?.Cancel(); + } + catch (ObjectDisposedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + _outbound.Writer.TryComplete(); + if (_runTask is not null) + { + try + { + await _runTask.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + await _stream.DisposeAsync().ConfigureAwait(false); + _tcp.Dispose(); + _writeLock.Dispose(); + VatpClientLogMessages.ConnectionClosed(_logger, _host, _port); + } + + private async Task PerformHelloExchangeAsync(CancellationToken cancellationToken) + { + await WriteFrameDirectAsync( + VatpFrameEncoder.EncodeHello(_limits.DefaultMaxFramePayload), + cancellationToken).ConfigureAwait(false); + + var frame = await ReadOneFrameAsync(cancellationToken).ConfigureAwait(false); + if (frame.Header.Type != VatpFrameType.Hello + || frame.Header.StreamId != VatpProtocol.ConnectionStreamId) + { + throw new VatpConnectionException("Expected server HELLO."); + } + + Span helloBytes = stackalloc byte[VatpProtocol.HelloPayloadLength]; + CopyPayload(frame.Payload, helloBytes); + if (!VatpHello.TryDecode(helloBytes, out var hello, out var error)) + { + throw new VatpConnectionException($"Invalid server HELLO: {error}"); + } + + return Math.Min(hello.MaxFramePayload, _limits.DefaultMaxFramePayload); + } + + private void StartRunLoop() + { + _runCts = new CancellationTokenSource(); + _runTask = RunAsync(_runCts.Token); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + var writer = RunWriterAsync(cancellationToken); + try + { + await RunReadLoopAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + MarkDead(ex.Message); + } + finally + { + _outbound.Writer.TryComplete(); + try + { + await writer.ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex; // observed; intentionally ignored at this boundary + } + + FailAllPending(VatpFetchResult.ConnectionFailure("VATP connection closed.")); + } + } + + private async Task RunWriterAsync(CancellationToken cancellationToken) + { + await foreach (var frame in _outbound.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + await WriteFrameDirectAsync(frame, cancellationToken).ConfigureAwait(false); + } + } + + private async Task RunReadLoopAsync(CancellationToken cancellationToken) + { + // Socket reads are independent of VATP framing: one ReadAsync may yield a partial + // frame, one frame, or many frames. Only the incomplete tail is retained, and that + // tail is bounded to one negotiated maximum frame. + var readBuffer = ArrayPool.Shared.Rent(32 * 1024); + var parseBuffer = ArrayPool.Shared.Rent(VatpProtocol.HeaderLengthBytes + (int)_maxFramePayload); + var buffered = 0; + try + { + while (!cancellationToken.IsCancellationRequested && !IsDead) + { + var maxIncomplete = VatpProtocol.HeaderLengthBytes + (int)_maxFramePayload; + if (buffered > maxIncomplete) + { + MarkDead("Read buffer exceeded negotiated frame size."); + return; + } + + int bytesRead; + try + { + using var ioCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ioCts.CancelAfter(_options.IoTimeout); + bytesRead = await _stream.ReadAsync(readBuffer.AsMemory(0, readBuffer.Length), ioCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + MarkDead("Read timed out."); + return; + } + + if (bytesRead == 0) + { + MarkDead("Peer closed connection."); + return; + } + + EnsureParseCapacity(ref parseBuffer, buffered + bytesRead); + readBuffer.AsSpan(0, bytesRead).CopyTo(parseBuffer.AsSpan(buffered)); + buffered += bytesRead; + + var consumed = 0; + while (consumed < buffered) + { + var candidate = new ReadOnlySequence(parseBuffer, consumed, buffered - consumed); + var parsed = VatpFrameParser.ParseOneFrame(in candidate, _maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, parsed.Error.ToString()); + MarkDead($"Invalid frame: {parsed.Error}"); + return; + } + + consumed += checked((int)parsed.ConsumedBytes); + if (!await HandleFrameAsync(parsed.Frame!.Value, cancellationToken).ConfigureAwait(false)) + { + return; + } + } + + if (consumed > 0) + { + parseBuffer.AsSpan(consumed, buffered - consumed).CopyTo(parseBuffer); + buffered -= consumed; + } + + // After dispatching every complete frame, only an incomplete frame may remain. + if (buffered > maxIncomplete) + { + MarkDead("Read buffer exceeded negotiated frame size."); + return; + } + } + } + finally + { + ArrayPool.Shared.Return(readBuffer); + ArrayPool.Shared.Return(parseBuffer); + } + } + + private async Task HandleFrameAsync(VatpParsedFrame frame, CancellationToken cancellationToken) + { + if (frame.Header.Type == VatpFrameType.Fail) + { + return HandleFailFrame(frame); + } + + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, "Unexpected connection-level frame."); + MarkDead("Unexpected connection-level frame."); + return false; + } + + PendingStream? pending; + ArticleTransferReceiveStream? receive; + lock (_gate) + { + if (!_pending.TryGetValue(frame.Header.StreamId, out pending)) + { + // Late META/DATA/END after local stream cleanup (e.g. fetch already + // completed while WINDOW/FAIL races drained) must not kill the connection. + VatpClientLogMessages.ProtocolViolation( + _logger, + _host, + _port, + $"Ignoring frame for inactive StreamId {frame.Header.StreamId}."); + return !IsDead; + } + + receive = pending.ReceiveStream; + } + + switch (frame.Header.Type) + { + case VatpFrameType.Meta: + VatpClientLogMessages.MetaReceived( + _logger, + frame.Header.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString()); + if (!receive.TryAcceptMeta(frame.Payload).Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "META rejected.")); + } + + break; + case VatpFrameType.Data: + { + // Receive credit is consumed on accept. Replenish locally and advertise + // the same amount via WINDOW so ArtSize may exceed InitialStreamWindowBytes + // without treating the initial window as an article-size ceiling. + var apply = receive.TryAcceptData(frame.Payload, frame.Header.HasFin); + if (!apply.Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "DATA rejected.")); + break; + } + + var payloadLength = checked((uint)frame.Payload.Length); + // Pair outbound WINDOW with local credit restore. Skip both when the + // stream was cancelled between accept and replenish (no cross-stream credit). + if (payloadLength > 0 && receive.TryAcceptWindow(payloadLength).Success) + { + await EnqueueFrameAsync( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, payloadLength), + cancellationToken).ConfigureAwait(false); + } + + if (receive.IsTerminal) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "DATA rejected.")); + } + + break; + } + + case VatpFrameType.End: + if (!receive.TryAcceptEnd().Success) + { + CompletePending( + frame.Header.StreamId, + ToFetchFailure(receive, "END rejected.")); + break; + } + + if (receive.TryTakeRecord(out var record)) + { + VatpClientLogMessages.TransferComplete( + _logger, + frame.Header.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString()); + CompletePending( + frame.Header.StreamId, + VatpFetchResult.FromSuccess( + record, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes)); + } + else + { + CompletePending( + frame.Header.StreamId, + VatpFetchResult.IncompleteOrMalformed( + "END received without consumable record.", + receive.Failure, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes)); + } + + break; + default: + VatpClientLogMessages.ProtocolViolation( + _logger, + _host, + _port, + $"Unexpected frame type {frame.Header.Type}."); + MarkDead("Unexpected frame type."); + return false; + } + + return !IsDead; + } + + private bool HandleFailFrame(VatpParsedFrame frame) + { + Span payload = stackalloc byte[(int)Math.Min(frame.Payload.Length, VatpProtocol.FailMaxPayloadLength)]; + CopyPayload(frame.Payload, payload); + if (!VatpControlPayload.TryDecodeFail(payload, out var errorCode, out var reason)) + { + MarkDead("Invalid FAIL payload."); + return false; + } + + var reasonText = reason.IsEmpty ? null : System.Text.Encoding.ASCII.GetString(reason); + if (frame.Header.StreamId == VatpProtocol.ConnectionStreamId) + { + MarkDead(reasonText ?? errorCode.ToString()); + return false; + } + + // Server may emit UnknownStream for WINDOW/CANCEL that arrive after it already + // removed a finished stream (common when ArtSize fits in the initial window). + // That race must not fail an in-flight or just-completed transfer, nor poison siblings. + if (errorCode == VatpErrorCode.UnknownStream) + { + return !IsDead; + } + + lock (_gate) + { + if (_pending.TryGetValue(frame.Header.StreamId, out var pending)) + { + VatpClientLogMessages.TransferFailed( + _logger, + frame.Header.StreamId, + pending.ReceiveStream.RequestId, + pending.ArticleId.ToLowerHexString(), + errorCode); + CompletePendingLocked( + frame.Header.StreamId, + VatpFetchResult.RemoteFailure( + reasonText, + errorCode, + pending.ReceiveStream.RequestId, + pending.ArticleId, + pending.ReceiveStream.ReceivedBytes)); + } + } + + return !IsDead; + } + + private void CompletePending(uint streamId, VatpFetchResult result) + { + lock (_gate) + { + CompletePendingLocked(streamId, result); + } + } + + private void CompletePendingLocked(uint streamId, VatpFetchResult result) + { + if (_pending.TryGetValue(streamId, out var pending)) + { + pending.Completion.TrySetResult(result); + } + } + + private VatpFetchResult ToFetchFailure(ArticleTransferReceiveStream receive, string detail) + { + VatpClientLogMessages.TransferFailed( + _logger, + receive.StreamId, + receive.RequestId, + receive.ExpectedArtId.ToLowerHexString(), + receive.Failure); + if (receive.Failure == VatpErrorCode.Cancelled) + { + return VatpFetchResult.Cancelled( + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes); + } + + return VatpFetchResult.IncompleteOrMalformed( + detail, + receive.Failure, + receive.RequestId, + receive.ExpectedArtId, + receive.ReceivedBytes); + } + + private void FailAllPending(VatpFetchResult result) + { + lock (_gate) + { + foreach (var (streamId, pending) in _pending.ToArray()) + { + pending.Completion.TrySetResult( + VatpFetchResult.ConnectionFailure( + result.Error, + pending.ReceiveStream.RequestId, + pending.ArticleId, + pending.ReceiveStream.ReceivedBytes)); + _pending.Remove(streamId); + _streamTable.TryRemove(streamId); + RecycleStreamIdLocked(streamId); + } + } + } + + private void MarkDead(string reason) + { + if (Interlocked.Exchange(ref _dead, 1) == 1) + { + return; + } + + VatpClientLogMessages.ProtocolViolation(_logger, _host, _port, reason); + FailAllPending(VatpFetchResult.ConnectionFailure(reason)); + } + + private async Task TrySendCancelAsync(uint streamId) + { + PendingStream? pending; + lock (_gate) + { + if (!_pending.TryGetValue(streamId, out pending) || pending.ReceiveStream.IsTerminal) + { + return; + } + + _ = pending.ReceiveStream.TryCancel(); + } + + VatpClientLogMessages.CancelSent( + _logger, + streamId, + pending!.ReceiveStream.RequestId, + pending.ArticleId.ToLowerHexString()); + + try + { + await EnqueueFrameAsync(VatpFrameEncoder.EncodeCancel(streamId), CancellationToken.None) + .ConfigureAwait(false); + } + catch (ChannelClosedException ex) + { + _ = ex; // observed; intentionally ignored at this boundary + } + } + + private async Task EnqueueFrameAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + if (IsDead) + { + throw new VatpConnectionException("Connection is dead."); + } + + await _outbound.Writer.WriteAsync(frame, cancellationToken).ConfigureAwait(false); + } + + private async Task WriteFrameDirectAsync(VatpFrameEncoder.EncodedFrame frame, CancellationToken cancellationToken) + { + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (frame.Header.Length > 0) + { + await _stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (frame.Payload.Length > 0) + { + await _stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _writeLock.Release(); + } + } + + private async Task ReadOneFrameAsync(CancellationToken cancellationToken) + { + var header = new byte[VatpProtocol.HeaderLengthBytes]; + await ReadExactAsync(header, cancellationToken).ConfigureAwait(false); + var parsedHeader = VatpFrameHeader.ReadFrom(header); + var payload = new byte[parsedHeader.PayloadLength]; + if (payload.Length > 0) + { + await ReadExactAsync(payload, cancellationToken).ConfigureAwait(false); + } + + return new VatpParsedFrame(parsedHeader, new ReadOnlySequence(payload)); + } + + private async Task ReadExactAsync(Memory buffer, CancellationToken cancellationToken) + { + var total = 0; + while (total < buffer.Length) + { + using var ioCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ioCts.CancelAfter(_options.IoTimeout); + var read = await _stream.ReadAsync(buffer[total..], ioCts.Token).ConfigureAwait(false); + if (read == 0) + { + throw new VatpConnectionException("Unexpected end of stream."); + } + + total += read; + } + } + + private uint AllocateStreamIdLocked() + { + // Do not reuse StreamIds on a live connection. Late WINDOW/END/FAIL for a just- + // finished stream must not be applied to a successor transfer that recycled the id. + // Concurrent capacity remains bounded by MaxStreamsPerConnection via the stream table. + if (_nextStreamId == 0) + { + _nextStreamId = 1; + } + + return _nextStreamId++; + } + + private void RecycleStreamIdLocked(uint streamId) + { + // Intentionally unused for wire StreamId values; retained as a call-site hook so + // stream-table removal stays paired with allocation cleanup. + _ = streamId; + } + + private static void EnsureParseCapacity(ref byte[] parseBuffer, int required) + { + if (required <= parseBuffer.Length) + { + return; + } + + var bigger = ArrayPool.Shared.Rent(Math.Max(parseBuffer.Length * 2, required)); + parseBuffer.AsSpan(0, Math.Min(parseBuffer.Length, required)).CopyTo(bigger); + ArrayPool.Shared.Return(parseBuffer); + parseBuffer = bigger; + } + + private static void CopyPayload(in ReadOnlySequence payload, Span destination) + { + if (payload.Length > destination.Length) + { + throw new InvalidOperationException("Payload buffer too small."); + } + + payload.CopyTo(destination); + } + + private sealed class PendingStream(ArticleTransferReceiveStream receiveStream, ArticleId articleId) + { + public ArticleTransferReceiveStream ReceiveStream { get; } = receiveStream; + + public ArticleId ArticleId { get; } = articleId; + + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + /// Connection setup failure surfaced to the pool. + internal sealed class VatpConnectionException : Exception + { + public VatpConnectionException(string message) + : base(message) + { + } + + public VatpConnectionException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnectionPool.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnectionPool.cs new file mode 100644 index 00000000..0143512a --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpConnectionPool.cs @@ -0,0 +1,246 @@ +using System.Collections.Concurrent; +using System.Net.Security; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Pools multiplexed VATP connections keyed by cache endpoint. + internal sealed class VatpConnectionPool : IAsyncDisposable + { + private readonly ILogger _logger; + private readonly VatpClientOptions _options; + private readonly ConcurrentDictionary _endpoints = new(); + private int _disposed; + + /// Optional certificate callback for tests only. + internal RemoteCertificateValidationCallback? ServerCertificateValidationCallback { get; set; } + + /// Optional TCP connect host override for tests (TLS TargetHost remains the URI host). + internal string? TestTcpConnectHost { get; set; } + + public VatpConnectionPool(ILogger logger) + : this(logger, new VatpClientOptions()) + { + } + + internal VatpConnectionPool(ILogger logger, VatpClientOptions options) + { + _logger = logger; + _options = options; + } + + internal async Task<(VatpConnection? Connection, VatpFetchResult? Failure)> AcquireAsync( + string host, + int port, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var key = new EndpointKey(host, port); + var pool = _endpoints.GetOrAdd(key, static _ => new EndpointPool()); + + var existing = pool.TryAcquireExisting(); + if (existing is not null) + { + return (existing, null); + } + + if (pool.ConnectionCount >= _options.MaxConnectionsPerEndpoint) + { + return pool.ConnectionCount == 0 + ? (null, VatpFetchResult.ConnectionFailure("No VATP connections available for endpoint.")) + : (null, VatpFetchResult.ConnectionFailure( + "All VATP connections to the endpoint are at stream capacity.")); + } + + await pool.CreateGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + existing = pool.TryAcquireExisting(); + if (existing is not null) + { + return (existing, null); + } + + if (pool.ConnectionCount >= _options.MaxConnectionsPerEndpoint) + { + return ( + null, + VatpFetchResult.ConnectionFailure( + "All VATP connections to the endpoint are at stream capacity.")); + } + + try + { + var created = await VatpConnection.ConnectAsync( + host, + port, + _logger, + _options, + ServerCertificateValidationCallback, + cancellationToken, + TestTcpConnectHost).ConfigureAwait(false); + pool.Add(created); + return (created, null); + } + catch (VatpConnectionException ex) + { + return (null, VatpFetchResult.ConnectionFailure(ex.Message)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch (Exception ex) + { + return (null, VatpFetchResult.ConnectionFailure(ex.Message)); + } + } + finally + { + pool.CreateGate.Release(); + } + } + + internal void Release(VatpConnection connection) + { + if (connection.IsDead) + { + RemoveDead(connection); + } + } + + internal void RemoveDead(VatpConnection connection) + { + var key = new EndpointKey(connection.Host, connection.Port); + if (_endpoints.TryGetValue(key, out var pool)) + { + pool.Remove(connection); + } + + _ = connection.DisposeAsync(); + } + + internal int GetEndpointConnectionCount(string host, int port) + { + var key = new EndpointKey(host, port); + return _endpoints.TryGetValue(key, out var pool) ? pool.ConnectionCount : 0; + } + + internal int GetEndpointAliveConnectionCount(string host, int port) + { + var key = new EndpointKey(host, port); + return _endpoints.TryGetValue(key, out var pool) ? pool.AliveConnectionCount : 0; + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + foreach (var pool in _endpoints.Values) + { + await pool.DisposeAsync().ConfigureAwait(false); + } + + _endpoints.Clear(); + } + + private readonly record struct EndpointKey(string Host, int Port); + + private sealed class EndpointPool + { + private readonly List _connections = []; + private readonly object _gate = new(); + + internal SemaphoreSlim CreateGate { get; } = new(1, 1); + + public int ConnectionCount + { + get + { + lock (_gate) + { + return _connections.Count; + } + } + } + + public int AliveConnectionCount + { + get + { + lock (_gate) + { + PurgeDeadLocked(); + return _connections.Count; + } + } + } + + public void Add(VatpConnection connection) + { + lock (_gate) + { + _connections.Add(connection); + } + } + + public void Remove(VatpConnection connection) + { + lock (_gate) + { + _connections.Remove(connection); + } + } + + public VatpConnection? TryAcquireExisting() + { + lock (_gate) + { + PurgeDeadLocked(); + foreach (var candidate in _connections) + { + if (candidate.HasStreamCapacity) + { + return candidate; + } + } + } + + return null; + } + + public async ValueTask DisposeAsync() + { + VatpConnection[] snapshot; + lock (_gate) + { + snapshot = _connections.ToArray(); + _connections.Clear(); + } + + CreateGate.Dispose(); + foreach (var connection in snapshot) + { + await connection.DisposeAsync().ConfigureAwait(false); + } + } + + private void PurgeDeadLocked() + { + for (var i = _connections.Count - 1; i >= 0; i--) + { + if (_connections[i].IsDead) + { + var dead = _connections[i]; + _connections.RemoveAt(i); + _ = dead.DisposeAsync(); + } + } + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchKind.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchKind.cs new file mode 100644 index 00000000..ce378d73 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchKind.cs @@ -0,0 +1,27 @@ +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Outcome classification for a VATP article fetch. + public enum VatpFetchKind + { + /// Canonical article received and validated. + Success, + + /// Remote peer rejected or failed the transfer (FAIL frame, open rejected). + RemoteTransferFailure, + + /// Local protocol violation or unexpected framing. + ProtocolFailure, + + /// Connect, TLS, or transport I/O failure. + ConnectionFailure, + + /// Operation cancelled by caller. + Cancelled, + + /// Article identity could not be used for OPEN. + InvalidArticleId, + + /// Transfer ended without a consumable canonical record. + IncompleteOrMalformedArticle, + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchResult.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchResult.cs new file mode 100644 index 00000000..745f41ef --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpFetchResult.cs @@ -0,0 +1,116 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// Result of . + public readonly record struct VatpFetchResult + { + private VatpFetchResult( + VatpFetchKind kind, + ArticleRecord record, + string? error, + VatpErrorCode? errorCode, + Guid? requestId, + ArticleId? articleId, + int acceptedDataBytes) + { + if (acceptedDataBytes < 0) + { + throw new ArgumentOutOfRangeException(nameof(acceptedDataBytes)); + } + + Kind = kind; + Record = record; + Error = error; + ErrorCode = errorCode; + RequestId = requestId; + ArticleId = articleId; + AcceptedDataBytes = acceptedDataBytes; + } + + /// Gets the outcome kind. + public VatpFetchKind Kind { get; } + + /// Gets the article when is . + public ArticleRecord Record { get; } + + /// Gets optional diagnostic text (never article payload). + public string? Error { get; } + + /// Gets the VATP error code when supplied by the peer or state machine. + public VatpErrorCode? ErrorCode { get; } + + /// Gets the OPEN RequestId when known. + public Guid? RequestId { get; } + + /// Gets the OPEN ArticleId when known. + public ArticleId? ArticleId { get; } + + /// + /// Gets the number of DATA payload bytes copied into the receive stream before this result was built. + /// + /// + /// Zero when no receive stream existed or no DATA payload had been copied. + /// When a stream exists, this is that stream's copied DATA count at completion. + /// + public int AcceptedDataBytes { get; } + + /// Creates a success result. + public static VatpFetchResult FromSuccess( + ArticleRecord record, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new(VatpFetchKind.Success, record, null, null, requestId, articleId, acceptedDataBytes); + + /// Creates a remote transfer failure. + public static VatpFetchResult RemoteFailure( + string? error, + VatpErrorCode? errorCode, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new(VatpFetchKind.RemoteTransferFailure, default, error, errorCode, requestId, articleId, acceptedDataBytes); + + /// Creates a protocol failure. + public static VatpFetchResult ProtocolFailure( + string? error, + Guid? requestId = null, + ArticleId? articleId = null, + int acceptedDataBytes = 0) => + new(VatpFetchKind.ProtocolFailure, default, error, null, requestId, articleId, acceptedDataBytes); + + /// Creates a connection failure. + public static VatpFetchResult ConnectionFailure( + string? error, + Guid? requestId = null, + ArticleId? articleId = null, + int acceptedDataBytes = 0) => + new(VatpFetchKind.ConnectionFailure, default, error, null, requestId, articleId, acceptedDataBytes); + + /// Creates a cancelled result. + public static VatpFetchResult Cancelled(Guid requestId, ArticleId articleId, int acceptedDataBytes = 0) => + new(VatpFetchKind.Cancelled, default, null, VatpErrorCode.Cancelled, requestId, articleId, acceptedDataBytes); + + /// Creates an invalid ArticleId result. + public static VatpFetchResult InvalidArticleId(string? error) => + new(VatpFetchKind.InvalidArticleId, default, error, null, null, null, 0); + + /// Creates an incomplete or malformed article result. + public static VatpFetchResult IncompleteOrMalformed( + string? error, + VatpErrorCode? errorCode, + Guid requestId, + ArticleId articleId, + int acceptedDataBytes = 0) => + new( + VatpFetchKind.IncompleteOrMalformedArticle, + default, + error, + errorCode, + requestId, + articleId, + acceptedDataBytes); + } +} diff --git a/src/VectorNNTPD.TransitService/Transport/Vatp/VatpTlsClient.cs b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpTlsClient.cs new file mode 100644 index 00000000..335997f8 --- /dev/null +++ b/src/VectorNNTPD.TransitService/Transport/Vatp/VatpTlsClient.cs @@ -0,0 +1,56 @@ +using System.Net.Security; +using System.Security.Authentication; + +namespace VectorNNTPD.TransitService.Transport.Vatp +{ + /// + /// Shared outbound TLS client options and handshake for VATP cache connections. + /// + internal static class VatpTlsClient + { + internal static SslClientAuthenticationOptions CreateClientOptions( + string host, + RemoteCertificateValidationCallback? serverCertificateValidationCallback) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + var options = new SslClientAuthenticationOptions + { + TargetHost = host, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }; + if (serverCertificateValidationCallback is not null) + { + options.RemoteCertificateValidationCallback = serverCertificateValidationCallback; + } + + return options; + } + + internal static async Task AuthenticateAsClientAsync( + Stream inner, + string host, + TimeSpan timeout, + RemoteCertificateValidationCallback? serverCertificateValidationCallback, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(inner); + var ssl = new SslStream(inner, leaveInnerStreamOpen: false); + try + { + using var tlsCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + tlsCts.CancelAfter(timeout); + await ssl.AuthenticateAsClientAsync( + CreateClientOptions(host, serverCertificateValidationCallback), + tlsCts.Token) + .ConfigureAwait(false); + return ssl; + } + // codeql[cs/catch-of-all-exceptions] Service/process boundary must observe every non-fatal failure without aborting the host. + catch + { + await ssl.DisposeAsync().ConfigureAwait(false); + throw; + } + } + } +} diff --git a/src/VectorNNTPD.TransitService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.TransitService/VectorNNTPD.RabbitMQ.json new file mode 100644 index 00000000..934ea129 --- /dev/null +++ b/src/VectorNNTPD.TransitService/VectorNNTPD.RabbitMQ.json @@ -0,0 +1,38 @@ +{ + "RabbitMQ": { + "ChannelLeaseTimeoutSeconds": 60, + "ChannelPoolSize": 512, + "ConnectionBlockedTimeoutSeconds": 30, + "ConnectionScaleDownIdleSeconds": 300, + "DegradedThreshold": 0.75, + "EnableSsl": false, + "Hosts": [ + "198.18.0.3" + ], + "Management": { + "BaseUrl": "http://198.18.0.3:15672", + "RequestTimeoutSeconds": 5 + }, + "MaxConnections": 16, + "MaximumShutdownDrainTimeoutSeconds": 30, + "MaxPendingLeaseWaiters": 1024, + "MinConnections": 4, + "MinimumConnectionLifetimeSeconds": 300, + "NetworkRecoveryIntervalSeconds": 5, + "PoolReconnectBaseDelayMs": 250, + "PoolReconnectMaxDelayMs": 30000, + "Port": 5672, + "PublishConfirmTimeoutSeconds": 10, + "RequestedChannelMax": 2047, + "RequestedHeartbeatSeconds": 60, + "RpcTimeoutSeconds": 30, + "ScaleDownCooldownSeconds": 30, + "SocketTimeoutSeconds": 30, + "UnhealthyThreshold": 5, + "VirtualHost": "/", + "WorkRequestMaxPayloadBytes": 1024, + "ArticleWorkDeliveryLimit": 5, + "ArticleWorkQueueMaxLength": 10000, + "ArticleWorkQueueMessageTtlMilliseconds": 60000 + } +} diff --git a/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.csproj b/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.csproj new file mode 100644 index 00000000..0754b490 --- /dev/null +++ b/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.csproj @@ -0,0 +1,64 @@ + + + + + + net10.0 + Exe + true + VectorNNTPD.TransitService + VectorNNTPD.TransitService + NNTP transit-server host and implementation (VectorNNTPD.TransitService). + false + x64 + + + + + + + + + + + + + PreserveNewest + PreserveNewest + + + PreserveNewest + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json b/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json new file mode 100644 index 00000000..e6e8b456 --- /dev/null +++ b/src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json @@ -0,0 +1,2987 @@ +{ + "Moderation": { + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", + "RetrievedFrom": "InnUrl", + "Url": "https://github.com/InterNetNews/inn/blob/main/samples/moderators" + } + }, + "Email": { + "Enabled": false, + "DefaultFrom": "noreply@usenet.ninja", + "EnvelopeSender": "noreply@usenet.ninja", + "Smtp": { + "Host": "", + "Port": 587, + "Security": "StartTls", + "Username": "", + "Password": "", + "RequireTlsForAuthentication": true, + "ConnectTimeout": "00:00:15", + "CommandTimeout": "00:00:30", + "MaxAttempts": 3, + "InitialRetryDelay": "00:00:02", + "MaximumRetryDelay": "00:01:00" + }, + "Spool": { + "Directory": "spool/smtp", + "ShutdownTimeout": "00:00:15", + "ScanInterval": "00:00:01" + } + }, + "Control": { + "PgpAuthorities": { + "Authorities": [ + { + "AdminGroup": "aioe.system", + "Authorizations": [ + { + "From": "usenet@aioe.org", + "Message": "checkgroups", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "newgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "rmgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + } + ], + "Contact": "usenet@aioe.org", + "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8", + "KeyUrl": "http://news.aioe.org/hierarchy/aioe.txt", + "Name": "AIOE", + "SyncableServer": "nntp.aioe.org", + "Url": "http://news.aioe.org/hierarchy/" + }, + { + "AdminGroup": "alabama.config", + "Authorizations": [ + { + "From": "jpc@suespammers.org", + "Message": "checkgroups", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "newgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "rmgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + } + ], + "Contact": "jpc@suespammers.org", + "Name": "ALABAMA & HSV" + }, + { + "AdminGroup": "at.usenet.gruppen", + "Authorizations": [ + { + "From": "control@usenet.backbone.at", + "Message": "checkgroups", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "newgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "rmgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + } + ], + "KeyUrl": "http://www.usenet.at/pgpkey.asc", + "Name": "AT", + "Url": "http://www.usenet.at/" + }, + { + "AdminGroup": "aus.net.news", + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "AUS", + "Url": "http://aus.news-admin.org/" + }, + { + "AdminGroup": "be.announce", + "Authorizations": [ + { + "From": "group-admin@usenet.be", + "Message": "checkgroups", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "newgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "rmgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + } + ], + "Contact": "be-hierarchy-admin@usenet.be", + "KeyFingerprint": "302A459470DE1FD5818C5864D2F70871", + "KeyUrl": "http://usenet.be/be.announce.newgroups.asc", + "Name": "BE", + "Url": "http://usenet.be/" + }, + { + "AdminGroup": "belwue.infos", + "Authorizations": [ + { + "From": "news@news.belwue.de", + "Message": "checkgroups", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "newgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "rmgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + } + ], + "Contact": "news@belwue.de", + "KeyFingerprint": "2EFF2B8A891FE5AA6F89672450B542D9", + "KeyUrl": "http://www.belwue.de/fileadmin/belwue/Dokumente/Newsserver/belwue-hir-control.asc", + "Name": "BELWUE", + "Url": "http://www.belwue.de/produkte/dienste/dienste-usenet1.html" + }, + { + "AdminGroup": "bionet.general", + "Authorizations": [ + { + "From": "Biosci-control-key@net.bio.net", + "Message": "checkgroups", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "newgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "rmgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + } + ], + "KeyFingerprint": "EBC0F1BA260BC6D6FB8DEDC4AE5D1054", + "Name": "BIONET", + "Url": "http://www.bio.net/" + }, + { + "AdminGroup": "bit.admin", + "Authorizations": [ + { + "From": "bit@newsadmin.com", + "Message": "checkgroups", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "newgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "rmgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + } + ], + "Name": "BIT", + "Url": "http://www.newsadmin.com/bit/bit.htm" + }, + { + "AdminGroup": "bln.misc", + "Authorizations": [ + { + "From": "news@fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + } + ], + "Contact": "news@fu-berlin.de", + "KeyFingerprint": "2B24B5C78A4861407BCD5A88281D90763F368EA0", + "KeyUrl": "https://ftp.fu-berlin.de/doc/news/bln/bln-key.txt", + "Name": "BLN", + "Url": "https://ftp.fu-berlin.de/doc/news/bln/bln-info.txt" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "BNE", + "Url": "http://bne.news-admin.org/" + }, + { + "Contact": "bofh-control@lists.killfile.org", + "KeyFingerprint": "40B58A561E006152083E38B3CEF569807DC1A266", + "Name": "BOFH" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "CANB", + "Url": "http://canb.news-admin.org/" + }, + { + "Authorizations": [ + { + "From": "felix.rauch@nice.ch", + "Message": "checkgroups", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "newgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "rmgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + } + ], + "Contact": "ch-news-admin@use-net.ch", + "KeyFingerprint": "7180D68CA7DE2C70624A486ED99602DF", + "KeyUrl": "http://www.use-net.ch/Usenet/adminkey.html", + "Name": "CH", + "Url": "http://www.use-net.ch/Usenet/" + }, + { + "Contact": "koordination@cl-netz.de", + "KeyUrl": "http://www.cl-netz.de/control.txt", + "Name": "CL", + "Url": "http://www.cl-netz.de/" + }, + { + "AdminGroup": "clari.net.admin", + "Authorizations": [ + { + "From": "cl*@clarinet.com", + "Message": "rmgroup", + "Newsgroups": "clari.*", + "VerificationIdentity": "ClariNet.Group" + } + ], + "Contact": "support@clari.net", + "KeyUrl": "http://www.clari.net/tech/clarikey.txt", + "Name": "CLARI" + }, + { + "AdminGroup": "cn.announce", + "Authorizations": [ + { + "From": "control@bentium.com", + "Message": "checkgroups", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "newgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "rmgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + } + ], + "KeyFingerprint": "6297EE33F71625C1A49E47BAC53E5E9E", + "Name": "CN", + "Url": "http://news.yaako.com/" + }, + { + "AdminGroup": "cn.bbs.admin.announce", + "Authorizations": [ + { + "From": "control@cn-bbs.org", + "Message": "checkgroups", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "newgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "rmgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + } + ], + "Name": "CN.BBS", + "Url": "http://bbs.cn.news-admin.org/" + }, + { + "AdminGroup": "news.announce.newgroups", + "Authorizations": [ + { + "From": "group-admin@isc.org", + "Message": "checkgroups", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "newgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "rmgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + } + ], + "Contact": "board@big-8.org", + "KeyFingerprint": "8E181BEF0BE638962C1A5C98FAFE7B550C18C8B7", + "Name": "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK", + "Url": "https://www.big-8.org/" + }, + { + "AdminGroup": "de.admin.news.announce", + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "checkgroups", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "newgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Contact": "moderator@dana.de", + "KeyFingerprint": "5BB05288BF55194F667DC2AE16262825", + "KeyUrl": "http://www.dana.de/pgp/dana.txt", + "Name": "DE", + "Url": "http://www.dana.de/" + }, + { + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.alt.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Name": "DE.ALT" + }, + { + "AdminGroup": "demon.news", + "Authorizations": [ + { + "From": "newsmaster@demon.net", + "Message": "checkgroups", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "newgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "rmgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + } + ], + "Contact": "newsmaster@demon.net", + "KeyUrl": "ftp://ftp.demon.co.uk/pub/news/doc/demon.news.txt", + "Name": "DEMON" + }, + { + "Authorizations": [ + { + "From": "news@news.dknet.dk", + "Message": "checkgroups", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "newgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "rmgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + } + ], + "KeyFingerprint": "7CB2C750F37D5D738CEE2E3F558072FF", + "KeyUrl": "http://www.usenet.dk/dk-admin/pubkey.html", + "Name": "DK", + "Url": "http://www.usenet.dk/dk-admin/" + }, + { + "AdminGroup": "easynet.support", + "Authorizations": [ + { + "From": "newsmaster@easynet.net", + "Message": "checkgroups", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "newgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "rmgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + } + ], + "Contact": "Christiaan Keet ", + "Name": "EASYNET" + }, + { + "Authorizations": [ + { + "From": "news@news.ut.ee", + "Message": "checkgroups", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "newgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "rmgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + } + ], + "Contact": "usenet@news.ut.ee", + "KeyUrl": "http://news.ut.ee/pubkey.asc", + "Name": "EE", + "Url": "http://news.ut.ee/" + }, + { + "AdminGroup": "eug.config", + "Authorizations": [ + { + "From": "newsadmin@efn.org", + "Message": "checkgroups", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "newgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "rmgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + } + ], + "Name": "EFN & EUG" + }, + { + "AdminGroup": "england.news.policy", + "Authorizations": [ + { + "From": "admin@england.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "newgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + } + ], + "Contact": "admin@england.news-admin.org", + "KeyFingerprint": "DA3EC20146E561CBA24309CA136D311F", + "Name": "ENGLAND" + }, + { + "AdminGroup": "es.news.anuncios", + "Authorizations": [ + { + "From": "moderador@corus-es.org", + "Message": "checkgroups", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "newgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "rmgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + } + ], + "Contact": "moderador@corus-es.org", + "KeyUrl": "http://www.corus-es.org/docs/esnews.asc", + "Name": "ES", + "Url": "http://www.corus-es.org/docs/es_newsadmins_faq.txt" + }, + { + "Authorizations": [ + { + "From": "mod-ena@ennui.org", + "Message": "checkgroups", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "newgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "rmgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + } + ], + "Contact": "", + "KeyUrl": "http://ennui.org/esp/mod-ena.asc", + "Name": "ESP", + "Url": "http://ennui.org/esp/" + }, + { + "Authorizations": [ + { + "From": "news@eternal-september.org", + "Message": "checkgroups", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "newgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "rmgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + } + ], + "Contact": "news@eternal-september.org", + "KeyFingerprint": "B79A2BC67D5AFF7918E2AC914BC125F1", + "KeyUrl": "http://www.eternal-september.org/control/pgpkey.txt", + "Name": "ETERNAL-SEPTEMBER", + "Url": "http://www.eternal-september.org/" + }, + { + "AdminGroup": "europa.usenet.admin", + "Authorizations": [ + { + "From": "group-admin@usenet.eu.org", + "Message": "checkgroups", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "newgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "rmgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + } + ], + "KeyFingerprint": "3A05A849FB16292575E3DEBB69E01DB4", + "KeyUrl": "http://www.europa.usenet.eu.org/pgp/index.html", + "Name": "EUROPA", + "Url": "http://www.europa.usenet.eu.org/" + }, + { + "AdminGroup": "ffm.admin", + "Authorizations": [ + { + "From": "ffm.admin@arcor.de", + "Message": "checkgroups", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "newgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "rmgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + } + ], + "KeyFingerprint": "53A082626FC781C9CF53AB00A3F8C211", + "KeyUrl": "ftp://ftp.arcor-online.net/pub/news/PGPKEY.FFM", + "Name": "FFM", + "Url": "http://ffm.arcornews.de/" + }, + { + "AdminGroup": "fido.belg.news", + "Authorizations": [ + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "checkgroups", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "newgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "rmgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + } + ], + "Name": "FIDO.BELG" + }, + { + "Authorizations": [ + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + } + ], + "KeyUrl": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/PGP-Key", + "Name": "FIDO.GER", + "Url": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/fido.ger-info.english" + }, + { + "AdminGroup": "fido7.postmasters", + "Authorizations": [ + { + "From": "newgroups-request@fido7.org", + "Message": "checkgroups", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "newgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "rmgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + } + ], + "KeyUrl": "http://www.fido7.org/pgpcontrol.html", + "Name": "FIDO7", + "Url": "http://www.fido7.org/" + }, + { + "AdminGroup": "fj.news.announce", + "Authorizations": [ + { + "From": "committee@fj-news.org", + "Message": "checkgroups", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "newgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "rmgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + } + ], + "Contact": "committee@fj-news.org", + "KeyUrl": "http://www.is.tsukuba.ac.jp/~yas/fj/fj.asc", + "Name": "FJ", + "Url": "http://www.fj-news.org/index.html.en" + }, + { + "AdminGroup": "flora.general", + "Authorizations": [ + { + "From": "news@flora.ottawa.on.ca", + "Message": "checkgroups", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "newgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "rmgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + } + ], + "Contact": "russell@flora.org", + "Name": "FLORA" + }, + { + "AdminGroup": "fr.usenet.forums.annonces", + "Authorizations": [ + { + "From": "control@usenet-fr.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "newgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + } + ], + "KeyFingerprint": "D5F369B297573622015354E7FA423E89970EB10B", + "KeyUrl": "https://www.usenet-fr.net/pgp-fr-2020.txt", + "Name": "FR", + "Url": "https://www.usenet-fr.net/" + }, + { + "AdminGroup": "france.admin.evolutions", + "Authorizations": [ + { + "From": "control@usenet-france.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "newgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + } + ], + "Contact": "control@usenet-france.news.eu.org", + "Name": "FRANCE" + }, + { + "AdminGroup": "gnu.gnusenet.config", + "Authorizations": [ + { + "From": "usenet@gnu.org", + "Message": "checkgroups", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "newgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "rmgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + } + ], + "KeyUrl": "http://www.gnu.org/usenet/usenet-pgp-key.txt", + "Name": "GNU", + "Url": "http://www.gnu.org/usenet/usenet.html" + }, + { + "Authorizations": [ + { + "From": "news@gnuu.de", + "Message": "rmgroup", + "Newsgroups": "gnuu.*", + "VerificationIdentity": "news@gnuu.de" + } + ], + "KeyUrl": "http://www.gnuu.de/config/PGPKEY.GNUU", + "Name": "GNUU", + "Url": "http://www.gnuu.de/" + }, + { + "AdminGroup": "grisbi.admin", + "Authorizations": [ + { + "From": "grisbi-control@grisbi.org", + "Message": "checkgroups", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "newgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "rmgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + } + ], + "Contact": "newsmaster@grisbi.org", + "KeyFingerprint": "EB350C030080BD2A7E0CA4C9F2C62A3DC86CC6E1", + "KeyUrl": "http://news.grisbi.org/public-key.asc", + "Name": "GRISBI", + "Url": "http://news.grisbi.org/" + }, + { + "Authorizations": [ + { + "From": "news@zhaum.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "newgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + } + ], + "Name": "HACKTIC" + }, + { + "AdminGroup": "hamburg.koordination", + "Authorizations": [ + { + "From": "hamburg@steering-group.net", + "Message": "checkgroups", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "newgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "rmgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + } + ], + "Contact": "hamburg@steering-group.net", + "KeyFingerprint": "3EE70CBB6E0194EE456FC557F4B9548E", + "KeyUrl": "http://www.steering-group.net/hamburg/hamburg.koordination.txt", + "Name": "HAMBURG", + "Url": "http://www.steering-group.net/hamburg/" + }, + { + "AdminGroup": "hamster.de.config", + "Authorizations": [ + { + "From": "hamster-control@snafu.de", + "Message": "checkgroups", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "newgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "rmgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + } + ], + "Contact": "hamster-contact@snafu.de", + "KeyFingerprint": "1275A9428AD61F776ACFB40C79155F93", + "Name": "HAMSTER" + }, + { + "AdminGroup": "han.news.admin", + "Authorizations": [ + { + "From": "newgroups-request@usenet.or.kr", + "Message": "checkgroups", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "newgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "rmgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + } + ], + "Contact": "newgroups-request@usenet.or.kr", + "KeyUrl": "ftp://ftp.usenet.or.kr/pub/korea/usenet/pgp/PGPKEY.han", + "Name": "HAN" + }, + { + "AdminGroup": "hiv.config", + "Authorizations": [ + { + "From": "news@hivnet.org", + "Message": "checkgroups", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "newgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "rmgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + } + ], + "Contact": "news@hivnet.org", + "KeyFingerprint": "5DD60EDC1E2DEA0BB0564DD65253D7A4", + "Name": "HIV" + }, + { + "AdminGroup": "houston.usenet.config", + "Authorizations": [ + { + "From": "news@academ.com", + "Message": "checkgroups", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "newgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "rmgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + } + ], + "Name": "HOUSTON" + }, + { + "AdminGroup": "hr.news.admin", + "Authorizations": [ + { + "From": "newsmaster@carnet.hr", + "Message": "checkgroups", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "newgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "rmgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + } + ], + "Contact": "newsmaster@carnet.hr", + "KeyFingerprint": "0EE574FB1C407ADB0AACA52F71921BA3ED63AD9A", + "KeyUrl": "http://newsfeed.carnet.hr/control/key.txt", + "Name": "HR", + "SyncableServer": "news.carnet.hr", + "Url": "http://newsfeed.carnet.hr/control/" + }, + { + "Authorizations": [ + { + "From": "news-admin@humanityquest.com", + "Message": "checkgroups", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "newgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "rmgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + } + ], + "Contact": "news-admin@humanityquest.com", + "KeyFingerprint": "BA3DB306B6F552AABA8F32F08C4F504016F9C046", + "KeyUrl": "http://www.humanityquest.com/projects/newsgroups/PGP.htm", + "Name": "HUMANITYQUEST", + "Url": "http://www.humanityquest.com/projects/newsgroups/" + }, + { + "AdminGroup": "hun.admin.news", + "Authorizations": [ + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "checkgroups", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "newgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "rmgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + } + ], + "KeyUrl": "http://gatling.ikk.sztaki.hu/~kissg/news/hun.admin.news.asc", + "Name": "HUN", + "Url": "http://gatling.ikk.sztaki.hu/~kissg/news/" + }, + { + "AdminGroup": "ie.news.group", + "Authorizations": [ + { + "From": "control@usenet.ie", + "Message": "checkgroups", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "newgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "rmgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + } + ], + "Contact": "control@usenet.ie", + "Name": "IE" + }, + { + "AdminGroup": "is.isnet", + "Authorizations": [ + { + "From": "group-admin@usenet.is", + "Message": "checkgroups", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "newgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "rmgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + } + ], + "Contact": "IS Group Admins ", + "KeyFingerprint": "33328D461E5E1C7F48608E72E53ECAEA", + "KeyUrl": "http://www.usenet.is/group-admin.asc", + "Name": "IS", + "Url": "http://www.usenet.is/" + }, + { + "AdminGroup": "isu.newgroups", + "Authorizations": [ + { + "From": "news@news.isu.edu.tw", + "Message": "checkgroups", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "newgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "rmgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + } + ], + "Contact": "news@news.isu.edu.tw", + "KeyUrl": "http://news.isu.edu.tw/isu.asc", + "Name": "ISU", + "Url": "http://news.isu.edu.tw/" + }, + { + "AdminGroup": "it.news.annunci", + "Authorizations": [ + { + "From": "gcn@news.nic.it", + "Message": "checkgroups", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "newgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "rmgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + } + ], + "Contact": "gcn@news.nic.it", + "KeyFingerprint": "94A4F7B54696D6C7A673F298C48CD0E0", + "KeyUrl": "http://www.news.nic.it/pgp.txt", + "Name": "IT", + "Url": "http://www.news.nic.it/" + }, + { + "AdminGroup": "italia.announce.newgroups", + "Authorizations": [ + { + "From": "news@news.cineca.it", + "Message": "checkgroups", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "newgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "rmgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + } + ], + "Contact": "news@news.cineca.it", + "KeyFingerprint": "0FBB7162DA5D5DB8D586FC2802671A6B", + "KeyUrl": "http://news.cineca.it/italia/italia-pgp.txt", + "Name": "ITALIA", + "Url": "http://news.cineca.it/italia/" + }, + { + "AdminGroup": "japan.admin.announce", + "Authorizations": [ + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "checkgroups", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "newgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "rmgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + } + ], + "Contact": "Tsuneo Tanaka ", + "KeyFingerprint": "6AFA1947691B107438534B1BD8BA3E85", + "KeyUrl": "http://grex.cyberspace.org/~tt/japan.admin.announce.asc", + "Name": "JAPAN", + "Url": "http://www.asahi-net.or.jp/~AE5T-KSN/japan-e.html" + }, + { + "AdminGroup": "jlug.config", + "Authorizations": [ + { + "From": "news@linux.or.jp", + "Message": "checkgroups", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "newgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "rmgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + } + ], + "Contact": "news@linux.or.jp", + "KeyUrl": "http://www.linux.or.jp/pgpkey/news", + "Name": "JLUG", + "Url": "http://www.linux.or.jp/community/news/index.html" + }, + { + "AdminGroup": "ka.admin", + "Contact": "usenet@karlsruhe.org", + "KeyFingerprint": "DE19BB2576198117F067D223E8C87C90", + "KeyUrl": "http://www.karlsruhe.org/pubkey-news.karlsruhe.org.asc", + "Name": "KA", + "Url": "http://www.karlsruhe.org/" + }, + { + "Authorizations": [ + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "checkgroups", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + }, + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "rmgroup", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + } + ], + "Name": "KANTO" + }, + { + "AdminGroup": "kassel.admin", + "Authorizations": [ + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + } + ], + "Name": "KASSEL" + }, + { + "AdminGroup": "linux.admin.news", + "Authorizations": [ + { + "From": "linux-admin@bofh.it", + "Message": "checkgroups", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "newgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "rmgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + } + ], + "Contact": "Marco d'Itri ", + "KeyFingerprint": "81B327994FCE32D11BC9010DBBB32E41", + "Name": "LINUX" + }, + { + "AdminGroup": "malta.config", + "Authorizations": [ + { + "From": "cmeli@cis.um.edu.mt", + "Message": "checkgroups", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "newgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "rmgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + } + ], + "Contact": "cmeli@cis.um.edu.mt", + "KeyFingerprint": "2017015CF0D01A42E41330580B1448A6", + "KeyUrl": "http://www.cis.um.edu.mt/news-malta/PGP.PUBLICKEY", + "Name": "MALTA", + "Url": "http://www.malta.news-admin.org/" + }, + { + "AdminGroup": "maus.info", + "Authorizations": [ + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + } + ], + "KeyFingerprint": "8252C77026B972A1379855983F26623E", + "Name": "MAUS" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "MELB", + "Url": "http://melb.news-admin.org/" + }, + { + "AdminGroup": "mensa.config", + "Authorizations": [ + { + "From": "usenet@newsgate.mensa.org", + "Message": "checkgroups", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "newgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "rmgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + } + ], + "Contact": "usenet@newsgate.mensa.org", + "KeyFingerprint": "52B9396385D908068E197344973C5005DC7DB7A7", + "Name": "MENSA" + }, + { + "AdminGroup": "microsoft.public.news.server", + "Authorizations": [ + { + "From": "control-microsoft@trigofacile.com", + "Message": "checkgroups", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "newgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "rmgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + } + ], + "Contact": "control-microsoft@trigofacile.com", + "KeyFingerprint": "DF705FC9F615D52E02DBA3CB63A98D13E60E2FAA", + "KeyUrl": "http://www.trigofacile.com/divers/usenet/clefs/pgpkey-microsoft.asc", + "Name": "MICROSOFT", + "SyncableServer": "msnews.microsoft.com", + "Url": "http://www.trigofacile.com/divers/usenet/clefs/index.htm" + }, + { + "AdminGroup": "milw.config", + "Authorizations": [ + { + "From": "milw@usenet.mil.wi.us", + "Message": "checkgroups", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "newgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "rmgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + } + ], + "Contact": "milw@usenet.mil.wi.us", + "KeyFingerprint": "6E9B9F7098AB9CE5C3C00582215BF49E", + "KeyUrl": "http://usenet.mil.wi.us/pgpkey", + "Name": "MILW", + "Url": "http://usenet.mil.wi.us/" + }, + { + "AdminGroup": "muc.admin", + "Authorizations": [ + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "checkgroups", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "newgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "rmgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + } + ], + "KeyFingerprint": "43C70E7C45C706E0BD6F76CE07395E66", + "KeyUrl": "http://home.arcor.de/andreas-barth/muc-admin.html", + "Name": "MUC" + }, + { + "AdminGroup": "netscape.public.admin", + "Authorizations": [ + { + "From": "news@netscape.com", + "Message": "checkgroups", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "newgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "rmgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + } + ], + "Contact": "news@netscape.com", + "KeyFingerprint": "B78055121F9C170B8666AD3BDB6835EC", + "Name": "NETSCAPE", + "Url": "http://www.mozilla.org/community.html" + }, + { + "AdminGroup": "nippon.news.group", + "Authorizations": [ + { + "From": "news@gcd.org", + "Message": "rmgroup", + "Newsgroups": "nippon.*", + "VerificationIdentity": "nippon.news.group" + } + ], + "KeyFingerprint": "BCCF15CDB13CDFB3C3DE356F2FF746DB", + "KeyUrl": "http://www.gcd.org/news/nippon/", + "Name": "NIPPON", + "Url": "http://www.gcd.org/news/nippon/" + }, + { + "AdminGroup": "nl.newsgroups", + "Authorizations": [ + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "checkgroups", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "newgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "rmgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + } + ], + "Contact": "nl-admin@nic.surfnet.nl", + "KeyFingerprint": "45200BD5A121EA7CEFB2956C25754D27", + "Name": "NL", + "Url": "http://nl.news-admin.org/info/nladmin.html" + }, + { + "Authorizations": [ + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "news@kink.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + } + ], + "KeyFingerprint": "6B62EB534D5D2F9635D9C89CB0650E4C", + "Name": "NL-ALT" + }, + { + "Authorizations": [ + { + "From": "news@nl.linux.org", + "Message": "checkgroups", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "newgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "rmgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + } + ], + "KeyFingerprint": "63DCB2510AF3DD72C2BDC6FDC1C544CF", + "Name": "NLO", + "SyncableServer": "news.nl.linux.org", + "Url": "http://mail.nl.linux.org/" + }, + { + "AdminGroup": "nntpworld.config", + "Authorizations": [ + { + "From": "usenet@nntpworld.net", + "Message": "checkgroups", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "newgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "rmgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + } + ], + "Contact": "usenet@nntpworld.net", + "KeyFingerprint": "C7BBE0567A8A5A8609B5E329E1BB8CE1", + "KeyUrl": "http://www.nntpworld.net/nntpworld.asc", + "Name": "NNTPWORLD", + "Url": "http://www.nntpworld.net/" + }, + { + "AdminGroup": "no.usenet.admin", + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "checkgroups", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "newgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + } + ], + "KeyUrl": "http://www.usenet.no/pgp-key.txt", + "Name": "NO", + "Url": "http://www.usenet.no/" + }, + { + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.alt.*", + "VerificationIdentity": "no-hir-control" + } + ], + "Name": "NO.ALT" + }, + { + "AdminGroup": "nrw.admin.announce", + "Authorizations": [ + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "checkgroups", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "newgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "rmgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + } + ], + "Contact": "moderator@nrw.usenetverwaltung.de", + "KeyFingerprint": "134A80FED634B464AF32083F620EB1E2", + "KeyUrl": "http://nrw.usenetverwaltung.de/pgp/nrw.asc", + "Name": "NRW", + "Url": "http://nrw.usenetverwaltung.de/" + }, + { + "AdminGroup": "nz.net.announce", + "Authorizations": [ + { + "From": "root@usenet.net.nz", + "Message": "checkgroups", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "newgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "rmgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + } + ], + "Contact": "root@usenet.net.nz", + "KeyFingerprint": "07DF48AAD0EDAA881670C591653D1A28", + "Name": "NZ", + "Url": "http://www.faqs.org/faqs/usenet/nz-news-hierarchy" + }, + { + "Authorizations": [ + { + "From": "news@news2.open-news-network.org", + "Message": "checkgroups", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "newgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "rmgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + } + ], + "Name": "OPENNEWS", + "Url": "http://www.open-news-network.org/" + }, + { + "AdminGroup": "openwatcom.contributors", + "Authorizations": [ + { + "From": "admin@openwatcom.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "newgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + } + ], + "Contact": "admin@openwatcom.news-admin.org", + "KeyUrl": "http://cmeerw.org/files/openwatcom/pgp-openwatcom.asc", + "Name": "OPENWATCOM", + "SyncableServer": "news.openwatcom.org", + "Url": "http://www.openwatcom.org/" + }, + { + "Authorizations": [ + { + "From": "*@opera.com", + "Message": "checkgroups", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "newgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "rmgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + } + ], + "Contact": "usenet@opera.com", + "Name": "OPERA", + "SyncableServer": "news.opera.com" + }, + { + "Authorizations": [ + { + "From": "postmaster@upb.de", + "Message": "checkgroups", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "newgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "rmgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + } + ], + "Contact": "news@uni-paderborn.de", + "Name": "PBINFO" + }, + { + "Authorizations": [ + { + "From": "newsadmin@perl.org", + "Message": "checkgroups", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "newgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "rmgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + } + ], + "Contact": "newsadmin@perl.org", + "KeyFingerprint": "438FD1BA4DCC3B1ABED82BCC32988A7D8B2ACFBB", + "KeyUrl": "http://www.nntp.perl.org/about/newsadmin@perl.org.pgp", + "Name": "PERL", + "Url": "http://www.nntp.perl.org/about/" + }, + { + "AdminGroup": "pgh.config", + "Authorizations": [ + { + "From": "pgh-config@psc.edu", + "Message": "checkgroups", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "newgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "rmgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + } + ], + "Name": "PGH" + }, + { + "Authorizations": [ + { + "From": "news@postgresql.org", + "Message": "checkgroups", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "newgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "rmgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + } + ], + "Contact": "news@postgresql.org", + "KeyUrl": "http://news.hub.org/gpg_public_keys.html", + "Name": "PGSQL", + "Url": "http://news.hub.org/gpg_public_keys.html" + }, + { + "AdminGroup": "pl.news.admin", + "Authorizations": [ + { + "From": "michalj@*fuw.edu.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + } + ], + "KeyUrl": "http://www.usenet.pl/doc/news-pl-new-site-faq.html#pgp", + "Name": "PL", + "Url": "http://www.usenet.pl/doc/news-pl-new-site-faq.html" + }, + { + "AdminGroup": "pt.internet.usenet", + "Authorizations": [ + { + "From": "pmelo@*.inescc.pt", + "Message": "checkgroups", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "newgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "rmgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + } + ], + "KeyUrl": "http://www.usenet-pt.org/control@usenet-pt.org.asc", + "Name": "PT", + "Url": "http://www.usenet-pt.org/" + }, + { + "AdminGroup": "relcom.netnews", + "Authorizations": [ + { + "From": "coord@*.relcom.ru", + "Message": "checkgroups", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "newgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "rmgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + } + ], + "KeyUrl": "ftp://ftp.relcom.ru/pub/relcom/netinfo/coordpubkey.txt", + "Name": "RELCOM", + "Url": "ftp://ftp.relcom.ru/pub/relcom/netinfo/" + }, + { + "AdminGroup": "rocksolid.shared.rocksolid", + "Authorizations": [ + { + "From": "usenet@novabbs.com", + "Message": "checkgroups", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "newgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "rmgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + } + ], + "Contact": "usenet@novabbs.com", + "KeyFingerprint": "FF846D730B72CC0CB69B64802A1C655BF6D10B13", + "KeyUrl": "http://news.novabbs.com/hierarchy/rocksolid.txt", + "Name": "ROCKSOLID", + "Url": "http://news.novabbs.com/hierarchy/" + }, + { + "AdminGroup": "saar.admin.news", + "Authorizations": [ + { + "From": "control@saar-admin-news.de", + "Message": "checkgroups", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "newgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "rmgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + } + ], + "KeyUrl": "http://www.saar-admin-news.de/saar-control.asc", + "Name": "SAAR", + "Url": "http://www.saar-admin-news.de/" + }, + { + "AdminGroup": "sat.usenet.config", + "Authorizations": [ + { + "From": "satgroup@endicor.com", + "Message": "checkgroups", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "newgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "rmgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + } + ], + "Contact": "satgroup@endicor.com", + "Name": "SAT" + }, + { + "AdminGroup": "schule.admin", + "Authorizations": [ + { + "From": "newsctrl@schule.de", + "Message": "checkgroups", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "newgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "rmgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + } + ], + "Contact": "schule-admin@roxel.ms.sub.org", + "KeyFingerprint": "6406F0AEE146850CBDCA0E538B1E73D2", + "KeyUrl": "http://www.afaik.de/usenet/admin/schule/control/schule.asc", + "Name": "SCHULE", + "Url": "http://home.pages.de/~schule-admin/" + }, + { + "AdminGroup": "scot.newsgroups.discuss", + "Authorizations": [ + { + "From": "control@scot.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "newgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + } + ], + "KeyUrl": "http://scot.news-admin.org/signature.html", + "Name": "SCOT", + "Url": "http://scot.news-admin.org/" + }, + { + "AdminGroup": "scout.admin", + "Authorizations": [ + { + "From": "control@news.scoutnet.org", + "Message": "checkgroups", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "newgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "rmgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + } + ], + "KeyUrl": "http://news.scoutnet.org/scout-pgpkey.asc", + "Name": "SCOUT", + "Url": "http://news.scoutnet.org/" + }, + { + "AdminGroup": "se.internet.news.meddelanden", + "Authorizations": [ + { + "From": "usenet@usenet-se.net", + "Message": "checkgroups", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "newgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "rmgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + } + ], + "Contact": "usenet@usenet-se.net", + "KeyFingerprint": "6803F0FD0CC34E696F0D0C603C586396", + "Name": "SE" + }, + { + "AdminGroup": "sfnet.ryhmat+listat", + "Authorizations": [ + { + "From": "sfnet@*cs.tut.fi", + "Message": "checkgroups", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "newgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "rmgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + } + ], + "Contact": "sfnet@cs.tut.fi", + "KeyFingerprint": "DE7933C2D359D12844E56A0CB6E30E536933A636", + "Name": "SFNET", + "Url": "http://www.cs.tut.fi/sfnet/" + }, + { + "AdminGroup": "si.news.announce.newsgroups", + "Authorizations": [ + { + "From": "news-admin@arnes.si", + "Message": "checkgroups", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "newgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "rmgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + } + ], + "KeyUrl": "http://www.arnes.si/news/config/", + "Name": "SI", + "Url": "http://www.arnes.si/news/config/" + }, + { + "AdminGroup": "staroffice.admin", + "Authorizations": [ + { + "From": "news@stardivision.de", + "Message": "checkgroups", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "newgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "rmgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + } + ], + "Contact": "news@starnews.sun.com", + "KeyFingerprint": "C63E816F2A19D3847251F91BE3B9B2C9", + "Name": "STAROFFICE", + "SyncableServer": "starnews.sun.com" + }, + { + "AdminGroup": "stgt.net", + "Authorizations": [ + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "checkgroups", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "newgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "rmgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + } + ], + "Contact": "news@news.uni-stuttgart.de", + "KeyFingerprint": "BAA40A548FF5F51E724851AE0951DE44", + "KeyUrl": "http://news.uni-stuttgart.de/hierarchie/stgt/stgt-control.txt", + "Name": "STGT", + "Url": "http://news.uni-stuttgart.de/hierarchie/stgt/" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "SYD", + "Url": "http://syd.news-admin.org/" + }, + { + "AdminGroup": "szaf.admin", + "Authorizations": [ + { + "From": "hirtenrat@szaf.org", + "Message": "rmgroup", + "Newsgroups": "szaf.*", + "VerificationIdentity": "Hirtenrat" + } + ], + "Contact": "hirtenrat@szaf.org", + "KeyUrl": "http://news.szaf.org/szaf/szaf-key.txt", + "Name": "SZAF" + }, + { + "AdminGroup": "thur.net.news.groups", + "Authorizations": [ + { + "From": "usenet@thur.de", + "Message": "checkgroups", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "newgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "rmgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + } + ], + "Contact": "usenet@thur.de", + "KeyFingerprint": "7E3D731393D4CA7839DE3CE737EE22F1", + "Name": "THUR", + "SyncableServer": "news.thur.de", + "Url": "http://www.thur.de/thurnet/old/thurnews.html" + }, + { + "Authorizations": [ + { + "From": "usenet@agate.berkeley.edu", + "Message": "checkgroups", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "newgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "rmgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + } + ], + "Contact": "Chris van den Berg ", + "KeyFingerprint": "96B88E9A9809377D0EEC8188DB9029BF", + "KeyUrl": "http://www.net.berkeley.edu/usenet/usenet.asc", + "Name": "UCB", + "Url": "http://www.net.berkeley.edu/usenet/" + }, + { + "AdminGroup": "ufra.admin", + "KeyFingerprint": "F7AD96D87A3F7E84020C839ADB8FEBB8", + "Name": "UFRA" + }, + { + "AdminGroup": "uk.net.news.announce", + "Authorizations": [ + { + "From": "control@usenet.org.uk", + "Message": "checkgroups", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "newgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "rmgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + } + ], + "KeyUrl": "http://www.usenet.org.uk/newsadmins.html", + "Name": "UK", + "Url": "http://www.usenet.org.uk/" + }, + { + "AdminGroup": "un.public.usenet.admin", + "Authorizations": [ + { + "From": "news@news.itu.int", + "Message": "checkgroups", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "newgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "rmgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + } + ], + "Name": "UN" + }, + { + "AdminGroup": "us.config", + "Authorizations": [ + { + "From": "us-control@lists.killfile.org", + "Message": "checkgroups", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "newgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "rmgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + } + ], + "Contact": "us-control@lists.killfile.org", + "KeyFingerprint": "BB96EB2CCFD075C8E9DEC2C21DA29D87B73CAF1B", + "Name": "US" + }, + { + "AdminGroup": "uw.newsgroups", + "Authorizations": [ + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "checkgroups", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "newgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "rmgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + } + ], + "Name": "UW", + "SyncableServer": "news.uwaterloo.ca" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "WA", + "Url": "http://wa.news-admin.org/" + }, + { + "AdminGroup": "wales.usenet.config", + "Authorizations": [ + { + "From": "control@wales-usenet.org", + "Message": "checkgroups", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "newgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "rmgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + } + ], + "Contact": "committee@wales-usenet.org", + "KeyFingerprint": "2D9EDEDF12DA345C49E1EE28E3AB0DAD", + "KeyUrl": "http://www.wales-usenet.org/english/newsadmin.txt", + "Name": "WALES", + "Url": "http://www.wales-usenet.org/" + }, + { + "AdminGroup": "x-privat.info", + "Authorizations": [ + { + "From": "dmitry@x-privat.org", + "Message": "checkgroups", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "newgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "rmgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + } + ], + "Contact": "dmitry@x-privat.org", + "KeyFingerprint": "9B0A7E682780C796476B039051056843", + "KeyUrl": "http://www.x-privat.org/dmitry.asc", + "Name": "X-PRIVAT", + "Url": "http://www.x-privat.org/" + }, + { + "AdminGroup": "z-netz.koordination.user+sysops", + "Authorizations": [ + { + "From": "teko@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + } + ], + "Contact": "teko@dinoex.sub.org", + "KeyUrl": "ftp://ftp.dinoex.de/pub/keys/z-netz.koordination.user+sysops.asc", + "Name": "Z-NETZ" + } + ], + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl", + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl" + } + } + }, + "Nntpd": { + "AcmeStateDir": "certs/", + "AllowCleartextAuth": true, + "BindAddress": [ + "198.18.0.66", + "2c0f:f030:1442:501:198:18:0:66" + ], + "BindPort": 1199, + "BindPortTls": 5633, + "GracefulShutdownTimeout": "00:01:00", + "HistoryTime": "02:00:00", + "IdleTime": 300, + "LogDir": "logs/", + "MailComplaintsTo": "abuse@usenet.ninja", + "MaxArticleSize": 5242880, + "ProxyHosts": [], + "ServerId": 1, + "StartupTimeout": null, + "StopHostOnUnexpectedServiceTermination": true, + "Top1000": [ + "top1000@anthologeek.net", + "cknipe@opticnetworks.net" + ], + "Transit": { + "StreamOutstandingArticleDepth": 8 + }, + "Systemd": { + "EnableWatchdog": true, + "NotifyReadyOnApplicationRunning": true, + "NotifyStoppingOnApplicationShutdown": true, + "ReportLifecycleStatus": true, + "WatchdogIntervalFraction": 0.5 + }, + "ArticleIngestion": { + "MaxArticleBytes": 4194304, + "MinWorkers": 2, + "MaxWorkers": 32, + "MaxPublishConcurrency": 32, + "OverviewDbWorkQueueMemoryLimit": 1073741824, + "OverviewDbMinPublisherWorkers": 2, + "OverviewDbMaxPublisherWorkers": 32, + "OverviewDbPublisherBatchSize": 100, + "OverviewDbPublisherShutdownSeconds": 2, + "ScaleIntervalSeconds": 2, + "ScaleUpPressureThreshold": 0.4, + "ScaleDownPressureThreshold": 0.1, + "ScaleUpConsecutiveIntervals": 2, + "ScaleDownConsecutiveIntervals": 3 + }, + "TransitQueueMemoryLimit": 4294967296 + }, + "NntpDb": { + "StartupTimeout": "00:00:15" + }, + "Redis": { + "Host": [ "198.18.0.70" ], + "Port": 6379 + }, + "Serilog": { + "Enrich": [ "FromLogContext" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information", + "System": "Warning", + "VectorNNTPD.TransitService": "Debug" + } + }, + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File", "Serilog.Sinks.Async" ], + "WriteTo": [ + { + "Args": { + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "restrictedToMinimumLevel": "Information" + }, + "Name": "Console" + }, + { + "Args": { + "blockWhenFull": true, + "bufferSize": 50000, + "configure": [ + { + "Args": { + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "path": "logs/pending-.log", + "restrictedToMinimumLevel": "Debug", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Name": "File" + } + ] + }, + "Name": "Async" + } + ], + "News": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/news-.log", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Inpaths": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/inpaths-.log", + "retainedFileCountLimit": 1, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs index 4f44bd5a..b2ec10e1 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs @@ -92,8 +92,13 @@ public void Disabled_profiler_records_nothing() [Fact] public void Enabled_profiler_tracks_active_and_stages() { + // Install only a throwaway instance for IsEnabled. Ambient pipeline hooks from + // parallel tests must not mutate the profiler whose Active/Record counts we assert. + WithInstalled( + new ArticleWorkStageProfiler(enabled: true), + static () => Assert.True(ArticleWorkStageProfiler.IsEnabled)); + var profiler = new ArticleWorkStageProfiler(enabled: true); - WithInstalled(profiler, static () => Assert.True(ArticleWorkStageProfiler.IsEnabled)); profiler.BeginActive(); Assert.Equal(1, profiler.Active); var started = System.Diagnostics.Stopwatch.GetTimestamp(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs index d57e24e2..cf59e53a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs @@ -140,15 +140,25 @@ public void Environment_example_requires_only_the_nntpdb_connection_string_place }); } - /// Windows release checks reject the Linux environment-file example. + /// + /// Publish-Release and CI require each Linux package to include its unit and environment-file + /// example, and reject those Linux-only files from non-Linux (including win-x64) packages. + /// [Fact] public void Windows_release_validation_rejects_the_linux_environment_example() { var script = File.ReadAllText(RepositoryFiles.Under("tools", "Publish-Release.ps1")); var workflow = File.ReadAllText(RepositoryFiles.Under(".github", "workflows", "build.yml")); - Assert.Contains("Missing environment-file example in the linux-x64 release", script, StringComparison.Ordinal); - Assert.Contains("Environment-file example must not be packaged for ${Rid}", script, StringComparison.Ordinal); + // linux-x64: package must ship the role's unit + env example (and README). + Assert.Contains("Missing Linux package file in", script, StringComparison.Ordinal); + Assert.Contains("LinuxEnvExample", script, StringComparison.Ordinal); + Assert.Contains("backfiller.env.example", script, StringComparison.Ordinal); + + // Non-linux RIDs: Linux unit/env must not appear in the package. + Assert.Contains("Linux-only file must not be packaged for ${Rid}", script, StringComparison.Ordinal); + + // Windows CI package validation forbids shipping the Linux install files. Assert.Contains( "'vectornntp-backfiller.service', 'backfiller.env.example'", workflow, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs index fca35075..e388bbee 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -9,7 +9,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress { /// - /// End-to-end stress orchestrator: Article Work publish → RabbitMQ reply → TLS ARTICLE delivery. + /// End-to-end stress orchestrator: Article Work publish → RabbitMQ reply → TLS ARTICLE delivery. /// /// /// The RabbitMQ reply is routing metadata only. Article payload bytes are measured exclusively @@ -202,6 +202,7 @@ await File.WriteAllTextAsync( var disposeReplyClientOnFailure = true; try + // codeql[cs/complex-block] Stress harness orchestration keeps preflight/publish steps inline for report clarity. { // Live retrieval preflight runs after the reply consumer is ready and before any throughput publish. if (_options.ArticlePreflightCount > 0) diff --git a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs index 7a3d70b8..af384a73 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs @@ -329,7 +329,7 @@ public void Materialize_PreservesHeaderSeparatorStyleAndBodyBytesAcrossSupported "From: user@example.test", "Subject: separator-variant", ], - $"body-{separator.Length}x{((int)separator[0]).ToString()}\r", + $"body-{separator.Length}x{(int)separator[0]}\r", separator); var parse = parser.Parse(originalArticle); diff --git a/tests/VectorNNTPD.Common.Tests/Articles/YEnc/YEncHardwareCrcEquivalenceTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/YEnc/YEncHardwareCrcEquivalenceTests.cs index 3e27d735..1ead893b 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/YEnc/YEncHardwareCrcEquivalenceTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/YEnc/YEncHardwareCrcEquivalenceTests.cs @@ -173,9 +173,8 @@ private static byte[] Encode(byte[] decoded) { var output = new List(decoded.Length + (decoded.Length / 32)); var lineCount = 0; - foreach (var value in decoded) + foreach (var encoded in decoded.Select(static value => unchecked((byte)(value + 42)))) { - var encoded = unchecked((byte)(value + 42)); if (encoded is 0 or 9 or 10 or 13 or 32 or 46 or 61) { output.Add((byte)'='); diff --git a/tests/VectorNNTPD.Common.Tests/Cloudflare/CloudflareJsonContractTests.cs b/tests/VectorNNTPD.Common.Tests/Cloudflare/CloudflareJsonContractTests.cs index 8a876244..03e8a707 100644 --- a/tests/VectorNNTPD.Common.Tests/Cloudflare/CloudflareJsonContractTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Cloudflare/CloudflareJsonContractTests.cs @@ -356,6 +356,7 @@ protected override Task SendAsync( { RequestCount++; var payload = _payloads.Dequeue(); + // codeql[cs/local-not-disposed] HttpResponseMessage ownership transfers to HttpClient; the caller disposes the response. return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(payload, Encoding.UTF8, "application/json"), diff --git a/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationJsonConfigurationTests.cs b/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationJsonConfigurationTests.cs index b2969f4e..8280c24f 100644 --- a/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationJsonConfigurationTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationJsonConfigurationTests.cs @@ -255,11 +255,13 @@ private static void TryDelete(string path) { Directory.Delete(path, recursive: true); } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } } diff --git a/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationLocalPathResolutionTests.cs b/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationLocalPathResolutionTests.cs index 2170d215..38ac62ea 100644 --- a/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationLocalPathResolutionTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Configuration/ApplicationLocalPathResolutionTests.cs @@ -304,8 +304,9 @@ private static void TryDelete(string path) { Directory.Delete(path, recursive: true); } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } diff --git a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs index c187ba3a..9fd72198 100644 --- a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs @@ -89,8 +89,9 @@ public void Shipped_sample_binds_and_present_host_primary_json_omits_the_section { root.Delete(recursive: true); } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } } @@ -141,8 +142,9 @@ public void Environment_variable_and_command_line_override_RabbitMq_json() { root.Delete(recursive: true); } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } } diff --git a/tests/VectorNNTPD.Common.Tests/Logging/GzipRollingFileSinkTests.cs b/tests/VectorNNTPD.Common.Tests/Logging/GzipRollingFileSinkTests.cs index 73d9161a..bde223ff 100644 --- a/tests/VectorNNTPD.Common.Tests/Logging/GzipRollingFileSinkTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Logging/GzipRollingFileSinkTests.cs @@ -376,8 +376,9 @@ public void Dispose() { Directory.Delete(Path, recursive: true); } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } } diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientPropertiesTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientPropertiesTests.cs index fa960e33..06791f34 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientPropertiesTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientPropertiesTests.cs @@ -121,8 +121,8 @@ public void ResolveClientMachineName_PreservesReportedHost() /// The string value stored at . private static string RequireString(IDictionary properties, string key) { - Assert.True(properties.ContainsKey(key)); - return Assert.IsType(properties[key]); + Assert.True(properties.TryGetValue(key, out var value)); + return Assert.IsType(value); } } } diff --git a/tests/VectorNNTPD.Common.Tests/TestDoubles/CommonTestSupport.cs b/tests/VectorNNTPD.Common.Tests/TestDoubles/CommonTestSupport.cs index 9a1ce177..252a5755 100644 --- a/tests/VectorNNTPD.Common.Tests/TestDoubles/CommonTestSupport.cs +++ b/tests/VectorNNTPD.Common.Tests/TestDoubles/CommonTestSupport.cs @@ -40,11 +40,13 @@ public void Dispose() Directory.Delete(Path, recursive: true); } } - catch (IOException) + catch (IOException ex) { + _ = ex; // observed; best-effort temp directory cleanup } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) { + _ = ex; // observed; best-effort temp directory cleanup } } } diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteDurableStoreTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteDurableStoreTests.cs new file mode 100644 index 00000000..acfd0b01 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteDurableStoreTests.cs @@ -0,0 +1,123 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteDurableStoreTests +{ + [Fact] + public void ConsumeResult_RejectsNegativeValues() + { + Assert.Throws(() => + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining: -1, consumed: 0)); + Assert.Throws(() => + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining: 0, consumed: -1)); + } + + [Theory] + [InlineData(1000, 100, 900, 100)] + [InlineData(1000, 1000, 0, 1000)] + [InlineData(1000, 1500, 0, 1000)] + [InlineData(0, 50, 0, 0)] + public async Task InMemory_Consume_ClampsAtZero( + long start, + long bytes, + long remaining, + long consumed) + { + var store = new InMemoryAccountByteDurableStore(); + store.SeedByteAccount("alice", start); + var result = await store.ConsumeAsync("alice", bytes); + Assert.Equal(AccountByteConsumeStatus.Consumed, result.Status); + Assert.Equal(remaining, result.Remaining); + Assert.Equal(consumed, result.Consumed); + Assert.True(result.Remaining >= 0); + Assert.True(result.Consumed >= 0); + } + + [Fact] + public async Task InMemory_Consume_RejectsNegativeBytes() + { + var store = new InMemoryAccountByteDurableStore(); + store.SeedByteAccount("alice", 10); + await Assert.ThrowsAsync(() => store.ConsumeAsync("alice", -1).AsTask()); + } + + [Fact] + public async Task InMemory_MissingAccount_DoesNotConsume() + { + var store = new InMemoryAccountByteDurableStore(); + store.SeedByteAccount("rate", 100); + var consumed = await store.ConsumeAsync("rate", 10); + Assert.Equal(AccountByteConsumeStatus.Consumed, consumed.Status); + Assert.Equal(90, consumed.Remaining); + Assert.Equal(AccountByteConsumeStatus.AccountNotFound, (await store.ConsumeAsync("ghost", 10)).Status); + Assert.Equal(AccountByteConsumeStatus.Consumed, (await store.QueryRemainingAsync("rate")).Status); + Assert.Equal(AccountByteConsumeStatus.AccountNotFound, (await store.QueryRemainingAsync("ghost")).Status); + } + + [Fact] + public async Task InMemory_ConcurrentDecrements_NeverNegative() + { + var store = new InMemoryAccountByteDurableStore(); + store.SeedByteAccount("alice", 1000); + await Task.WhenAll(Enumerable.Range(0, 20).Select(_ => store.ConsumeAsync("alice", 80).AsTask())); + Assert.Equal(0, store.Remaining("alice")); + } + + [Fact] + public async Task FakeNntpDb_ConsumeAndQuery_MatchInMemorySemantics() + { + var factory = new FakeNntpDbConnectionFactory(); + factory.Users["alice"] = MemoryNntpUserRecordStore.Create("alice", "pw", byteLimit: 1000); + factory.Users["rate"] = MemoryNntpUserRecordStore.Create("rate", "pw", rateLimitBps: 2_400, byteLimit: 50); + await using var connection = (FakeNntpDbConnection)await factory.OpenAsync("Server=x;Database=y", CancellationToken.None); + + var consumed = await connection.ConsumeAccountBytesAsync("alice", 100, CancellationToken.None); + Assert.Equal(900, consumed.Remaining); + Assert.Equal(100, consumed.Consumed); + Assert.Equal(900, (await connection.QueryAccountByteRemainingAsync("alice", CancellationToken.None)).Remaining); + + var over = await connection.ConsumeAccountBytesAsync("alice", 5000, CancellationToken.None); + Assert.Equal(0, over.Remaining); + Assert.Equal(900, over.Consumed); + + var zero = await connection.ConsumeAccountBytesAsync("alice", 10, CancellationToken.None); + Assert.Equal(0, zero.Remaining); + Assert.Equal(0, zero.Consumed); + + var rate = await connection.ConsumeAccountBytesAsync("rate", 1, CancellationToken.None); + Assert.Equal(AccountByteConsumeStatus.Consumed, rate.Status); + Assert.Equal(49, rate.Remaining); + Assert.Equal( + AccountByteConsumeStatus.AccountNotFound, + (await connection.ConsumeAccountBytesAsync("ghost", 1, CancellationToken.None)).Status); + } + + [Theory] + [InlineData(-5L, 0L)] + [InlineData(0L, 0L)] + [InlineData(10L, 10L)] + public void ConvertByteLimit_NeverNegative(long input, long expected) + { + Assert.Equal(expected, MySqlNntpDbConnection.ConvertByteLimit(input)); + } + + [Fact] + public void ConvertByteLimit_SaturatesUnsignedOverflow() + { + Assert.Equal(long.MaxValue, MySqlNntpDbConnection.ConvertByteLimit(ulong.MaxValue)); + Assert.Equal(0, MySqlNntpDbConnection.ConvertByteLimit(null)); + } + + [Fact] + public void ConsumeSql_UsesCaseClamp_NotUncheckedSubtract() + { + Assert.Contains("WHEN account_byte_limit > @bytes THEN account_byte_limit - @bytes", NntpUserQueries.ConsumeAccountBytes, StringComparison.Ordinal); + Assert.Contains("ELSE 0 END", NntpUserQueries.ConsumeAccountBytes, StringComparison.Ordinal); + Assert.DoesNotContain("account_type", NntpUserQueries.ConsumeAccountBytes, StringComparison.Ordinal); + Assert.DoesNotContain("account_byte_limit = account_byte_limit - @bytes", NntpUserQueries.ConsumeAccountBytes, StringComparison.Ordinal); + Assert.Contains("FOR UPDATE", NntpUserQueries.SelectByteQuotaForUpdate, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteEngineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteEngineTests.cs new file mode 100644 index 00000000..b46e82b8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteEngineTests.cs @@ -0,0 +1,167 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteEngineTests +{ + [Fact] + public void Apply_MissingKey_InitializesFromMysqlAfter_WithoutSubtractingConsumed() + { + var engine = new AccountByteEngine(); + Assert.Equal(900, engine.Apply("k", "batch-a", consumed: 100, mysqlRemainingAfter: 900)); + Assert.Equal(900, engine.Observe("k")); + Assert.True(engine.WasApplied("k", "batch-a")); + } + + [Theory] + [InlineData(1000, 100, 900, 900)] + [InlineData(1000, 1000, 0, 0)] + [InlineData(1000, 1500, 0, 0)] + [InlineData(0, 50, 0, 0)] + public void Apply_ExistingKey_FloorsToMysqlAndNeverGoesNegative( + long start, + long consumed, + long mysqlAfter, + long expected) + { + var engine = new AccountByteEngine(); + engine.Write("k", start); + Assert.Equal(expected, engine.Apply("k", "b1", consumed, mysqlAfter)); + Assert.True(engine.Observe("k") >= 0); + } + + [Fact] + public void Apply_StaleHighRedis_FloorsToMysqlRemaining() + { + var engine = new AccountByteEngine(); + engine.Write("k", 8_000); + Assert.Equal(100, engine.Apply("k", "b1", consumed: 100, mysqlRemainingAfter: 100)); + } + + [Fact] + public void Apply_RedisBelowMysql_NeverIncreases() + { + var engine = new AccountByteEngine(); + engine.Write("k", 50); + Assert.Equal(50, engine.Apply("k", "b1", consumed: 10, mysqlRemainingAfter: 200)); + } + + [Fact] + public void Apply_SameBatchTwice_DoesNotChangeRemaining() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + } + + [Fact] + public void Apply_SameBatchTenTimes_DoesNotChangeRemaining() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + for (var i = 0; i < 9; i++) + { + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + } + } + + [Fact] + public void Apply_BatchAThenB_FloorsToLowestMysqlAfter() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(900, engine.Apply("k", "A", 100, 900)); + Assert.Equal(800, engine.Apply("k", "B", 100, 800)); + } + + [Fact] + public void Apply_BatchBThenA_DoesNotRaiseAndDoesNotDoubleCount() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(800, engine.Apply("k", "B", 100, 800)); + Assert.Equal(800, engine.Apply("k", "A", 100, 900)); + } + + [Fact] + public void Apply_DuplicateAfterOppositeOrder_Unchanged() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(800, engine.Apply("k", "B", 100, 800)); + Assert.Equal(800, engine.Apply("k", "A", 100, 900)); + Assert.Equal(800, engine.Apply("k", "A", 100, 900)); + Assert.Equal(800, engine.Apply("k", "B", 100, 800)); + } + + [Fact] + public void Apply_MissingKey_RepeatedSameBatch_StaysAtMysqlAfter() + { + var engine = new AccountByteEngine(); + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + Assert.Equal(900, engine.Apply("k", "batch-a", 100, 900)); + Assert.Equal(900, engine.Observe("k")); + } + + [Fact] + public void Apply_NegativeInputs_ClampToZero() + { + var engine = new AccountByteEngine(); + Assert.Equal(0, engine.Apply("k", "b1", consumed: -5, mysqlRemainingAfter: -9)); + Assert.Equal(0, engine.Observe("k")); + } + + [Fact] + public void Apply_RequiresBatchId() + { + var engine = new AccountByteEngine(); + Assert.Throws(() => engine.Apply("k", "", 1, 1)); + Assert.Throws(() => engine.Apply("k", " ", 1, 1)); + } + + [Fact] + public void Observe_Missing_ReturnsSentinel() + { + var engine = new AccountByteEngine(); + Assert.Equal(AccountByteKeys.Missing, engine.Observe("missing")); + } + + [Fact] + public void ConcurrentApply_NeverNegativeAndNeverIncreases() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1_000); + Parallel.For(0, 20, i => engine.Apply("k", "n" + i, 30, mysqlRemainingAfter: 400)); + Assert.Equal(400, engine.Observe("k")); + } + + [Fact] + public void Apply_EvictedBatchMark_RetryStillDoesNotChangeRemaining() + { + var engine = new AccountByteEngine(); + engine.Write("k", 10_000); + Assert.Equal(9_000, engine.Apply("k", "oldest", 1_000, 9_000)); + var remaining = 9_000L; + for (var i = 0; i < AccountByteBatchId.MaxRetainedMarks; i++) + { + remaining -= 1; + Assert.Equal(remaining, engine.Apply("k", "n" + i, 1, remaining)); + } + + Assert.False(engine.WasApplied("k", "oldest")); + Assert.Equal(remaining, engine.Apply("k", "oldest", 1_000, 9_000)); + Assert.Equal(remaining, engine.Observe("k")); + } + + [Fact] + public void Keys_AreAccountSha256_WithoutTtlNamespace() + { + var key = System.Text.Encoding.ASCII.GetString(AccountByteKeys.Create("alice")); + Assert.StartsWith("nntpd:bytes:", key, StringComparison.Ordinal); + Assert.Equal(12 + 64, key.Length); + Assert.NotEqual(AccountByteKeys.Create("alice"), AccountByteKeys.Create("bob")); + Assert.Equal(AccountByteKeys.Create("alice"), AccountByteKeys.Create("alice")); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteExhaustionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteExhaustionTests.cs new file mode 100644 index 00000000..fbb47840 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteExhaustionTests.cs @@ -0,0 +1,429 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteExhaustionTests +{ + [Fact] + public async Task Authinfo_ZeroRemaining_Still281_NextCommand400AndClose() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 0); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + await using var duplex = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 0), tracker); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.True(tracker.IsExhausted("alice")); + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(run.IsCompleted); + } + + [Fact] + public async Task Authinfo_StaleCachedPositiveByteLimit_CannotResurrectExhaustedAccount() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 0); + await cluster.ApplyAsync("alice", "seed", 0, 0); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + await using var duplex = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 99_999), tracker); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(tracker.IsExhausted("alice")); + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Authinfo_TopUpWithoutInvalidation_Stays400_UntilRedisDeleted() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 0); + await cluster.ApplyAsync("alice", "seed", 0, 0); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + + await using (var duplex = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 0), tracker)) + { + var session = duplex.CreateSession(); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("DATE"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + durable.SeedByteAccount("alice", 1000); + await using (var stillExhausted = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 1000), tracker)) + { + var session = stillExhausted.CreateSession(); + var run = session.RunAsync(); + await stillExhausted.ReadGreetingAsync(); + await stillExhausted.WriteClientLineAsync("AUTHINFO USER alice"); + await stillExhausted.ReadClientLineAsync(); + await stillExhausted.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await stillExhausted.ReadClientLineAsync(), StringComparison.Ordinal); + await stillExhausted.WriteClientLineAsync("DATE"); + Assert.Equal("400 Service temporarily unavailable", await stillExhausted.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + await using var reenabled = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 1000), tracker); + var live = reenabled.CreateSession(); + var liveRun = live.RunAsync(); + await reenabled.ReadGreetingAsync(); + await reenabled.WriteClientLineAsync("AUTHINFO USER alice"); + await reenabled.ReadClientLineAsync(); + await reenabled.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await reenabled.ReadClientLineAsync(), StringComparison.Ordinal); + await reenabled.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await reenabled.ReadClientLineAsync(), StringComparison.Ordinal); + await reenabled.WriteClientLineAsync("QUIT"); + await reenabled.ReadClientLineAsync(); + await liveRun; + } + + [Fact] + public async Task ExactZeroAfterReconcile_RejectsNextCommandOnAllLocalSessions() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 100); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + tracker.CreateSink("alice").ObserveCopied(100); + await tracker.ReconcileAsync(); + Assert.True(tracker.IsExhausted("alice")); + Assert.Equal(0, durable.Remaining("alice")); + + await using var duplex = new ExhaustionDuplex(CreateProvider("alice", byteLimit: 100), tracker); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("DATE"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task InFlightResponse_IsNotTruncated_WhenExhaustionIsMarkedAfterWriteStarts() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 10_000); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + writer.SetByteSink(tracker.CreateSink("alice")); + await writer.WriteMultilineStartAsync(100, "Help text follows"); + tracker.MarkExhausted("alice"); + await writer.WriteMultilineDataAsync("line-one"); + await writer.WriteMultilineEndAsync(); + var expected = "100 Help text follows\r\nline-one\r\n.\r\n"; + var buffer = new byte[expected.Length]; + var read = 0; + while (read < expected.Length) + { + var result = await pipe.Reader.ReadAsync(); + var slice = result.Buffer.Slice(0, Math.Min(result.Buffer.Length, expected.Length - read)); + slice.CopyTo(buffer.AsSpan(read, (int)slice.Length)); + read += (int)slice.Length; + pipe.Reader.AdvanceTo(slice.End); + } + + Assert.Equal(expected, Encoding.ASCII.GetString(buffer)); + Assert.True(tracker.IsExhausted("alice")); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task Authinfo_ZeroBytes_2400Bps_AttachesBoth_WithoutWriteOnObserve() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 0); + var bytes = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var rates = new AccountRateAllocator(); + var admission = new InMemorySessionStateTracker(rates); + var cap = new FakeCap(); + await using var duplex = new ExhaustionDuplex( + CreateProvider("alice", byteLimit: 0, rateLimitBps: 2_400), + bytes); + var session = duplex.CreateSession(admission, rates, cap); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.True(bytes.IsExhausted("alice")); + Assert.Equal(2_400, session.AccountPolicy!.RateLimitBps); + Assert.Equal(300, cap.MaxSendBytesPerSecond); + Assert.Equal(0, cluster.ApplyCalls); + Assert.Equal(0, durable.ConsumeCalls); + Assert.True(cluster.ObserveCalls >= 1); + Assert.True(durable.QueryCalls >= 1); + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(300, cap.MaxSendBytesPerSecond); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task ZeroBytes_PositiveRate_Still281Then400() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 0); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + await using var duplex = new ExhaustionDuplex( + CreateProvider("alice", byteLimit: 0, rateLimitBps: 10_000_000), + tracker); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER alice"); + await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(tracker.IsExhausted("alice")); + Assert.Equal(10_000_000, session.AccountPolicy!.RateLimitBps); + await duplex.WriteClientLineAsync("DATE"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static INntpAuthenticationProvider CreateProvider( + string user, + long byteLimit, + int rateLimitBps = 0) + { + var record = MemoryNntpUserRecordStore.Create(user, "secret", rateLimitBps: rateLimitBps, byteLimit: byteLimit); + var policy = NntpAccountPolicy.FromRecord(record); + return new ScriptedProvider(user, policy); + } + + private sealed class ScriptedProvider : INntpAuthenticationProvider + { + private readonly string _user; + private readonly NntpAccountPolicy _policy; + + public ScriptedProvider(string user, NntpAccountPolicy policy) + { + _user = user; + _policy = policy; + } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + if (username == _user && password == "secret") + { + return ValueTask.FromResult(NntpAuthenticationResult.Success( + username, + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false), + _policy)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class ExhaustionDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly INntpAuthenticationProvider _provider; + private readonly IAccountByteAccountant _accountant; + + public ExhaustionDuplex(INntpAuthenticationProvider provider, IAccountByteAccountant accountant) + { + _provider = provider; + _accountant = accountant; + } + + public NntpSession CreateSession( + ISessionStateTracker? admission = null, + IAccountRateAllocator? rates = null, + IOutboundRateCap? outboundRate = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + outboundRate); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: _provider, + sessionAdmission: admission, + accountBytes: _accountant, + accountRates: rates); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("201 ", line, StringComparison.Ordinal); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection( + PipeReader input, + PipeWriter output, + ConnectionClientIdentity identity, + IOutboundRateCap? outboundRate = null) + { + Input = input; + Output = output; + ClientIdentity = identity; + OutboundRate = outboundRate; + } + + public IOutboundRateCap? OutboundRate { get; } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class FakeCap : IOutboundRateCap + { + public long MaxSendBytesPerSecond { get; private set; } + + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => MaxSendBytesPerSecond = bytesPerSecond; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteIdempotencyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteIdempotencyTests.cs new file mode 100644 index 00000000..53f91263 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteIdempotencyTests.cs @@ -0,0 +1,127 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +/// +/// Production + script dispatch (FakeRedis → engine). +/// These cases must stay aligned with live Lua EVAL tests. +/// +public sealed class AccountByteIdempotencyTests +{ + [Fact] + public async Task SameBatch_AppliedTwice_RedisUnchanged() + { + var store = CreateStore(); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task SameBatch_AppliedTenTimes_RedisUnchanged() + { + var store = CreateStore(); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + for (var i = 0; i < 9; i++) + { + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + } + + Assert.Equal(900, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task BatchAThenB_FloorsToDurable800() + { + var store = CreateStore(); + Assert.Equal(900, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task BatchBThenA_DoesNotExceedDurable800() + { + var store = CreateStore(); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task ADuplicatedAfterB_Unchanged() + { + var store = CreateStore(); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task BDuplicatedAfterA_Unchanged() + { + var store = CreateStore(); + Assert.Equal(900, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task AmbiguousResult_RetrySameBatch_DoesNotConsumeTwice() + { + var redis = new FakeRedisService(); + var store = new RedisAccountByteStore(redis); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task MissingKey_RepeatedSameBatch_InitializesOnce() + { + var store = CreateStore(); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task RedisBelowMysql_NeverRepairedUpward() + { + var redis = new FakeRedisService(); + var store = new RedisAccountByteStore(redis); + var key = System.Text.Encoding.UTF8.GetString(AccountByteKeys.Create("alice")); + redis.Database.AccountByteEngine.Write(key, 800); + Assert.Equal(800, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task RedisAboveMysql_FloorsDown() + { + var redis = new FakeRedisService(); + var store = new RedisAccountByteStore(redis); + var key = System.Text.Encoding.UTF8.GetString(AccountByteKeys.Create("alice")); + redis.Database.AccountByteEngine.Write(key, 50_000); + Assert.Equal(900, await store.ApplyAsync("alice", "batch-a", 100, 900)); + Assert.Equal(900, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task ConcurrentNodes_WithDuplicateRetries_FinalIsMinMysqlAfter() + { + var redis = new FakeRedisService(); + var store = new RedisAccountByteStore(redis); + await Task.WhenAll( + store.ApplyAsync("alice", "A", 100, 900).AsTask(), + store.ApplyAsync("alice", "B", 100, 800).AsTask(), + store.ApplyAsync("alice", "A", 100, 900).AsTask(), + store.ApplyAsync("alice", "B", 100, 800).AsTask()); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + private static RedisAccountByteStore CreateStore() => new(new FakeRedisService()); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteInvariantTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteInvariantTests.cs new file mode 100644 index 00000000..b227fa8d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteInvariantTests.cs @@ -0,0 +1,212 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +/// +/// Production-contract locks for the thirteen AccountBytes invariants. +/// +public sealed class AccountByteInvariantTests +{ + [Fact] + public async Task Mysql_NeverGoesBelowZero() + { + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 100); + var first = await durable.ConsumeAsync("alice", 150); + Assert.Equal(0, first.Remaining); + Assert.Equal(100, first.Consumed); + var second = await durable.ConsumeAsync("alice", 50); + Assert.Equal(0, second.Remaining); + Assert.Equal(0, second.Consumed); + Assert.Equal(0, durable.Remaining("alice")); + } + + [Fact] + public void Redis_NeverGoesBelowZero() + { + var engine = new AccountByteEngine(); + Assert.Equal(0, engine.Apply("k", "b1", consumed: 50, mysqlRemainingAfter: -9)); + engine.Write("k", 10); + Assert.Equal(0, engine.Apply("k", "b2", consumed: 1_000, mysqlRemainingAfter: 0)); + Assert.True(engine.Observe("k") >= 0); + } + + [Fact] + public void RedisApply_NeverIncreasesRemaining() + { + var engine = new AccountByteEngine(); + engine.Write("k", 50); + Assert.Equal(50, engine.Apply("k", "b1", consumed: 10, mysqlRemainingAfter: 800)); + Assert.Equal(50, engine.Apply("k", "b1", consumed: 10, mysqlRemainingAfter: 800)); + } + + [Fact] + public void RedisApply_DoesNotDecreaseBecauseOfConsumed() + { + var engine = new AccountByteEngine(); + engine.Write("k", 1000); + Assert.Equal(900, engine.Apply("k", "batch-a", consumed: 100, mysqlRemainingAfter: 900)); + Assert.Equal(900, engine.Apply("k", "batch-a", consumed: 100, mysqlRemainingAfter: 900)); + Assert.Equal(900, engine.Apply("k", "other", consumed: 500, mysqlRemainingAfter: 900)); + } + + [Fact] + public void EvictedBatchMark_ReplayIsHarmless() + { + var engine = new AccountByteEngine(); + engine.Write("k", 10_000); + Assert.Equal(9_000, engine.Apply("k", "oldest", 1_000, 9_000)); + var remaining = 9_000L; + for (var i = 0; i < AccountByteBatchId.MaxRetainedMarks; i++) + { + remaining -= 1; + Assert.Equal(remaining, engine.Apply("k", "n" + i, 1, remaining)); + } + + Assert.False(engine.WasApplied("k", "oldest")); + Assert.Equal(remaining, engine.Apply("k", "oldest", 1_000, 9_000)); + } + + [Fact] + public async Task OutOfOrderNodes_CannotIncreaseEffectiveQuota() + { + var store = new RedisAccountByteStore(new VectorNNTPD.ReaderService.Tests.TestDoubles.FakeRedisService()); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task MissingRedisKey_CannotRestoreAboveMysqlRemaining() + { + var store = new RedisAccountByteStore(new VectorNNTPD.ReaderService.Tests.TestDoubles.FakeRedisService()); + Assert.Equal(AccountByteKeys.Missing, await store.ObserveAsync("alice")); + Assert.Equal(400, await store.ApplyAsync("alice", "batch-a", 100, 400)); + Assert.Equal(400, await store.ApplyAsync("alice", "batch-a", 100, 400)); + Assert.Equal(400, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task Crash_NeverReplaysCommittedMysqlBatch() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 1000); + var first = new AccountByteTracker(durable, cluster, NullLogger.Instance); + first.CreateSink("alice").ObserveCopied(100); + cluster.Unavailable = true; + await first.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(1, durable.ConsumeCalls); + + var restarted = new AccountByteTracker(durable, cluster, NullLogger.Instance); + cluster.Unavailable = false; + await restarted.ReconcileAsync(); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(AccountByteKeys.Missing, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task Crash_MayLosePending_NeverDoubleCharges() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 1000); + var first = new AccountByteTracker(durable, cluster, NullLogger.Instance); + first.CreateSink("alice").ObserveCopied(250); + Assert.Equal(250, first.PendingBytes("alice")); + Assert.Equal(0, durable.ConsumeCalls); + + var restarted = new AccountByteTracker(durable, cluster, NullLogger.Instance); + await restarted.ReconcileAsync(); + Assert.Equal(0, durable.ConsumeCalls); + Assert.Equal(1000, durable.Remaining("alice")); + } + + [Fact] + public async Task MysqlTopUp_HiddenWhileRedisKeyExists() + { + var (tracker, durable, cluster) = CreateTracker(); + durable.SeedByteAccount("alice", 800); + await cluster.ApplyAsync("alice", "seed", 0, 800); + Assert.Equal(800, await tracker.ObserveRemainingAsync("alice")); + durable.SeedByteAccount("alice", 9_000); + Assert.Equal(800, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task MysqlTopUp_VisibleOnlyAfterRedisKeyIsGone() + { + var (tracker, durable, cluster) = CreateTracker(); + durable.SeedByteAccount("alice", 800); + await cluster.ApplyAsync("alice", "seed", 0, 800); + durable.SeedByteAccount("alice", 9_000); + cluster.Engine.Delete(InMemoryAccountByteStore.EncodingKey("alice")); + Assert.Equal(9_000, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task StaleHighRedis_IsFlooredOnObserve_SoLaterTopUpStaysHidden() + { + var (tracker, durable, cluster) = CreateTracker(); + durable.SeedByteAccount("alice", 0); + await cluster.ApplyAsync("alice", "stale", 0, 1_000); + Assert.Equal(0, await tracker.ObserveRemainingAsync("alice")); + Assert.Equal(0, await cluster.ObserveAsync("alice")); + durable.SeedByteAccount("alice", 5_000); + Assert.Equal(0, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public void Writer_HasNoStoreIo() + { + var source = File.ReadAllText( + Path.Join(FindRepositoryRoot(), "src", "VectorNNTPD.ReaderService", "Session", "CommandProcessor", "NntpResponseWriter.cs")); + Assert.Contains("_output.Advance(toCopy);", source, StringComparison.Ordinal); + Assert.Contains("ObserveCopied(toCopy)", source, StringComparison.Ordinal); + Assert.DoesNotContain("IAccountByteStore", source, StringComparison.Ordinal); + Assert.DoesNotContain("INntpDbConnection", source, StringComparison.Ordinal); + Assert.DoesNotContain("ScriptEvaluate", source, StringComparison.Ordinal); + Assert.DoesNotContain("ConsumeAsync", source, StringComparison.Ordinal); + Assert.DoesNotContain("QueryRemainingAsync", source, StringComparison.Ordinal); + } + + [Fact] + public void SessionAttach_DoesNotUseCachedByteLimitForExhaustion() + { + var source = File.ReadAllText( + Path.Join(FindRepositoryRoot(), "src", "VectorNNTPD.ReaderService", "Session", "NntpSession.cs")); + Assert.Contains("ObserveRemainingAsync(policy.Username", source, StringComparison.Ordinal); + Assert.Contains("if (remaining is <= 0)", source, StringComparison.Ordinal); + Assert.Contains("ClearExhausted(policy.Username)", source, StringComparison.Ordinal); + Assert.DoesNotContain("policy.ByteLimit < 0", source, StringComparison.Ordinal); + Assert.DoesNotContain("policy.ByteLimit <= 0", source, StringComparison.Ordinal); + } + + private static (AccountByteTracker Tracker, InMemoryAccountByteDurableStore Durable, InMemoryAccountByteStore Cluster) CreateTracker() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + return (tracker, durable, cluster); + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Join(directory.FullName, "VectorNNTPD.slnx"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Repository root was not found."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteLuaStaticAuditTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteLuaStaticAuditTests.cs new file mode 100644 index 00000000..329bfe15 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteLuaStaticAuditTests.cs @@ -0,0 +1,214 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteLuaStaticAuditTests +{ + [Fact] + public async Task ProductionStore_DispatchesExactAccountByteScripts() + { + var redis = new RecordingRedis(); + var store = new RedisAccountByteStore(redis); + _ = await store.ApplyAsync("alice", "batch-a", 10, 90); + Assert.Same(AccountByteScripts.Apply, redis.LastScript); + _ = await store.ObserveAsync("alice"); + Assert.Same(AccountByteScripts.Observe, redis.LastScript); + _ = await store.DeleteAsync("alice"); + Assert.Same(AccountByteScripts.Delete, redis.LastScript); + } + + [Fact] + public void Scripts_NeverSetTtl_NeverIncrement_AndMarkBatchIds() + { + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("EXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PEXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PERSIST", script, StringComparison.Ordinal); + Assert.DoesNotContain("INCR", script, StringComparison.Ordinal); + Assert.DoesNotContain("INCRBY", script, StringComparison.Ordinal); + Assert.DoesNotContain("EVAL", script, StringComparison.Ordinal); + } + + Assert.Contains("HEXISTS", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.Contains("b:' .. batchId", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.Contains("if next > mysql then", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.Contains("HLEN", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.Contains("HDEL", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.DoesNotContain("current - consumed", AccountByteScripts.Apply, StringComparison.Ordinal); + Assert.Contains("HGET", AccountByteScripts.Observe, StringComparison.Ordinal); + Assert.Contains("return -1", AccountByteScripts.Observe, StringComparison.Ordinal); + Assert.DoesNotContain("HSET", AccountByteScripts.Observe, StringComparison.Ordinal); + Assert.Contains("DEL", AccountByteScripts.Delete, StringComparison.Ordinal); + Assert.DoesNotContain("HSET", AccountByteScripts.Delete, StringComparison.Ordinal); + Assert.DoesNotContain("INCR", AccountByteScripts.Delete, StringComparison.Ordinal); + } + + [Fact] + public void Scripts_UseOnlyHashCommands() + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "EXISTS", "HGET", "HSET", "HEXISTS", "HLEN", "HKEYS", "HDEL", "DEL", + }; + foreach (var script in AllScripts()) + { + foreach (var command in EnumerateRedisCalls(script)) + { + Assert.Contains(command, allowed); + } + } + } + + [Fact] + public async Task FakeRedis_DispatchesAccountByteScriptsToEngine() + { + var redis = new FakeRedisService(); + var store = new RedisAccountByteStore(redis); + Assert.Equal(900, await store.ApplyAsync("alice", "A", 100, 900)); + Assert.Equal(800, await store.ApplyAsync("alice", "B", 100, 800)); + Assert.Equal(800, await store.ObserveAsync("alice")); + redis.Database.AccountByteEngine.Write( + System.Text.Encoding.UTF8.GetString(AccountByteKeys.Create("alice")), + 50); + Assert.Equal(50, await store.ApplyAsync("alice", "C", 10, 200)); + } + + [Fact] + public void Engine_AndLua_ShareMissingSentinel() + { + Assert.Equal(-1, AccountByteKeys.Missing); + Assert.Contains("return -1", AccountByteScripts.Observe, StringComparison.Ordinal); + } + + [Fact] + public void Writer_CountsOnlyAfterAdvance_AndNeverTouchesStores() + { + var source = File.ReadAllText( + Path.Join(FindRepositoryRoot(), "src", "VectorNNTPD.ReaderService", "Session", "CommandProcessor", "NntpResponseWriter.cs")); + Assert.Contains("_output.Advance(toCopy);", source, StringComparison.Ordinal); + Assert.Contains("ObserveCopied(toCopy)", source, StringComparison.Ordinal); + Assert.DoesNotContain("ObserveCopied(request.Payload.Length)", source, StringComparison.Ordinal); + Assert.DoesNotContain("IAccountByteStore", source, StringComparison.Ordinal); + Assert.DoesNotContain("INntpDbConnection", source, StringComparison.Ordinal); + Assert.DoesNotContain("Deflate", source, StringComparison.Ordinal); + Assert.DoesNotContain("ScriptEvaluate", source, StringComparison.Ordinal); + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Join(directory.FullName, "VectorNNTPD.slnx"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Repository root was not found."); + } + + private static IEnumerable AllScripts() + { + yield return AccountByteScripts.Apply; + yield return AccountByteScripts.Observe; + yield return AccountByteScripts.Delete; + } + + private static IEnumerable EnumerateRedisCalls(string script) + { + const string prefix = "redis.call('"; + var start = 0; + while (true) + { + var at = script.IndexOf(prefix, start, StringComparison.Ordinal); + if (at < 0) + { + yield break; + } + + var commandStart = at + prefix.Length; + var commandEnd = script.IndexOf('\'', commandStart); + Assert.True(commandEnd > commandStart); + yield return script[commandStart..commandEnd]; + start = commandEnd + 1; + } + } + + private sealed class RecordingRedis : IRedisService + { + public string? LastScript { get; private set; } + + public IRedisDatabase Database => new RecordingDatabase(this); + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + private sealed class RecordingDatabase : IRedisDatabase + { + private readonly RecordingRedis _owner; + + public RecordingDatabase(RecordingRedis owner) + { + _owner = owner; + } + + public Task PingAsync(CancellationToken cancellationToken = default) => + Task.FromResult(TimeSpan.FromMilliseconds(1)); + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + _owner.LastScript = script; + return ValueTask.FromResult(0L); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisIdempotencyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisIdempotencyTests.cs new file mode 100644 index 00000000..e26140ad --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisIdempotencyTests.cs @@ -0,0 +1,145 @@ +using StackExchange.Redis; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +/// +/// Live EVAL of production for retry +/// and concurrent-node idempotency. Not a fake engine path. +/// +public sealed class AccountByteRedisIdempotencyTests : IClassFixture +{ + private readonly AccountBytesRedisIntegrationFixture _redis; + + public AccountByteRedisIdempotencyTests(AccountBytesRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public async Task SameBatch_AppliedTwice_RedisUnchanged() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(RedisType.Hash, await _redis.KeyTypeAsync(account)); + Assert.True(await _redis.BatchMarkExistsAsync(account, "batch-a")); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(900, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task SameBatch_AppliedTenTimes_RedisUnchanged() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + for (var i = 0; i < 9; i++) + { + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + } + + Assert.Equal(900, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task BatchAThenB_FloorsToDurable800() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, "A", 100, 900)); + Assert.Equal(800, await _redis.ApplyAsync(account, "B", 100, 800)); + Assert.Equal(800, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task BatchBThenA_DoesNotExceedDurable800() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(800, await _redis.ApplyAsync(account, "B", 100, 800)); + Assert.Equal(800, await _redis.ApplyAsync(account, "A", 100, 900)); + Assert.Equal(800, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ADuplicatedAfterB_Unchanged() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(800, await _redis.ApplyAsync(account, "B", 100, 800)); + Assert.Equal(800, await _redis.ApplyAsync(account, "A", 100, 900)); + Assert.Equal(800, await _redis.ApplyAsync(account, "A", 100, 900)); + Assert.Equal(800, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task BDuplicatedAfterA_Unchanged() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, "A", 100, 900)); + Assert.Equal(800, await _redis.ApplyAsync(account, "B", 100, 800)); + Assert.Equal(800, await _redis.ApplyAsync(account, "B", 100, 800)); + Assert.Equal(800, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task AmbiguousResult_RetrySameBatch_DoesNotConsumeTwice() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, "lost-reply", 100, 900)); + Assert.Equal(900, await _redis.ApplyAsync(account, "lost-reply", 100, 900)); + Assert.True(await _redis.BatchMarkExistsAsync(account, "lost-reply")); + Assert.Equal(900, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task MissingKey_RepeatedSameBatch_InitializesOnce() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(900, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task RedisBelowMysql_NeverRepairedUpward() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedAsync(account, 800); + Assert.Equal(800, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(800, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task RedisAboveMysql_FloorsDown() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedAsync(account, 50_000); + Assert.Equal(900, await _redis.ApplyAsync(account, "batch-a", 100, 900)); + Assert.Equal(900, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ConcurrentNodes_WithDuplicateRetries_FinalIsMinMysqlAfter() + { + var account = await _redis.CreateAccountAsync(); + await Task.WhenAll( + _redis.ApplyAsync(account, "A", 100, 900).AsTask(), + _redis.ApplyAsync(account, "B", 100, 800).AsTask(), + _redis.ApplyAsync(account, "A", 100, 900).AsTask(), + _redis.ApplyAsync(account, "B", 100, 800).AsTask()); + Assert.Equal(800, await _redis.ObserveAsync(account)); + Assert.True(await _redis.BatchMarkExistsAsync(account, "A")); + Assert.True(await _redis.BatchMarkExistsAsync(account, "B")); + await _redis.DeleteKeyAsync(account); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisLuaIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisLuaIntegrationTests.cs new file mode 100644 index 00000000..8c46bb3a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisLuaIntegrationTests.cs @@ -0,0 +1,97 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +/// +/// Live EVAL of against the dedicated test Redis. +/// +public sealed class AccountByteRedisLuaIntegrationTests : IClassFixture +{ + private readonly AccountBytesRedisIntegrationFixture _redis; + + public AccountByteRedisLuaIntegrationTests(AccountBytesRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public void ConnectsToAuthorizedTestRedis() + { + Assert.True(_redis.IsConfigured); + Assert.Equal(SessionStateRedisIntegration.AuthorizedEndpoint, _redis.Endpoint); + Assert.False(string.IsNullOrWhiteSpace(_redis.ServerVersion)); + } + + [SessionStateRedisIntegrationFact] + public async Task Apply_MissingKey_InitializesFromMysqlAfter() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(900, await _redis.ApplyAsync(account, 100, 900)); + Assert.Equal(900, await _redis.ObserveAsync(account)); + Assert.Null(await _redis.KeyTimeToLiveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Apply_ConcurrentNodes_NeverIncreaseOrGoNegative() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(8_000, await _redis.ApplyAsync(account, 0, 8_000)); + var first = _redis.ApplyAsync(account, 300, 7_700); + var second = _redis.ApplyAsync(account, 500, 7_200); + await Task.WhenAll(first.AsTask(), second.AsTask()); + var remaining = await _redis.ObserveAsync(account); + Assert.True(remaining is >= 0 and <= 7_200); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Apply_StaleHigh_FloorsToMysql() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedAsync(account, 50_000); + Assert.Equal(100, await _redis.ApplyAsync(account, 100, 100)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Apply_RedisAhead_DoesNotRestoreQuota() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedAsync(account, 40); + Assert.Equal(40, await _redis.ApplyAsync(account, 10, 800)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Observe_Missing_ReturnsSentinel_AndDoesNotCreate() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Apply_Zero_StaysZero() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(0, await _redis.ApplyAsync(account, 10, 0)); + Assert.Equal(0, await _redis.ApplyAsync(account, 25, 0)); + Assert.Equal(0, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task MissingKeyAfterDelete_ReinitializesFromDurableRemaining() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(500, await _redis.ApplyAsync(account, 100, 500)); + await _redis.DeleteWithoutFixtureTrackingAsync(account); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(400, await _redis.ApplyAsync(account, 100, 400)); + await _redis.DeleteKeyAsync(account); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisTopUpTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisTopUpTests.cs new file mode 100644 index 00000000..f1e5d40d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteRedisTopUpTests.cs @@ -0,0 +1,72 @@ +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +/// Live production Lua DELETE / APPLY / OBSERVE for quota top-up invalidation. +public sealed class AccountByteRedisTopUpTests : IClassFixture +{ + private readonly AccountBytesRedisIntegrationFixture _redis; + + public AccountByteRedisTopUpTests(AccountBytesRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public async Task TopUpWithoutDelete_EffectiveStaysAtRedis() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(50, await _redis.ApplyAsync(account, "seed", 50, 50)); + Assert.Equal(50, await _redis.ApplyAsync(account, "top", 0, 1000)); + Assert.Equal(50, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task DeleteThenObserve_IsMissing_ApplyInitializesToMysql() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(50, await _redis.ApplyAsync(account, "seed", 50, 50)); + Assert.Equal(1, await _redis.DeleteAsync(account)); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(1000, await _redis.ApplyAsync(account, "after", 0, 1000)); + Assert.Equal(1000, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ExhaustedRedisZero_TopUpMysql_StaysZeroUntilDelete() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(0, await _redis.ApplyAsync(account, "seed", 0, 0)); + Assert.Equal(0, await _redis.ApplyAsync(account, "top", 0, 1000)); + Assert.Equal(0, await _redis.ObserveAsync(account)); + Assert.Equal(1, await _redis.DeleteAsync(account)); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(1000, await _redis.ApplyAsync(account, "init", 0, 1000)); + Assert.Equal(1000, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task MissingKey_ApplyNeverInitializesAboveMysql() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(400, await _redis.ApplyAsync(account, "init", 100, 400)); + Assert.Equal(400, await _redis.ObserveAsync(account)); + await _redis.DeleteKeyAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task DeleteIsIdempotent_AndDoesNotCreate() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(0, await _redis.DeleteAsync(account)); + Assert.Equal(AccountByteKeys.Missing, await _redis.ObserveAsync(account)); + Assert.Equal(0, await _redis.DeleteAsync(account)); + await _redis.DeleteKeyAsync(account); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteServiceTests.cs new file mode 100644 index 00000000..1d8071d4 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteServiceTests.cs @@ -0,0 +1,89 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteServiceTests +{ + [Fact] + public async Task SessionStateService_ReconcilesBytesOnInterval_ThenStopDoesFinalReconcile() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 1000); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + tracker.CreateSink("alice").ObserveCopied(25); + var clock = new ControllableTimeProvider(); + var service = new SessionStateService( + new NoOpLeaseManager(), + tracker, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + + await service.StartAsync(CancellationToken.None); + Assert.Equal("SessionState", service.Name); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (durable.ConsumeCalls < 1) + { + wait.Token.ThrowIfCancellationRequested(); + if (!clock.HasScheduledTimers) + { + await Task.Yield(); + continue; + } + + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + } + + Assert.Equal(975, durable.Remaining("alice")); + tracker.CreateSink("alice").ObserveCopied(25); + await service.StopAsync(CancellationToken.None); + Assert.Equal(950, durable.Remaining("alice")); + Assert.True(durable.ConsumeCalls >= 2); + } + + [Fact] + public async Task SessionStateService_StartIsIdempotent() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var service = new SessionStateService( + new NoOpLeaseManager(), + tracker, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, durable.ConsumeCalls); + } + + [Fact] + public void Defaults_ShareSessionStateTenSecondCycle() + { + Assert.Equal(SessionStateDefaults.RenewalPeriod, AccountByteDefaults.ReconciliationPeriod); + Assert.Equal(TimeSpan.FromSeconds(10), AccountByteDefaults.ReconciliationPeriod); + } + + [Fact] + public void AccountByteService_DoesNotExist() + { + var assembly = typeof(SessionStateService).Assembly; + Assert.Null(assembly.GetType("VectorNNTPD.ReaderService.AccountBytes.AccountByteService")); + Assert.Null(assembly.GetType("VectorNNTPD.ReaderService.SessionState.BytesAccounting.AccountByteService")); + Assert.Null(assembly.GetType("VectorNNTPD.ReaderService.SessionState.AccountByteService")); + } + + private sealed class NoOpLeaseManager : ISessionStateLeaseManager + { + public ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTopUpTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTopUpTests.cs new file mode 100644 index 00000000..31506041 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTopUpTests.cs @@ -0,0 +1,131 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteTopUpTests +{ + [Fact] + public async Task TopUpWithoutInvalidation_KeepsEffectiveAtRedis() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 100); + await cluster.ApplyAsync("alice", "seed", 0, 50); + Assert.Equal(50, await tracker.ObserveRemainingAsync("alice")); + durable.SeedByteAccount("alice", 1000); + Assert.Equal(50, await tracker.ObserveRemainingAsync("alice")); + Assert.Equal(50, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task DeleteThenObserve_ExposesDurableTopUp() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 100); + await cluster.ApplyAsync("alice", "seed", 0, 50); + durable.SeedByteAccount("alice", 1000); + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + Assert.Equal(AccountByteKeys.Missing, await cluster.ObserveAsync("alice")); + Assert.Equal(1000, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task DeleteThenApply_InitializesRedisToCurrentMysql() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 100); + await cluster.ApplyAsync("alice", "seed", 0, 50); + durable.SeedByteAccount("alice", 1000); + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + Assert.Equal(1000, await cluster.ApplyAsync("alice", "after-topup", 0, 1000)); + Assert.Equal(1000, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task ConsumptionAfterTopUpInvalidation_BehavesNormally() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 100); + await cluster.ApplyAsync("alice", "seed", 0, 50); + durable.SeedByteAccount("alice", 1000); + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + tracker.CreateSink("alice").ObserveCopied(100); + await tracker.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + Assert.Equal(900, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task StaleRedisAfterTopUp_NeverIncreasesEffectiveBeforeInvalidation() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 100); + await cluster.ApplyAsync("alice", "seed", 0, 50); + durable.SeedByteAccount("alice", 1000); + Assert.Equal(50, await tracker.ObserveRemainingAsync("alice")); + Assert.Equal(50, await cluster.ApplyAsync("alice", "later", 10, 1000)); + Assert.Equal(50, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task ExplicitInvalidationThenObserve_ExposesNewDurableQuota() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 100); + await tracker.DeleteAccountByteStateAsync("alice"); + durable.SeedByteAccount("alice", 1000); + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + Assert.Equal(1000, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task MissingRedis_NeverInitializesAboveCurrentMysql() + { + var store = new InMemoryAccountByteStore(); + Assert.Equal(AccountByteKeys.Missing, await store.ObserveAsync("alice")); + Assert.Equal(400, await store.ApplyAsync("alice", "init", 100, 400)); + Assert.Equal(400, await store.ObserveAsync("alice")); + } + + [Fact] + public async Task ExhaustedAccount_TopUpStaysZeroUntilInvalidation() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 0); + await cluster.ApplyAsync("alice", "seed", 0, 0); + tracker.MarkExhausted("alice"); + Assert.Equal(0, await tracker.ObserveRemainingAsync("alice")); + Assert.True(tracker.IsExhausted("alice")); + + durable.SeedByteAccount("alice", 1000); + Assert.Equal(0, await tracker.ObserveRemainingAsync("alice")); + Assert.True(tracker.IsExhausted("alice")); + + Assert.True(await tracker.DeleteAccountByteStateAsync("alice")); + Assert.False(tracker.IsExhausted("alice")); + Assert.Equal(1000, await tracker.ObserveRemainingAsync("alice")); + Assert.Equal(AccountByteKeys.Missing, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task DeleteWhenRedisUnavailable_DoesNotClearExhausted() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 0); + await cluster.ApplyAsync("alice", "seed", 0, 0); + tracker.MarkExhausted("alice"); + cluster.Unavailable = true; + Assert.False(await tracker.DeleteAccountByteStateAsync("alice")); + Assert.True(tracker.IsExhausted("alice")); + Assert.Equal(0, cluster.Engine.Observe(InMemoryAccountByteStore.EncodingKey("alice"))); + } + + private static (AccountByteTracker Tracker, InMemoryAccountByteDurableStore Durable, InMemoryAccountByteStore Cluster) Create() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + return (tracker, durable, cluster); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTrackerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTrackerTests.cs new file mode 100644 index 00000000..fe50262e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteTrackerTests.cs @@ -0,0 +1,347 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteTrackerTests +{ + [Fact] + public void Types_UseSessionStateBytesAccountingNamespace() + { + Assert.Equal("VectorNNTPD.ReaderService.SessionState.BytesAccounting", typeof(IAccountByteAccountant).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState.BytesAccounting", typeof(AccountByteTracker).Namespace); + Assert.StartsWith("VectorNNTPD.ReaderService.SessionState", typeof(AccountByteTracker).Namespace); + Assert.Null(typeof(SessionStateService).Assembly.GetType("VectorNNTPD.ReaderService.AccountBytes.AccountByteService")); + } + + [Fact] + public void OneSession_AccumulatesPending_WithoutStoreIo() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 10_000); + tracker.CreateSink("alice").ObserveCopied(100); + tracker.CreateSink("alice").ObserveCopied(50); + Assert.Equal(150, tracker.PendingBytes("alice")); + Assert.Equal(0, durable.ConsumeCalls); + Assert.Equal(0, cluster.ApplyCalls); + } + + [Fact] + public void MultipleSessionsSameAccount_SharePending() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 10_000); + tracker.CreateSink("alice").ObserveCopied(10); + tracker.CreateSink("alice").ObserveCopied(20); + Assert.Equal(30, tracker.PendingBytes("alice")); + } + + [Fact] + public void MultipleAccounts_AreIndependent() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 10_000); + durable.SeedByteAccount("bob", 10_000); + tracker.CreateSink("alice").ObserveCopied(7); + tracker.CreateSink("bob").ObserveCopied(9); + Assert.Equal(7, tracker.PendingBytes("alice")); + Assert.Equal(9, tracker.PendingBytes("bob")); + } + + [Fact] + public void ConcurrentWriters_SumExactly() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 1_000_000); + var sink = tracker.CreateSink("alice"); + Parallel.For(0, 100, _ => sink.ObserveCopied(3)); + Assert.Equal(300, tracker.PendingBytes("alice")); + } + + [Fact] + public void EmptySinkName_IsIgnored() + { + var (tracker, _, _) = Create(); + tracker.CreateSink("").ObserveCopied(50); + tracker.CreateSink(" ").ObserveCopied(50); + Assert.Equal(0, tracker.PendingBytes("")); + } + + [Fact] + public async Task EmptyBatch_DoesNoStoreWork() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + await tracker.ReconcileAsync(); + Assert.Equal(0, durable.ConsumeCalls); + Assert.Equal(0, cluster.ApplyCalls); + } + + [Fact] + public async Task Reconcile_OneAccountBatch_MysqlThenRedis() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(100); + await tracker.ReconcileAsync(); + Assert.Equal(0, tracker.PendingBytes("alice")); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(1, cluster.ApplyCalls); + } + + [Fact] + public async Task Reconcile_MultipleLocalSessions_OneAccountUpdate() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(40); + tracker.CreateSink("alice").ObserveCopied(60); + await tracker.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + Assert.Equal(1, durable.ConsumeCalls); + } + + [Fact] + public async Task Reconcile_DecrementLargerThanRemaining_StopsAtZero() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(1500); + await tracker.ReconcileAsync(); + Assert.Equal(0, durable.Remaining("alice")); + Assert.Equal(0, await cluster.ObserveAsync("alice")); + Assert.True(tracker.IsExhausted("alice")); + } + + [Fact] + public async Task Reconcile_AlreadyZero_RemainsZero() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 0); + tracker.CreateSink("alice").ObserveCopied(80); + await tracker.ReconcileAsync(); + Assert.Equal(0, durable.Remaining("alice")); + Assert.Equal(0, await cluster.ObserveAsync("alice")); + Assert.True(tracker.IsExhausted("alice")); + } + + [Fact] + public async Task TwoNodes_ConsumeSameAccount_DurableReflectsBoth() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 8_000); + var nodeA = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var nodeB = new AccountByteTracker(durable, cluster, NullLogger.Instance); + nodeA.CreateSink("alice").ObserveCopied(300); + nodeB.CreateSink("alice").ObserveCopied(500); + await nodeA.ReconcileAsync(); + await nodeB.ReconcileAsync(); + Assert.Equal(7_200, durable.Remaining("alice")); + Assert.Equal(7_200, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task TwoNodes_LostRedisReplies_RetryDoesNotDoubleCharge() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 1000); + var nodeA = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var nodeB = new AccountByteTracker(durable, cluster, NullLogger.Instance); + nodeA.CreateSink("alice").ObserveCopied(100); + nodeB.CreateSink("alice").ObserveCopied(100); + cluster.HideNextApplyResults = 2; + await nodeA.ReconcileAsync(); + await nodeB.ReconcileAsync(); + Assert.Equal(800, durable.Remaining("alice")); + Assert.Equal(2, durable.ConsumeCalls); + Assert.Equal(800, await cluster.ObserveAsync("alice")); + await nodeA.ReconcileAsync(); + await nodeB.ReconcileAsync(); + await nodeA.ReconcileAsync(); + await nodeB.ReconcileAsync(); + Assert.Equal(2, durable.ConsumeCalls); + Assert.Equal(800, durable.Remaining("alice")); + Assert.Equal(800, await cluster.ObserveAsync("alice")); + Assert.False(nodeA.MysqlCommitted("alice")); + Assert.False(nodeB.MysqlCommitted("alice")); + } + + [Fact] + public async Task MysqlUnavailable_KeepsInFlight_DoesNotChargeRedis() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(100); + durable.Unavailable = true; + await tracker.ReconcileAsync(); + Assert.Equal(100, tracker.InFlightBytes("alice")); + Assert.False(tracker.MysqlCommitted("alice")); + Assert.Equal(1000, durable.Remaining("alice")); + Assert.Equal(0, cluster.ApplyCalls); + durable.Unavailable = false; + await tracker.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task PendingDuringCommittedBatch_StaysPending_UntilApplyCompletes() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1_000); + tracker.CreateSink("alice").ObserveCopied(25); + await tracker.CommitDurableAsync(); + Assert.True(tracker.MysqlCommitted("alice")); + Assert.Equal(975, durable.Remaining("alice")); + tracker.CreateSink("alice").ObserveCopied(25); + await tracker.CommitDurableAsync(); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(25, tracker.PendingBytes("alice")); + Assert.Equal(25, tracker.InFlightBytes("alice")); + await tracker.ApplyCommittedAsync(); + Assert.False(tracker.MysqlCommitted("alice")); + Assert.Equal(25, tracker.PendingBytes("alice")); + await tracker.ReconcileAsync(); + Assert.Equal(950, durable.Remaining("alice")); + Assert.Equal(950, await cluster.ObserveAsync("alice")); + Assert.Equal(0, tracker.PendingBytes("alice")); + Assert.Equal(2, durable.ConsumeCalls); + } + + [Fact] + public async Task RedisSuccessWithLostResponse_RetriesSameBatch_DoesNotDoubleFloor() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(100); + cluster.HideNextApplyResults = 1; + await tracker.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(1, durable.ConsumeCalls); + Assert.True(tracker.MysqlCommitted("alice")); + Assert.NotNull(tracker.CurrentBatchId("alice")); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + var batchId = tracker.CurrentBatchId("alice"); + await tracker.ReconcileAsync(); + Assert.Equal(1, durable.ConsumeCalls); + Assert.NotNull(batchId); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + Assert.False(tracker.MysqlCommitted("alice")); + Assert.Null(tracker.CurrentBatchId("alice")); + } + + [Fact] + public async Task RedisUnavailableAfterMysql_RetriesRedisOnly() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 1000); + tracker.CreateSink("alice").ObserveCopied(100); + cluster.Unavailable = true; + await tracker.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(1, durable.ConsumeCalls); + Assert.True(tracker.MysqlCommitted("alice")); + Assert.Equal(100, tracker.InFlightBytes("alice")); + cluster.Unavailable = false; + await tracker.ReconcileAsync(); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(900, await cluster.ObserveAsync("alice")); + Assert.False(tracker.MysqlCommitted("alice")); + Assert.Equal(0, tracker.InFlightBytes("alice")); + } + + [Fact] + public async Task RestartAfterMysql_DoesNotReplayBatch() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + durable.SeedByteAccount("alice", 1000); + var first = new AccountByteTracker(durable, cluster, NullLogger.Instance); + first.CreateSink("alice").ObserveCopied(100); + cluster.Unavailable = true; + await first.ReconcileAsync(); + Assert.Equal(900, durable.Remaining("alice")); + + var restarted = new AccountByteTracker(durable, cluster, NullLogger.Instance); + cluster.Unavailable = false; + await restarted.ReconcileAsync(); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(900, durable.Remaining("alice")); + Assert.Equal(AccountByteKeys.Missing, await cluster.ObserveAsync("alice")); + } + + [Fact] + public async Task ObserveRemaining_PrefersMinOfRedisAndMysql() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 500); + await cluster.ApplyAsync("alice", AccountByteBatchId.Create(), 0, 800); + Assert.Equal(500, await tracker.ObserveRemainingAsync("alice")); + cluster.Engine.Write(InMemoryAccountByteStore.EncodingKey("alice"), 200); + Assert.Equal(200, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task ObserveRemaining_MissingRedis_UsesMysql() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 321); + Assert.Equal(321, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task ObserveRemaining_SeededAccount_UsesMysql() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 999); + Assert.Equal(999, await tracker.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task Consume_AppliesToEverySeededAccount() + { + var (tracker, durable, cluster) = Create(); + durable.SeedByteAccount("alice", 500); + tracker.CreateSink("alice").ObserveCopied(40); + await tracker.ReconcileAsync(); + Assert.Equal(460, durable.Remaining("alice")); + Assert.Equal(0, tracker.InFlightBytes("alice")); + Assert.Equal(1, cluster.ApplyCalls); + } + + [Fact] + public async Task MissingAccount_DropsInFlight() + { + var (tracker, _, cluster) = Create(); + tracker.CreateSink("ghost").ObserveCopied(10); + await tracker.ReconcileAsync(); + Assert.Equal(0, tracker.InFlightBytes("ghost")); + Assert.Equal(0, cluster.ApplyCalls); + } + + [Fact] + public void MarkExhausted_IsVisibleToAllLocalSinks() + { + var (tracker, durable, _) = Create(); + durable.SeedByteAccount("alice", 10); + _ = tracker.CreateSink("alice"); + Assert.False(tracker.IsExhausted("alice")); + tracker.MarkExhausted("alice"); + Assert.True(tracker.IsExhausted("alice")); + } + + private static (AccountByteTracker Tracker, InMemoryAccountByteDurableStore Durable, InMemoryAccountByteStore Cluster) Create() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + var tracker = new AccountByteTracker(durable, cluster, NullLogger.Instance); + return (tracker, durable, cluster); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteWireAccountingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteWireAccountingTests.cs new file mode 100644 index 00000000..52bf50cc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/AccountBytes/AccountByteWireAccountingTests.cs @@ -0,0 +1,190 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.AccountBytes; + +public sealed class AccountByteWireAccountingTests +{ + [Fact] + public async Task StatusLine_IncludesCrLf_Once() + { + await using var harness = await WireHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(200, "Hello"); + Assert.Equal("200 Hello\r\n".Length, harness.Sink.Copied); + Assert.Equal("200 Hello\r\n", await harness.ReadExactAsciiAsync("200 Hello\r\n".Length)); + } + + [Fact] + public async Task Multiline_IncludesFramingAndTerminator() + { + await using var harness = await WireHarness.CreateAsync(); + await harness.Writer.WriteMultilineStartAsync(100, "Help text follows"); + await harness.Writer.WriteMultilineDataAsync(" one"); + await harness.Writer.WriteMultilineEndAsync(); + var expected = "100 Help text follows\r\n one\r\n.\r\n"; + Assert.Equal(expected.Length, harness.Sink.Copied); + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + } + + [Fact] + public async Task DotStuffing_IsCountedOnTheWire() + { + await using var harness = await WireHarness.CreateAsync(); + await harness.Writer.WriteMultilineDataAsync(".secret"); + await harness.Writer.WriteMultilineEndAsync(); + var expected = "..secret\r\n.\r\n"; + Assert.Equal(expected.Length, harness.Sink.Copied); + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + } + + [Fact] + public async Task ArticlePayload_IsCountedOnce_WithTerminator() + { + await using var harness = await WireHarness.CreateAsync(); + var destuffed = "From: a@b\r\n\r\nbody\r\n"u8.ToArray(); + await harness.Writer.WriteArticleAsync(destuffed, NntpArticleTxFraming.PeerTakeThis("")); + Assert.True(harness.Sink.Copied > destuffed.Length); + var wire = await harness.ReadExactAsciiAsync(harness.Sink.Copied); + Assert.EndsWith(".\r\n", wire, StringComparison.Ordinal); + Assert.Equal(wire.Length, harness.Sink.Copied); + } + + [Fact] + public async Task PipeWriterCapacity_IsNotCounted() + { + var output = new CapacitySpyWriter(); + await using var writer = new NntpResponseWriter(output); + var sink = new RecordingSink(); + writer.SetByteSink(sink); + await writer.WriteLineAsync(200, "Hi"); + Assert.True(output.LastGetMemory > output.LastAdvance); + Assert.Equal(output.LastAdvance, sink.Copied); + Assert.Equal("200 Hi\r\n".Length, sink.Copied); + } + + [Fact] + public async Task DirectSpeedTestWrite_IsCountedAfterAdvance() + { + await using var harness = await WireHarness.CreateAsync(); + await using (var lease = await harness.Writer.AcquireDirectTxPipeAsync()) + { + await lease.WriteAndFlushAsync("SPEED"u8.ToArray()); + } + + Assert.Equal(5, harness.Sink.Copied); + } + + [Fact] + public void ObserveCopied_IgnoresNonPositive() + { + var durable = new InMemoryAccountByteDurableStore(); + var cluster = new InMemoryAccountByteStore(); + var tracker = new AccountByteTracker(durable, cluster, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + durable.SeedByteAccount("alice", 100); + var sink = tracker.CreateSink("alice"); + sink.ObserveCopied(0); + sink.ObserveCopied(-3); + Assert.Equal(0, tracker.PendingBytes("alice")); + } + + private sealed class RecordingSink : IAccountByteSink + { + public int Copied { get; private set; } + + public void ObserveCopied(int bytes) + { + if (bytes > 0) + { + Copied += bytes; + } + } + } + + private sealed class WireHarness : IAsyncDisposable + { + private WireHarness(Pipe pipe, NntpResponseWriter writer, RecordingSink sink) + { + Pipe = pipe; + Writer = writer; + Sink = sink; + } + + public Pipe Pipe { get; } + + public NntpResponseWriter Writer { get; } + + public RecordingSink Sink { get; } + + public static async Task CreateAsync() + { + var pipe = new Pipe(); + var writer = new NntpResponseWriter(pipe.Writer); + var sink = new RecordingSink(); + writer.SetByteSink(sink); + await Task.Yield(); + return new WireHarness(pipe, writer, sink); + } + + public async Task ReadExactAsciiAsync(int count) + { + var buffer = new byte[count]; + var read = 0; + while (read < count) + { + var result = await Pipe.Reader.ReadAsync(); + var copied = result.Buffer.Slice(0, Math.Min(result.Buffer.Length, count - read)); + copied.CopyTo(buffer.AsSpan(read, (int)copied.Length)); + read += (int)copied.Length; + Pipe.Reader.AdvanceTo(copied.End); + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + await Writer.DisposeAsync(); + await Pipe.Writer.CompleteAsync(); + await Pipe.Reader.CompleteAsync(); + } + } + + private sealed class CapacitySpyWriter : PipeWriter + { + private readonly Pipe _pipe = new(); + + public int LastGetMemory { get; private set; } + + public int LastAdvance { get; private set; } + + public override bool CanGetUnflushedBytes => true; + + public override long UnflushedBytes => _pipe.Writer.UnflushedBytes; + + public override void Advance(int bytes) + { + LastAdvance = bytes; + _pipe.Writer.Advance(bytes); + } + + public override Memory GetMemory(int sizeHint = 0) + { + var memory = _pipe.Writer.GetMemory(Math.Max(sizeHint, 64 * 1024)); + LastGetMemory = memory.Length; + return memory; + } + + public override Span GetSpan(int sizeHint = 0) => GetMemory(sizeHint).Span; + + public override void CancelPendingFlush() => _pipe.Writer.CancelPendingFlush(); + + public override void Complete(Exception? exception = null) => _pipe.Writer.Complete(exception); + + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) => + _pipe.Writer.FlushAsync(cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeEngineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeEngineTests.cs new file mode 100644 index 00000000..f8b7a9a6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeEngineTests.cs @@ -0,0 +1,853 @@ +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Acme; + +public sealed class AcmeConfigurationTests +{ + internal const string TestPfxPassword = "unit-test-pfx-password"; + + [Fact] + public void Defaults_MatchSpecification() + { + var options = new NntpdOptions(); + Assert.Equal(NntpdOptions.DefaultAcmeDirectoryUrl, options.AcmeDirectoryUrl); + Assert.Equal(NntpdOptions.DefaultAcmeStateDir, options.AcmeStateDir); + Assert.Equal(30, options.AcmeRenewalThresholdDays); + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.Contains("staging", options.AcmeDirectoryUrl, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void MissingEmailAndPassword_WithTlsDisabled_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void MissingEmail_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = TestPfxPassword; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingPassword_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = string.Empty; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void WhitespaceOnlyPassword_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = " "; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void NonPositiveRenewalThreshold_FailsSharedConfiguration() + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 0, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ])); + Assert.Contains("acmerenewalthresholddays", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void RenewalThresholdAboveNinety_IsAcceptedBySharedConfiguration() + { + var configuration = NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 91, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ]); + Assert.Equal(91, configuration.AcmeRenewalThresholdDays); + } + + [Fact] + public void CustomDirectoryAndStateDir_Accepted() + { + var options = TestHostFactory.CreateValidOptions(); + options.AcmeDirectoryUrl = "https://acme.example.test/directory"; + options.AcmeStateDir = "var/acme-state"; + options.AcmeRenewalThresholdDays = 15; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void HttpDirectoryUrl_IsAcceptedBySharedConfiguration() + { + var configuration = NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "http://acme.example.test/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ]); + Assert.Equal("http://acme.example.test/directory", configuration.AcmeDirectoryUrl); + } + + [Fact] + public void EnvironmentVariable_PasswordName_IsExact() + { + Assert.Equal("VECTOR__ACMECERTIFICATEPASSWORD", NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + } +} + +public sealed class CertificateIdentitiesTests +{ + [Fact] + public void ForFqdn_ReturnsExactSanPair() + { + var ids = CertificateIdentities.ForFqdn("nntpd01.usenet.ninja"); + Assert.Equal(new[] { "nntpd01.usenet.ninja", "news.usenet.ninja" }, ids); + } + + [Fact] + public void NewsTypo_IsNotUsed() + { + Assert.DoesNotContain("ninaja", CertificateIdentities.NewsHostname, StringComparison.Ordinal); + Assert.Equal("news.usenet.ninja", CertificateIdentities.NewsHostname); + } +} + +public sealed class AccountStoreTests +{ + [Fact] + public void FirstSave_ThenLoad_ReusesAccount_AsPkcs8Der() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + Assert.Null(store.Load()); + + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + var state = new AcmeAccountState( + "https://acme.example/acme/acct/1", + NntpdOptions.DefaultAcmeDirectoryUrl, + key); + store.Save(state); + + var loaded = store.Load(); + Assert.NotNull(loaded); + Assert.Equal(state.AccountUri, loaded.AccountUri); + Assert.True(key.AsSpan().SequenceEqual(loaded.PrivateKeyDer)); + Assert.False(DerCrypto.LooksLikePem(loaded.PrivateKeyDer)); + Assert.False(File.ReadAllText(AcmePaths.AccountKeyPath(dir.Path)).Contains("BEGIN", StringComparison.Ordinal)); + } + + [Fact] + public void EnsureRegistered_ReusesExisting_WithoutCallingRegister() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + store.Save(new AcmeAccountState("https://acme.example/acme/acct/1", "https://dir.example/directory", key)); + + var registerCalls = 0; + var result = store.EnsureRegistered( + "https://dir.example/directory", + () => throw new InvalidOperationException("should not generate"), + _ => + { + registerCalls++; + return ("x", ""); + }); + + Assert.Equal(0, registerCalls); + Assert.Equal("https://acme.example/acme/acct/1", result.AccountUri); + } + + [Fact] + public void EnsureRegistered_FirstStartup_PersistsAccount() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + var registerCalls = 0; + + var result = store.EnsureRegistered( + "https://dir.example/directory", + () => key, + k => + { + registerCalls++; + Assert.True(k.AsSpan().SequenceEqual(key)); + return ("https://acme.example/acme/acct/9", "body"); + }); + + Assert.Equal(1, registerCalls); + Assert.Equal("https://acme.example/acme/acct/9", result.AccountUri); + Assert.False(File.Exists(AcmePaths.AccountPendingKeyPath(dir.Path))); + Assert.NotNull(store.Load()); + } + + [Fact] + public void IncompleteAccount_ThrowsWithoutDeleting() + { + using var dir = new TempAcmeDir(); + AcmePaths.EnsureStateLayout(dir.Path); + File.WriteAllBytes(AcmePaths.AccountKeyPath(dir.Path), [1, 2, 3]); + var store = new AccountStore(dir.Path); + Assert.Throws(() => store.Load()); + Assert.True(File.Exists(AcmePaths.AccountKeyPath(dir.Path))); + } +} + +public sealed class PfxStoreAndValidationTests +{ + private static readonly string[] RequiredDomains = ["nntpd01.usenet.ninja", "news.usenet.ninja"]; + + [Fact] + public void SaveAndLoad_UsesBinaryPfx_AndSurvivesRestart() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + + store.Save(material); + Assert.False(DerCrypto.LooksLikePem(material.PfxBytes)); + + var store2 = CreateStore(dir.Path); + var loaded = store2.Load(); + Assert.NotNull(loaded); + Assert.False(DerCrypto.LooksLikePem(loaded.PfxBytes)); + + var paths = store2.Paths(); + Assert.EndsWith("certificate.pfx", paths.PfxPath, StringComparison.OrdinalIgnoreCase); + var onDisk = File.ReadAllBytes(paths.PfxPath); + Assert.False(DerCrypto.LooksLikePem(onDisk)); + Assert.True(onDisk.Length > 0); + + using var cert = PfxCrypto.LoadCertificate(loaded.PfxBytes, AcmeConfigurationTests.TestPfxPassword); + Assert.True(cert.HasPrivateKey); + var sans = cert.Extensions.OfType().Single() + .EnumerateDnsNames().Select(n => n.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sans); + Assert.Contains("news.usenet.ninja", sans); + } + + [Fact] + public void IncorrectPassword_FailsValidation() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + "wrong-password", + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.StartsWith("parse_failed", status.Reason, StringComparison.Ordinal); + Assert.DoesNotContain("wrong-password", status.Reason, StringComparison.Ordinal); + } + + [Fact] + public void IncompleteGeneration_WithoutComplete_IsIgnored() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var genId = Guid.NewGuid().ToString("N"); + var genDir = AcmePaths.GenerationDir(dir.Path, "nntpd01.usenet.ninja", genId); + Directory.CreateDirectory(genDir); + var paths = AcmePaths.GenerationCertificatePaths(dir.Path, "nntpd01.usenet.ninja", genId); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + File.WriteAllBytes(paths.PfxPath, material.PfxBytes); + Assert.Null(store.Load()); + } + + [Fact] + public void FailedReplacement_PreservesKnownGood() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var first = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + store.Save(first); + var firstPaths = store.Paths(); + + var orphan = Guid.NewGuid().ToString("N"); + Directory.CreateDirectory(AcmePaths.GenerationDir(dir.Path, "nntpd01.usenet.ninja", orphan)); + store.Recover(); + Assert.Equal(firstPaths.PfxPath, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + [Fact] + public void InvalidPfxBytes_DoNotBecomeCurrent() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var good = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + store.Save(good); + var currentBefore = store.Paths().PfxPath; + + Assert.ThrowsAny(() => + store.Save(new CertificateMaterial( + [1, 2, 3, 4], + RequiredDomains, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow.AddDays(30)))); + + Assert.Equal(currentBefore, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + [Fact] + public void Validate_WrongSan_Rejects() + { + var material = TestCertificateFactory.CreateMaterial( + ["wrong.example"], + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.Equal("missing_san", status.Reason); + } + + [Fact] + public void Validate_RenewalThreshold_Detected() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.True(status.Usable); + Assert.True(status.DueForRenewal); + } + + [Fact] + public void Validate_OutsideThreshold_NotDue() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.True(status.Usable); + Assert.False(status.DueForRenewal); + } + + [Fact] + public void Validate_Expired_Rejects() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(-1), + notBefore: DateTimeOffset.UtcNow.AddDays(-40)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.Equal("expired", status.Reason); + } + + private static CertificateStore CreateStore(string path) => + new(path, "nntpd01.usenet.ninja", AcmeConfigurationTests.TestPfxPassword, RequiredDomains, TimeSpan.FromDays(30)); +} + +public sealed class CertificateManagerTests +{ + private static readonly string[] RequiredDomains = ["nntpd01.usenet.ninja", "news.usenet.ninja"]; + + [Fact] + public async Task Ensure_ReusesValidCertificate_WithoutIssuance() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90)); + store.Save(material); + + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + var ensured = await manager.EnsureCertificateAsync(CancellationToken.None); + Assert.Equal(0, issuer.IssueCallCount); + Assert.True(Math.Abs((material.NotAfter - ensured.NotAfter).TotalSeconds) < 2); + Assert.Contains("news.usenet.ninja", ensured.Domains); + } + + [Fact] + public async Task Ensure_IssuesWhenAbsent() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + _ = await manager.EnsureCertificateAsync(CancellationToken.None); + Assert.Equal(1, issuer.IssueCallCount); + using var cert = manager.CreateTlsCertificate(); + Assert.True(cert.HasPrivateKey); + var sans = cert.Extensions.OfType().Single() + .EnumerateDnsNames().Select(n => n.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sans); + Assert.Contains("news.usenet.ninja", sans); + Assert.Equal(2, sans.Count); + Assert.True(File.Exists(store.Paths().PfxPath)); + } + + [Fact] + public async Task RenewIfDue_SkipsWhenNotDue() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + Assert.False(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(0, issuer.IssueCallCount); + } + + [Fact] + public async Task RenewIfDue_RenewsWithinThreshold() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10))); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + Assert.True(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(1, issuer.IssueCallCount); + } + + [Fact] + public async Task FailedIssuance_PreservesPriorPfx() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var prior = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10)); + store.Save(prior); + var priorPath = store.Paths().PfxPath; + var issuer = new FakeCertificateIssuer { ThrowOnIssue = new AcmeOrderException("boom", "fail") }; + var manager = CreateManager(dir.Path, store, issuer); + + Assert.False(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(priorPath, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + private static CertificateManager CreateManager( + string stateDir, + CertificateStore store, + ICertificateIssuer issuer) => + new( + "nntpd01.usenet.ninja", + stateDir, + NntpdOptions.DefaultAcmeDirectoryUrl, + store, + issuer, + AcmeConfigurationTests.TestPfxPassword, + TimeSpan.FromDays(30), + NullLogger.Instance); +} + +public sealed class Dns01SolverTests +{ + [Fact] + public async Task PlaceWaitCleanup_CreatesAndRemovesTxt() + { + using var dir = new TempAcmeDir(); + var cf = new FakeCloudflareDnsClient(); + var resolver = new FakeTxtResolver(); + var solver = new Dns01Solver( + cf, + "zone-1", + resolver, + "nntpd01.usenet.ninja", + dir.Path, + propagationTimeout: TimeSpan.FromSeconds(2), + propagationInterval: TimeSpan.FromMilliseconds(10)); + + var spec = new Dns01ChallengeSpec("nntpd01.usenet.ninja", "validation-token"); + resolver.Set(spec.RecordName, spec.Validation); + + await solver.PlaceAsync([spec], CancellationToken.None); + Assert.Equal(1, cf.CreateCallCount); + var created = cf.Snapshot().Single(r => r.Type == CloudflareDnsRecordTypes.TXT); + Assert.Equal(Dns01Solver.ChallengeTtlSeconds, created.Ttl); + Assert.False(created.Proxied); + + await solver.WaitPropagatedAsync([spec], CancellationToken.None); + await solver.CleanupAsync(CancellationToken.None); + Assert.Equal(1, cf.DeleteCallCount); + Assert.Empty(cf.Snapshot()); + } + + [Fact] + public async Task CancellationDuringPlace_Propagates() + { + using var dir = new TempAcmeDir(); + var cf = new FakeCloudflareDnsClient(); + cf.OnMutate = async ct => + { + await Task.Delay(Timeout.Infinite, ct); + }; + var solver = new Dns01Solver(cf, "zone-1", new FakeTxtResolver(), "nntpd01.usenet.ninja", dir.Path); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(20)); + await Assert.ThrowsAnyAsync(() => + solver.PlaceAsync([new Dns01ChallengeSpec("nntpd01.usenet.ninja", "tok")], cts.Token)); + } +} + +public sealed class AcmeLifecycleServiceTests +{ + [Fact] + public async Task TlsDisabled_DoesNotInitializeAcme_OrRequireCertificate() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + options.AcmeStateDir = dir.Path; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + + var factory = new AcmeComponentFactory( + Options.Create(options), + new FakeCloudflareDnsClient(), + new FakeHttpClientFactory(), + NullLoggerFactory.Instance); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.AcmeInitialized); + Assert.False(Directory.Exists(Path.Join(dir.Path, "account"))); + Assert.False(Directory.Exists(Path.Join(dir.Path, "live"))); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + } + + [Fact] + public async Task TlsEnabled_EnsureRuns_AndProviderAvailable() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = AcmeConfigurationTests.TestPfxPassword; + options.AcmeStateDir = dir.Path; + + var domains = CertificateIdentities.ForFqdn(options.Fqdn); + var store = new CertificateStore( + dir.Path, + options.Fqdn, + options.AcmeCertificatePassword, + domains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + domains, + options.AcmeCertificatePassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + + var manager = new CertificateManager( + options.Fqdn, + dir.Path, + options.AcmeDirectoryUrl, + store, + new FakeCertificateIssuer(), + options.AcmeCertificatePassword, + TimeSpan.FromDays(30), + NullLogger.Instance); + + var factory = new InjectedManagerFactory(manager, Options.Create(options)); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.True(service.AcmeInitialized); + Assert.NotNull(manager.CurrentMaterial); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + } + + [Fact] + public async Task TlsEnabled_MissingCertificateAndFailingIssuer_FailsStartup() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = AcmeConfigurationTests.TestPfxPassword; + options.AcmeStateDir = dir.Path; + + var domains = CertificateIdentities.ForFqdn(options.Fqdn); + var manager = new CertificateManager( + options.Fqdn, + dir.Path, + options.AcmeDirectoryUrl, + new CertificateStore( + dir.Path, + options.Fqdn, + options.AcmeCertificatePassword, + domains, + TimeSpan.FromDays(30)), + new FakeCertificateIssuer { ThrowOnIssue = new AcmeOrderException("fail", "x") }, + options.AcmeCertificatePassword, + TimeSpan.FromDays(30), + NullLogger.Instance); + + var factory = new InjectedManagerFactory(manager, Options.Create(options)); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await service.DisposeAsync(); + } + + private sealed class InjectedManagerFactory : AcmeComponentFactory + { + private readonly CertificateManager _manager; + + public InjectedManagerFactory(CertificateManager manager, IOptions options) + : base( + options, + new FakeCloudflareDnsClient(), + new FakeHttpClientFactory(), + NullLoggerFactory.Instance) + { + _manager = manager; + } + + internal override CertificateManager? GetOrCreateManager() => _manager; + } +} + +internal sealed class FakeCertificateIssuer : ICertificateIssuer +{ + public int IssueCallCount { get; private set; } + + public Exception? ThrowOnIssue { get; set; } + + public Task IssueAsync( + IReadOnlyList domains, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IssueCallCount++; + if (ThrowOnIssue is not null) + { + throw ThrowOnIssue; + } + + return Task.FromResult( + TestCertificateFactory.CreateMaterial( + domains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + } +} + +internal sealed class FakeTxtResolver : IAuthoritativeTxtResolver +{ + private readonly Dictionary> _map = new(StringComparer.OrdinalIgnoreCase); + + public void Set(string name, params string[] values) => _map[name] = values; + + public Task> LookupTxtAsync(string name, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult( + _map.TryGetValue(name, out var values) ? values : Array.Empty()); + } +} + +internal sealed class FakeHttpClientFactory : IHttpClientFactory +{ + public HttpClient CreateClient(string name) => new(); +} + +internal sealed class TempAcmeDir : IDisposable +{ + public string Path { get; } = System.IO.Path.Join( + System.IO.Path.GetTempPath(), + "vectornntp-acme-" + Guid.NewGuid().ToString("N")); + + public TempAcmeDir() => Directory.CreateDirectory(Path); + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Best-effort test cleanup. + } + } +} + +internal static class TestCertificateFactory +{ + public static CertificateMaterial CreateMaterial( + IReadOnlyList dnsNames, + string password, + DateTimeOffset notAfter, + DateTimeOffset? notBefore = null) + { + var before = notBefore ?? DateTimeOffset.UtcNow.AddDays(-1); + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + dnsNames[0], + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + foreach (var name in dnsNames) + { + sanBuilder.AddDnsName(name); + } + + request.CertificateExtensions.Add(sanBuilder.Build()); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, + critical: false)); + + using var cert = request.CreateSelfSigned(before.UtcDateTime, notAfter.UtcDateTime); + var pfx = PfxCrypto.ExportPfx(cert, intermediateCertificates: [], password); + return new CertificateMaterial( + pfx, + dnsNames.Select(n => n.ToLowerInvariant()).OrderBy(n => n).ToArray(), + before, + notAfter); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeOrderReadinessTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeOrderReadinessTests.cs new file mode 100644 index 00000000..24631ec8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeOrderReadinessTests.cs @@ -0,0 +1,207 @@ +using VectorNNTPD.Common.Acme; + +namespace VectorNNTPD.ReaderService.Tests.Acme; + +public sealed class AcmeOrderReadinessTests +{ + [Fact] + public void Evaluate_Pending_Continues() + { + var view = new AcmeOrderReadiness.OrderView( + "Pending", + [ + new AcmeOrderReadiness.AuthorizationView("nntpd01.usenet.ninja", "Pending", null, null, null), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null), + ]); + + var decision = AcmeOrderReadiness.Evaluate(view, out var diagnostic); + Assert.Equal(AcmeOrderReadiness.Decision.Continue, decision); + Assert.Null(diagnostic); + } + + [Fact] + public void Evaluate_Ready_IsReady() + { + var view = new AcmeOrderReadiness.OrderView( + "Ready", + [ + new AcmeOrderReadiness.AuthorizationView("nntpd01.usenet.ninja", "Valid", null, null, null), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null), + ]); + + Assert.Equal(AcmeOrderReadiness.Decision.Ready, AcmeOrderReadiness.Evaluate(view, out _)); + } + + [Fact] + public void Evaluate_InvalidAuthorization_SurfacesDnsProblem() + { + var view = new AcmeOrderReadiness.OrderView( + "Invalid", + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + "Invalid", + "urn:ietf:params:acme:error:dns", + "DNS problem: NXDOMAIN looking up TXT for _acme-challenge.nntpd01.usenet.ninja", + 400), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null), + ]); + + var decision = AcmeOrderReadiness.Evaluate(view, out var diagnostic); + Assert.Equal(AcmeOrderReadiness.Decision.Invalid, decision); + Assert.NotNull(diagnostic); + Assert.Contains("nntpd01.usenet.ninja", diagnostic, StringComparison.Ordinal); + Assert.Contains("urn:ietf:params:acme:error:dns", diagnostic, StringComparison.Ordinal); + Assert.Contains("NXDOMAIN", diagnostic, StringComparison.Ordinal); + Assert.Contains("type=", diagnostic, StringComparison.Ordinal); + Assert.Contains("detail=", diagnostic, StringComparison.Ordinal); + } + + [Fact] + public async Task WaitUntilReady_PendingThenReady_DoesNotThrow() + { + var polls = 0; + await AcmeOrderReadiness.WaitUntilReadyAsync( + _ => + { + polls++; + if (polls < 3) + { + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null)])); + } + + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Ready", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null)])); + }, + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask); + + Assert.True(polls >= 3); + } + + [Fact] + public async Task WaitUntilReady_Invalid_ThrowsAuthorizationInvalidWithDetails() + { + var ex = await Assert.ThrowsAsync(() => + AcmeOrderReadiness.WaitUntilReadyAsync( + _ => Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + "Invalid", + "urn:ietf:params:acme:error:dns", + "DNS problem: SERVFAIL", + 400), + ])), + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask)); + + Assert.Equal("authorization_invalid", ex.Category); + Assert.Contains("urn:ietf:params:acme:error:dns", ex.Message, StringComparison.Ordinal); + Assert.Contains("SERVFAIL", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("AcmeRequestException", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task WaitUntilReady_Timeout_ThrowsBoundedTimeout() + { + var delayCalls = 0; + var ex = await Assert.ThrowsAsync(() => + AcmeOrderReadiness.WaitUntilReadyAsync( + _ => Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null)])), + timeout: TimeSpan.FromMilliseconds(50), + interval: TimeSpan.FromMilliseconds(10), + CancellationToken.None, + delayAsync: async (delay, ct) => + { + delayCalls++; + await Task.Delay(delay, ct); + })); + + Assert.Equal("authorization_timeout", ex.Category); + Assert.Contains("order=Pending", ex.Message, StringComparison.Ordinal); + Assert.True(delayCalls >= 1); + } + + [Fact] + public async Task WaitUntilReady_Cancellation_ThrowsPromptly() + { + using var cts = new CancellationTokenSource(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waitTask = AcmeOrderReadiness.WaitUntilReadyAsync( + async ct => + { + gate.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + return new AcmeOrderReadiness.OrderView("Pending", []); + }, + timeout: TimeSpan.FromMinutes(1), + interval: TimeSpan.FromMilliseconds(5), + cts.Token, + delayAsync: (delay, ct) => Task.Delay(delay, ct)); + + await gate.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waitTask); + } +} + +/// +/// Documents the required issue sequence: finalize must not run while the order is still Pending. +/// Exercised via which calls after challenge submit. +/// +public sealed class AcmeIssuanceSequenceTests +{ + [Fact] + public async Task ReadinessGate_PreventsFinalizeWhilePending_ThenAllowsWhenReady() + { + var generateAllowed = false; + var states = new Queue(["Pending", "Pending", "Ready"]); + + await AcmeOrderReadiness.WaitUntilReadyAsync( + _ => + { + var status = states.Dequeue(); + Assert.False(generateAllowed); + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + status, + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + status == "Ready" ? "Valid" : "Pending", + null, + null, + null), + new AcmeOrderReadiness.AuthorizationView( + "news.usenet.ninja", + status == "Ready" ? "Valid" : "Pending", + null, + null, + null), + ])); + }, + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask); + + generateAllowed = true; + Assert.Empty(states); + Assert.True(generateAllowed); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeProblemDiagnosticsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeProblemDiagnosticsTests.cs new file mode 100644 index 00000000..4c0b2e5b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Acme/AcmeProblemDiagnosticsTests.cs @@ -0,0 +1,70 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Acme.Protocol; + +namespace VectorNNTPD.ReaderService.Tests.Acme; + +public sealed class AcmeProblemDiagnosticsTests +{ + [Fact] + public void FormatProblemFields_PreservesStructuredFields() + { + var formatted = AcmeProblemDiagnostics.FormatProblemFields( + preface: null, + type: "urn:ietf:params:acme:error:orderNotReady", + status: 403, + identifier: "nntpd01.usenet.ninja", + detail: "Order's status (\"pending\") is not acceptable for finalization", + subproblems: null); + + Assert.Contains("type=urn:ietf:params:acme:error:orderNotReady", formatted, StringComparison.Ordinal); + Assert.Contains("status=403", formatted, StringComparison.Ordinal); + Assert.Contains("identifier=nntpd01.usenet.ninja", formatted, StringComparison.Ordinal); + Assert.Contains("detail=", formatted, StringComparison.Ordinal); + Assert.Contains("pending", formatted, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void FormatException_RedactsPemAndLongTokens() + { + var pem = "oops -----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY----- trailing"; + var ex = new AcmeCaException( + "server error", + "urn:ietf:params:acme:error:serverInternal", + pem + " " + new string('A', 100), + statusCode: 500); + var formatted = AcmeProblemDiagnostics.FormatException(ex); + Assert.Contains("[redacted-pem]", formatted, StringComparison.Ordinal); + Assert.Contains("[redacted-token]", formatted, StringComparison.Ordinal); + Assert.DoesNotContain("BEGIN PRIVATE KEY", formatted, StringComparison.Ordinal); + } + + [Fact] + public void FailureSanitizer_IncludesAcmeOrderDiagnosticMessage() + { + var ex = new AcmeOrderException( + "challenge_failed", + "type=urn:ietf:params:acme:error:dns status=400 identifier=nntpd01.usenet.ninja detail=NXDOMAIN"); + var sanitized = AcmeFailureSanitizer.Sanitize(ex); + Assert.Contains("challenge_failed", sanitized, StringComparison.Ordinal); + Assert.Contains("urn:ietf:params:acme:error:dns", sanitized, StringComparison.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sanitized, StringComparison.Ordinal); + Assert.DoesNotContain("PRIVATE KEY", sanitized, StringComparison.Ordinal); + } + + [Fact] + public void ChallengeFailedWrapper_UsesFormattedProtocolDetails_NotTypeNameOnly() + { + var protocol = new AcmeCaException( + "Error processing request", + "urn:ietf:params:acme:error:dns", + "DNS problem: NXDOMAIN looking up TXT", + statusCode: 400); + var wrapped = new AcmeOrderException("challenge_failed", AcmeProblemDiagnostics.FormatException(protocol)); + + Assert.Equal("challenge_failed", wrapped.Category); + Assert.Contains("type=urn:ietf:params:acme:error:dns", wrapped.Message, StringComparison.Ordinal); + Assert.DoesNotContain("challenge_failed: AcmeException", wrapped.Message, StringComparison.Ordinal); + Assert.DoesNotContain("challenge_failed: AcmeCaException", wrapped.Message, StringComparison.Ordinal); + Assert.Equal(wrapped.Message, AcmeFailureSanitizer.Sanitize(wrapped)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs new file mode 100644 index 00000000..262be9a5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs @@ -0,0 +1,60 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Acme; + +public sealed class NntpdAcmeFilesystemParityTests +{ + [Fact] + public void Nntpd_certificate_identity_uses_common_fqdn_paths() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + options.DnsSuffix = "usenet.ninja"; + var fqdn = CertificateIdentities.NormalizeFqdn(options.Fqdn); + Assert.Equal("nntpd01.usenet.ninja", fqdn); + Assert.Equal( + new[] { "nntpd01.usenet.ninja", CertificateIdentities.NewsHostname }, + CertificateIdentities.ForFqdn(fqdn)); + + const string certs = "certs-root"; + Assert.Equal( + Path.Join(certs, "live", fqdn, "current"), + AcmePaths.CurrentGenerationPointerPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, "gens"), + AcmePaths.GenerationsDir(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, "dns01"), + AcmePaths.Dns01RecoveryDir(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "journal", fqdn + ".json"), + AcmePaths.TransactionJournalPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, ".issuance.lock"), + AcmePaths.IssuanceLockPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "account", "private_key.der"), + AcmePaths.AccountKeyPath(certs)); + Assert.DoesNotContain( + Path.Join("live", "gens"), + AcmePaths.GenerationsDir(certs, fqdn), + StringComparison.Ordinal); + Assert.DoesNotContain( + Path.Join("live", "current"), + AcmePaths.CurrentGenerationPointerPath(certs, fqdn), + StringComparison.Ordinal); + } + + [Fact] + public void Nntpd_uses_shared_acme_account_environment_variable() + { + Assert.Equal("VECTOR__ACMEACCOUNT", AcmeCloudflareOptions.AcmeAccountEnvironmentVariable); + Assert.Equal( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + NntpdOptions.AcmeAccountEnvironmentVariable); + Assert.DoesNotContain("NNTPD", NntpdOptions.AcmeAccountEnvironmentVariable, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ApplicationVersionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ApplicationVersionTests.cs new file mode 100644 index 00000000..831214f0 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ApplicationVersionTests.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; +using System.Reflection; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests; + +public sealed class ApplicationVersionTests +{ + [Fact] + public void CompiledVersion_IsDateTimeFourPartNumeric_WithoutSourceRevision() + { + AssertDateTimeApplicationVersion(typeof(NntpdLoggingExtensions).Assembly); + } + + private static void AssertDateTimeApplicationVersion(Assembly assembly) + { + var version = assembly.GetName().Version; + Assert.NotNull(version); + Assert.Equal(1, version.Major); + Assert.Equal(0, version.Minor); + Assert.Equal((DateTime.Now.Date - new DateTime(2000, 1, 1)).Days, version.Build); + Assert.InRange(version.Revision, 0, 43199); + + var expected = $"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"; + Assert.Matches(@"^1\.0\.\d+\.\d+$", expected); + + var informational = assembly.GetCustomAttribute(); + Assert.NotNull(informational); + Assert.Equal(expected, informational.InformationalVersion); + Assert.DoesNotContain("+", informational.InformationalVersion, StringComparison.Ordinal); + Assert.DoesNotContain("git", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("commit", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("branch", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("-", informational.InformationalVersion, StringComparison.Ordinal); + + var file = FileVersionInfo.GetVersionInfo(assembly.Location); + Assert.Equal(expected, file.FileVersion); + Assert.Equal(expected, file.ProductVersion); + + const string copyright = "© 2026 Chris Knipe "; + var copyrightAttribute = assembly.GetCustomAttribute(); + Assert.NotNull(copyrightAttribute); + Assert.Equal(copyright, copyrightAttribute.Copyright); + Assert.Equal(copyright, file.LegalCopyright); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs new file mode 100644 index 00000000..e3e53f7c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs @@ -0,0 +1,371 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class ArticleIngestionQueueTests +{ + [Fact] + public void DefaultMemoryLimit_IsOneGibibyte() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions()); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, queue.MemoryLimitBytes); + Assert.Equal(1_073_741_824L, queue.MemoryLimitBytes); + } + + [Fact] + public void ConfiguredMemoryLimit_IsHonoured() + { + var options = new NntpdOptions + { + TransitQueueMemoryLimit = 4096, + ArticleIngestion = new ArticleIngestionOptions(), + }; + var queue = new ArticleIngestionQueue(Options.Create(options)); + Assert.Equal(4096, queue.MemoryLimitBytes); + } + + [Fact] + public async Task ArticleSmallerThanBudget_IsAdmitted() + { + var article = Article("", InboundArticleProducer.IHave); + var queue = CreateQueue(article.Payload.Length + 16); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + Assert.Equal(article.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + } + + [Fact] + public async Task MultipleArticles_ConsumeBudget() + { + var a = Article(""); + var b = Article(""); + var c = Article(""); + var total = a.Payload.Length + b.Payload.Length + c.Payload.Length; + var queue = CreateQueue(total); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(c, CancellationToken.None)); + Assert.Equal(total, queue.QueuedBytes); + Assert.Equal(3, queue.Count); + Assert.Equal(total, queue.PeakQueuedBytes); + Assert.Equal(3, queue.PeakCount); + } + + [Fact] + public async Task Admission_BlocksWhenBudgetExhausted_AndResumesAfterRelease() + { + var full = Article(""); + var wait = Article(""); + var queue = CreateQueue(full.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(full, CancellationToken.None)); + + var blocked = queue.EnqueueAsync(wait, CancellationToken.None); + await WaitForWaitersAsync(queue, 1); + + Assert.False(blocked.IsCompleted); + Assert.Equal(full.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + + var first = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("", first!.MessageId); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Accepted, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(wait.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ConcurrentProducers_CannotOversubscribeBudget() + { + const int producers = 16; + const int admitted = 4; + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * admitted; + var queue = CreateQueue(limit); + var results = new ArticleEnqueueResult[producers]; + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = new Task[producers]; + for (var i = 0; i < producers; i++) + { + var index = i; + tasks[index] = Task.Run(async () => + { + await start.Task.ConfigureAwait(false); + results[index] = await queue + .EnqueueAsync(Article($""), CancellationToken.None) + .ConfigureAwait(false); + }); + } + + start.SetResult(); + + await WaitUntilAsync( + () => queue.Count == admitted && queue.DebugWaiterCount == producers - admitted, + TimeSpan.FromSeconds(5)); + + Assert.Equal(limit, queue.QueuedBytes); + Assert.Equal(admitted, queue.Count); + Assert.True(queue.QueuedBytes <= limit); + Assert.Equal(admitted, tasks.Count(static t => t.IsCompleted)); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + for (var i = 0; i < producers; i++) + { + Assert.NotNull(await queue.DequeueAsync(safety.Token)); + } + + await Task.WhenAll(tasks).WaitAsync(safety.Token); + Assert.All(results, static r => Assert.Equal(ArticleEnqueueResult.Accepted, r)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task ExactBoundaryArticle_Fits() + { + var edge = Article(""); + var queue = CreateQueue(edge.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(edge, CancellationToken.None)); + Assert.Equal(edge.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.False(queue.TryProbeCapacity()); + } + + [Fact] + public async Task ArticleExceedingBudget_IsRejected() + { + var big = Article(""); + var queue = CreateQueue(big.Payload.Length - 1); + Assert.Equal(ArticleEnqueueResult.Rejected, await queue.EnqueueAsync(big, CancellationToken.None)); + Assert.False(queue.TryEnqueue(Article(""))); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task FailedEnqueueAfterReservation_ReleasesBytes() + { + var article = Article(""); + var queue = CreateQueue(article.Payload.Length + 16); + queue.CompleteChannelWithoutMarkingUnavailableForTests(); + Assert.Equal(ArticleEnqueueResult.Unavailable, await queue.EnqueueAsync(article, CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task Cancellation_ReleasesReservation() + { + var held = Article(""); + var cancel = Article(""); + var queue = CreateQueue(held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(held, CancellationToken.None)); + + using var cts = new CancellationTokenSource(); + var blocked = queue.EnqueueAsync(cancel, cts.Token); + await WaitForWaitersAsync(queue, 1); + cts.Cancel(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ShutdownDrain_DoesNotLeakReservations() + { + var a = Article(""); + var b = Article(""); + var c = Article(""); + var queue = CreateQueue(a.Payload.Length + b.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + + var blocked = queue.EnqueueAsync(c, CancellationToken.None); + await WaitForWaitersAsync(queue, 1); + + queue.Complete(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await queue.EnqueueAsync(Article(""), CancellationToken.None)); + + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.False(queue.IsAccepting); + } + + [Fact] + public async Task QueuedBytes_ReturnToZeroAfterConsume() + { + var a = Article(""); + var b = Article(""); + var total = a.Payload.Length + b.Payload.Length; + var queue = CreateQueue(total); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + Assert.Equal(total, queue.QueuedBytes); + Assert.NotNull(await queue.DequeueAsync(CancellationToken.None)); + Assert.NotNull(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(total, queue.PeakQueuedBytes); + } + + [Fact] + public void TryProbeCapacity_IsFalseWhenBudgetIsExhausted() + { + var full = Article(""); + var queue = CreateQueue(full.Payload.Length); + Assert.True(queue.TryProbeCapacity()); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(full)); + Assert.False(queue.TryProbeCapacity()); + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.Equal(full.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + } + + [Fact] + public void TryAdmit_DoesNotWait_WhenBudgetIsExhausted() + { + var held = Article(""); + var queue = CreateQueue(held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + var started = Environment.TickCount64; + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.True(Environment.TickCount64 - started < 250); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + } + + [Fact] + public void TryAdmit_RejectedArticle_DoesNotReserve() + { + var big = Article(""); + var queue = CreateQueue(big.Payload.Length - 1); + Assert.Equal(ArticleEnqueueResult.Rejected, queue.TryAdmit(big)); + Assert.Equal(0, queue.QueuedBytes); + Assert.True(queue.TryProbeCapacity()); + } + + [Fact] + public void TryAdmit_ConcurrentProducers_CannotOversubscribe() + { + const int producers = 16; + const int admitted = 4; + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * admitted; + var queue = CreateQueue(limit); + var results = new ArticleEnqueueResult[producers]; + Parallel.For(0, producers, i => + { + results[i] = queue.TryAdmit(Article($"")); + }); + + Assert.Equal(admitted, results.Count(static r => r == ArticleEnqueueResult.Accepted)); + Assert.Equal(producers - admitted, results.Count(static r => r == ArticleEnqueueResult.Full)); + Assert.Equal(limit, queue.QueuedBytes); + Assert.Equal(admitted, queue.Count); + Assert.True(queue.QueuedBytes <= limit); + } + + [Fact] + public void TryAdmit_RejectedAndFull_IncrementAdmissionFailures() + { + var fit = Article(""); + var huge = Article("", body: "body\r\n" + new string('Z', 80) + "\r\n"); + Assert.True(huge.Payload.Length > fit.Payload.Length); + var queue = CreateQueue(fit.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Rejected, queue.TryAdmit(huge)); + Assert.Equal(1, queue.AdmissionFailureCount); + + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(fit)); + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.Equal(2, queue.AdmissionFailureCount); + Assert.Equal(0, queue.AdmissionWaitTicks); + } + + [Fact] + public async Task Dequeue_IsFifo() + { + var sample = Article("<0@fifo>"); + var queue = CreateQueue(sample.Payload.Length * 20); + for (var i = 0; i < 10; i++) + { + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(Article($"<{i}@fifo>"))); + } + + for (var i = 0; i < 10; i++) + { + Assert.Equal($"<{i}@fifo>", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + } + + queue.Complete(); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ProducerIdentityAndOrder_RemainIntact() + { + var first = Article("", InboundArticleProducer.IHave); + var second = Article(""); + var queue = CreateQueue(first.Payload.Length + second.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(second, CancellationToken.None)); + + var dequeuedFirst = await queue.DequeueAsync(CancellationToken.None); + var dequeuedSecond = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("", dequeuedFirst!.MessageId); + Assert.Equal(InboundArticleProducer.IHave, dequeuedFirst.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, dequeuedFirst.Record.ParseStatus); + Assert.Equal("", dequeuedSecond!.MessageId); + Assert.Equal(InboundArticleProducer.TakeThis, dequeuedSecond.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, dequeuedSecond.Record.ParseStatus); + Assert.Equal(0, queue.QueuedBytes); + } + + private static ArticleIngestionQueue CreateQueue(long memoryLimit) => + new(new ArticleIngestionOptions(), memoryLimit); + + private static InboundArticle Article( + string messageId, + InboundArticleProducer producer = InboundArticleProducer.TakeThis, + string body = "body\r\n") => + CanonicalArticleText.CreateQueued(messageId, producer, body); + + private static Task WaitForWaitersAsync(ArticleIngestionQueue queue, int count) => + WaitUntilAsync(() => queue.DebugWaiterCount >= count, TimeSpan.FromSeconds(5)); + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + await Task.Delay(1, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs new file mode 100644 index 00000000..52fe73fe --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs @@ -0,0 +1,116 @@ +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class ArticleRecordIngressTests +{ + private static readonly NntpArticleParser Parser = new("nntpd01.usenet.ninja"); + + [Fact] + public void TryCreateFromDestuffed_ValidArticle_ReturnsCanonicalRecord() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("", "one\r\ntwo\r\n")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var record = created.Record; + Assert.Equal(ArticleParseStatus.CanonicalV1, record.ParseStatus); + Assert.Equal(record.ArtData.Length, record.ArtSize); + Assert.Equal(2, record.ArtLines); + Assert.NotEqual(ArticleType.None, record.ArtType); + Assert.True(record.MessageId.SequenceEqual(""u8)); + Assert.True(record.Fields.MessageId.IsPresent); + Assert.True(record.Fields.Newsgroups.IsPresent); + Assert.True(record.ArtData.Span.Slice(record.Fields.MessageId.Offset, record.Fields.MessageId.Length) + .SequenceEqual(""u8)); + } + + [Fact] + public void TryCreateFromStuffedWire_DestuffsOnce_ThenFactory() + { + var destuffed = CanonicalArticleText.Destuffed("", ".hidden\r\n"); + var stuffed = destuffed.Replace(".hidden\r\n", "..hidden\r\n", StringComparison.Ordinal); + var created = ArticleRecordIngress.TryCreateFromStuffedWire( + Parser, + Encoding.ASCII.GetBytes(stuffed), + 64 * 1024); + Assert.True(created.IsAccepted); + var text = Encoding.ASCII.GetString(created.Record.ArtData.Span); + Assert.Contains("\r\n\r\n..hidden\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("\r\n\r\n.hidden\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public void TryCreateFromDestuffed_MissingDate_DoesNotProduceRecord() + { + var destuffed = "Message-ID: \r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\n\r\nbody\r\n"u8.ToArray(); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.False(created.IsAccepted); + Assert.Equal(NntpArticleParseFailureCode.MissingOrInvalidDate, created.ParseFailure); + Assert.Equal(ArticleParseStatus.None, created.Record.ParseStatus); + Assert.Equal(0, created.Record.ArtSize); + } + + [Fact] + public void CreateQueued_PayloadAliasesArtData() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.True(inbound.Payload.Equals(created.Record.ArtData)); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.Equal(InboundArticleProducer.TakeThis, inbound.Producer); + } + + [Fact] + public void CreateQueued_IHave_IsCanonicalV1() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.IHave); + Assert.Equal(InboundArticleProducer.IHave, inbound.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(created.Record.ArtData)); + } + + [Fact] + public void CreateQueued_RejectsNonCanonicalRecord() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var invalid = new ArticleRecord( + created.Record.ArtId, + created.Record.ArtHash, + created.Record.ArtType, + created.Record.ArtLines, + created.Record.CanonicalUtc, + ArticleParseStatus.None, + created.Record.ArtData.ToArray(), + created.Record.Fields); + Assert.Throws(() => + ArticleRecordIngress.CreateQueued( + "", + invalid, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.Post)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs new file mode 100644 index 00000000..d90e5ddd --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs @@ -0,0 +1,117 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class ArticleTypeClassifierTests +{ + [Fact] + public void PlainText_IsDefault() + { + var type = ArticleTypeClassifier.Classify("From: a@b\r\nSubject: hi\r\n"u8, "hello\r\n"u8); + Assert.Equal(ArticleType.Default, type); + } + + [Fact] + public void YbeginLine_IsYEncodedAndBinary() + { + var type = ArticleTypeClassifier.Classify( + "Subject: part\r\n"u8, + "=ybegin line=128 size=12 name=a.bin\r\n"u8); + Assert.True(type.HasFlag(ArticleType.YEncoded)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void YbeginPart_IsYEncodedPartial() + { + var type = ArticleTypeClassifier.Classify( + default, + "=ybegin part=1 line=128 size=99 name=a.bin\r\n"u8); + Assert.True(type.HasFlag(ArticleType.YEncoded)); + Assert.True(type.HasFlag(ArticleType.Partial)); + } + + [Fact] + public void MimeAndBase64Headers_ClassifyWithoutBodyScan() + { + var type = ArticleTypeClassifier.Classify( + "Mime-Version: 1.0\r\nContent-Type: text/plain\r\nContent-Transfer-Encoding: base64\r\n"u8, + default); + Assert.True(type.HasFlag(ArticleType.Mime)); + Assert.True(type.HasFlag(ArticleType.Base64)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void UuencodeBegin_IsUuEncode() + { + var type = ArticleTypeClassifier.Classify(default, "begin 644 file.dat\r\n"u8); + Assert.True(type.HasFlag(ArticleType.UuEncode)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void ContentLength_ParsesDestuffedHint() + { + Assert.Equal(12, ArticleTypeClassifier.TryParseContentLength("Content-Length: 12"u8)); + Assert.Equal(-1, ArticleTypeClassifier.TryParseContentLength("Subject: 12"u8)); + } + + [Fact] + public void YencSize_IsDecodedSizeNotWire() + { + Assert.Equal(12345, ArticleTypeClassifier.TryParseYencSize("=ybegin line=128 size=12345 name=a.bin"u8)); + Assert.Equal(-1, ArticleTypeClassifier.TryParseYencSize("=ybegin line=128 name=a.bin"u8)); + } + + [Fact] + public void YEncoded_MatchesTransitYencBit() + { + Assert.Equal((int)VectorNNTPD.ReaderService.Configuration.TransitMessageTypes.Yenc, (int)ArticleType.YEncoded); + } + + [Fact] + public void CommonArticleTypeBits_MatchTransitMessageTypes() + { + Assert.Equal((int)TransitMessageTypes.None, (int)ArticleType.None); + Assert.Equal((int)TransitMessageTypes.Default, (int)ArticleType.Default); + Assert.Equal((int)TransitMessageTypes.Control, (int)ArticleType.Control); + Assert.Equal((int)TransitMessageTypes.Cancel, (int)ArticleType.Cancel); + Assert.Equal((int)TransitMessageTypes.Mime, (int)ArticleType.Mime); + Assert.Equal((int)TransitMessageTypes.Binary, (int)ArticleType.Binary); + Assert.Equal((int)TransitMessageTypes.UuEncode, (int)ArticleType.UuEncode); + Assert.Equal((int)TransitMessageTypes.Base64, (int)ArticleType.Base64); + Assert.Equal((int)TransitMessageTypes.Yenc, (int)ArticleType.YEncoded); + Assert.Equal((int)TransitMessageTypes.BommaNews, (int)ArticleType.BommaNews); + Assert.Equal((int)TransitMessageTypes.UniData, (int)ArticleType.UniData); + Assert.Equal((int)TransitMessageTypes.Multipart, (int)ArticleType.Multipart); + Assert.Equal((int)TransitMessageTypes.Html, (int)ArticleType.Html); + Assert.Equal((int)TransitMessageTypes.PostScript, (int)ArticleType.PostScript); + Assert.Equal((int)TransitMessageTypes.BinHex, (int)ArticleType.BinHex); + Assert.Equal((int)TransitMessageTypes.Partial, (int)ArticleType.Partial); + Assert.Equal((int)TransitMessageTypes.PgpMessage, (int)ArticleType.PgpMessage); + } + + [Fact] + public void IhaveDestuff_UsesCommonClassifier() + { + var article = IhaveArticleInterpreter.DestuffToArticle( + "Control: cancel \r\n\r\nbody\r\n"u8, + 64 * 1024); + Assert.True(article.Type.HasFlag(ArticleType.Control)); + Assert.True(article.Type.HasFlag(ArticleType.Cancel)); + } + + [Fact] + public void ArticleSize_IsDestuffedCompleteArticle() + { + var headers = Encoding.ASCII.GetBytes("Subject: hi\r\n"); + var body = Encoding.ASCII.GetBytes("body\r\n"); + var article = new Article(headers, body, ArticleType.Default); + Assert.Equal(headers.Length + 2 + body.Length, article.Size); + Assert.Equal("Subject: hi\r\n\r\nbody\r\n", Encoding.ASCII.GetString(article.Payload.Span)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs new file mode 100644 index 00000000..436ff814 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs @@ -0,0 +1,113 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class IhaveArticleInterpreterTests +{ + [Fact] + public void DestuffToArticle_UnstuffsOnce_AndMatchesCanonical() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var article = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + Assert.Equal("Subject: d\r\n", Encoding.ASCII.GetString(article.Headers.Span)); + Assert.Equal(".foo\r\n", Encoding.ASCII.GetString(article.Body.Span)); + Assert.Equal("Subject: d\r\n\r\n.foo\r\n"u8.ToArray(), article.Payload.ToArray()); + Assert.Equal(article.Payload.Length, article.Size); + } + + [Fact] + public void DestuffToArticle_DoesNotDestuffTwice() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var once = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + var twice = IhaveArticleInterpreter.DestuffToArticle(once.Payload.Span, 64 * 1024); + Assert.Equal(".foo\r\n", Encoding.ASCII.GetString(once.Body.Span)); + Assert.Equal("foo\r\n", Encoding.ASCII.GetString(twice.Body.Span)); + } + + [Fact] + public void DestuffToArticle_Yenc_IsNotDecoded() + { + var wire = "Subject: y\r\n\r\n=ybegin line=128 size=4 name=a.bin\r\n=ypart begin=1 end=4\r\n)ab=\r\n=yend size=4\r\n"u8; + var article = IhaveArticleInterpreter.DestuffToArticle(wire, 64 * 1024); + Assert.True(article.Type.HasFlag(ArticleType.YEncoded)); + Assert.Contains("=ybegin", Encoding.ASCII.GetString(article.Body.Span), StringComparison.Ordinal); + } + + [Fact] + public void DestuffThenRestuff_PreservesDotStuffingOnOutbound() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var article = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + var wire = ArticleWireReconstructor.RestuffArticle(article.Payload.Span); + Assert.Equal("Subject: d\r\n\r\n..foo\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public async Task SpoolWriter_ConsumesCanonicalRecordsWithoutReparse() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = new ArticleIngestionOptions() }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + var ihave = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave, ".foo\r\n"); + var takeThis = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, ".foo\r\n"); + var post = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post, ".foo\r\n"); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(ihave, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(takeThis, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(post, CancellationToken.None)); + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(ArticleParseStatus.CanonicalV1, ihave.Record.ParseStatus); + Assert.Equal(ArticleParseStatus.CanonicalV1, takeThis.Record.ParseStatus); + Assert.Equal(ArticleParseStatus.CanonicalV1, post.Record.ParseStatus); + Assert.True(ihave.Payload.Equals(ihave.Record.ArtData)); + Assert.True(takeThis.Payload.Equals(takeThis.Record.ArtData)); + Assert.True(post.Payload.Equals(post.Record.ArtData)); + } + + [Fact] + public async Task SpoolWriter_CanonicalV1Record_IsNotReparsedOrCopied() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = new ArticleIngestionOptions() }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("", "one\r\n"))); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Record.ArtData.Equals(created.Record.ArtData)); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs new file mode 100644 index 00000000..93e5c511 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs @@ -0,0 +1,723 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Post-queue news disposition and INN news writer contracts. +public sealed class IncomingSpoolNewsLogTests +{ + [Fact] + public async Task NormalAcceptedArticle_ProducesPlus() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "alt.test"), + "", + news, + Catalogue("alt.test")); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, evt.Disposition); + Assert.True(evt.MessageId.Span.SequenceEqual(""u8)); + Assert.True(evt.Feed.IsEmpty); + Assert.True(evt.Sites.IsEmpty); + Assert.True(evt.Size > 0); + } + + [Fact] + public async Task AcceptedJunkArticle_ProducesJ() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "unknown.un.carried"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(Uncarried("unknown.un.carried"), Encoding.ASCII.GetString(evt.Reason.Span)); + } + + [Fact] + public async Task PeerOnlyCatalogueHit_ProducesJWithPeerOnlyReason() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "junk.local"), + "", + news, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + wantTrash: false, + logTrash: true); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(PeerOnlyReason("junk.local"), Encoding.ASCII.GetString(evt.Reason.Span)); + } + + [Fact] + public async Task NamedInboundFeed_IsCopiedOntoAcceptedEvent_AndFormattedWithoutSession() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + feed: "BlueWorldHosting"u8.ToArray()); + await RunWorkerAsync(inbound, news, Catalogue("alt.test")); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, evt.Disposition); + Assert.True(evt.Feed.Span.SequenceEqual("BlueWorldHosting"u8)); + Assert.Equal(inbound.Payload.Length, evt.Size); + var line = FormatNews(in evt); + Assert.Contains(" + BlueWorldHosting " + evt.Size + " ?", line, StringComparison.Ordinal); + Assert.DoesNotContain(" + ? ", line, StringComparison.Ordinal); + Assert.EndsWith(" ?\n", line, StringComparison.Ordinal); + } + + [Fact] + public async Task NamedInboundFeed_Junk_KeepsPeerAndOmitsOutboundSites() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.un.carried", + feed: "BlueWorldHosting"u8.ToArray()); + await RunWorkerAsync( + inbound, + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.True(evt.Feed.Span.SequenceEqual("BlueWorldHosting"u8)); + var line = FormatNews(in evt); + Assert.Contains(" j BlueWorldHosting " + evt.Size + " " + Uncarried("unknown.un.carried"), line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public async Task WantTrash_DoesNotRewriteOriginalNewsgroupsHeader() + { + var news = new RecordingNewsLogWriter(); + const string groups = "unknown.un.carried"; + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: groups), + "", + news, + Catalogue("alt.test")); + + var keep = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, keep.Disposition); + Assert.Equal(Uncarried(groups), Encoding.ASCII.GetString(keep.Reason.Span)); + } + + [Fact] + public async Task WantTrash_AppliesToIhaveUnknownGroup() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.IHave, + newsgroups: "not.carried.here"); + await RunWorkerAsync( + inbound, + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var ihaveJunk = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, ihaveJunk.Disposition); + Assert.Equal(Uncarried("not.carried.here"), Encoding.ASCII.GetString(ihaveJunk.Reason.Span)); + Assert.Contains("Newsgroups: not.carried.here"u8, inbound.Payload.Span); + } + + [Fact] + public async Task LogTrashFalse_SuppressesJunkLine_WithoutRejecting() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "unknown.group"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + Assert.Empty(news.Events); + Assert.Equal(0, news.WriteCalls); + } + + [Fact] + public async Task LogTrashFalse_StillLogsNormalAcceptedArticles() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "alt.test"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + var stillPlus = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, stillPlus.Disposition); + Assert.True(stillPlus.Reason.IsEmpty); + } + + [Fact] + public async Task MessageId_IsActualArticleMessageId_NotArticleId() + { + const string id = ""; + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed(id, newsgroups: "alt.test")); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued(id, created, Identity(), DateTimeOffset.UtcNow, InboundArticleProducer.TakeThis), + news, + Catalogue("alt.test")); + + Assert.True(Assert.Single(news.Events).MessageId.Span.SequenceEqual(Encoding.ASCII.GetBytes(id))); + Assert.True(created.MessageId.SequenceEqual(Encoding.ASCII.GetBytes(id))); + Span artId = stackalloc byte[ArticleId.Length]; + created.ArtId.CopyTo(artId); + Assert.False(news.Events[0].MessageId.Span.SequenceEqual(artId)); + } + + [Fact] + public async Task TakeThisCanonicalRecord_IsNotReparsedToGenerateNews() + { + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, news, Catalogue("alt.test")); + + Assert.True(inbound.Record.ArtData.Equals(created.ArtData)); + Assert.True(news.Events[0].MessageId.Span.SequenceEqual(created.MessageId)); + Assert.True( + news.Events[0].MessageId.Span.SequenceEqual( + inbound.Record.ArtData.Span.Slice( + inbound.Record.Fields.MessageId.Offset, + inbound.Record.Fields.MessageId.Length))); + } + + [Fact] + public async Task Post_IsNeverClassifiedAsWantTrashJunk() + { + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed("", newsgroups: "unknown.group")); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.Post), + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var post = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, post.Disposition); + Assert.NotEqual(NewsLogDisposition.Junk, post.Disposition); + Assert.True(post.Reason.IsEmpty); + } + + [Fact] + public async Task NewsEvent_IsProducedAfterEnqueue_NotDuringReceiveAdmission() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + Assert.Equal(0, news.WriteCalls); + + var writer = CreateWriter(queue, news, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(1, news.WriteCalls); + Assert.Equal(1, news.FlushCalls); + Assert.Equal(NewsLogDisposition.Accepted, Assert.Single(news.Events).Disposition); + } + + [Fact] + public async Task Shutdown_FlushesPendingNewsEntries() + { + using var dir = new TempDir(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using var news = new SerilogNewsLogWriter(configuration); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + LogDir = dir.Path, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + news.Dispose(); + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + } + + [Fact] + public async Task NewsWriterFailure_DoesNotReprocessOrSkipPersister() + { + var throwing = new ThrowingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed(""), + "", + throwing, + Catalogue("alt.test")); + + Assert.Equal(1, throwing.WriteCalls); + } + + [Fact] + public void Classify_CarriedNonJunkGroup_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "alt.test"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"))); + } + + [Fact] + public void Classify_UnknownGroup_WantTrash_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Equal(Uncarried("unknown.group"), reason); + } + + [Fact] + public void Classify_MultipleGroups_OneCarriedNonJunk_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group,alt.test,other.unknown"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Null(reason); + } + + [Fact] + public void Classify_MixedCarriedAndPeerOnly_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "alt.test,junk.local"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify( + inbound, + TrashOn(), + CatalogueWith( + ("alt.test", NewsgroupPostingStatus.Allowed), + ("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Null(reason); + } + + [Fact] + public void Classify_MixedUnknownAndPeerOnly_ReportsOnlyPeerOnlyGroups() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group,junk.local,other.unknown"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + TrashOn(), + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.local"), reason); + Assert.DoesNotContain("unknown.group", reason, StringComparison.Ordinal); + Assert.DoesNotContain("other.unknown", reason, StringComparison.Ordinal); + } + + [Fact] + public void Classify_MultiplePeerOnlyGroups_ReportsHeaderOrder() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.beta, junk.alpha"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith( + ("junk.alpha", NewsgroupPostingStatus.PeerOnly), + ("junk.beta", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.beta", "junk.alpha"), reason); + } + + [Fact] + public void Classify_MultipleUnknownGroups_WantTrash_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.one,unknown.two"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Equal(Uncarried("unknown.one", "unknown.two"), reason); + Assert.DoesNotContain("alt.test", reason, StringComparison.Ordinal); + } + + [Fact] + public void Classify_PeerOnlyJunkGroup_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.local"), reason); + } + + [Fact] + public void Classify_WantTrashFalse_UnknownGroup_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + Catalogue("alt.test"), + out var reason)); + Assert.Null(reason); + } + + [Fact] + public void TryCreateEvent_WantTrashUnknown_CarriesNewsgroupNotCarried() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.one,unknown.two"); + Assert.True( + IngressNewsDisposition.TryCreateEvent( + inbound, + TrashOn(), + Catalogue("alt.test"), + new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero), + out var evt)); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(Uncarried("unknown.one", "unknown.two"), Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.Equal( + $"Jan 5 00:00:00.000 j ? {evt.Size} newsgroup not carried: unknown.one, unknown.two\n", + FormatNews(in evt)); + } + + [Fact] + public void TryCreateEvent_PeerOnly_CarriesPeerOnlyReason() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.True( + IngressNewsDisposition.TryCreateEvent( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero), + out var evt)); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(PeerOnlyReason("junk.local"), Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.Equal( + $"Jan 5 00:00:00.000 j ? {evt.Size} peer-only: junk.local\n", + FormatNews(in evt)); + } + + [Fact] + public void IsUncarriedWantTrashRejection_WantTrashFalse_UnknownGroup_IsTrue() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.True( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false, LogTrash = false }, + Catalogue("alt.test"), + out var reason)); + Assert.Equal(Uncarried("unknown.group"), reason); + } + + [Fact] + public void IsUncarriedWantTrashRejection_MultipleUnknown_ReportsHeaderOrder() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.two,unknown.one"); + Assert.True( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false, LogTrash = false }, + Catalogue("alt.test"), + out var reason)); + Assert.Equal(Uncarried("unknown.two", "unknown.one"), reason); + } + + [Fact] + public void IsUncarriedWantTrashRejection_WantTrashTrue_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.IHave, + newsgroups: "unknown.group"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = true }, + Catalogue("alt.test"))); + } + + [Fact] + public void IsUncarriedWantTrashRejection_PeerOnly_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)))); + } + + [Fact] + public void IsUncarriedWantTrashRejection_PostOrMissingCatalogue_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.Post, + newsgroups: "unknown.group"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false }, + Catalogue("alt.test"))); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"), + new TransitOptions { WantTrash = false }, + catalogue: null)); + } + + [Fact] + public void Classify_WithoutCatalogue_IsAcceptedRatherThanInventedJunk() + { + var created = CreateRecord(CanonicalArticleText.Destuffed("", newsgroups: "unknown.group")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, new TransitOptions { WantTrash = true }, catalogue: null)); + } + + private static async Task RunTakeThisAsync( + string destuffed, + string messageId, + INewsLogWriter news, + INewsgroupCatalogue catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var created = CreateRecord(destuffed); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + messageId, + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + news, + catalogue, + wantTrash, + logTrash); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + INewsLogWriter news, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, news, catalogue, wantTrash, logTrash); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + INewsLogWriter news, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions(), + Transit = new TransitOptions { WantTrash = wantTrash, LogTrash = logTrash }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: catalogue); + + private static ArticleRecord CreateRecord(string destuffed) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(destuffed)); + Assert.True(created.IsAccepted); + return created.Record; + } + + private static string FormatNews(in NewsLogEvent evt) + { + var timestamp = evt.Timestamp == default + ? new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero) + : evt.Timestamp; + var buffer = new byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + return Encoding.ASCII.GetString(buffer.AsSpan(0, written)); + } + + private static string Uncarried(params string[] groups) => + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, groups); + + private static string PeerOnlyReason(params string[] groups) => + IngressNewsReasons.WithGroups(IngressNewsReasons.PeerOnly, groups); + + private static TransitOptions TrashOn() => new() { WantTrash = true, LogTrash = true }; + + private static INewsgroupCatalogue Catalogue(params string[] groups) => + CatalogueWith(groups.Select(static name => (name, NewsgroupPostingStatus.Allowed)).ToArray()); + + private static INewsgroupCatalogue CatalogueWith(params (string Name, NewsgroupPostingStatus Status)[] groups) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + groups.Select(static g => + new NewsgroupDefinition(g.Name, string.Empty, 2, 1, g.Status)).ToArray())); + + private static ConnectionClientIdentity Identity() => + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-news-stop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs new file mode 100644 index 00000000..1b1b52da --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs @@ -0,0 +1,298 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles.OverviewDb; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Ingestion-worker OverviewDB handoff success, requeue, and topology contracts. +[Collection(SerilogCollection.Name)] +public sealed class IncomingSpoolOverviewHandoffTests +{ + [Fact] + public async Task Topology_DeclaresOverviewDbClassicQueue_WithoutExchangeOrBinding() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + var overview = Assert.Single( + connection.QueueDeclarations, + static queue => queue.Name == OverviewDbTopology.QueueName); + Assert.Equal("overviewdb.queue", overview.Name); + Assert.True(overview.Durable); + Assert.False(overview.Exclusive); + Assert.False(overview.AutoDelete); + var arguments = overview.Arguments ?? new Dictionary(StringComparer.Ordinal); + Assert.True(arguments.TryGetValue(OverviewDbTopology.QueueTypeArgumentName, out var queueType)); + Assert.Equal(OverviewDbTopology.QuorumQueueType, queueType); + Assert.False(arguments.ContainsKey("x-message-ttl")); + Assert.False(arguments.ContainsKey("x-expires")); + Assert.Single(arguments); + Assert.DoesNotContain(connection.ExchangeDeclarations, static e => e.Name == OverviewDbTopology.QueueName); + Assert.DoesNotContain(connection.BindingDeclarations, static b => b.Queue == OverviewDbTopology.QueueName); + } + + [Fact] + public async Task PublishSuccess_CompletesWorkerAndPersistsArticle() + { + var overview = new RecordingOverviewDbHandoffPublisher(); + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(1, overview.AttemptCount); + var payload = Assert.Single(overview.Payloads); + var decoded = OverviewArticleV1Codec.Decode(payload); + Assert.True(Encoding.ASCII.GetBytes(inbound.MessageId).AsSpan().SequenceEqual(decoded.MessageId)); + Assert.Equal(2, decoded.Newsgroups.Count); + Assert.True(Encoding.ASCII.GetBytes("alt.test").AsSpan().SequenceEqual(decoded.Newsgroups[0])); + Assert.True(Encoding.ASCII.GetBytes("rec.test").AsSpan().SequenceEqual(decoded.Newsgroups[1])); + Assert.Equal((uint)inbound.Record.ArtSize, decoded.Bytes); + Assert.Equal((uint)inbound.Record.ArtLines, decoded.Lines); + } + + [Fact] + public async Task Worker_RecordsOneItemOnPipelineMetrics() + { + var overview = new RecordingOverviewDbHandoffPublisher(); + var pipeline = new IngestionPipelineMetrics(); + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview, pipeline); + + var snapshot = pipeline.CaptureInterval(); + Assert.Equal(1, snapshot.WorkerItems); + Assert.Equal(1, snapshot.ToPublishStart.Count); + Assert.Equal(1, snapshot.Encode.Count); + Assert.Equal(1, snapshot.News.Count); + Assert.True(snapshot.BusyTicks > 0); + } + + [Fact] + public async Task PublishFailure_RequeuesUntilConfirm_ThenCompletes() + { + var overview = new RecordingOverviewDbHandoffPublisher { RemainingFailures = 1 }; + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public async Task UnroutableReturn_RequeuesSameInboundArticle_UntilConfirm() + { + var overview = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new InvalidOperationException( + "RabbitMQ returned the OverviewDB handoff as unroutable."), + }; + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public async Task PublishFailure_ArticlePersistsBeforeConfirm_AndWorkItemRetries() + { + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var overview = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + BlockOnFailure = block, + }; + var diagnostics = new SignalingFeedDiagnostics(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, overview, diagnostics); + await writer.StartAsync(CancellationToken.None); + var inbound = CreateArticle(); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var end = await diagnostics.Ended.Task.WaitAsync(cts.Token); + + // Article path continues after OverviewDB work enqueue; Rabbit confirm is background. + Assert.False(end.Persisted); + await overview.FirstAttempt.WaitAsync(cts.Token); + Assert.Equal(1, overview.AttemptCount); + Assert.Empty(overview.Payloads); + + block.TrySetResult(); + using var retryWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.AttemptCount < 2 || overview.Payloads.Count < 1) + { + retryWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, retryWait.Token); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public void Worker_HasNoOverviewDbRpcOrDatabaseDependency() + { + var ctor = typeof(IncomingSpoolWriterService).GetConstructors().Single(); + var types = ctor.GetParameters().Select(static p => p.ParameterType).ToArray(); + Assert.Contains(typeof(IOverviewDbHandoffPublisher), types); + Assert.DoesNotContain(types, static t => t.Name.Contains("Persister", StringComparison.Ordinal)); + Assert.DoesNotContain(types, static t => t == typeof(IRabbitMqRpcChannel)); + Assert.DoesNotContain(types, static t => t.Name.Contains("NntpDb", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("MySql", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("Http", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("Grpc", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Host_RegistersOverviewDbHandoffPublisher_NotAnRpcClient() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var publisher = host.Services.GetRequiredService(); + Assert.IsType(publisher); + Assert.Same(IngestionPipelineMetrics.Shared, host.Services.GetRequiredService()); + Assert.IsNotType(publisher); + var services = host.Services.GetServices().Select(static s => s.GetType()).ToArray(); + Assert.DoesNotContain(services, static t => t.Name.Contains("OverviewDbClient", StringComparison.Ordinal)); + Assert.DoesNotContain(services, static t => t.Name.Contains("OverviewDbRpc", StringComparison.Ordinal)); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + RecordingOverviewDbHandoffPublisher overview, + IngestionPipelineMetrics? pipelineMetrics = null) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, overview, pipelineMetrics: pipelineMetrics); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await overview.Confirmed.WaitAsync(cts.Token); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + IOverviewDbHandoffPublisher overview, + IFeedDiagnostics? feedDiagnostics = null, + IngestionPipelineMetrics? pipelineMetrics = null) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + MaxPublishConcurrency = 1, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + feedDiagnostics: feedDiagnostics, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("rec.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)])), + overviewHandoff: overview, + pipelineMetrics: pipelineMetrics); + + private static InboundArticle CreateArticle() => + CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + body: "overview-body\r\n", + newsgroups: "alt.test,rec.test", + subject: "overview-subject", + from: "poster@example.test", + references: ""); + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + /// Signals spool-work completion for tests. + private sealed class SignalingFeedDiagnostics : IFeedDiagnostics + { + /// Completes when the worker finishes one spool-work attempt. + public TaskCompletionSource<(int Bytes, bool Persisted)> Ended { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + public bool IsEnabled => false; + + /// + public void OnRejected(string peerName, string remote) + { + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) => null; + + /// + public void OnReleased(FeedSessionProbe? probe) + { + } + + /// + public void RecordTcpBytes(int bytes) + { + } + + /// + public void BeginSpoolWork() + { + } + + /// + public void EndSpoolWork(int bytes, bool persisted) => Ended.TrySetResult((bytes, persisted)); + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) => + FeedDiagnosticsSnapshot.Empty; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs new file mode 100644 index 00000000..a95eabb9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs @@ -0,0 +1,299 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Post-queue Path-survey capture from canonical ArticleRecord.Path. +public sealed class IncomingSpoolPathSurveyTests +{ + [Fact] + public async Task CanonicalArticleRecordPath_IsWritten_WithoutSynthesis() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, news, paths, Catalogue("alt.test")); + + var observed = Assert.Single(paths.Paths); + Assert.True(observed.AsSpan().SequenceEqual(inbound.Record.Path)); + var line = Encoding.Latin1.GetString(FormatPath(observed)); + Assert.Equal("Path: " + Encoding.Latin1.GetString(inbound.Record.Path) + "\n", line); + Assert.DoesNotContain("giganews", line, StringComparison.Ordinal); + Assert.StartsWith("Path: ", line, StringComparison.Ordinal); + Assert.Equal(1, news.WriteCalls); + Assert.Equal(NewsLogDisposition.Accepted, Assert.Single(news.Events).Disposition); + } + + [Fact] + public async Task MultipleArticles_ProduceSequentialPathRecords() + { + var paths = new RecordingPathSurveyWriter(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(second, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(2, paths.WriteCalls); + Assert.True(paths.Paths[0].AsSpan().SequenceEqual(first.Record.Path)); + Assert.True(paths.Paths[1].AsSpan().SequenceEqual(second.Record.Path)); + Assert.Equal(1, paths.FlushCalls); + } + + [Fact] + public async Task JunkWithLogTrashFalse_StillWritesPath_AndOmitsNews() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.un.carried"); + await RunWorkerAsync( + inbound, + news, + paths, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + Assert.Equal(0, news.WriteCalls); + var observed = Assert.Single(paths.Paths); + Assert.True(observed.AsSpan().SequenceEqual(inbound.Record.Path)); + } + + [Fact] + public async Task PathWriterFailure_DoesNotSkipNewsOrPersister() + { + var throwing = new ThrowingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed(""), + "", + news, + throwing, + Catalogue("alt.test")); + + Assert.Equal(1, throwing.WriteCalls); + Assert.Equal(1, news.WriteCalls); + } + + [Fact] + public async Task NewsWriterFailure_StillWritesPathAndPersists() + { + var throwingNews = new ThrowingNewsLogWriter(); + var paths = new RecordingPathSurveyWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, throwingNews, paths, Catalogue("alt.test")); + + Assert.Equal(1, throwingNews.WriteCalls); + Assert.True(Assert.Single(paths.Paths).AsSpan().SequenceEqual(inbound.Record.Path)); + } + + [Fact] + public async Task PathIsNotWrittenUntilAfterOverviewConfirm() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + Assert.Equal(0, paths.WriteCalls); + Assert.Equal(0, news.WriteCalls); + + var writer = CreateWriter(queue, news, paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(1, paths.WriteCalls); + Assert.Equal(1, news.WriteCalls); + } + + [Fact] + public async Task LargeSequentialInput_IsCountedWithoutRetainingTheStream() + { + var paths = new CountingPathSurveyWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 64 }); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + const int count = 32; + long expectedBytes = 0; + for (var i = 0; i < count; i++) + { + var inbound = CanonicalArticleText.CreateQueued( + $"", + InboundArticleProducer.TakeThis); + expectedBytes += inbound.Record.Path.Length; + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(count, paths.WriteCalls); + Assert.Equal(expectedBytes, paths.Bytes); + Assert.Null(paths.GetType().GetProperty("Paths")); + } + + [Fact] + public async Task Shutdown_FlushesPathSurveyFile() + { + using var dir = new TempDir(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using var paths = new SerilogPathSurveyWriter(configuration); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + paths.Dispose(); + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: " + Encoding.Latin1.GetString(inbound.Record.Path) + "\n", text); + } + + private static async Task RunTakeThisAsync( + string destuffed, + string messageId, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var created = CreateRecord(destuffed); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + messageId, + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + news, + paths, + catalogue, + wantTrash, + logTrash); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, news, paths, catalogue, wantTrash, logTrash); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions(), + Transit = new TransitOptions { WantTrash = wantTrash, LogTrash = logTrash }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: catalogue, + pathSurvey: paths); + + private static ArticleRecord CreateRecord(string destuffed) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(destuffed)); + Assert.True(created.IsAccepted); + return created.Record; + } + + private static byte[] FormatPath(ReadOnlySpan path) + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + PathSurveyLineFormatter.Write(buffer, path); + return buffer; + } + + private static INewsgroupCatalogue Catalogue(params string[] groups) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + groups.Select(static name => + new NewsgroupDefinition(name, string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)).ToArray())); + + private static ConnectionClientIdentity Identity() => + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-inpaths-stop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs new file mode 100644 index 00000000..34c22de9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs @@ -0,0 +1,161 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// +/// Demonstrates that article workers are not serialized behind OverviewDB +/// publisher-confirm latency after the intermediate work-queue handoff, and that +/// the publisher stage itself pipelines confirms asynchronously. +/// +public sealed class IngestionThroughputCeilingTests +{ + private static readonly TimeSpan ConfirmLatency = TimeSpan.FromMilliseconds(18); + + [Fact] + public async Task ArticlePersistPath_IsNotSerializedBehindPublisherConfirmLatency() + { + const int articleCount = 32; + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = articleCount + 8 }, + transitQueueMemoryLimit: 64 * 1024 * 1024); + var overview = new RecordingOverviewDbHandoffPublisher + { + ConfirmDelay = ConfirmLatency, + OutstandingWindow = 100, + }; + using var done = new CountdownEvent(articleCount); + var paths = new CountingPathSurvey(done); + + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 4, + MaxWorkers = 4, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + OverviewDbPublisherBatchSize = 100, + ScaleIntervalSeconds = 3600, + }, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed), + ])), + overviewHandoff: overview, + pathSurvey: paths); + + await writer.StartAsync(CancellationToken.None); + for (var i = 0; i < articleCount; i++) + { + var inbound = CanonicalArticleText.CreateQueued( + $"", + InboundArticleProducer.TakeThis, + body: "throughput-body\r\n", + newsgroups: "alt.test"); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync(inbound, CancellationToken.None)); + } + + var sw = Stopwatch.StartNew(); + Assert.True(done.Wait(TimeSpan.FromSeconds(5)), "Timed out waiting for Path survey."); + sw.Stop(); + + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(articleCount * ConfirmLatency.TotalMilliseconds / 2), + $"Article persist took {sw.Elapsed.TotalMilliseconds:F0} ms; expected well under serial confirm ceiling."); + + using var confirmWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.Payloads.Count < articleCount) + { + confirmWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, confirmWait.Token); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + Assert.Equal(articleCount, overview.Payloads.Count); + Assert.Equal(articleCount, paths.WriteCalls); + } + + [Fact] + public async Task PublisherStage_PipelinesConfirms_WithOutstandingWindow() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 16 * 1024 * 1024); + var overview = new RecordingOverviewDbHandoffPublisher + { + ConfirmDelay = ConfirmLatency, + OutstandingWindow = 100, + }; + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + OverviewDbPublisherBatchSize = 100, + OverviewDbPublisherShutdownSeconds = 5, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, overview, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + const int count = 40; + var sw = Stopwatch.StartNew(); + for (var i = 0; i < count; i++) + { + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem([(byte)i], $""), + CancellationToken.None)); + } + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.Payloads.Count < count) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + sw.Stop(); + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(count * ConfirmLatency.TotalMilliseconds / 2), + $"Publisher stage took {sw.Elapsed.TotalMilliseconds:F0} ms; expected pipelined throughput."); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private sealed class CountingPathSurvey(CountdownEvent done) : IPathSurveyWriter + { + private int _writeCalls; + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + done.Signal(); + } + + public void Flush() + { + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs new file mode 100644 index 00000000..83056fcc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs @@ -0,0 +1,328 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Ingestion worker-pool scaling and lifecycle contracts. +public sealed class IngestionWorkerPoolTests +{ + [Fact] + public async Task Pool_StartsAtMinimumWorkers_AndNeverExceedsMaximum() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 2, + MaxWorkers = 3, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.1, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance, + samplePressure: static () => new IngestionPressureSnapshot(0, 0, 1, 0, 0)); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 2, TimeSpan.FromSeconds(2)); + Assert.Equal(2, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests( + new IngestionPressureSnapshot(1.0, 1, 1, 10, 1), + cts.Token); + Assert.Equal(3, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests( + new IngestionPressureSnapshot(1.0, 1, 1, 10, 1), + cts.Token); + Assert.Equal(3, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, pool.WorkerCount); + } + + [Fact] + public async Task Pool_ScalesDownOnlyAfterSustainedLowPressure_AndNotBelowMinimum() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 3, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.2, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 2, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 1, TimeSpan.FromSeconds(2)); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + Assert.Equal(3, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(3, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(2, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Pool_Hysteresis_ResetsWhenPressureIsInBand() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 4, + ScaleUpPressureThreshold = 0.6, + ScaleDownPressureThreshold = 0.2, + ScaleUpConsecutiveIntervals = 3, + ScaleDownConsecutiveIntervals = 3, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 1, TimeSpan.FromSeconds(2)); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0.4, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + Assert.Equal(0, pool.ScaleUpCount); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + Assert.Equal(2, pool.WorkerCount); + Assert.Equal(1, pool.ScaleUpCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task MultipleWorkers_DrainQueueWithoutDuplicateOrLoss() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 1024 * 1024); + var processed = new ConcurrentBag(); + var options = new ArticleIngestionOptions + { + MinWorkers = 4, + MaxWorkers = 4, + ScaleIntervalSeconds = 3600, + }; + + for (var i = 0; i < 40; i++) + { + var article = CanonicalArticleText.CreateQueued( + $"<{i}@pool.test>", + InboundArticleProducer.TakeThis, + body: $"body-{i}\r\n"); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + (article, _, _) => + { + processed.Add(article.MessageId); + return Task.CompletedTask; + }, + options, + NullLogger.Instance); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => processed.Count == 40, TimeSpan.FromSeconds(5)); + Assert.Equal(40, processed.Count); + Assert.Equal(40, processed.Distinct(StringComparer.Ordinal).Count()); + Assert.Equal(0, queue.Count); + + queue.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task GracefulShutdown_CancellationBetweenItems_PersistsArticlesWhenCountReadsEmpty() + { + var queue = new CountReportsEmptyQueue(); + var processed = new ConcurrentBag(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + for (var i = 0; i < 3; i++) + { + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + CanonicalArticleText.CreateQueued( + $"<{i}@shutdown.test>", + InboundArticleProducer.IHave, + "body\r\n"), + CancellationToken.None)); + } + + var pool = new IngestionWorkerPool( + queue, + async (article, _, _) => + { + if (entered.TrySetResult()) + { + await release.Task.ConfigureAwait(false); + } + + processed.Add(article.MessageId); + }, + options, + NullLogger.Instance); + + using var cts = new CancellationTokenSource(); + var run = pool.RunAsync(cts.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + queue.Complete(); + await cts.CancelAsync(); + release.TrySetResult(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(3, processed.Count); + Assert.Equal(3, processed.Distinct(StringComparer.Ordinal).Count()); + } + + [Fact] + public void PressureSnapshot_UsesByteUtilisationAndWaitingProducers() + { + var fromBytes = new IngestionPressureSnapshot(0.5, 50, 100, 3, 0); + Assert.Equal(0.5, fromBytes.Pressure); + + var waiting = IngestionPressureSnapshot.FromQueue( + new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 100)); + Assert.Equal(0, waiting.Pressure); + } + + /// + /// Queue whose stays 0 while accepted articles remain readable. + /// + /// + /// Models the approximate count window: an article can already be in the channel + /// before advances. Shutdown must not treat 0 as empty. + /// + private sealed class CountReportsEmptyQueue : IArticleIngestionQueue + { + private readonly ConcurrentQueue _items = new(); + private int _accepting = 1; + + public long MemoryLimitBytes => 1024 * 1024; + + public long QueuedBytes => 0; + + public long PeakQueuedBytes => 0; + + public int MaxArticleBytes => 1024 * 1024; + + public int Count => 0; + + public int PeakCount => 0; + + public bool IsAccepting => Volatile.Read(ref _accepting) != 0; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + cancellationToken.ThrowIfCancellationRequested(); + if (!IsAccepting) + { + return new ValueTask(ArticleEnqueueResult.Unavailable); + } + + _items.Enqueue(article); + return new ValueTask(ArticleEnqueueResult.Accepted); + } + + public bool TryProbeCapacity() => IsAccepting; + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + if (!IsAccepting) + { + return ArticleEnqueueResult.Unavailable; + } + + _items.Enqueue(article); + return ArticleEnqueueResult.Accepted; + } + + public bool TryEnqueue(InboundArticle article) => + TryAdmit(article) == ArticleEnqueueResult.Accepted; + + public void Complete() => Interlocked.Exchange(ref _accepting, 0); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_items.TryDequeue(out var article)) + { + return new ValueTask(article); + } + + return IsAccepting + ? throw new InvalidOperationException("Test queue was read while empty and still accepting.") + : new ValueTask((InboundArticle?)null); + } + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs new file mode 100644 index 00000000..89088668 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs @@ -0,0 +1,104 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Session.Commands.Posting; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class IngressNewsReasonsTests +{ + [Fact] + public void ForParseFailure_MapsSourceBackedCauses() + { + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.YEncDecodingFailed)); + Assert.Equal( + IngressNewsReasons.MessageIdInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.InvalidMessageId)); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.MissingOrInvalidDate)); + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.ArticleTooLarge)); + } + + [Fact] + public void ForExistingRejectDetail_MapsIhaveAndTakethisStrings() + { + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForExistingRejectDetail("rejected too large")); + Assert.Equal( + IngressNewsReasons.ArticleTypeNotPermitted, + IngressNewsReasons.ForExistingRejectDetail("rejected article type")); + Assert.Equal( + IngressNewsReasons.QueueCapacityExceeded, + IngressNewsReasons.ForExistingRejectDetail("rejected exceeds queue budget")); + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record YEncDecodingFailed")); + Assert.Equal( + IngressNewsReasons.MessageIdInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record InvalidMessageId")); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record MissingOrInvalidDate")); + } + + [Fact] + public void ForPostingFailure_KeepsPostFilterClosed_AndMapsKnownCauses() + { + Assert.Equal( + "closed", + IngressNewsReasons.ForPostingFailure(new PostingFailure(PostingFailureCategory.PolicyRejected, "closed"))); + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded"))); + Assert.Equal( + IngressNewsReasons.ArticleTypeNotPermitted, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PolicyRejected, "arttype-capability"))); + Assert.Equal( + IngressNewsReasons.QueueCapacityExceeded, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PersistenceFailure, nameof(ArticleEnqueueResult.Rejected)))); + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForPostingFailure( + new PostingFailure( + PostingFailureCategory.PolicyRejected, + NntpArticleParseFailureCode.YEncDecodingFailed.ToString()))); + Assert.Equal( + IngressNewsReasons.NewsgroupNotCarried, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PolicyRejected, "unknown newsgroup"))); + } + + [Fact] + public void WithGroups_JoinsAlreadySelectedNames_WithoutInventingThem() + { + Assert.Equal( + "newsgroup not carried", + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, [])); + Assert.Equal( + "newsgroup not carried: alt.example.foo", + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, ["alt.example.foo"])); + Assert.Equal( + "peer-only: alt.foo, alt.bar", + IngressNewsReasons.WithGroups(IngressNewsReasons.PeerOnly, ["alt.foo", "alt.bar"])); + } + + [Fact] + public void ForArticleRecord_UsesParseCodeNotEnumName() + { + var created = ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.YEncDecodingFailed); + Assert.Equal(IngressNewsReasons.YEncodingInvalid, IngressNewsReasons.ForArticleRecord(in created)); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForMaterializeFailure(NntpArticleCanonicalFailureCode.MissingSelectedDateHeader)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs new file mode 100644 index 00000000..ecd47e75 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs @@ -0,0 +1,419 @@ +using System.Text; +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Exact INN news line format from innd(8) LOGGING / ARTlog. +public sealed class NewsLogLineFormatterTests +{ + private static readonly DateTimeOffset ExampleTimestamp = + new(2026, 9, 28, 8, 18, 41, 398, TimeSpan.Zero); + + [Fact] + public void HasOutboundSiteField_IsTrueOnlyForAccepted() + { + Assert.True( + new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Junk, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Rejected, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Moderated, ""u8.ToArray()).HasOutboundSiteField); + } + + [Fact] + public void Write_Accepted_NamedPeer_UsesInboundFeed_Size_AndOutboundPlaceholder() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + var line = Format(in evt, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + line); + Assert.DoesNotContain(" + ? ", line, StringComparison.Ordinal); + Assert.True(evt.HasOutboundSiteField); + } + + [Fact] + public void Write_Accepted_Giganews_EmitsOutboundPlaceholder() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + size: 793311); + Assert.Equal( + "Sep 28 08:18:41.398 + giganews 793311 ?\n", + Format(in evt, ExampleTimestamp)); + Assert.True(evt.HasOutboundSiteField); + } + + [Fact] + public void Write_Junk_Giganews_OmitsOutboundPlaceholder_ThenReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + reason: "newsgroup not carried: alt.binaries.encryptnzb.hotel"u8.ToArray(), + size: 741116); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal( + "Sep 28 08:18:41.398 j giganews 741116 newsgroup not carried: alt.binaries.encryptnzb.hotel\n", + line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_Giganews_OmitsOutboundPlaceholder_ThenReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 793311); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal( + "Sep 28 08:18:41.398 - giganews 793311 yEncoding invalid\n", + line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Moderated_Giganews_OmitsOutboundPlaceholder_AndHasNoReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + size: 123456); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal("Sep 28 08:18:41.398 m giganews 123456\n", line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmptySites_OnJunkRejectedModerated_DoesNotEmitOutboundPlaceholder() + { + var timestamp = ExampleTimestamp; + ReadOnlyMemory emptySites = ReadOnlyMemory.Empty; + var junk = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + reason: "newsgroup not carried: alt.binaries.encryptnzb.hotel"u8.ToArray(), + size: 741116); + var rejected = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 793311); + var moderated = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + size: 123456); + Assert.Equal( + "Sep 28 08:18:41.398 j giganews 741116 newsgroup not carried: alt.binaries.encryptnzb.hotel\n", + Format(in junk, timestamp)); + Assert.Equal( + "Sep 28 08:18:41.398 - giganews 793311 yEncoding invalid\n", + Format(in rejected, timestamp)); + Assert.Equal( + "Sep 28 08:18:41.398 m giganews 123456\n", + Format(in moderated, timestamp)); + Assert.DoesNotContain("?", Format(in junk, timestamp), StringComparison.Ordinal); + Assert.DoesNotContain("?", Format(in rejected, timestamp), StringComparison.Ordinal); + Assert.DoesNotContain("?", Format(in moderated, timestamp), StringComparison.Ordinal); + } + + [Fact] + public void Write_SitesOnJunkRejectedModerated_AreNotRendered() + { + var timestamp = ExampleTimestamp; + var sites = "peer.example"u8.ToArray(); + var junk = Format( + new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + var rejected = Format( + new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + var moderated = Format( + new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + Assert.Equal("Sep 28 08:18:41.398 j giganews 1\n", junk); + Assert.Equal("Sep 28 08:18:41.398 - giganews 1\n", rejected); + Assert.Equal("Sep 28 08:18:41.398 m giganews 1\n", moderated); + Assert.DoesNotContain("peer.example", junk, StringComparison.Ordinal); + Assert.DoesNotContain("peer.example", rejected, StringComparison.Ordinal); + Assert.DoesNotContain("peer.example", moderated, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_NamedPeer_OmitsOutboundSites_ThenReason() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var id = ""u8.ToArray(); + var withoutReason = new NewsLogEvent( + NewsLogDisposition.Junk, + id, + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + Assert.Equal( + "Sep 27 00:04:04.293 j BlueWorldHosting 1584\n", + Format(in withoutReason, timestamp)); + Assert.DoesNotContain("?", Format(in withoutReason, timestamp), StringComparison.Ordinal); + + var withReason = new NewsLogEvent( + NewsLogDisposition.Junk, + id, + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: unknown.un.carried"u8.ToArray(), + size: 1584); + var line = Format(in withReason, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 j BlueWorldHosting 1584 newsgroup not carried: unknown.un.carried\n", + line); + Assert.DoesNotContain(" 1584 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_NamedPeer_OmitsOutboundSites_ThenReason() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 1584); + var line = Format(in evt, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 - BlueWorldHosting 1584 yEncoding invalid\n", + line); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_IsSelfContained_DoesNotRequireSessionOrConnection() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + Format(in evt, timestamp)); + } + + [Fact] + public void Write_Accepted_MatchesInnFieldOrder_FeedThenMessageId() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "?"u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:41.839 + ? 0 ?\n", line); + } + + [Fact] + public void Write_Junk_WithoutReason_DoesNotInventOne() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("newsgroup not carried", line, StringComparison.Ordinal); + Assert.DoesNotContain("peer-only", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_SerializesSuppliedReason_InInnFieldOrder() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: alt.example.foo"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 newsgroup not carried: alt.example.foo\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_PeerOnly_SerializesSuppliedReason() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "peer-only: alt.example.foo"u8.ToArray()); + Assert.Equal( + "Jan 5 00:00:00.000 j ? 0 peer-only: alt.example.foo\n", + Format(in evt, timestamp)); + } + + [Fact] + public void Write_Junk_DoesNotDeriveGroupNames() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: supplied.group"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 newsgroup not carried: supplied.group\n", line); + Assert.DoesNotContain("Newsgroups", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_SerializesSuppliedUncarriedGroupReason() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + timestamp: timestamp, + responseCode: 437, + reason: "newsgroup not carried: alt.example.foo"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 - ? 0 newsgroup not carried: alt.example.foo\n", line); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Moderated_UsesM_NotPlusOrUnknown() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:41.839 m ? 0\n", line); + Assert.DoesNotContain(" + ", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.Equal((byte)'m', (byte)NewsLogDisposition.Moderated); + Assert.NotEqual((byte)'?', (byte)NewsLogDisposition.Moderated); + } + + [Fact] + public void Write_RejectedYEncoding_OmitsResponseCode_AndUsesInnFieldOrder() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + "<23k82@bar.net>"u8.ToArray(), + timestamp: timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:54.638 - ? <23k82@bar.net> 0 yEncoding invalid\n", line); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_DoesNotRenderResponseCode() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + timestamp: timestamp, + responseCode: 437, + reason: "article too large"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:54.638 - ? 0 article too large\n", line); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_FutureSites_AreAppendedWithoutInventingThem() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "news.server.fr"u8.ToArray(), + "a.peer other.server.org"u8.ToArray(), + timestamp); + var line = Format(in evt, timestamp); + Assert.Equal( + "Aug 25 13:37:41.839 + news.server.fr 0 a.peer other.server.org\n", + line); + } + + private static string Format(in NewsLogEvent evt, DateTimeOffset timestamp) + { + Span buffer = stackalloc byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + return Encoding.ASCII.GetString(buffer[..written]); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsTestConfiguration.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsTestConfiguration.cs new file mode 100644 index 00000000..749426fd --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/NewsTestConfiguration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +internal static class NewsTestConfiguration +{ + public static IConfiguration Create( + string directory, + string rollingInterval = "Day", + int retainedFileCountLimit = 14, + bool buffered = false, + long? fileSizeLimitBytes = null, + bool rollOnFileSizeLimit = false, + int bufferSize = 1000, + bool blockWhenFull = true) + { + Directory.CreateDirectory(directory); + return new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = directory, + [NntpdNewsLogging.SectionName + ":path"] = Path.Join(directory, "news-.log"), + [NntpdNewsLogging.SectionName + ":rollingInterval"] = rollingInterval, + [NntpdNewsLogging.SectionName + ":retainedFileCountLimit"] = retainedFileCountLimit.ToString(), + [NntpdNewsLogging.SectionName + ":buffered"] = buffered ? "true" : "false", + [NntpdNewsLogging.SectionName + ":rollOnFileSizeLimit"] = rollOnFileSizeLimit ? "true" : "false", + [NntpdNewsLogging.SectionName + ":fileSizeLimitBytes"] = fileSizeLimitBytes?.ToString(), + [NntpdNewsLogging.SectionName + ":bufferSize"] = bufferSize.ToString(), + [NntpdNewsLogging.SectionName + ":blockWhenFull"] = blockWhenFull ? "true" : "false", + }) + .Build(); + } + + public static string ReadNewsFile(string directory) + { + var files = Directory.GetFiles(directory, "news*") + .Where(static path => !path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (files.Length == 0) + { + return string.Empty; + } + + using var stream = new FileStream(files[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs new file mode 100644 index 00000000..354b6fff --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs @@ -0,0 +1,352 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// OverviewDB async publisher-confirm and AMQP property contracts. +public sealed class OverviewDbHandoffPublisherTests +{ + [Fact] + public async Task PublishAsync_SetsRequiredAmqpProperties_AndUsesDefaultExchange() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 10 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("overview-one"u8.ToArray(), ""), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("overview-two"u8.ToArray(), ""), CancellationToken.None); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Empty(connection.RpcChannels); + Assert.Empty(connection.PublishChannels); + var channel = Assert.Single(connection.AsyncConfirmPublishChannels); + Assert.Equal(2, channel.Publications.Count); + + Assert.All( + channel.Publications, + publication => + { + Assert.Equal(OverviewDbTopology.DefaultExchange, publication.Exchange); + Assert.Equal(OverviewDbTopology.QueueName, publication.RoutingKey); + Assert.Equal(nntpd.Fqdn, publication.AppId); + Assert.Equal(OverviewDbTopology.ExpirationMilliseconds, publication.Expiration); + Assert.True(publication.Persistent); + Assert.True(publication.Mandatory); + Assert.True(Guid.TryParse(publication.MessageId, out var messageId)); + Assert.NotEqual(Guid.Empty, messageId); + }); + + Assert.NotEqual(channel.Publications[0].MessageId, channel.Publications[1].MessageId); + Assert.Equal("overview-one"u8.ToArray(), channel.Publications[0].Body); + Assert.Equal("overview-two"u8.ToArray(), channel.Publications[1].Body); + } + + [Fact] + public void CreateHandoffProperties_SetsPerMessageExpirationAndSequenceHeader() + { + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + publishSequenceNumber: 42); + + Assert.Equal("2000", properties.Expiration); + Assert.Equal("11111111-1111-1111-1111-111111111111", properties.MessageId); + Assert.Equal("nntpd01.usenet.ninja", properties.AppId); + Assert.Equal(DeliveryModes.Persistent, properties.DeliveryMode); + Assert.True(properties.Persistent); + Assert.NotNull(properties.Headers); + // UInt64 is not a valid AMQP table value in RabbitMQ.Client 7.2.2. + Assert.IsType(properties.Headers[Constants.PublishSequenceNumberHeader]); + Assert.Equal("42", properties.Headers[Constants.PublishSequenceNumberHeader]); + } + + [Theory] + [InlineData(0UL)] + [InlineData(1UL)] + [InlineData(42UL)] + [InlineData(9_007_199_254_740_992UL)] // 2^53 — past float mantissa exactness + [InlineData(ulong.MaxValue)] + public void PublishSequenceHeader_RoundTripsLosslessThroughAmqpSupportedRepresentations( + ulong sequence) + { + var formatted = RabbitMqClientAsyncConfirmPublishChannel.FormatPublishSequenceNumberHeader( + sequence); + Assert.Equal(sequence.ToString(System.Globalization.CultureInfo.InvariantCulture), formatted); + + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + sequence); + Assert.True(OverviewDbHandoffPublisher.TryReadPublishSequence(properties, out var fromString)); + Assert.Equal(sequence, fromString); + + // Broker Basic.Return typically echoes longstr ('S') as byte[]. + var asBytes = System.Text.Encoding.ASCII.GetBytes(formatted); + Assert.True( + OverviewDbHandoffPublisher.TryParsePublishSequenceNumberHeader(asBytes, out var fromBytes)); + Assert.Equal(sequence, fromBytes); + + // RabbitMQ.Client HandleReturn also accepts long when the value fits. + if (sequence <= long.MaxValue) + { + Assert.True( + OverviewDbHandoffPublisher.TryParsePublishSequenceNumberHeader( + (long)sequence, + out var fromLong)); + Assert.Equal(sequence, fromLong); + } + } + + [Fact] + public void PublishSequenceHeader_UInt64IsNotUsedAsAmqpTableValue() + { + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + ulong.MaxValue); + + var header = Assert.IsType( + properties.Headers![Constants.PublishSequenceNumberHeader]); + Assert.Equal("18446744073709551615", header); + Assert.NotEqual(typeof(ulong), header.GetType()); + } + + [Fact] + public async Task BasicReturn_WithStringSequenceHeader_CorrelatesCorrectWorkItem() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var first = new OverviewDbWorkItem("one"u8.ToArray(), ""); + var second = new OverviewDbWorkItem("two"u8.ToArray(), ""); + await publisher.PublishAsync(first, CancellationToken.None); + await publisher.PublishAsync(second, CancellationToken.None); + + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + // Return only sequence 2 (string header via FakeRabbitMq); ack both. + await channel.RaiseReturnAsync(2, second.Payload); + await channel.RaiseAckAsync(1, multiple: false); + await channel.RaiseAckAsync(2, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(second, failed); + Assert.False(publisher.TryDequeuePublishFailure(out _)); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task PublishAsync_DoesNotWaitForConfirm_BeforeReturning() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + connection.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = true; + channel.AutoAckDelay = TimeSpan.FromMilliseconds(50); + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 100 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < 20; i++) + { + await publisher.PublishAsync( + new OverviewDbWorkItem([(byte)i], $"<{i}@test>"), + CancellationToken.None); + } + + sw.Stop(); + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(200), + $"PublishAsync serialized behind confirms: {sw.Elapsed.TotalMilliseconds:F0} ms"); + Assert.True(publisher.OutstandingCount > 0); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + } + + [Fact] + public async Task OutstandingWindow_BoundsUnconfirmedPublishes() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + connection.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = false; + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 2 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("a"u8.ToArray(), ""), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("b"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(2, publisher.OutstandingCount); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => publisher.PublishAsync(new OverviewDbWorkItem("c"u8.ToArray(), ""), cts.Token)); + + var channel = Assert.Single(connection.AsyncConfirmPublishChannels); + await channel.RaiseAckAsync(2, multiple: true); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task MultipleAck_SettlesAllSequencesUpToDeliveryTag() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 10 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("1"u8.ToArray(), "<1@test>"), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("2"u8.ToArray(), "<2@test>"), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("3"u8.ToArray(), "<3@test>"), CancellationToken.None); + Assert.Equal(3, publisher.OutstandingCount); + + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseAckAsync(3, multiple: true); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task Nack_SurfacesFailureForRequeue() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var item = new OverviewDbWorkItem("nack"u8.ToArray(), ""); + await publisher.PublishAsync(item, CancellationToken.None); + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseNackAsync(1, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(item, failed); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task BasicReturn_ThenAck_SurfacesFailure_NotSilentSuccess() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var item = new OverviewDbWorkItem("ret"u8.ToArray(), ""); + await publisher.PublishAsync(item, CancellationToken.None); + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseReturnAsync(1, item.Payload); + await channel.RaiseAckAsync(1, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(item, failed); + Assert.False(publisher.TryDequeuePublishFailure(out _)); + } + + [Fact] + public async Task AbandonOutstanding_ReleasesWindow_WithoutInfiniteRetry() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 1 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("x"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(1, publisher.OutstandingCount); + publisher.AbandonOutstanding(); + Assert.Equal(0, publisher.OutstandingCount); + + await publisher.PublishAsync(new OverviewDbWorkItem("y"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(1, publisher.OutstandingCount); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs new file mode 100644 index 00000000..71faa618 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Async outstanding-confirmation window contracts for OverviewDB publishing. +public sealed class OverviewDbPublishConcurrencyTests +{ + [Fact] + public async Task OutstandingWindow_AllowsPipelinedPublishes_OnSingleChannel() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = true; + channel.AutoAckDelay = TimeSpan.FromMilliseconds(18); + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 100 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + const int count = 50; + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < count; i++) + { + await publisher.PublishAsync( + new OverviewDbWorkItem([(byte)i], $"<{i}@test>"), + CancellationToken.None); + } + + sw.Stop(); + + // Serial confirm model would need >= 50 * 18ms = 900ms. Pipelined write path must be far lower. + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(count * 18 / 2), + $"Pipelined publish took {sw.Elapsed.TotalMilliseconds:F0} ms; expected well under serial confirm ceiling."); + + Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + Assert.Equal(count, factory.LastConnection.AsyncConfirmPublishChannels[0].Publications.Count); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs new file mode 100644 index 00000000..dec8535e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs @@ -0,0 +1,340 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client.Exceptions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// OverviewDB intermediate work-queue and publisher-pool contracts. +public sealed class OverviewDbPublisherStageTests +{ + [Fact] + public async Task WorkQueue_BackpressuresWhenByteBudgetExhausted() + { + var queue = new OverviewDbWorkQueue(memoryLimitBytes: 10); + var first = new OverviewDbWorkItem(new byte[8], ""); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + var second = new OverviewDbWorkItem(new byte[8], ""); + await Assert.ThrowsAnyAsync( + () => queue.EnqueueAsync(second, cts.Token).AsTask()); + Assert.True(queue.WaitingProducerCount >= 0); + Assert.Equal(1, queue.Count); + } + + [Fact] + public async Task PublisherPool_PublishesAndAwaitsConfirm_WithoutArticleQueue() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + MaxPublishConcurrency = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool( + work, + publisher, + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("abc"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(1, publisher.AttemptCount); + Assert.Single(publisher.Payloads); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_RequeuesWorkItemOnPublishFailure() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher { RemainingFailures = 1 }; + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("retry"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(2, publisher.AttemptCount); + Assert.Single(publisher.Payloads); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_WireFormattingException_IsPublishFailure_NotQueueUnavailable() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new WireFormattingException( + "Value of type 'UInt64' cannot appear as table value"), + }; + var logger = new CollectingLogger(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, logger); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem("wire"u8.ToArray(), ""), + CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(2, publisher.AttemptCount); + Assert.Contains( + logger.Messages, + m => m.Contains("OverviewDB RabbitMQ handoff failed", StringComparison.Ordinal)); + Assert.DoesNotContain( + logger.Messages, + m => m.Contains("queue is unavailable", StringComparison.Ordinal)); + Assert.Contains( + logger.Messages, + m => m.Contains("was requeued", StringComparison.Ordinal)); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_RequeueUnavailable_OnlyWhenWorkQueueUnavailable() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new WireFormattingException("local serialize failure"), + BlockOnFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var logger = new CollectingLogger(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, logger); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem("gone"u8.ToArray(), ""), + CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.AttemptCount < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + // Complete the work queue before the publish failure requeue runs. + work.Complete(); + publisher.BlockOnFailure.TrySetResult(); + + using var logWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!logger.Messages.Exists( + m => m.Contains("queue is unavailable", StringComparison.Ordinal))) + { + logWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, logWait.Token); + } + + Assert.Contains( + logger.Messages, + m => m.Contains("OverviewDB RabbitMQ handoff failed", StringComparison.Ordinal)); + Assert.Contains( + logger.Messages, + m => m.Contains("queue is unavailable", StringComparison.Ordinal)); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_ScalesIndependentlyOfArticleWorkerBounds() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + OverviewDbMinPublisherWorkers = 2, + OverviewDbMaxPublisherWorkers = 4, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.1, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool( + work, + new RecordingOverviewDbHandoffPublisher(), + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal(2, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + Assert.Equal(3, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_DrainsQueuedWorkOnComplete_BeforeExit() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var publisher = new GatedOverviewDbHandoffPublisher(gate); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("drain-a"u8.ToArray(), ""), CancellationToken.None)); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("drain-b"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Started < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + work.Complete(); + gate.TrySetResult(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, publisher.ConfirmCount); + Assert.Equal(0, work.Count); + } + + private sealed class GatedOverviewDbHandoffPublisher(TaskCompletionSource gate) : IOverviewDbHandoffPublisher + { + private int _started; + private int _confirms; + + public int Started => Volatile.Read(ref _started); + + public int ConfirmCount => Volatile.Read(ref _confirms); + + public int OutstandingCount => 0; + + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + Interlocked.Increment(ref _started); + await gate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _confirms); + } + + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + item = null!; + return false; + } + + public void AbandonOutstanding() + { + } + } + + private sealed class CollectingLogger : ILogger + { + private readonly object _gate = new(); + + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + lock (_gate) + { + Messages.Add(formatter(state, exception)); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs new file mode 100644 index 00000000..40fddb49 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs @@ -0,0 +1,78 @@ +using System.Text; +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// Exact INN-style Path-survey line from canonical ArticleRecord.Path bytes. +public sealed class PathSurveyLineFormatterTests +{ + [Fact] + public void Write_CopiesCanonicalPathBytes_WithPrefixAndLf() + { + var path = "news.usenet.ninja!nntpd01.usenet.ninja!peer.example"u8; + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + var written = PathSurveyLineFormatter.Write(buffer, path); + Assert.Equal(buffer.Length, written); + Assert.Equal("Path: news.usenet.ninja!nntpd01.usenet.ninja!peer.example\n", Encoding.Latin1.GetString(buffer)); + } + + [Fact] + public void Write_PreservesExactPathBytes_WithoutInventingHops() + { + var path = "foo!bar!not-for-mail"u8; + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + PathSurveyLineFormatter.Write(buffer, path); + var line = Encoding.Latin1.GetString(buffer); + Assert.Equal("Path: foo!bar!not-for-mail\n", line); + Assert.DoesNotContain("giganews", line, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd01", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmptyPath_FollowsArticleRecordEmptySpanSemantics() + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength([])]; + var written = PathSurveyLineFormatter.Write(buffer, []); + Assert.Equal(7, written); + Assert.Equal("Path: \n", Encoding.Latin1.GetString(buffer)); + } + + [Fact] + public void Write_MultipleCalls_AreIndependentSequentialRecords() + { + using var stream = new MemoryStream(); + WriteOne(stream, "alpha!one"u8); + WriteOne(stream, "beta!two"u8); + Assert.Equal("Path: alpha!one\nPath: beta!two\n", Encoding.Latin1.GetString(stream.ToArray())); + } + + [Fact] + public void Write_LargeSequentialInput_DoesNotMaterializeTheCompleteStream() + { + var path = "hop.example!not-for-mail"u8; + var required = PathSurveyLineFormatter.RequiredLength(path); + var buffer = new byte[required]; + const int count = 20_000; + using var stream = new FileStream( + Path.Join(Path.GetTempPath(), "vectornntp-inpaths-seq-" + Guid.NewGuid().ToString("N")), + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 64 * 1024, + FileOptions.DeleteOnClose); + for (var i = 0; i < count; i++) + { + var written = PathSurveyLineFormatter.Write(buffer, path); + stream.Write(buffer.AsSpan(0, written)); + } + + Assert.Equal((long)required * count, stream.Length); + } + + private static void WriteOne(Stream stream, ReadOnlySpan path) + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + var written = PathSurveyLineFormatter.Write(buffer, path); + stream.Write(buffer.AsSpan(0, written)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs new file mode 100644 index 00000000..10367975 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +internal static class PathSurveyTestConfiguration +{ + public static IConfiguration Create( + string directory, + string rollingInterval = "Day", + int retainedFileCountLimit = 1, + bool buffered = false, + long? fileSizeLimitBytes = null, + bool rollOnFileSizeLimit = false, + int bufferSize = 1000, + bool blockWhenFull = true) + { + Directory.CreateDirectory(directory); + return new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = directory, + [NntpdPathSurveyLogging.SectionName + ":path"] = Path.Join(directory, "inpaths-.log"), + [NntpdPathSurveyLogging.SectionName + ":rollingInterval"] = rollingInterval, + [NntpdPathSurveyLogging.SectionName + ":retainedFileCountLimit"] = retainedFileCountLimit.ToString(), + [NntpdPathSurveyLogging.SectionName + ":buffered"] = buffered ? "true" : "false", + [NntpdPathSurveyLogging.SectionName + ":rollOnFileSizeLimit"] = rollOnFileSizeLimit ? "true" : "false", + [NntpdPathSurveyLogging.SectionName + ":fileSizeLimitBytes"] = fileSizeLimitBytes?.ToString(), + [NntpdPathSurveyLogging.SectionName + ":bufferSize"] = bufferSize.ToString(), + [NntpdPathSurveyLogging.SectionName + ":blockWhenFull"] = blockWhenFull ? "true" : "false", + }) + .Build(); + } + + public static string ReadInpathsFile(string directory) + { + var files = Directory.GetFiles(directory, "inpaths*") + .Where(static path => !path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (files.Length == 0) + { + return string.Empty; + } + + using var stream = new FileStream(files[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs new file mode 100644 index 00000000..3ccee4e5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs @@ -0,0 +1,151 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// +/// After Path survey the ingestion worker discards the article and does not +/// persist or place it. +/// +public sealed class PostPathSurveyDiscardTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.Parse("2026-10-04T00:00:00Z"); + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task AfterPathSurvey_DoesNotPersistSelectOrPlace(InboundArticleProducer producer) + { + var observed = await RunAsync( + CanonicalArticleText.CreateQueued($"", producer)); + + Assert.Equal(0, observed.Registry.ActiveCalls); + Assert.Equal(0, observed.Placement.Calls); + Assert.Equal(1, observed.Paths.WriteCalls); + Assert.Single(observed.Overview.Payloads); + Assert.Equal(0, observed.Queue.Count); + } + + [Fact] + public async Task SubsequentArticle_IsProcessedAfterTheFirstIsDiscarded() + { + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var observed = await RunAsync(first, second); + + Assert.Equal(0, observed.Registry.ActiveCalls); + Assert.Equal(0, observed.Placement.Calls); + Assert.Equal(2, observed.Paths.WriteCalls); + Assert.Equal(2, observed.Overview.Payloads.Count); + Assert.Equal(0, observed.Queue.Count); + } + + private static async Task RunAsync(params InboundArticle[] articles) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var placement = new CountingPlacement(); + var registry = new CountingRegistry(EligibleRegistry()); + var paths = new CountingPathSurveyWriter(); + var overview = new RecordingOverviewDbHandoffPublisher(); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }, + }), + NullLogger.Instance, + overviewHandoff: overview, + pathSurvey: paths, + placement: placement, + placementRegistry: registry); + + await writer.StartAsync(CancellationToken.None); + foreach (var article in articles) + { + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + queue.Complete(); + // Confirms are recorded after PublishAsync returns, while the worker is still + // running. Waiting here, before StopAsync, observes that signal. StopAsync's + // own budget must not be the observation window: once it expires, StopAsync + // abandons outstanding OverviewDB work and the confirm never arrives. + using var confirmed = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await overview.WaitUntilConfirmedAsync(articles.Length, confirmed.Token); + using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await writer.StopAsync(stop.Token); + return new Observed(queue, placement, registry, paths, overview); + } + + private static StorageServerRegistry EligibleRegistry() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.example", 10_000, 0, 10_000, Now, 563), + Now); + return registry; + } + + private sealed record Observed( + ArticleIngestionQueue Queue, + CountingPlacement Placement, + CountingRegistry Registry, + CountingPathSurveyWriter Paths, + RecordingOverviewDbHandoffPublisher Overview); + + private sealed class CountingPlacement : IArticlePlacementClient + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _calls); + return ValueTask.FromResult(new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + } + } + + private sealed class CountingRegistry(StorageServerRegistry inner) : IStorageServerRegistry + { + private int _activeCalls; + + public int ActiveCalls => Volatile.Read(ref _activeCalls); + + public void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc) => + inner.ApplyAdvertisement(advertisement, receivedAtUtc); + + public bool TryGet(string fqdn, out StorageServerFleetEntry entry) => inner.TryGet(fqdn, out entry); + + public IReadOnlyList Snapshot() => inner.Snapshot(); + + public IReadOnlyList GetActive(DateTimeOffset utcNow) + { + Interlocked.Increment(ref _activeCalls); + return inner.GetActive(utcNow); + } + + public void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc) => + inner.ApplyLifecycle(announcement, receivedAtUtc); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/QueueDrain20Tests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/QueueDrain20Tests.cs new file mode 100644 index 00000000..d17e7468 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/QueueDrain20Tests.cs @@ -0,0 +1,438 @@ +using System.Collections.Concurrent; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +/// +/// True concurrent consumers. +/// Ownership is released inside dequeue; workers must not release again. +/// +public sealed class QueueDrain20Tests +{ + internal const int WorkerCount = 20; + + [Fact] + public async Task TwoWorkers_ConsumeEveryArticleExactlyOnce() + { + await DrainExactlyOnceAsync(workers: 2, articleCount: 100); + } + + [Fact] + public async Task TwentyWorkers_ConsumeEveryArticleExactlyOnce() + { + await DrainExactlyOnceAsync(workers: WorkerCount, articleCount: 200); + } + + [Fact] + public async Task TwentyWorkers_CallDequeueAsyncConcurrently_AndAllParticipate() + { + const int articleCount = 400; + var queue = CreateQueue(articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[WorkerCount]; + var workers = StartWorkers(queue, WorkerCount, seen, hits); + EnqueueUnique(queue, articleCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(articleCount, hits.Sum()); + Assert.True(hits.Count(static h => h > 0) > 1); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_WaitingThenEnqueue_AllExitAfterComplete() + { + const int articleCount = 200; + var queue = CreateQueue(articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[WorkerCount]; + var workers = StartWorkers(queue, WorkerCount, seen, hits); + + var ids = EnqueueUnique(queue, articleCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.True(ids.SetEquals(seen.Keys)); + Assert.Equal(articleCount, hits.Sum()); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ConcurrentConsumers_EachWorkerReceivesIncreasingFifoSubsequence() + { + const int articleCount = 200; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount, idPrefix: "seq"); + var perWorker = new List[WorkerCount]; + for (var i = 0; i < WorkerCount; i++) + { + perWorker[i] = []; + } + + var workers = new Task[WorkerCount]; + for (var i = 0; i < WorkerCount; i++) + { + var workerIndex = i; + workers[workerIndex] = Task.Run(async () => + { + while (await queue.DequeueAsync(CancellationToken.None) is { } article) + { + var start = article.MessageId.IndexOf('.') + 1; + var end = article.MessageId.IndexOf('@'); + var seq = int.Parse(article.MessageId.AsSpan(start, end - start)); + perWorker[workerIndex].Add(seq); + } + }); + } + + queue.Complete(); + await WaitWorkersAsync(workers); + + var all = new HashSet(); + foreach (var ordered in perWorker.Select(static bag => bag.ToArray())) + { + for (var i = 1; i < ordered.Length; i++) + { + Assert.True(ordered[i] > ordered[i - 1], "per-worker dequeue subsequence must be increasing FIFO"); + } + + foreach (var seq in ordered) + { + Assert.True(all.Add(seq)); + } + } + + Assert.Equal(articleCount, all.Count); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_AccountingReturnsToZero_AndPeaksMatchPrefill() + { + const int articleCount = 80; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount); + var expectedBytes = queue.QueuedBytes; + + Assert.Equal(expectedBytes, queue.QueuedBytes); + Assert.Equal(articleCount, queue.Count); + Assert.Equal(expectedBytes, queue.PeakQueuedBytes); + Assert.Equal(articleCount, queue.PeakCount); + + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.Equal(expectedBytes, queue.PeakQueuedBytes); + Assert.Equal(articleCount, queue.PeakCount); + Assert.False(queue.IsAccepting); + } + + [Fact] + public async Task TwentyWorkers_CompleteEmptyQueue_UnblocksEveryWorker() + { + var queue = CreateQueue(1); + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + } + + [Fact] + public async Task TwentyWorkers_Cancellation_UnblocksWithoutReservationLeak() + { + var queue = CreateQueue(1); + using var cts = new CancellationTokenSource(); + var workers = StartWorkers(queue, WorkerCount, cancellationToken: cts.Token); + + await cts.CancelAsync(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.True(queue.IsAccepting); + } + + [Fact] + public async Task CancelledConsumer_DoesNotCorruptAccounting_RemainingWorkersDrain() + { + const int articleCount = 120; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount); + + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + using var cancelled = new CancellationTokenSource(); + var tasks = new List(WorkerCount); + tasks.AddRange(StartWorkers(queue, 5, seen, cancellationToken: cancelled.Token)); + tasks.AddRange(StartWorkers(queue, 15, seen)); + + await cancelled.CancelAsync(); + queue.Complete(); + await WaitWorkersAsync(tasks.ToArray()); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task TwentyWorkers_CancelDuringDrain_DoesNotDoubleRelease() + { + const int articleCount = 100; + var queue = CreateQueue(articleCount); + var sample = Article(""); + EnqueueUnique(queue, articleCount); + + using var cts = new CancellationTokenSource(); + var consumed = new ConcurrentDictionary(StringComparer.Ordinal); + var workers = StartWorkers(queue, WorkerCount, consumed, cancellationToken: cts.Token); + await cts.CancelAsync(); + await WaitWorkersAsync(workers); + + var remainingBytes = queue.QueuedBytes; + var remainingCount = queue.Count; + Assert.True(remainingBytes >= 0); + Assert.True(remainingCount >= 0); + Assert.Equal(remainingCount * sample.Payload.Length, remainingBytes); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.All(consumed.Values, static count => Assert.Equal(1, count)); + + queue.Complete(); + var leftover = 0; + while (await queue.DequeueAsync(CancellationToken.None) is not null) + { + leftover++; + } + + Assert.Equal(remainingCount, leftover); + Assert.Equal(articleCount, consumed.Count + leftover); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_CompleteAfterPartialPrefill_DrainsAndRejectsNewAdmits() + { + var queue = CreateQueue(5); + EnqueueUnique(queue, 5, idPrefix: "held"); + + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(ArticleEnqueueResult.Unavailable, queue.TryAdmit(Article(""))); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.False(queue.IsAccepting); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + } + + [Fact] + public async Task ConcurrentProducersAndConsumers_CannotOversubscribeBudget() + { + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * 4L; + const int producers = 8; + const int consumers = 4; + const int perProducer = 8; + var queue = CreateWithByteLimit(limit); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var maxObserved = 0L; + var oversubscribed = 0; + using var observeCts = new CancellationTokenSource(); + var observer = Task.Run(async () => + { + while (!observeCts.Token.IsCancellationRequested) + { + var bytes = queue.QueuedBytes; + if (bytes > maxObserved) + { + maxObserved = bytes; + } + + if (bytes > limit) + { + Interlocked.Increment(ref oversubscribed); + } + + await Task.Yield(); + } + }); + + var consumerTasks = StartWorkers(queue, consumers, seen); + var producerTasks = new Task[producers]; + for (var p = 0; p < producers; p++) + { + var producer = p; + producerTasks[producer] = Task.Run(async () => + { + for (var i = 0; i < perProducer; i++) + { + var result = await queue.EnqueueAsync(Article($""), CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, result); + } + }); + } + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await Task.WhenAll(producerTasks).WaitAsync(safety.Token); + queue.Complete(); + await WaitWorkersAsync(consumerTasks); + await observeCts.CancelAsync(); + await observer.WaitAsync(safety.Token); + + Assert.Equal(0, oversubscribed); + Assert.True(maxObserved <= limit); + Assert.Equal(producers * perProducer, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task SuccessfulDequeue_ReleasesAccountingBeforeCallerOwnsArticle() + { + var article = Article(""); + var queue = CreateQueue(1); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(article)); + Assert.Equal(article.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + + var dequeued = await queue.DequeueAsync(CancellationToken.None); + Assert.Same(article, dequeued); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(article.Payload.Length, dequeued!.Payload.Length); + } + + private static async Task DrainExactlyOnceAsync(int workers, int articleCount) + { + var queue = CreateQueue(articleCount); + var ids = EnqueueUnique(queue, articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[workers]; + + var tasks = StartWorkers(queue, workers, seen, hits); + queue.Complete(); + await WaitWorkersAsync(tasks); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.True(ids.SetEquals(seen.Keys)); + Assert.Equal(articleCount, hits.Sum()); + Assert.True(hits.Count(static h => h > 0) >= 1); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + private static ArticleIngestionQueue CreateQueue(int articleSlots) + { + ArgumentOutOfRangeException.ThrowIfNegative(articleSlots); + return new ArticleIngestionQueue(new ArticleIngestionOptions(), 64L * 1024 * 1024); + } + + private static ArticleIngestionQueue CreateWithByteLimit(long memoryLimit) => + new(new ArticleIngestionOptions(), memoryLimit); + + private static HashSet EnqueueUnique( + ArticleIngestionQueue queue, + int articleCount, + string idPrefix = "q20") + { + var ids = new HashSet(articleCount, StringComparer.Ordinal); + for (var i = 0; i < articleCount; i++) + { + var id = $"<{idPrefix}.{i:D3}@ex.com>"; + Assert.True(ids.Add(id)); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(Article(id))); + } + + return ids; + } + + private static Task[] StartWorkers( + IArticleIngestionQueue queue, + int workers, + ConcurrentDictionary? seen = null, + int[]? hits = null, + CancellationToken cancellationToken = default) + { + var tasks = new Task[workers]; + for (var i = 0; i < workers; i++) + { + var workerIndex = i; + tasks[workerIndex] = Task.Run( + () => DrainAsync(queue, seen, hits, workerIndex, cancellationToken), + CancellationToken.None); + } + + return tasks; + } + + private static async Task DrainAsync( + IArticleIngestionQueue queue, + ConcurrentDictionary? seen, + int[]? hits, + int workerIndex, + CancellationToken cancellationToken) + { + while (true) + { + InboundArticle? article; + try + { + article = await queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + + if (article is null) + { + return; + } + + if (seen is not null) + { + seen.AddOrUpdate(article.MessageId, 1, static (_, count) => count + 1); + } + + if (hits is not null) + { + Interlocked.Increment(ref hits[workerIndex]); + } + } + } + + private static async Task WaitWorkersAsync(Task[] workers) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await Task.WhenAll(workers).WaitAsync(safety.Token); + } + + private static InboundArticle Article(string messageId) => + CanonicalArticleText.CreateQueued(messageId, InboundArticleProducer.IHave); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs new file mode 100644 index 00000000..2bc2dd13 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs @@ -0,0 +1,189 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class SerilogNewsLogWriterTests +{ + [Fact] + public void ReadSettings_HonoursConfiguredOperationalProperties() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create( + dir.Path, + rollingInterval: "Hour", + retainedFileCountLimit: 3, + buffered: false, + fileSizeLimitBytes: 1_048_576, + rollOnFileSizeLimit: true, + bufferSize: 250, + blockWhenFull: false); + var settings = NntpdNewsLogging.ReadSettings(configuration); + Assert.Equal(Path.Join(dir.Path, "news-.log"), settings.Path); + Assert.Equal(RollingInterval.Hour, settings.RollingInterval); + Assert.Equal(3, settings.RetainedFileCountLimit); + Assert.False(settings.Buffered); + Assert.True(settings.RollOnFileSizeLimit); + Assert.Equal(1_048_576, settings.FileSizeLimitBytes); + Assert.Equal(250, settings.BufferSize); + Assert.False(settings.BlockWhenFull); + Assert.Null(settings.FlushToDiskInterval); + } + + [Fact] + public void BindResolvedNewsPath_UsesLogDir_AndDoesNotExposeOutputTemplate() + { + using var dir = new TempLogDir(); + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = dir.Path, + ["Serilog:News:path"] = "logs/news-.log", + ["Serilog:News:outputTemplate"] = "HACK {Message}", + }); + NntpdNewsLogging.BindResolvedNewsPath(configuration, dir.Path); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(dir.Path), configuration["Serilog:News:path"]); + var settings = NntpdNewsLogging.ReadSettings(configuration); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(dir.Path), settings.Path); + Assert.Equal("HACK {Message}", configuration["Serilog:News:outputTemplate"]); + } + + [Fact] + public void Write_UsesConfiguredPath_AndInnFormatter() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Flush(); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.Contains(" + ? 0 ?", text, StringComparison.Ordinal); + Assert.DoesNotContain("HACK", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_NamedPeer_EmitsInboundFeedSizeAndOutboundPlaceholder() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(new NewsLogEvent( + NewsLogDisposition.Accepted, + Encoding.ASCII.GetBytes(""), + Encoding.ASCII.GetBytes("BlueWorldHosting"), + size: 1584)); + writer.Flush(); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains( + " + BlueWorldHosting 1584 ?", + text, + StringComparison.Ordinal); + Assert.DoesNotContain(" + ? ", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmitsCompleteLines_AndDisposeFlushesThem() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite", buffered: true); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Write(Accepted("")); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.EndsWith("\n", text, StringComparison.Ordinal); + Assert.Equal(2, text.Split('\n', StringSplitOptions.RemoveEmptyEntries).Length); + } + + [Fact] + public void DailyRolling_WritesDatedFileName() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Day"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Dispose(); + } + + var dated = Directory.GetFiles(dir.Path, "news-????????.log"); + Assert.Single(dated); + Assert.Contains("", NewsTestConfiguration.ReadNewsFile(dir.Path), StringComparison.Ordinal); + } + + [Fact] + public void ApplicationLogger_DoesNotReceiveNewsLines() + { + using var dir = new TempLogDir(); + var appSink = new CollectingSink(); + using var app = new LoggerConfiguration().WriteTo.Sink(appSink).CreateLogger(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Dispose(); + } + + app.Information("application diagnostic"); + Assert.DoesNotContain(appSink.Lines, static line => line.Contains("", StringComparison.Ordinal)); + Assert.Contains("", NewsTestConfiguration.ReadNewsFile(dir.Path), StringComparison.Ordinal); + } + + [Fact] + public void CustomNewsFileTypes_AreGone() + { + Assert.Null(Type.GetType("VectorNNTPD.ReaderService.ArticleIngestion.InnNewsLogWriter, VectorNNTPD.ReaderService")); + Assert.Null(Type.GetType("VectorNNTPD.ReaderService.Logging.NntpdLogCompression, VectorNNTPD.ReaderService")); + } + + private static NewsLogEvent Accepted(string messageId) => + new(NewsLogDisposition.Accepted, Encoding.ASCII.GetBytes(messageId)); + + private sealed class CollectingSink : ILogEventSink + { + public List Lines { get; } = []; + + public void Emit(LogEvent logEvent) => Lines.Add(logEvent.RenderMessage()); + } + + private sealed class TempLogDir : IDisposable + { + public TempLogDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-news-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs new file mode 100644 index 00000000..5e7f2799 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs @@ -0,0 +1,290 @@ +using System.IO.Compression; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.ArticleIngestion; + +public sealed class SerilogPathSurveyWriterTests +{ + [Fact] + public void ReadSettings_HonoursConfiguredOperationalProperties() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create( + dir.Path, + rollingInterval: "Hour", + retainedFileCountLimit: 1, + buffered: false, + fileSizeLimitBytes: 1_048_576, + rollOnFileSizeLimit: true, + bufferSize: 250, + blockWhenFull: false); + var settings = NntpdPathSurveyLogging.ReadSettings(configuration); + Assert.Equal(Path.Join(dir.Path, "inpaths-.log"), settings.Path); + Assert.Equal(RollingInterval.Hour, settings.RollingInterval); + Assert.Equal(1, settings.RetainedFileCountLimit); + Assert.False(settings.Buffered); + Assert.True(settings.RollOnFileSizeLimit); + Assert.Equal(1_048_576, settings.FileSizeLimitBytes); + Assert.Equal(250, settings.BufferSize); + Assert.False(settings.BlockWhenFull); + Assert.Null(settings.FlushToDiskInterval); + } + + [Fact] + public void BindResolvedPathSurveyPath_UsesLogDir_AndDoesNotExposeOutputTemplate() + { + using var dir = new TempLogDir(); + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = dir.Path, + ["Serilog:Inpaths:path"] = "logs/inpaths-.log", + ["Serilog:Inpaths:outputTemplate"] = "HACK {Message}", + }); + NntpdPathSurveyLogging.BindResolvedPathSurveyPath(configuration, dir.Path); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(dir.Path), configuration["Serilog:Inpaths:path"]); + var settings = NntpdPathSurveyLogging.ReadSettings(configuration); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(dir.Path), settings.Path); + Assert.Equal("HACK {Message}", configuration["Serilog:Inpaths:outputTemplate"]); + } + + [Fact] + public void Write_UsesConfiguredPath_AndPathFormatter() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("canonical.example!peer"u8); + writer.Flush(); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: canonical.example!peer\n", text); + Assert.DoesNotContain("HACK", text, StringComparison.Ordinal); + Assert.DoesNotContain("giganews", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_PreservesExactCanonicalPathBytes() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + var path = "news.usenet.ninja!nntpd01.usenet.ninja!BlueWorldHosting!not-for-mail"u8; + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write(path); + writer.Dispose(); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: " + Encoding.Latin1.GetString(path) + "\n", text); + } + + [Fact] + public void Write_EmptyPath_EmitsPathPrefixAndNewline() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write([]); + writer.Dispose(); + } + + Assert.Equal("Path: \n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void Write_EmitsCompleteLines_AndDisposeFlushesThem() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite", buffered: true); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("one.example"u8); + writer.Write("two.example"u8); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: one.example\nPath: two.example\n", text); + } + + [Fact] + public void SizeBasedRoll_HandsCompletedFileToHandler_BeforeGzip() + { + using var dir = new TempLogDir(); + var handler = new RecordingCompletedPathSurveyFileHandler(); + var configuration = PathSurveyTestConfiguration.Create( + dir.Path, + rollingInterval: "Infinite", + retainedFileCountLimit: 1, + buffered: false, + fileSizeLimitBytes: 80, + rollOnFileSizeLimit: true); + using (var writer = new SerilogPathSurveyWriter( + configuration, + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 40; i++) + { + writer.Write("aaaaaaaaaaaaaaaaaaaaaaaa"u8); + } + + writer.Dispose(); + } + + Assert.NotEmpty(handler.Completed); + Assert.All(handler.GzipExistedAtHandoff, static existed => Assert.False(existed)); + Assert.All(handler.Completed, static path => Assert.False(File.Exists(path))); + Assert.Empty(Directory.GetFiles(dir.Path, "inpaths*.gz")); + Assert.Single(Directory.GetFiles(dir.Path, "inpaths*.log")); + } + + [Fact] + public void DailyRolling_WritesDatedFileName() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Day"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("day.example"u8); + writer.Dispose(); + } + + var dated = Directory.GetFiles(dir.Path, "inpaths-????????.log"); + Assert.Single(dated); + Assert.Equal("Path: day.example\n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void ApplicationLogger_DoesNotReceivePathLines() + { + using var dir = new TempLogDir(); + var appSink = new CollectingSink(); + using var app = new LoggerConfiguration().WriteTo.Sink(appSink).CreateLogger(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("isolated.example"u8); + writer.Dispose(); + } + + app.Information("application diagnostic"); + Assert.DoesNotContain(appSink.Lines, static line => line.Contains("isolated.example", StringComparison.Ordinal)); + Assert.DoesNotContain(appSink.Lines, static line => line.StartsWith("Path: ", StringComparison.Ordinal)); + Assert.Equal("Path: isolated.example\n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void NewsLogger_DoesNotReceivePathSurveyLines() + { + using var dir = new TempLogDir(); + var pathConfig = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + var newsConfig = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var paths = new SerilogPathSurveyWriter(pathConfig)) + using (var news = new SerilogNewsLogWriter(newsConfig)) + { + paths.Write("survey-only.example"u8); + news.Write(new NewsLogEvent(NewsLogDisposition.Accepted, Encoding.ASCII.GetBytes(""))); + paths.Dispose(); + news.Dispose(); + } + + var pathText = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + var newsText = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Equal("Path: survey-only.example\n", pathText); + Assert.DoesNotContain("Path:", newsText, StringComparison.Ordinal); + Assert.Contains("", newsText, StringComparison.Ordinal); + Assert.DoesNotContain("", pathText, StringComparison.Ordinal); + Assert.DoesNotContain("survey-only.example", newsText, StringComparison.Ordinal); + } + + [Fact] + public void Writer_DoesNotRetainPathObservationsInACollection() + { + var fields = typeof(SerilogPathSurveyWriter).GetFields( + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain( + fields, + static field => + field.FieldType != typeof(Serilog.Core.Logger) + && typeof(System.Collections.IEnumerable).IsAssignableFrom(field.FieldType) + && field.FieldType != typeof(string)); + Assert.Single(fields); + } + + [Fact] + public void CreateLogger_UsesDistinctSourceContext() + { + Assert.Equal("VectorNNTPD.ReaderService.Inpaths", NntpdPathSurveyLogging.SourceContext); + Assert.NotEqual(NntpdNewsLogging.SourceContext, NntpdPathSurveyLogging.SourceContext); + } + + [Fact] + public void ProductionAppsettings_DeclaresInpathsBesideNews() + { + var json = File.ReadAllText(FindProductionAppsettings()); + Assert.Contains("\"Inpaths\"", json, StringComparison.Ordinal); + Assert.Contains("logs/inpaths-.log", json, StringComparison.Ordinal); + Assert.Contains("logs/news-.log", json, StringComparison.Ordinal); + } + + private static string FindProductionAppsettings() + { + var dir = AppContext.BaseDirectory; + while (dir is not null) + { + var candidate = Path.Join(dir, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = Directory.GetParent(dir)?.FullName; + } + + throw new InvalidOperationException("Production VectorNNTPD.ReaderService.json was not found."); + } + + private sealed class CollectingSink : ILogEventSink + { + public List Lines { get; } = []; + + public void Emit(LogEvent logEvent) => Lines.Add(logEvent.RenderMessage()); + } + + private sealed class TempLogDir : IDisposable + { + public TempLogDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-inpaths-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/AccountPolicyMatrixTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/AccountPolicyMatrixTests.cs new file mode 100644 index 00000000..a449f007 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/AccountPolicyMatrixTests.cs @@ -0,0 +1,192 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class AccountPolicyMatrixTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + + [Fact] + public void A_ByteAndRatePositive_BothEnforced() + { + var record = MemoryNntpUserRecordStore.Create( + "alice", + "x", + rateLimitBps: 2_400, + byteLimit: 1_000_000); + var policy = NntpAccountPolicy.FromRecord(record); + Assert.Equal(1_000_000, policy.ByteLimit); + Assert.Equal(2_400, policy.RateLimitBps); + Assert.True(policy.RequiresRateTracking); + Assert.Equal(300, AccountRateFormula.AccountBytesPerSecond(policy.RateLimitBps)); + + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount(policy.Username, 1); + rates.Register(policy.Username, "s1", cap, policy.RateLimitBps); + Assert.Equal(300, cap.MaxSendBytesPerSecond); + + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount(policy.Username, policy.ByteLimit); + var tracker = new AccountByteTracker(durable, new InMemoryAccountByteStore(), NullLogger.Instance); + tracker.CreateSink(policy.Username).ObserveCopied(100); + Assert.False(tracker.IsExhausted(policy.Username)); + Assert.Equal(300, cap.MaxSendBytesPerSecond); + } + + [Fact] + public void B_BytePositive_RateZero_QuotaOnly() + { + var policy = new NntpAccountPolicy("alice", 0, 1_000_000, 0, 0, "c"); + Assert.False(policy.RequiresRateTracking); + Assert.False(policy.RequiresAdmission); + Assert.Equal(0, AccountRateFormula.EqualShareCap(policy.RateLimitBps, 1)); + Assert.Equal(1_000_000, policy.ByteLimit); + } + + [Fact] + public void C_ByteZero_RatePositive_ExhaustedWithRatePolicy() + { + var policy = new NntpAccountPolicy("alice", 10_000_000, 0, 0, 0, "c"); + Assert.Equal(0, policy.ByteLimit); + Assert.True(policy.RequiresRateTracking); + Assert.Equal(1_250_000, AccountRateFormula.EqualShareCap(policy.RateLimitBps, 1)); + } + + [Fact] + public void D_BothZero_ExhaustedAndUnlimitedRate() + { + var policy = new NntpAccountPolicy("alice", 0, 0, 0, 0, "c"); + Assert.Equal(0, policy.ByteLimit); + Assert.False(policy.RequiresRateTracking); + Assert.Equal(0, AccountRateFormula.EqualShareCap(policy.RateLimitBps, 1)); + } + + [Fact] + public void E_MultipleSessions_ShareRateAndOneByteLedger() + { + var rates = new AccountRateAllocator(); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 1_000); + var bytes = new AccountByteTracker(durable, new InMemoryAccountByteStore(), NullLogger.Instance); + var caps = new FakeCap[2]; + for (var i = 0; i < 2; i++) + { + caps[i] = new FakeCap(); + rates.Register("alice", "s" + i, caps[i], 2_400); + bytes.CreateSink("alice").ObserveCopied(50); + } + + rates.ObserveClusterSessionCount("alice", 2); + Assert.Equal(150, caps[0].MaxSendBytesPerSecond); + Assert.Equal(150, caps[1].MaxSendBytesPerSecond); + Assert.Equal(100, bytes.PendingBytes("alice")); + } + + [Fact] + public async Task F_TwoNodes_RateSafetyAndByteLedgerCoexist() + { + var membership = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(); + var ratesB = new AccountRateAllocator(); + var nodeA = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01", + TimeProvider.System, + rates: ratesA); + var nodeB = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd02", + TimeProvider.System, + rates: ratesB); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 5_000); + var bytes = new AccountByteTracker(durable, new InMemoryAccountByteStore(), NullLogger.Instance); + + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a1", V4A, 10, 0, 2_400)); + var a1 = new FakeCap(); + ratesA.Register("alice", "a1", a1, 2_400); + bytes.CreateSink("alice").ObserveCopied(25); + + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b1", V4B, 10, 0, 2_400)); + var b1 = new FakeCap(); + ratesB.Register("alice", "b1", b1, 2_400); + bytes.CreateSink("alice").ObserveCopied(25); + + Assert.True( + AccountRateFormula.AllocatedBytesPerSecond(a1.MaxSendBytesPerSecond) + + AccountRateFormula.AllocatedBytesPerSecond(b1.MaxSendBytesPerSecond) + <= AccountRateFormula.AccountBytesPerSecond(2_400)); + Assert.Equal(50, bytes.PendingBytes("alice")); + Assert.Equal(5_000, durable.Remaining("alice")); + } + + [Fact] + public void G_SubEightBps_IsBlocked() + { + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount("alice", 1); + rates.Register("alice", "s1", cap, 7); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, cap.MaxSendBytesPerSecond); + } + + [Fact] + public async Task H_RateOnly_TracksSessionsWhenLimitsAreZero() + { + var node = new InMemorySessionStateTracker(new AccountRateAllocator()); + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", "s1", V4A, 0, 0, 2_400)); + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", "s2", V4A, 0, 0, 2_400)); + } + + [Fact] + public void I_SelectUserByName_OmitsAccountType_AndKeepsOrdinals() + { + Assert.DoesNotContain("account_type", NntpUserQueries.SelectUserByName, StringComparison.Ordinal); + Assert.Contains("account_rate_limit, account_byte_limit, account_session_limit, account_srcip_limit", NntpUserQueries.SelectUserByName, StringComparison.Ordinal); + Assert.Contains("account_art_type", NntpUserQueries.SelectUserByName, StringComparison.Ordinal); + Assert.DoesNotContain("account_type", NntpUserQueries.SelectByteQuotaForUpdate, StringComparison.Ordinal); + Assert.DoesNotContain("account_type", NntpUserQueries.SelectAccountByteRemaining, StringComparison.Ordinal); + Assert.StartsWith("SELECT account_byte_limit", NntpUserQueries.SelectByteQuotaForUpdate, StringComparison.Ordinal); + Assert.StartsWith("SELECT account_byte_limit", NntpUserQueries.SelectAccountByteRemaining, StringComparison.Ordinal); + } + + [Fact] + public void J_ConcurrentSinks_ShareAccountLedger() + { + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 1_000); + var tracker = new AccountByteTracker(durable, new InMemoryAccountByteStore(), NullLogger.Instance); + tracker.CreateSink("alice").ObserveCopied(10); + tracker.CreateSink("alice").ObserveCopied(20); + Assert.Equal(30, tracker.PendingBytes("alice")); + } + + [Fact] + public void NullLimits_MapToZero() + { + Assert.Equal(0, VectorNNTPD.ReaderService.NntpDb.MySqlNntpDbConnection.ConvertByteLimit(null)); + Assert.Equal(0, AccountRateFormula.AccountBytesPerSecond(0)); + } + + private sealed class FakeCap : IOutboundRateCap + { + public long MaxSendBytesPerSecond { get; private set; } + + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => MaxSendBytesPerSecond = bytesPerSecond; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/ArticleTypeAccessPolicyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ArticleTypeAccessPolicyTests.cs new file mode 100644 index 00000000..48512ffc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ArticleTypeAccessPolicyTests.cs @@ -0,0 +1,64 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Authentication; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class ArticleTypeAccessPolicyTests +{ + [Fact] + public void NullPolicy_IsUnrestricted() + { + Assert.True(ArticleTypeAccessPolicy.Allows(null, ArticleType.Binary | ArticleType.YEncoded)); + } + + [Fact] + public void AllFlags_AreUnrestricted() + { + var policy = new NntpAccountPolicy("alice", 0, 0, 0, 0, "c", ArticleTypeCapabilities.All); + Assert.True(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Binary)); + Assert.True(ArticleTypeAccessPolicy.Allows(policy, ArticleType.None)); + Assert.Equal(65535u, ArticleTypeCapabilities.AllValue); + Assert.Equal(ArticleTypeCapabilities.All, (ArticleType)ArticleTypeCapabilities.AllValue); + } + + [Fact] + public void TextOnly_RejectsBinary() + { + var policy = new NntpAccountPolicy( + "alice", + 0, + 0, + 0, + 0, + "c", + ArticleTypeCapabilities.TextOnly); + Assert.True(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Default)); + Assert.False(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Binary)); + Assert.False(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Default | ArticleType.Html)); + } + + [Fact] + public void FromRecord_CopiesCapability() + { + var record = new NntpUserRecord( + "alice", + "secret", + allowAuthPlain: true, + allowAuthScram256: false, + ReadOnlyMemory.Empty, + 0, + ReadOnlyMemory.Empty, + ReadOnlyMemory.Empty, + 0, + 0, + 0, + 0, + isEnabled: true, + "cust", + ArticleTypeCapabilities.TextOnly | ArticleType.Html); + var policy = NntpAccountPolicy.FromRecord(record); + Assert.Equal(record.AllowedArtTypes, policy.AllowedArtTypes); + Assert.True(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Default | ArticleType.Html)); + Assert.False(ArticleTypeAccessPolicy.Allows(policy, ArticleType.Binary)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/AuthenticatedSessionTeardownTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/AuthenticatedSessionTeardownTests.cs new file mode 100644 index 00000000..124398d1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/AuthenticatedSessionTeardownTests.cs @@ -0,0 +1,405 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class AuthenticatedSessionTeardownTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public async Task Quit_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, membership, node, _) = await StartAuthenticatedAsync(); + await using (harness) + { + await harness.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + await session.FinalizeAdmissionAsync(); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task RemoteEof_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, membership, node, _) = await StartAuthenticatedAsync(); + await using (harness) + { + await harness.CompleteClientInputAsync(); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(0, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + } + } + + [Fact] + public async Task ConnectionReset_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, _, node, _) = await StartAuthenticatedAsync(); + await using (harness) + { + await harness.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionReset)); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task SocketException_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, _, node, _) = await StartAuthenticatedAsync(); + await using (harness) + { + await harness.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionAborted)); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task IoException_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, _, node, _) = await StartAuthenticatedAsync(); + await using (harness) + { + await harness.CompleteClientInputAsync(new IOException("peer disappeared")); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task IdleTimeout_ReleasesAdmittedSessionOnce() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + await using var harness = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var session = harness.CreateSession( + commandIdleTimeout: TimeSpan.FromSeconds(1), + timeProvider: clock); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + await session.IdleWatchArmed; + clock.Advance(TimeSpan.FromSeconds(2)); + await run; + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + + [Fact] + public async Task Cancellation_ReleasesAdmittedSessionOnce() + { + var (harness, session, run, _, node, cts) = await StartAuthenticatedAsync(); + await using (harness) + { + await cts.CancelAsync(); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task MultipleTeardownSignals_ReleaseOnce() + { + var (harness, session, run, _, node, cts) = await StartAuthenticatedAsync(); + await using (harness) + { + var quit = harness.WriteClientLineAsync("QUIT"); + await cts.CancelAsync(); + session.RequestClose(); + await session.FinalizeAdmissionAsync(); + await harness.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionReset)); + await quit; + await run; + await session.FinalizeAdmissionAsync(); + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(1, node.ReleaseCalls); + } + } + + [Fact] + public async Task UnauthenticatedDisconnect_DoesNotRelease() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + await using var harness = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.CompleteClientInputAsync(); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(0, node.ReleaseCalls); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task AdmissionRejection_LeavesNoOwnership() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + await using var first = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + await using var second = await AuthHarness.CreateAsync(record, admission: node, clientIp: V6A); + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many sessions", await second.ReadClientLineAsync()); + await second.CompleteClientInputAsync(); + await run2; + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + Assert.Equal(0, node.ReleaseCalls); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await run1; + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task RedisUnavailableDuringRelease_FinalizesLocallyAndDropsRenewal() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1, srcIpLimit: 1); + await using var harness = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + + membership.Unavailable = true; + await harness.CompleteClientInputAsync(); + await run; + Assert.Equal(NntpSessionLifetime.Finalized, session.Lifetime); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + + await node.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(31)); + Assert.Equal(0, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task RenewalConcurrentWithFinalization_DoesNotResurrectOwnership() + { + var membership = new InMemorySessionStateStore + { + BlockRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1, srcIpLimit: 1); + await using var harness = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + + var eof = harness.CompleteClientInputAsync(); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (node.GetLocalSessionCount("alice") != 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + var renew = node.RenewLeasesAsync().AsTask(); + membership.BlockRelease.TrySetResult(); + await eof; + await run; + await renew; + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.Null(membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(1, node.ReleaseCalls); + } + + [Fact] + public async Task SameIpMultipleSessions_ReleaseDecrementsUntilSourceGone() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 5, srcIpLimit: 2); + await using var a = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + await using var b = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + await using var c = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var (s1, r1) = await AuthAsync(a); + var (s2, r2) = await AuthAsync(b); + var (s3, r3) = await AuthAsync(c); + Assert.Equal(3, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + + await a.CompleteClientInputAsync(); + await r1; + Assert.Equal(2, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + + await b.CompleteClientInputAsync(); + await r2; + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + + await c.CompleteClientInputAsync(); + await r3; + Assert.Equal(0, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.Equal(3, node.ReleaseCalls); + Assert.Equal(NntpSessionLifetime.Finalized, s1.Lifetime); + Assert.Equal(NntpSessionLifetime.Finalized, s2.Lifetime); + Assert.Equal(NntpSessionLifetime.Finalized, s3.Lifetime); + } + + [Fact] + public async Task ReleasingOneSourceIp_LeavesTheOther() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 5, srcIpLimit: 2); + await using var a = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + await using var b = await AuthHarness.CreateAsync(record, admission: node, clientIp: V6A); + var (_, r1) = await AuthAsync(a); + var (_, r2) = await AuthAsync(b); + + await a.CompleteClientInputAsync(); + await r1; + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + + await b.CompleteClientInputAsync(); + await r2; + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task ReleasingOneNode_DoesNotAffectTheOther() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 5, srcIpLimit: 2); + await using var a = await AuthHarness.CreateAsync(record, admission: node1, clientIp: V4A); + await using var b = await AuthHarness.CreateAsync(record, admission: node2, clientIp: V6A); + var (_, r1) = await AuthAsync(a); + var (_, r2) = await AuthAsync(b); + + await a.CompleteClientInputAsync(); + await r1; + Assert.Equal(0, membership.OwnerSessionCount("alice", node1.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", node2.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node2.OwnerId, NowMs())); + + await b.CompleteClientInputAsync(); + await r2; + } + + [Fact] + public async Task ReleaseAndAdmitRace_PreservesExactCapacity() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + await using var first = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + await using var second = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4B); + var (_, run1) = await AuthAsync(first); + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + + var eof = first.CompleteClientInputAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + var secondLine = await second.ReadClientLineAsync(); + await eof; + await run1; + if (secondLine.StartsWith("481 ", StringComparison.Ordinal)) + { + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await second.ReadClientLineAsync(), StringComparison.Ordinal); + } + else + { + Assert.StartsWith("281 ", secondLine, StringComparison.Ordinal); + } + + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + await second.CompleteClientInputAsync(); + await run2; + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + private static async Task<(AuthHarness Harness, NntpSession Session, Task Run, InMemorySessionStateStore Membership, DistributedSessionStateTracker Node, CancellationTokenSource Cts)> + StartAuthenticatedAsync() + { + // codeql[cs/local-not-disposed] CancellationTokenSource ownership transfers to the caller via the returned tuple. + var cts = new CancellationTokenSource(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 2, srcIpLimit: 2); + var harness = await AuthHarness.CreateAsync(record, admission: node, clientIp: V4A); + var session = harness.CreateSession(); + var run = session.RunAsync(cts.Token); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + return (harness, session, run, membership, node, cts); + } + + private static async Task<(NntpSession Session, Task Run)> AuthAsync(AuthHarness harness) + { + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + return (session, run); + } + + private static DistributedSessionStateTracker CreateNode( + string nodeId, + InMemorySessionStateStore membership, + TimeProvider? time = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/CachedNntpUserRecordStoreTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/CachedNntpUserRecordStoreTests.cs new file mode 100644 index 00000000..045d25bf --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/CachedNntpUserRecordStoreTests.cs @@ -0,0 +1,684 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class CachedNntpUserRecordStoreTests +{ + private const string SecretPassword = "super-secret-password-xyz"; + private const string CustomerId = "22222222-2222-2222-2222-222222222222"; + private static readonly byte[] ScramSalt = [1, 2, 3, 4, 5, 6, 7, 8]; + private static readonly byte[] ScramStoredKey = [9, 10, 11, 12, 13, 14, 15, 16]; + private static readonly byte[] ScramServerKey = [17, 18, 19, 20, 21, 22, 23, 24]; + + [Fact] + public async Task CacheMiss_SelectsMySql_AndPopulatesRedisWithTenSecondTtl() + { + var inner = new RecordingUserRecordStore(); + var record = CreateRecord("alice"); + inner.Add(record); + var redis = new FakeRedisService(); + var logger = new CollectingLogger(); + var cache = new CachedNntpUserRecordStore(inner, redis, logger); + + var loaded = await cache.TryGetUserAsync("alice"); + + AssertEqualRecord(record, loaded); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, redis.Database.SetCount); + Assert.Equal(TimeSpan.FromSeconds(10), redis.Database.LastSetExpiry); + Assert.True(redis.Database.TryGetStored(AccountCacheKeys.Create("alice"), out var payload, out var expiry)); + Assert.True(NntpAccountCacheCodec.TryDecode(payload, "alice", out var decoded)); + AssertEqualRecord(record, decoded); + Assert.True(expiry > DateTimeOffset.UtcNow.AddSeconds(8)); + Assert.True(expiry <= DateTimeOffset.UtcNow.AddSeconds(11)); + AssertNoSecrets(logger); + } + + [Fact] + public async Task CacheHit_DoesNotCallMySql() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + var logger = new CollectingLogger(); + redis.Database.Seed(AccountCacheKeys.Create("alice"), NntpAccountCacheCodec.Encode(CreateRecord("alice"))); + var cache = new CachedNntpUserRecordStore(inner, redis, logger); + + var loaded = await cache.TryGetUserAsync("alice"); + + AssertEqualRecord(CreateRecord("alice"), loaded); + Assert.Equal(0, inner.LookupCount); + Assert.Equal(0, redis.Database.SetCount); + Assert.Equal(1, redis.Database.GetCount); + AssertNoSecrets(logger); + } + + [Fact] + public async Task ConcurrentColdCache_SingleFlight_OneMySqlSelect() + { + var inner = new RecordingUserRecordStore(); + var record = CreateRecord("alice"); + inner.Add(record); + inner.Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var redis = new FakeRedisService(); + var logger = new CollectingLogger(); + var cache = new CachedNntpUserRecordStore(inner, redis, logger); + + var lookups = new Task[40]; + for (var i = 0; i < lookups.Length; i++) + { + lookups[i] = cache.TryGetUserAsync("alice").AsTask(); + } + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await inner.WaitStartedAsync("alice", safety.Token); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, cache.InFlightCount); + inner.Block.SetResult(); + var results = await Task.WhenAll(lookups); + + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, redis.Database.SetCount); + Assert.Equal(0, cache.InFlightCount); + foreach (var loaded in results) + { + AssertEqualRecord(record, loaded); + } + + AssertNoSecrets(logger); + } + + [Fact] + public async Task ConcurrentDifferentAccounts_DoNotShareOneGlobalLock() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + inner.Add(CreateRecord("bob", rateLimitBps: 240, byteLimit: 99)); + inner.Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + var alice = new Task[20]; + var bob = new Task[20]; + for (var i = 0; i < 20; i++) + { + alice[i] = cache.TryGetUserAsync("alice").AsTask(); + bob[i] = cache.TryGetUserAsync("bob").AsTask(); + } + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await inner.WaitStartedAsync("alice", safety.Token); + await inner.WaitStartedAsync("bob", safety.Token); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, inner.LookupsFor("bob")); + inner.Block.SetResult(); + var aliceResults = await Task.WhenAll(alice); + var bobResults = await Task.WhenAll(bob); + + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, inner.LookupsFor("bob")); + Assert.Equal(0, cache.InFlightCount); + foreach (var loaded in aliceResults) + { + AssertEqualRecord(CreateRecord("alice"), loaded); + } + + foreach (var loaded in bobResults) + { + AssertEqualRecord(CreateRecord("bob", rateLimitBps: 240, byteLimit: 99), loaded); + } + } + + [Fact] + public async Task RedisGetFailure_FallsBackToMySql_AndDoesNotTransientFailAuth() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + redis.Database.GetException = new RedisUnavailableException("get down"); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(cache, NullLogger.Instance); + + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + + Assert.True(result.Succeeded); + Assert.NotEqual(NntpAuthenticationFailureKind.TransientFailure, result.Failure); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.False(redis.IsUnavailable); + } + + [Fact] + public async Task RedisSetFailure_ReturnsMySqlRecord_AndDoesNotTransientFailAuth() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + redis.Database.SetException = new RedisUnavailableException("set down"); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(cache, NullLogger.Instance); + + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + + Assert.True(result.Succeeded); + Assert.NotEqual(NntpAuthenticationFailureKind.TransientFailure, result.Failure); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.False(redis.IsUnavailable); + } + + [Fact] + public async Task RedisUnavailable_FallsBackToMySql() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService { IsUnavailable = true }; + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + var loaded = await cache.TryGetUserAsync("alice"); + + AssertEqualRecord(CreateRecord("alice"), loaded); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(0, redis.Database.GetCount); + Assert.Equal(0, redis.Database.SetCount); + } + + [Fact] + public async Task MySqlFailureAfterMiss_PreservesExistingFailure() + { + var inner = new RecordingUserRecordStore + { + Exception = new NntpDbUnavailableException("nntpusers down"), + }; + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(cache, NullLogger.Instance); + + await Assert.ThrowsAsync(() => cache.TryGetUserAsync("alice").AsTask()); + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + + Assert.False(result.Succeeded); + Assert.Equal(NntpAuthenticationFailureKind.TransientFailure, result.Failure); + Assert.Equal(0, redis.Database.SetCount); + Assert.Equal(0, cache.InFlightCount); + } + + [Fact] + public void Codec_PreservesAllPolicyAndCredentialFields() + { + var record = new NntpUserRecord( + "alice", + SecretPassword, + allowAuthPlain: false, + allowAuthScram256: true, + ScramSalt, + scramIterations: 4096, + ScramStoredKey, + ScramServerKey, + rateLimitBps: 10_000_000, + byteLimit: long.MaxValue, + sessionLimit: 40, + srcIpLimit: 2, + isEnabled: true, + CustomerId); + + var payload = NntpAccountCacheCodec.Encode(record); + Assert.True(NntpAccountCacheCodec.TryDecode(payload, "alice", out var decoded)); + AssertEqualRecord(record, decoded); + Assert.False(NntpAccountCacheCodec.TryDecode(payload, "bob", out _)); + Assert.False(NntpAccountCacheCodec.TryDecode([0x02], "alice", out _)); + + foreach (var (rateBps, byteLimit) in new (int Rate, long Bytes)[] + { + (0, 0), + (1, 1), + (7, 0), + (8, 8), + (240, 10_000_000_000), + (int.MaxValue, long.MaxValue), + }) + { + var limited = CreateRecord("alice", rateLimitBps: rateBps, byteLimit: byteLimit); + var encoded = NntpAccountCacheCodec.Encode(limited); + Assert.True(NntpAccountCacheCodec.TryDecode(encoded, "alice", out var roundTrip)); + AssertEqualRecord(limited, roundTrip); + } + } + + [Fact] + public void RedisKey_IsSha256Hex_AndDoesNotContainAccountName() + { + var key = Encoding.ASCII.GetString(AccountCacheKeys.Create("alice")); + var hash = AccountCacheKeys.HashHex("alice"); + Assert.Equal("nntpd:account:" + hash, key); + Assert.Equal(64, hash.Length); + Assert.DoesNotContain("alice", key, StringComparison.OrdinalIgnoreCase); + Assert.Matches("^[0-9a-f]{64}$", hash); + } + + [Fact] + public async Task RedisGet_DoesNotExtendTtl() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + AssertEqualRecord(CreateRecord("alice"), await cache.TryGetUserAsync("alice")); + Assert.True(redis.Database.TryGetStored(AccountCacheKeys.Create("alice"), out _, out var expiryAfterSet)); + Assert.Equal(1, redis.Database.SetCount); + var lastExpiry = redis.Database.LastSetExpiry; + + AssertEqualRecord(CreateRecord("alice"), await cache.TryGetUserAsync("alice")); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, redis.Database.SetCount); + Assert.Equal(lastExpiry, redis.Database.LastSetExpiry); + Assert.True(redis.Database.TryGetStored(AccountCacheKeys.Create("alice"), out _, out var expiryAfterGet)); + Assert.Equal(expiryAfterSet, expiryAfterGet); + } + + [Fact] + public async Task RepeatedAuth_DoesNotBypassRedisExpiry() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + var store = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(store, NullLogger.Instance); + + for (var i = 0; i < 5; i++) + { + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + Assert.True(result.Succeeded); + } + + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, redis.Database.SetCount); + redis.Database.Expire(AccountCacheKeys.Create("alice")); + + var afterExpiry = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + Assert.True(afterExpiry.Succeeded); + Assert.Equal(2, inner.LookupsFor("alice")); + Assert.Equal(2, redis.Database.SetCount); + } + + [Fact] + public async Task RepeatedCacheHits_StillVerifyPasswordEachTime() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + var store = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(store, NullLogger.Instance); + + Assert.True((await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback)).Succeeded); + Assert.True((await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback)).Succeeded); + var rejected = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + "wrong-password", + IPAddress.Loopback); + + Assert.False(rejected.Succeeded); + Assert.Equal(NntpAuthenticationFailureKind.InvalidCredentials, rejected.Failure); + Assert.Equal(1, inner.LookupsFor("alice")); + } + + [Fact] + public async Task CachedZeroByteLimit_StillAuthenticates_AndDoesNotMutateCache() + { + var inner = new RecordingUserRecordStore(); + var record = CreateRecord("alice", rateLimitBps: 0, byteLimit: 0); + inner.Add(record); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(cache, NullLogger.Instance); + + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + SecretPassword, + IPAddress.Loopback); + + Assert.True(result.Succeeded); + Assert.Equal(0, result.Policy!.ByteLimit); + Assert.Equal(0, result.Policy.RateLimitBps); + Assert.True(redis.Database.TryGetStored(AccountCacheKeys.Create("alice"), out var payload, out _)); + Assert.True(NntpAccountCacheCodec.TryDecode(payload, "alice", out var cached)); + Assert.Equal(0, cached!.ByteLimit); + Assert.True(payload.AsSpan().SequenceEqual(NntpAccountCacheCodec.Encode(record))); + } + + [Fact] + public async Task SingleFlight_CancelledWaiter_DoesNotAbortSharedLookup() + { + var inner = new RecordingUserRecordStore(); + var record = CreateRecord("alice"); + inner.Add(record); + inner.Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + var winner = cache.TryGetUserAsync("alice").AsTask(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await inner.WaitStartedAsync("alice", safety.Token); + + using var waiterCts = new CancellationTokenSource(); + var cancelled = cache.TryGetUserAsync("alice", waiterCts.Token).AsTask(); + var kept = cache.TryGetUserAsync("alice").AsTask(); + waiterCts.Cancel(); + + await Assert.ThrowsAnyAsync(() => cancelled); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(1, cache.InFlightCount); + + inner.Block.SetResult(); + AssertEqualRecord(record, await winner); + AssertEqualRecord(record, await kept); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(0, cache.InFlightCount); + } + + [Fact] + public async Task FailedSharedLookup_RemovesGate_ForAllWaiters() + { + var inner = new RecordingUserRecordStore + { + Exception = new NntpDbUnavailableException("nntpusers down"), + Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + var first = cache.TryGetUserAsync("alice").AsTask(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await inner.WaitStartedAsync("alice", safety.Token); + var second = cache.TryGetUserAsync("alice").AsTask(); + Assert.Equal(1, cache.InFlightCount); + inner.Block.SetResult(); + + await Assert.ThrowsAsync(() => first); + await Assert.ThrowsAsync(() => second); + Assert.Equal(0, cache.InFlightCount); + Assert.Equal(1, inner.LookupsFor("alice")); + Assert.Equal(0, redis.Database.SetCount); + } + + [Fact] + public async Task CacheHit_StillRequiresPasswordVerification() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + redis.Database.Seed(AccountCacheKeys.Create("alice"), NntpAccountCacheCodec.Encode(CreateRecord("alice"))); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + var validator = new MySqlNntpCredentialValidator(cache, NullLogger.Instance); + + var result = await validator.ValidatePasswordAsync( + "AUTHINFO PASS", + "alice", + "wrong-password", + IPAddress.Loopback); + + Assert.False(result.Succeeded); + Assert.Equal(NntpAuthenticationFailureKind.InvalidCredentials, result.Failure); + Assert.Equal(0, inner.LookupCount); + } + + [Fact] + public async Task MissingAccount_IsNotCached() + { + var inner = new RecordingUserRecordStore(); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + Assert.Null(await cache.TryGetUserAsync("missing")); + Assert.Null(await cache.TryGetUserAsync("missing")); + Assert.Equal(2, inner.LookupsFor("missing")); + Assert.Equal(0, redis.Database.SetCount); + Assert.False(redis.Database.Contains(AccountCacheKeys.Create("missing"))); + } + + [Fact] + public async Task ExpiredCache_ReturnsToMySql() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + AssertEqualRecord(CreateRecord("alice"), await cache.TryGetUserAsync("alice")); + Assert.Equal(1, inner.LookupsFor("alice")); + redis.Database.Expire(AccountCacheKeys.Create("alice")); + + AssertEqualRecord(CreateRecord("alice"), await cache.TryGetUserAsync("alice")); + Assert.Equal(2, inner.LookupsFor("alice")); + Assert.Equal(2, redis.Database.SetCount); + } + + [Fact] + public async Task CorruptPayload_IsTreatedAsMiss() + { + var inner = new RecordingUserRecordStore(); + inner.Add(CreateRecord("alice")); + var redis = new FakeRedisService(); + redis.Database.Seed(AccountCacheKeys.Create("alice"), "not-a-cache-record"u8.ToArray()); + var cache = new CachedNntpUserRecordStore( + inner, + redis, + NullLogger.Instance); + + AssertEqualRecord(CreateRecord("alice"), await cache.TryGetUserAsync("alice")); + Assert.Equal(1, inner.LookupsFor("alice")); + } + + private static NntpUserRecord CreateRecord( + string name, + int rateLimitBps = 240, + long byteLimit = 10_000_000_000) => + new( + name, + SecretPassword, + allowAuthPlain: true, + allowAuthScram256: true, + ScramSalt, + scramIterations: 4096, + ScramStoredKey, + ScramServerKey, + rateLimitBps, + byteLimit, + sessionLimit: 40, + srcIpLimit: 2, + isEnabled: true, + CustomerId); + + private static void AssertEqualRecord(NntpUserRecord expected, NntpUserRecord? actual) + { + Assert.NotNull(actual); + Assert.Equal(expected.AccountName, actual.AccountName); + Assert.Equal(expected.AccountPassword, actual.AccountPassword); + Assert.Equal(expected.AllowAuthPlain, actual.AllowAuthPlain); + Assert.Equal(expected.AllowAuthScram256, actual.AllowAuthScram256); + Assert.True(expected.ScramSalt.Span.SequenceEqual(actual.ScramSalt.Span)); + Assert.Equal(expected.ScramIterations, actual.ScramIterations); + Assert.True(expected.ScramStoredKey.Span.SequenceEqual(actual.ScramStoredKey.Span)); + Assert.True(expected.ScramServerKey.Span.SequenceEqual(actual.ScramServerKey.Span)); + Assert.Equal(expected.RateLimitBps, actual.RateLimitBps); + Assert.Equal(expected.ByteLimit, actual.ByteLimit); + Assert.Equal(expected.SessionLimit, actual.SessionLimit); + Assert.Equal(expected.SrcIpLimit, actual.SrcIpLimit); + Assert.Equal(expected.IsEnabled, actual.IsEnabled); + Assert.Equal(expected.CustomerId, actual.CustomerId); + Assert.Equal(expected.AllowedArtTypes, actual.AllowedArtTypes); + } + + private static void AssertNoSecrets(CollectingLogger logger) + { + var text = string.Join('\n', logger.Messages); + Assert.DoesNotContain(SecretPassword, text, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToHexString(ScramSalt), text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(Convert.ToHexString(ScramStoredKey), text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(Convert.ToHexString(ScramServerKey), text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(Encoding.UTF8.GetString(ScramSalt), text, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd:account:", text, StringComparison.Ordinal); + } + + private sealed class RecordingUserRecordStore : INntpUserRecordStore + { + private readonly ConcurrentDictionary _users = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _lookups = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _started = new(StringComparer.Ordinal); + + public TaskCompletionSource? Block { get; set; } + + public Exception? Exception { get; set; } + + public int LookupCount => _lookups.Values.Sum(); + + public void Add(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + _users[record.AccountName] = record; + } + + public int LookupsFor(string accountName) => + _lookups.TryGetValue(accountName, out var count) ? count : 0; + + public Task WaitStartedAsync(string accountName, CancellationToken cancellationToken) => + _started.GetOrAdd( + accountName, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)) + .Task + .WaitAsync(cancellationToken); + + public async ValueTask TryGetUserAsync( + string accountName, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + _lookups.AddOrUpdate(accountName, 1, static (_, count) => count + 1); + _started.GetOrAdd( + accountName, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)) + .TrySetResult(); + if (Block is not null) + { + await Block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Exception is not null) + { + throw Exception; + } + + return _users.TryGetValue(accountName, out var record) ? record : null; + } + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static NullScope Instance { get; } = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/CramMd5MechanismTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/CramMd5MechanismTests.cs new file mode 100644 index 00000000..09ca5b99 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/CramMd5MechanismTests.cs @@ -0,0 +1,74 @@ +using System.Text; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class CramMd5MechanismTests +{ + [Fact] + public void CreateChallenge_IsRfc2195MsgIdWithNntpdFqdn() + { + var fqdn = NntpdOptions.FormatFqdn(1, "usenet.ninja"); + var clock = new ControllableTimeProvider(); + clock.Advance(TimeSpan.FromSeconds(1_700_000_000)); + var challenge = CramMd5Mechanism.CreateChallenge(fqdn, clock); + + Assert.True(CramMd5Mechanism.IsRfc2195Challenge(challenge)); + Assert.StartsWith("<", challenge, StringComparison.Ordinal); + Assert.True(challenge.Contains(".1700000000@", StringComparison.Ordinal)); + Assert.EndsWith("@nntpd01.usenet.ninja>", challenge, StringComparison.Ordinal); + Assert.Equal(-1, challenge.IndexOf(' ')); + } + + [Fact] + public void CreateChallenge_IsUniqueBetweenCalls() + { + var fqdn = NntpdOptions.FormatFqdn(7, "usenet.ninja"); + var first = CramMd5Mechanism.CreateChallenge(fqdn); + var second = CramMd5Mechanism.CreateChallenge(fqdn); + Assert.NotEqual(first, second); + Assert.EndsWith("@nntpd07.usenet.ninja>", first, StringComparison.Ordinal); + Assert.EndsWith("@nntpd07.usenet.ninja>", second, StringComparison.Ordinal); + } + + [Fact] + public void HmacInput_IsDecodedNativeChallenge_NotBase64WireToken() + { + const string password = "tanstaaftanstaaf"; + var native = CramMd5Mechanism.CreateChallenge(NntpdOptions.FormatFqdn(1, "usenet.ninja")); + var wire = Convert.ToBase64String(Encoding.ASCII.GetBytes(native)); + var decoded = Encoding.ASCII.GetString(Convert.FromBase64String(wire)); + Assert.Equal(native, decoded); + + var secret = CramMd5Mechanism.SecretFromPassword(password); + var digest = CramMd5Mechanism.ComputeHexDigest(secret, native); + Assert.Equal(digest, CramMd5Mechanism.ComputeHexDigest(secret, decoded)); + Assert.NotEqual(digest, CramMd5Mechanism.ComputeHexDigest(secret, wire)); + Assert.True(CramMd5Mechanism.Verify("tim", "tim " + digest, native, secret)); + Assert.False(CramMd5Mechanism.Verify("tim", "tim " + digest, wire, secret)); + } + + [Fact] + public void Rfc2195Example_ProducesPublishedDigest() + { + const string native = "<1896.697170952@postoffice.reston.mci.net>"; + const string secret = "tanstaaftanstaaf"; + var digest = CramMd5Mechanism.ComputeHexDigest(Encoding.ASCII.GetBytes(secret), native); + Assert.Equal("b913a602c7eda7a495b4e6e7334d3890", digest); + Assert.True( + CramMd5Mechanism.Verify( + "tim", + "tim b913a602c7eda7a495b4e6e7334d3890", + native, + Encoding.ASCII.GetBytes(secret))); + } + + [Fact] + public void CreateChallenge_RejectsInvalidFqdn() + { + Assert.Throws(() => CramMd5Mechanism.CreateChallenge("host@evil")); + Assert.Throws(() => CramMd5Mechanism.CreateChallenge(" ")); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpReaderAuthenticationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpReaderAuthenticationTests.cs new file mode 100644 index 00000000..d92fe8c1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpReaderAuthenticationTests.cs @@ -0,0 +1,1398 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Tests.Session; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpReaderAuthenticationTests +{ + [Fact] + public async Task UserPass_Succeeds_AndCachesPolicy() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("alice", session.Authentication.Username); + Assert.True(session.Authorization.AuthorizedReader); + Assert.True(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.ControlCancelPermitted); + Assert.NotNull(session.AccountPolicy); + Assert.Equal(0, session.AccountPolicy!.RateLimitBps); + Assert.True(session.AccountPolicy.ByteLimit > 0); + Assert.Equal(ArticleTypeCapabilities.All, session.AccountPolicy.AllowedArtTypes); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task UserPass_CachesTextOnlyArtTypeCapability() + { + await using var harness = await AuthHarness.CreateAsync( + MemoryNntpUserRecordStore.Create( + "alice", + "secret", + allowedArtTypes: ArticleTypeCapabilities.TextOnly)); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal(ArticleTypeCapabilities.TextOnly, session.AccountPolicy!.AllowedArtTypes); + Assert.True(ArticleTypeAccessPolicy.CanPostArticleType(session, ArticleType.Default)); + Assert.False(ArticleTypeAccessPolicy.CanPostArticleType(session, ArticleType.Binary)); + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task UnknownUser_AndDisabled_AndWrongPassword_AllReturn481() + { + await using var harness = await AuthHarness.CreateAsync( + MemoryNntpUserRecordStore.Create("alice", "secret"), + MemoryNntpUserRecordStore.Create("disabled", "secret", enabled: false)); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER missing"); + Assert.StartsWith("381 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Equal("missing", session.PendingAuthUsername); + + await harness.WriteClientLineAsync("AUTHINFO USER disabled"); + Assert.StartsWith("381 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("381 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowAuthPlainDisabled_RejectsPasswordMechanisms() + { + await using var harness = await AuthHarness.CreateAsync( + MemoryNntpUserRecordStore.Create("alice", "secret", allowPlain: false)); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task MissingAccountArtTypeColumn_Returns503_Not481() + { + var store = new MemoryNntpUserRecordStore(); + store.Add(MemoryNntpUserRecordStore.Create("alice", "secret")); + store.Exception = new NntpDbUnavailableException( + "MySQL nntpusers lookup failed.", + new InvalidOperationException("Unknown column 'account_art_type'")); + await using var harness = await AuthHarness.CreateAsync(store); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("503 Temporary authentication failure", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task BackendFailure_Returns503_Not481() + { + var store = new MemoryNntpUserRecordStore(); + store.Add(MemoryNntpUserRecordStore.Create("alice", "secret")); + store.Exception = new NntpDbUnavailableException("down"); + await using var harness = await AuthHarness.CreateAsync(store); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("503 Temporary authentication failure", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task SessionLimit_RejectsSecondSession_ThenReleasesOnClose() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + var admission = new InMemorySessionStateTracker(); + await using var first = await AuthHarness.CreateAsync(record, admission: admission); + await using var second = await AuthHarness.CreateAsync(record, admission: admission); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.WriteClientLineAsync("AUTHINFO USER alice"); + await first.ReadClientLineAsync(); + await first.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await first.ReadClientLineAsync(), StringComparison.Ordinal); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many sessions", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await run1; + + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await second.ReadClientLineAsync(), StringComparison.Ordinal); + + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run2; + } + + [Fact] + public async Task SourceIpLimit_RejectsDistinctAddress() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", srcIpLimit: 1); + var admission = new InMemorySessionStateTracker(); + await using var first = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("203.0.113.10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("203.0.113.11")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many source addresses", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task SourceIpLimit_SameAddressAdditionalSessions_AreAccepted() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", srcIpLimit: 1); + var admission = new InMemorySessionStateTracker(); + var ip = IPAddress.Parse("203.0.113.10"); + await using var first = await AuthHarness.CreateAsync(record, admission: admission, clientIp: ip); + await using var second = await AuthHarness.CreateAsync(record, admission: admission, clientIp: ip); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await second.ReadClientLineAsync()); + Assert.True(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task DistributedAdmissionUnavailable_Returns503() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", srcIpLimit: 1); + var membership = new InMemorySessionStateStore { Unavailable = true }; + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + await using var harness = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("192.0.2.10")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("503 Temporary authentication failure", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.AccountPolicy); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DistributedSourceIpLimit_SameAddressAdditionalSessions_AreAccepted() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", srcIpLimit: 1); + var membership = new InMemorySessionStateStore(); + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + var ip = IPAddress.Parse("192.0.2.10"); + await using var first = await AuthHarness.CreateAsync(record, admission: admission, clientIp: ip); + await using var second = await AuthHarness.CreateAsync(record, admission: admission, clientIp: ip); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await second.ReadClientLineAsync()); + Assert.True(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task DistributedSourceIpLimit_RejectsIpv6WhenIpv4IsActive() + { + var record = MemoryNntpUserRecordStore.Create("a", "secret", srcIpLimit: 1); + var membership = new InMemorySessionStateStore(); + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + await using var first = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("192.0.2.10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("2001:db8::10")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("a", "secret"); + Assert.True(session1.Authentication.IsAuthenticated); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER a"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many source addresses", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + Assert.Null(session2.AccountPolicy); + Assert.False(session2.Authorization.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task DistributedSourceIpLimit_RejectsIpv4WhenIpv6IsActive() + { + var record = MemoryNntpUserRecordStore.Create("a", "secret", srcIpLimit: 1); + var membership = new InMemorySessionStateStore(); + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + await using var first = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("2001:db8::10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("192.0.2.10")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("a", "secret"); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER a"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many source addresses", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task IdleAuthenticatedSession_KeepsClusterSlotWithoutFurtherCommands() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01", + clock); + await using var first = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("192.0.2.10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("198.51.100.20")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + Assert.True(session1.Authentication.IsAuthenticated); + + for (var i = 0; i < 4; i++) + { + clock.Advance(TimeSpan.FromSeconds(10)); + await admission.RenewLeasesAsync(); + } + + Assert.Equal(1, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many sessions", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await run1; + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run2; + } + + [Fact] + public async Task DistributedSessionLimit_IsClusterWide() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", sessionLimit: 1); + var membership = new InMemorySessionStateStore(); + var nodeA = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + var nodeB = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd02"); + await using var first = await AuthHarness.CreateAsync( + record, + admission: nodeA, + clientIp: IPAddress.Parse("192.0.2.10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: nodeB, + clientIp: IPAddress.Parse("2001:db8::10")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + Assert.True(session1.Authentication.IsAuthenticated); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many sessions", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + Assert.Null(session2.AccountPolicy); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await run1; + + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await second.ReadClientLineAsync()); + Assert.True(session2.Authentication.IsAuthenticated); + + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run2; + } + + [Fact] + public async Task DistributedReauthentication_DoesNotLeakSourceSlot() + { + var record = MemoryNntpUserRecordStore.Create("alice", "secret", srcIpLimit: 1); + var membership = new InMemorySessionStateStore(); + var admission = new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + "nntpd01"); + await using var first = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("192.0.2.10")); + await using var second = await AuthHarness.CreateAsync( + record, + admission: admission, + clientIp: IPAddress.Parse("2001:db8::10")); + + var session1 = first.CreateSession(); + var run1 = session1.RunAsync(); + await first.ReadGreetingAsync(); + await first.AuthenticateAsync("alice", "secret"); + await first.WriteClientLineAsync("AUTHINFO USER alice"); + Assert.StartsWith("502 ", await first.ReadClientLineAsync(), StringComparison.Ordinal); + + var session2 = second.CreateSession(); + var run2 = session2.RunAsync(); + await second.ReadGreetingAsync(); + await second.WriteClientLineAsync("AUTHINFO USER alice"); + await second.ReadClientLineAsync(); + await second.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Too many source addresses", await second.ReadClientLineAsync()); + Assert.False(session2.Authentication.IsAuthenticated); + + await first.WriteClientLineAsync("QUIT"); + await first.ReadClientLineAsync(); + await second.WriteClientLineAsync("QUIT"); + await second.ReadClientLineAsync(); + await run1; + await run2; + } + + [Fact] + public async Task NewsmasterUsername_DoesNotFallThroughToMysql() + { + var options = new NntpdOptions + { + NewsmasterUser = "alice", + NewsmasterPassword = "newsmaster-secret", + }; + var store = new MemoryNntpUserRecordStore(); + store.Add(MemoryNntpUserRecordStore.Create("alice", "mysql-secret")); + await using var harness = await AuthHarness.CreateAsync(store, newsmaster: options); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS mysql-secret"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.Equal(0, store.LookupCount); + + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS newsmaster-secret"); + Assert.StartsWith("281 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authorization.ControlCancelPermitted); + Assert.Equal(0, store.LookupCount); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task FailedAuth_ClearsIdentity_AndDoesNotLogPassword() + { + var recording = new RecordingLoggerFactory(); + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(loggerFactory: recording); + session.ApplySuccessfulAuthentication( + "stale", + new NntpAuthorization(true, true, false, true, false), + NntpAccountPolicy.FromRecord(MemoryNntpUserRecordStore.Create("stale", "x"))); + session.ApplyFailedAuthentication(); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.AccountPolicy); + + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.WriteClientLineAsync("AUTHINFO USER alice"); + await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync("AUTHINFO PASS not-the-password"); + Assert.StartsWith("481 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("not-the-password", joined, StringComparison.Ordinal); + Assert.DoesNotContain("secret", joined, StringComparison.Ordinal); + } + + [Fact] + public async Task FakeNntpDb_LookupAndBackendFailure() + { + var factory = new FakeNntpDbConnectionFactory(); + factory.Users["alice"] = MemoryNntpUserRecordStore.Create("alice", "secret"); + await using var connection = (FakeNntpDbConnection)await factory.OpenAsync("Server=test;", CancellationToken.None); + var found = await connection.QueryUserAccountAsync("alice", CancellationToken.None); + Assert.NotNull(found); + Assert.Null(await connection.QueryUserAccountAsync("missing", CancellationToken.None)); + + connection.QueryUserException = new NntpDbUnavailableException("down"); + await Assert.ThrowsAsync( + async () => await connection.QueryUserAccountAsync("alice", CancellationToken.None)); + } + + [Fact] + public async Task AuthenticatedAccount_ReceivesBothByteAndRatePolicies() + { + await using var harness = await AuthHarness.CreateAsync( + MemoryNntpUserRecordStore.Create("alice", "secret", rateLimitBps: 240, byteLimit: 1000)); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + await harness.AuthenticateAsync("alice", "secret"); + Assert.Equal(1000, session.AccountPolicy!.ByteLimit); + Assert.Equal(240, session.AccountPolicy.RateLimitBps); + Assert.True(session.AccountPolicy.RequiresRateTracking); + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } +} + +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpSaslAuthenticationTests +{ + [Fact] + public async Task Capabilities_AdvertiseImplementedMechanismsOnly() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await harness.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO USER SASL", caps); + Assert.Contains("SASL PLAIN LOGIN SCRAM-SHA-256 CRAM-MD5", caps); + Assert.DoesNotContain(caps, static c => c.Contains("DIGEST-MD5", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Contains("GSSAPI", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Contains("EXTERNAL", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Contains("SCRAM-SHA-1", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Contains("PLUS", StringComparison.Ordinal)); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Plain_WithInitialResponse_Succeeds() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var token = Convert.ToBase64String("\0alice\0secret"u8.ToArray()); + await harness.WriteClientLineAsync($"AUTHINFO SASL PLAIN {token}"); + Assert.StartsWith("281 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal("alice", session.Authentication.Username); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Plain_ContinuationAndCancel() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL PLAIN"); + Assert.Equal("383 =", await harness.ReadClientLineAsync()); + Assert.True(session.HasSaslExchange); + + await harness.WriteClientLineAsync("*"); + Assert.Equal("481 Authentication cancelled", await harness.ReadClientLineAsync()); + Assert.False(session.HasSaslExchange); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("AUTHINFO SASL PLAIN"); + Assert.Equal("383 =", await harness.ReadClientLineAsync()); + await harness.WriteClientLineAsync(Convert.ToBase64String("\0alice\0secret"u8.ToArray())); + Assert.StartsWith("281 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Login_Succeeds_AndInvalidBase64Is504() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL PLAIN !!!not-base64!!!"); + Assert.Equal("504 Base64 encoding error", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("AUTHINFO SASL LOGIN"); + var loginUserChallenge = await harness.ReadClientLineAsync(); + Assert.Equal("383 VXNlcm5hbWU6", loginUserChallenge); + Assert.False(loginUserChallenge.StartsWith("334 ", StringComparison.Ordinal)); + await harness.WriteClientLineAsync(Convert.ToBase64String("alice"u8.ToArray())); + var loginPasswordChallenge = await harness.ReadClientLineAsync(); + Assert.Equal("383 UGFzc3dvcmQ6", loginPasswordChallenge); + await harness.WriteClientLineAsync(Convert.ToBase64String("secret"u8.ToArray())); + var loginSuccess = await harness.ReadClientLineAsync(); + Assert.StartsWith("281 ", loginSuccess, StringComparison.Ordinal); + Assert.False(loginSuccess.StartsWith("235 ", StringComparison.Ordinal)); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task UnsupportedMechanisms_Return503() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + foreach (var mechanism in new[] { "DIGEST-MD5", "SCRAM-SHA-1", "SCRAM-SHA-256-PLUS", "EXTERNAL", "GSSAPI" }) + { + await harness.WriteClientLineAsync($"AUTHINFO SASL {mechanism}"); + Assert.Equal("503 Mechanism not supported", await harness.ReadClientLineAsync()); + } + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CramMd5_ValidAndInvalid() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var challengeLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", challengeLine, StringComparison.Ordinal); + Assert.False(challengeLine.StartsWith("334 ", StringComparison.Ordinal)); + var challenge = challengeLine[4..]; + var native = Encoding.ASCII.GetString(Convert.FromBase64String(challenge)); + Assert.True(CramMd5Mechanism.IsRfc2195Challenge(native)); + Assert.EndsWith("@nntpd01.usenet.ninja>", native, StringComparison.Ordinal); + await harness.WriteClientLineAsync(CramResponse("alice", "secret", challenge)); + var cramSuccess = await harness.ReadClientLineAsync(); + Assert.Equal("281 Authentication accepted", cramSuccess); + Assert.False(cramSuccess.StartsWith("235 ", StringComparison.Ordinal)); + Assert.Equal("alice", session.Authentication.Username); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CramMd5_WrongResponse_Returns481() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var challengeLine = await harness.ReadClientLineAsync(); + await harness.WriteClientLineAsync(CramResponse("alice", "wrong", challengeLine[4..])); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CramMd5_ChallengeIsUniqueAndUsesNntpdFqdn() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var firstLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", firstLine, StringComparison.Ordinal); + var firstNative = Encoding.ASCII.GetString(Convert.FromBase64String(firstLine[4..])); + Assert.True(CramMd5Mechanism.IsRfc2195Challenge(firstNative)); + Assert.StartsWith("<", firstNative, StringComparison.Ordinal); + Assert.EndsWith("@nntpd01.usenet.ninja>", firstNative, StringComparison.Ordinal); + await harness.WriteClientLineAsync("*"); + Assert.Equal("481 Authentication cancelled", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var secondLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", secondLine, StringComparison.Ordinal); + var secondNative = Encoding.ASCII.GetString(Convert.FromBase64String(secondLine[4..])); + Assert.True(CramMd5Mechanism.IsRfc2195Challenge(secondNative)); + Assert.EndsWith("@nntpd01.usenet.ninja>", secondNative, StringComparison.Ordinal); + Assert.NotEqual(firstNative, secondNative); + Assert.NotEqual(firstLine, secondLine); + + await harness.WriteClientLineAsync("*"); + Assert.StartsWith("481 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CramMd5_InvalidBase64Continuation_Returns504() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + Assert.StartsWith("383 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("abcd=efg"); + Assert.Equal("504 Base64 encoding error", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ScramSha256_CompletesRfcExchange() + { + const string password = "pencil"; + var salt = Convert.FromHexString("4142434445464748494A4B4C4D4E4F50"); + var keys = ScramClient.Derive(password, salt, 4096); + var record = MemoryNntpUserRecordStore.Create( + "user", + password, + scramSalt: salt, + scramIterations: 4096, + scramStoredKey: keys.StoredKey, + scramServerKey: keys.ServerKey); + await using var harness = await AuthHarness.CreateAsync(record); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + const string clientFirst = "n,,n=user,r=fyko+d2lbbFgONRv9qkxdawL"; + await harness.WriteClientLineAsync($"AUTHINFO SASL SCRAM-SHA-256 {Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFirst))}"); + var serverFirstLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", serverFirstLine, StringComparison.Ordinal); + Assert.DoesNotContain("s=", serverFirstLine, StringComparison.Ordinal); + Assert.DoesNotContain("i=", serverFirstLine, StringComparison.Ordinal); + var serverFirst = Encoding.UTF8.GetString(Convert.FromBase64String(serverFirstLine[4..])); + Assert.Contains("s=", serverFirst, StringComparison.Ordinal); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + var success = await harness.ReadClientLineAsync(); + Assert.StartsWith("283 ", success, StringComparison.Ordinal); + Assert.False(success.StartsWith("235 ", StringComparison.Ordinal)); + Assert.False(success.StartsWith("281 ", StringComparison.Ordinal)); + var serverFinal = Encoding.UTF8.GetString(Convert.FromBase64String(success[4..])); + Assert.StartsWith("v=", serverFinal, StringComparison.Ordinal); + Assert.Equal("user", session.Authentication.Username); + Assert.DoesNotContain(Convert.ToHexString(keys.StoredKey), success, StringComparison.Ordinal); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Scram_DisabledFlag_FailsWithoutAdvertisingKeys() + { + var salt = RandomNumberGenerator.GetBytes(16); + var keys = ScramClient.Derive("pencil", salt, 4096); + var record = MemoryNntpUserRecordStore.Create( + "user", + "pencil", + allowScram: false, + scramSalt: salt, + scramIterations: 4096, + scramStoredKey: keys.StoredKey, + scramServerKey: keys.ServerKey); + await using var harness = await AuthHarness.CreateAsync(record); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + const string clientFirst = "n,,n=user,r=clientnonce"; + await harness.WriteClientLineAsync($"AUTHINFO SASL SCRAM-SHA-256 {Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFirst))}"); + var serverFirstLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", serverFirstLine, StringComparison.Ordinal); + var serverFirst = Encoding.UTF8.GetString(Convert.FromBase64String(serverFirstLine[4..])); + Assert.DoesNotContain(Convert.ToBase64String(salt), serverFirst, StringComparison.Ordinal); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Login_Cancellation_Returns481_AndNextCommandIsParsed() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL LOGIN"); + Assert.Equal("383 VXNlcm5hbWU6", await harness.ReadClientLineAsync()); + await harness.WriteClientLineAsync("*"); + Assert.Equal("481 Authentication cancelled", await harness.ReadClientLineAsync()); + Assert.False(session.HasSaslExchange); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + _ = await harness.ReadMultilineBodyAsync(); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowAuthPlainDisabled_BlocksPlainLoginAndCram() + { + await using var harness = await AuthHarness.CreateAsync( + MemoryNntpUserRecordStore.Create("alice", "secret", allowPlain: false)); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync($"AUTHINFO SASL PLAIN {Convert.ToBase64String("\0alice\0secret"u8.ToArray())}"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("AUTHINFO SASL LOGIN"); + Assert.Equal("383 VXNlcm5hbWU6", await harness.ReadClientLineAsync()); + await harness.WriteClientLineAsync(Convert.ToBase64String("alice"u8.ToArray())); + Assert.Equal("383 UGFzc3dvcmQ6", await harness.ReadClientLineAsync()); + await harness.WriteClientLineAsync(Convert.ToBase64String("secret"u8.ToArray())); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var challengeLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", challengeLine, StringComparison.Ordinal); + await harness.WriteClientLineAsync(CramResponse("alice", "secret", challengeLine[4..])); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Plain_EmptyFields_Returns481() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync($"AUTHINFO SASL PLAIN {Convert.ToBase64String("\0\0"u8.ToArray())}"); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task SaslChallenges_AreNotCopiedIntoDebugLogs() + { + const string password = "pencil"; + var salt = Convert.FromHexString("4142434445464748494A4B4C4D4E4F50"); + var keys = ScramClient.Derive(password, salt, 4096); + var record = MemoryNntpUserRecordStore.Create( + "user", + password, + scramSalt: salt, + scramIterations: 4096, + scramStoredKey: keys.StoredKey, + scramServerKey: keys.ServerKey); + var recording = new RecordingLoggerFactory(); + await using var harness = await AuthHarness.CreateAsync(record); + var session = harness.CreateSession(loggerFactory: recording); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + const string clientFirst = "n,,n=user,r=fyko+d2lbbFgONRv9qkxdawL"; + await harness.WriteClientLineAsync($"AUTHINFO SASL SCRAM-SHA-256 {Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFirst))}"); + var serverFirstLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", serverFirstLine, StringComparison.Ordinal); + Assert.DoesNotContain("s=", serverFirstLine, StringComparison.Ordinal); + await harness.WriteClientLineAsync("*"); + Assert.StartsWith("481 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + + await harness.WriteClientLineAsync("AUTHINFO SASL CRAM-MD5"); + var cramLine = await harness.ReadClientLineAsync(); + Assert.StartsWith("383 ", cramLine, StringComparison.Ordinal); + await harness.WriteClientLineAsync("*"); + Assert.StartsWith("481 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain(Convert.ToBase64String(salt), joined, StringComparison.Ordinal); + Assert.DoesNotContain(serverFirstLine[4..], joined, StringComparison.Ordinal); + Assert.DoesNotContain(cramLine[4..], joined, StringComparison.Ordinal); + Assert.DoesNotContain(password, joined, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToHexString(keys.StoredKey), joined, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToHexString(keys.ServerKey), joined, StringComparison.Ordinal); + } + + [Fact] + public async Task Sasl_CleartextDisabled_Returns483() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(allowCleartextAuth: false); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync("AUTHINFO SASL PLAIN"); + Assert.StartsWith("483 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.WriteClientLineAsync("CAPABILITIES"); + await harness.ReadClientLineAsync(); + var caps = await harness.ReadMultilineBodyAsync(); + Assert.DoesNotContain(caps, static c => c.Contains("SASL", StringComparison.Ordinal)); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task LoginAndCram_InitialResponse_Returns482() + { + await using var harness = await AuthHarness.CreateAsync(MemoryNntpUserRecordStore.Create("alice", "secret")); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + await harness.WriteClientLineAsync($"AUTHINFO SASL LOGIN {Convert.ToBase64String("alice"u8.ToArray())}"); + Assert.Equal("482 SASL protocol error", await harness.ReadClientLineAsync()); + await harness.WriteClientLineAsync($"AUTHINFO SASL CRAM-MD5 {Convert.ToBase64String("alice"u8.ToArray())}"); + Assert.Equal("482 SASL protocol error", await harness.ReadClientLineAsync()); + Assert.False(session.HasSaslExchange); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + private static string CramResponse(string username, string password, string challenge) + { + var challengeBytes = Convert.FromBase64String(challenge); +#pragma warning disable CA5351 + using var hmac = new System.Security.Cryptography.HMACMD5(Encoding.ASCII.GetBytes(password)); +#pragma warning restore CA5351 + var hex = Convert.ToHexString(hmac.ComputeHash(challengeBytes)).ToLowerInvariant(); + return Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username} {hex}")); + } +} + +internal sealed class AuthHarness : IAsyncDisposable +{ + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly INntpAuthenticationProvider _provider; + private readonly NntpSaslService _sasl; + private readonly ISessionStateTracker _admission; + private readonly IPAddress _clientIp; + private readonly bool _isTls; + + private AuthHarness( + INntpAuthenticationProvider provider, + NntpSaslService sasl, + ISessionStateTracker admission, + IPAddress clientIp, + bool isTls) + { + _provider = provider; + _sasl = sasl; + _admission = admission; + _clientIp = clientIp; + _isTls = isTls; + } + + public static Task CreateAsync( + params NntpUserRecord[] users) => + CreateAsync(Users(users)); + + public static Task CreateAsync( + NntpUserRecord record, + ISessionStateTracker? admission = null, + IPAddress? clientIp = null) => + CreateAsync(Users(record), admission: admission, clientIp: clientIp); + + public static Task CreateAsync( + MemoryNntpUserRecordStore store, + ISessionStateTracker? admission = null, + IPAddress? clientIp = null, + NntpdOptions? newsmaster = null, + bool isTls = false, + ScramDummyVerifier? dummyScram = null) + { + var validator = new MySqlNntpCredentialValidator(store, NullLogger.Instance); + var newsmasterProvider = newsmaster is null + ? DenyAllNntpAuthenticationProvider.Instance + : NewsmasterNntpAuthenticationProvider.Create(newsmaster); + var provider = new CompositeNntpAuthenticationProvider( + newsmasterProvider, + newsmaster?.NewsmasterUser, + validator); + var nntpd = newsmaster is not null && !string.IsNullOrWhiteSpace(newsmaster.Fqdn) + ? newsmaster + : new NntpdOptions { ServerId = 1, DnsSuffix = "usenet.ninja" }; + var sasl = new NntpSaslService(provider, validator, Options.Create(nntpd), dummyScram); + return Task.FromResult(new AuthHarness( + provider, + sasl, + admission ?? new InMemorySessionStateTracker(), + clientIp ?? IPAddress.Loopback, + isTls)); + } + + public NntpSession CreateSession( + bool allowCleartextAuth = true, + ILoggerFactory? loggerFactory = null, + TimeSpan? commandIdleTimeout = null, + TimeProvider? timeProvider = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientIp, 119)), + isTls: _isTls); + return new NntpSession( + connection, + loggerFactory?.CreateLogger() ?? NullLogger.Instance, + authenticationProvider: _provider, + allowCleartextAuth: allowCleartextAuth, + loggerFactory: loggerFactory, + sessionAdmission: _admission, + saslService: _sasl, + commandIdleTimeout: commandIdleTimeout, + timeProvider: timeProvider); + } + + public async Task AuthenticateAsync(string username, string password) + { + await WriteClientLineAsync($"AUTHINFO USER {username}"); + Assert.StartsWith("381 ", await ReadClientLineAsync(), StringComparison.Ordinal); + await WriteClientLineAsync($"AUTHINFO PASS {password}"); + Assert.StartsWith("281 ", await ReadClientLineAsync(), StringComparison.Ordinal); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("20", line, StringComparison.Ordinal); + } + + public Task CompleteClientInputAsync(Exception? error = null) => + _clientToServer.Writer.CompleteAsync(error).AsTask(); + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private static MemoryNntpUserRecordStore Users(params NntpUserRecord[] records) + { + var store = new MemoryNntpUserRecordStore(); + foreach (var record in records) + { + store.Add(record); + } + + return store; + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection( + PipeReader input, + PipeWriter output, + ConnectionClientIdentity identity, + bool isTls) + { + Input = input; + Output = output; + ClientIdentity = identity; + IsTls = isTls; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls { get; } + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} + +internal sealed class RecordingLoggerFactory : ILoggerFactory +{ + public ConcurrentBag Messages { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) => new RecordingLogger(Messages); + + public void Dispose() + { + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpUsersAccountArtTypeMigrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpUsersAccountArtTypeMigrationTests.cs new file mode 100644 index 00000000..d1589d25 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/NntpUsersAccountArtTypeMigrationTests.cs @@ -0,0 +1,98 @@ +using System.Globalization; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +[Collection("NntpDbPostFilter")] +public sealed class NntpUsersAccountArtTypeMigrationTests +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + + public NntpUsersAccountArtTypeMigrationTests(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [Fact] + public void OfficialMigrationScript_IsAdditiveAndIdempotent() + { + var path = NntpUsersArtTypeMigrationScript.FindPath(); + Assert.EndsWith( + Path.Join("docs", "schema", "nntpusers-account-art-type.sql"), + path, + StringComparison.OrdinalIgnoreCase); + var sql = File.ReadAllText(path); + Assert.Contains("ALTER TABLE nntpusers", sql, StringComparison.Ordinal); + Assert.Contains("ADD COLUMN account_art_type", sql, StringComparison.Ordinal); + Assert.Contains("INT UNSIGNED NOT NULL DEFAULT 65535", sql, StringComparison.Ordinal); + Assert.Contains("information_schema.columns", sql, StringComparison.Ordinal); + Assert.Contains("DELIMITER $$", sql, StringComparison.Ordinal); + Assert.DoesNotContain("DROP TABLE", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("CREATE TABLE nntpusers", sql, StringComparison.OrdinalIgnoreCase); + Assert.Contains("DEFAULT 65535", NntpUserQueries.AddAccountArtTypeColumn, StringComparison.Ordinal); + Assert.DoesNotContain("IF NOT EXISTS", NntpUserQueries.AddAccountArtTypeColumn, StringComparison.Ordinal); + Assert.Equal(65535u, ArticleTypeCapabilities.AllValue); + var statements = NntpUsersArtTypeMigrationScript.ReadStatements(); + Assert.Equal(4, statements.Count); + Assert.Equal("DROP PROCEDURE IF EXISTS nntpusers_add_account_art_type", statements[0]); + Assert.StartsWith("CREATE PROCEDURE nntpusers_add_account_art_type()", statements[1], StringComparison.Ordinal); + Assert.Equal("CALL nntpusers_add_account_art_type()", statements[2]); + Assert.Equal("DROP PROCEDURE nntpusers_add_account_art_type", statements[3]); + } + + [NntpDbIntegrationFact] + public async Task OfficialMigration_IsSafeToApplyTwice_AndDefaultsExistingRows() + { + RequireReady(); + await _fixture.ExecuteRawAsync( + """ + CREATE TABLE IF NOT EXISTS nntpusers ( + account_name CHAR(32) NOT NULL PRIMARY KEY + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + """); + await _fixture.ExecuteRawAsync( + "INSERT IGNORE INTO nntpusers (account_name) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')"); + foreach (var statement in NntpUsersArtTypeMigrationScript.ReadStatements()) + { + await _fixture.ExecuteRawAsync(statement); + } + + foreach (var statement in NntpUsersArtTypeMigrationScript.ReadStatements()) + { + await _fixture.ExecuteRawAsync(statement); + } + + var columnType = await _fixture.ExecuteRawScalarAsync( + """ + SELECT COLUMN_TYPE FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'nntpusers' + AND column_name = 'account_art_type' + """); + Assert.Equal("int unsigned", Convert.ToString(columnType, CultureInfo.InvariantCulture)); + + var defaultValue = await _fixture.ExecuteRawScalarAsync( + "SELECT account_art_type FROM nntpusers WHERE account_name = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'"); + Assert.Equal( + ArticleTypeCapabilities.AllValue, + Convert.ToUInt32(defaultValue, CultureInfo.InvariantCulture)); + } + + private void RequireReady() + { + if (_fixture.IsConfigured) + { + return; + } + + if (NntpDbIntegration.TryGetConnectionString() is null) + { + return; + } + + Assert.Fail(_fixture.SkipReason ?? "PostFilter MySQL fixture is not configured."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramAntiEnumerationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramAntiEnumerationTests.cs new file mode 100644 index 00000000..467534cc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramAntiEnumerationTests.cs @@ -0,0 +1,388 @@ +using System.Text; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Tests.Session; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class ScramAntiEnumerationTests +{ + private const string ClientNonce = "fyko+d2lbbFgONRv9qkxdawL"; + private const string RealPassword = "pencil"; + + [Fact] + public async Task UnknownAccount_Receives383_Then481_WithoutIdentity() + { + var dummy = CreateInjectedDummy(); + await using var harness = await AuthHarness.CreateAsync( + new MemoryNntpUserRecordStore(), + dummyScram: dummy); + await AssertDummyPathAsync(harness, "definitely-missing-user", dummy); + } + + [Fact] + public async Task DisabledAccount_FollowsUnknownAccountPath() + { + var dummy = CreateInjectedDummy(); + var real = CreateRealRecord("disabled-user", enabled: true); + var disabled = MemoryNntpUserRecordStore.Create( + "disabled-user", + RealPassword, + enabled: false, + scramSalt: real.ScramSalt, + scramIterations: real.ScramIterations, + scramStoredKey: real.ScramStoredKey, + scramServerKey: real.ScramServerKey); + await using var harness = await AuthHarness.CreateAsync(Users(disabled), dummyScram: dummy); + var serverFirst = await AssertDummyPathAsync(harness, "disabled-user", dummy); + Assert.DoesNotContain(Convert.ToBase64String(disabled.ScramSalt.ToArray()), serverFirst, StringComparison.Ordinal); + } + + [Fact] + public async Task ScramDisabledAccount_FollowsUnknownAccountPath() + { + var dummy = CreateInjectedDummy(); + var real = CreateRealRecord("noscram-user"); + var denied = MemoryNntpUserRecordStore.Create( + "noscram-user", + RealPassword, + allowScram: false, + scramSalt: real.ScramSalt, + scramIterations: real.ScramIterations, + scramStoredKey: real.ScramStoredKey, + scramServerKey: real.ScramServerKey); + await using var harness = await AuthHarness.CreateAsync(Users(denied), dummyScram: dummy); + var serverFirst = await AssertDummyPathAsync(harness, "noscram-user", dummy); + Assert.DoesNotContain(Convert.ToBase64String(denied.ScramSalt.ToArray()), serverFirst, StringComparison.Ordinal); + } + + [Fact] + public async Task MissingScramMaterial_FollowsUnknownAccountPath() + { + var dummy = CreateInjectedDummy(); + var missing = MemoryNntpUserRecordStore.Create("bare-user", RealPassword, allowScram: true); + Assert.False(missing.HasScramMaterial); + await using var harness = await AuthHarness.CreateAsync(Users(missing), dummyScram: dummy); + await AssertDummyPathAsync(harness, "bare-user", dummy); + } + + [Fact] + public async Task RealValidScramAccount_StillSucceeds() + { + var dummy = CreateInjectedDummy(); + var record = CreateRealRecord("user"); + var keys = ScramClient.Derive(RealPassword, record.ScramSalt.ToArray(), record.ScramIterations); + await using var harness = await AuthHarness.CreateAsync(Users(record), dummyScram: dummy); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var clientFirst = ClientFirst("user"); + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var serverFirstLine = await harness.ReadClientLineAsync(); + var serverFirst = DecodeChallenge(serverFirstLine); + Assert.Contains(Convert.ToBase64String(record.ScramSalt.ToArray()), serverFirst, StringComparison.Ordinal); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + var success = await harness.ReadClientLineAsync(); + Assert.StartsWith("283 ", success, StringComparison.Ordinal); + Assert.Equal("user", session.Authentication.Username); + Assert.True(session.Authentication.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedReader); + Assert.True(session.Authorization.PostingPermitted); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task RealAccountWrongPassword_Uses383Then481() + { + var dummy = CreateInjectedDummy(); + var record = CreateRealRecord("user"); + var wrong = ScramClient.Derive("wrong-password", record.ScramSalt.ToArray(), record.ScramIterations); + await using var harness = await AuthHarness.CreateAsync(Users(record), dummyScram: dummy); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var clientFirst = ClientFirst("user"); + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var serverFirstLine = await harness.ReadClientLineAsync(); + AssertServerFirstShape(serverFirstLine, ClientNonce); + var serverFirst = DecodeChallenge(serverFirstLine); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, wrong.ClientKey, wrong.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ObservableServerFirst_HasSameShape_AcrossNonAuthenticatingPaths() + { + var dummy = CreateInjectedDummy(); + var real = CreateRealRecord("real-user"); + var disabled = MemoryNntpUserRecordStore.Create( + "disabled-user", + RealPassword, + enabled: false, + scramSalt: real.ScramSalt, + scramIterations: real.ScramIterations, + scramStoredKey: real.ScramStoredKey, + scramServerKey: real.ScramServerKey); + var denied = MemoryNntpUserRecordStore.Create( + "noscram-user", + RealPassword, + allowScram: false, + scramSalt: real.ScramSalt, + scramIterations: real.ScramIterations, + scramStoredKey: real.ScramStoredKey, + scramServerKey: real.ScramServerKey); + var store = Users(real, disabled, denied); + await using var harness = await AuthHarness.CreateAsync(store, dummyScram: dummy); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + foreach (var clientFirst in new[] { "missing-user", "disabled-user", "noscram-user", "real-user" }.Select(ClientFirst)) + { + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var line = await harness.ReadClientLineAsync(); + AssertServerFirstShape(line, ClientNonce); + await harness.WriteClientLineAsync("*"); + Assert.StartsWith("481 ", await harness.ReadClientLineAsync(), StringComparison.Ordinal); + } + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DummyVerifierProof_CannotAuthenticate() + { + var dummy = CreateInjectedDummy(); + await using var harness = await AuthHarness.CreateAsync( + Users(CreateRealRecord("real-user")), + dummyScram: dummy); + await AssertDummyPathAsync(harness, "missing-user", dummy, completeWithDummyKeys: true); + } + + [Fact] + public async Task DummyFailure_DoesNotGrantPrivilegesOrUsername() + { + var dummy = CreateInjectedDummy(); + await using var harness = await AuthHarness.CreateAsync(new MemoryNntpUserRecordStore(), dummyScram: dummy); + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var clientFirst = ClientFirst("ghost"); + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var serverFirst = DecodeChallenge(await harness.ReadClientLineAsync()); + var dummyKeys = KeysFromDummy(dummy); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, dummyKeys.ClientKey, dummyKeys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.ControlCancelPermitted); + Assert.False(session.HasSaslExchange); + Assert.Null(session.AccountPolicy); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DummyPath_DoesNotLogAccountStateOrSecrets() + { + var dummy = CreateInjectedDummy(); + var recording = new RecordingLoggerFactory(); + await using var harness = await AuthHarness.CreateAsync(new MemoryNntpUserRecordStore(), dummyScram: dummy); + var session = harness.CreateSession(loggerFactory: recording); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var clientFirst = ClientFirst("ghost"); + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var serverFirstLine = await harness.ReadClientLineAsync(); + var serverFirst = DecodeChallenge(serverFirstLine); + var dummyKeys = KeysFromDummy(dummy); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, dummyKeys.ClientKey, dummyKeys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("unknown account", joined, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("disabled account", joined, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("not provisioned", joined, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(Convert.ToBase64String(dummy.Credential.Salt.ToArray()), joined, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToHexString(dummy.Credential.StoredKey.ToArray()), joined, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToHexString(dummy.Credential.ServerKey.ToArray()), joined, StringComparison.Ordinal); + Assert.DoesNotContain(serverFirstLine[4..], joined, StringComparison.Ordinal); + Assert.DoesNotContain("dummy-secret", joined, StringComparison.Ordinal); + } + + [Fact] + public void TryGetAuthorized_RejectsDisabledDeniedAndMalformed() + { + var usable = CreateRealRecord("alice"); + Assert.True(ScramStoredCredential.TryGetAuthorized(usable, out var real)); + Assert.Equal(usable.ScramIterations, real.IterationCount); + + Assert.False(ScramStoredCredential.TryGetAuthorized(null, out _)); + var disabled = MemoryNntpUserRecordStore.Create( + "alice", + RealPassword, + enabled: false, + scramSalt: usable.ScramSalt, + scramIterations: usable.ScramIterations, + scramStoredKey: usable.ScramStoredKey, + scramServerKey: usable.ScramServerKey); + Assert.False(ScramStoredCredential.TryGetAuthorized(disabled, out _)); + + var denied = MemoryNntpUserRecordStore.Create( + "alice", + RealPassword, + allowScram: false, + scramSalt: usable.ScramSalt, + scramIterations: usable.ScramIterations, + scramStoredKey: usable.ScramStoredKey, + scramServerKey: usable.ScramServerKey); + Assert.False(ScramStoredCredential.TryGetAuthorized(denied, out _)); + + var shortKey = MemoryNntpUserRecordStore.Create( + "alice", + RealPassword, + scramSalt: usable.ScramSalt, + scramIterations: 4096, + scramStoredKey: usable.ScramStoredKey[..16], + scramServerKey: usable.ScramServerKey); + Assert.False(ScramStoredCredential.TryGetAuthorized(shortKey, out _)); + } + + private static async Task AssertDummyPathAsync( + AuthHarness harness, + string username, + ScramDummyVerifier dummy, + bool completeWithDummyKeys = true) + { + var session = harness.CreateSession(); + var run = session.RunAsync(); + await harness.ReadGreetingAsync(); + + var clientFirst = ClientFirst(username); + await harness.WriteClientLineAsync(ScramStart(clientFirst)); + var serverFirstLine = await harness.ReadClientLineAsync(); + AssertServerFirstShape(serverFirstLine, ClientNonce); + var serverFirst = DecodeChallenge(serverFirstLine); + Assert.Contains(Convert.ToBase64String(dummy.Credential.Salt.ToArray()), serverFirst, StringComparison.Ordinal); + Assert.Contains("i=4096", serverFirst, StringComparison.Ordinal); + + var keys = completeWithDummyKeys + ? KeysFromDummy(dummy) + : ScramClient.Derive("unrelated", dummy.Credential.Salt.ToArray(), ScramDummyVerifier.IterationCount); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + await harness.WriteClientLineAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFinal))); + Assert.Equal("481 Authentication failed", await harness.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + Assert.False(session.HasSaslExchange); + + await harness.WriteClientLineAsync("QUIT"); + await harness.ReadClientLineAsync(); + await run; + return serverFirst; + } + + private static void AssertServerFirstShape(string line, string clientNonce) + { + Assert.StartsWith("383 ", line, StringComparison.Ordinal); + var payload = line[4..]; + var bytes = Convert.FromBase64String(payload); + var serverFirst = Encoding.UTF8.GetString(bytes); + Assert.StartsWith("r=" + clientNonce, serverFirst, StringComparison.Ordinal); + Assert.Contains(",s=", serverFirst, StringComparison.Ordinal); + Assert.Contains(",i=", serverFirst, StringComparison.Ordinal); + var salt = Attribute(serverFirst, 's'); + Assert.True(Convert.FromBase64String(salt).Length >= 8); + Assert.True(int.TryParse(Attribute(serverFirst, 'i'), out var iterations) && iterations > 0); + var nonce = Attribute(serverFirst, 'r'); + Assert.True(nonce.Length > clientNonce.Length); + } + + private static ScramDummyVerifier CreateInjectedDummy() + { + var salt = Convert.FromHexString("5152535455565758595A5B5C5D5E5F60"); + var keys = ScramClient.Derive("dummy-secret", salt, ScramDummyVerifier.IterationCount); + return ScramDummyVerifier.Create( + new ScramStoredCredential(salt, ScramDummyVerifier.IterationCount, keys.StoredKey, keys.ServerKey)); + } + + private static NntpUserRecord CreateRealRecord(string name, bool enabled = true) + { + var salt = Convert.FromHexString("4142434445464748494A4B4C4D4E4F50"); + var keys = ScramClient.Derive(RealPassword, salt, 4096); + return MemoryNntpUserRecordStore.Create( + name, + RealPassword, + enabled: enabled, + scramSalt: salt, + scramIterations: 4096, + scramStoredKey: keys.StoredKey, + scramServerKey: keys.ServerKey); + } + + private static MemoryNntpUserRecordStore Users(params NntpUserRecord[] records) + { + var store = new MemoryNntpUserRecordStore(); + foreach (var record in records) + { + store.Add(record); + } + + return store; + } + + private static ScramClient.Keys KeysFromDummy(ScramDummyVerifier dummy) => + ScramClient.Derive("dummy-secret", dummy.Credential.Salt.ToArray(), dummy.Credential.IterationCount); + + private static string ClientFirst(string username) => $"n,,n={username},r={ClientNonce}"; + + private static string ScramStart(string clientFirst) => + $"AUTHINFO SASL SCRAM-SHA-256 {Convert.ToBase64String(Encoding.UTF8.GetBytes(clientFirst))}"; + + private static string DecodeChallenge(string line) => + Encoding.UTF8.GetString(Convert.FromBase64String(line[4..])); + + private static string Attribute(string message, char key) + { + foreach (var part in message.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (part.Length >= 2 && part[0] == key && part[1] == '=') + { + return part[2..]; + } + } + + throw new InvalidOperationException($"Missing SCRAM attribute {key}."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramMechanismTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramMechanismTests.cs new file mode 100644 index 00000000..b734e436 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Authentication/ScramMechanismTests.cs @@ -0,0 +1,143 @@ +using System.Security.Cryptography; +using System.Text; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Authentication.Sasl; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Authentication; + +public sealed class ScramMechanismTests +{ + [Fact] + public void ValidExchange_ReturnsServerSignature() + { + const string password = "pencil"; + const int iterations = 4096; + var salt = Convert.FromHexString("4142434445464748494A4B4C4D4E4F50"); + var keys = ScramClient.Derive(password, salt, iterations); + var credential = new ScramStoredCredential(salt, iterations, keys.StoredKey, keys.ServerKey); + const string clientFirst = "n,,n=user,r=fyko+d2lbbFgONRv9qkxdawL"; + var (state, serverFirst) = ScramMechanism.Begin(clientFirst, credential); + Assert.Contains("s=", serverFirst, StringComparison.Ordinal); + Assert.Contains("i=4096", serverFirst, StringComparison.Ordinal); + + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + var serverFinal = state.TryFinish(clientFinal); + Assert.NotNull(serverFinal); + Assert.StartsWith("v=", serverFinal, StringComparison.Ordinal); + Assert.Equal(serverFinal, ScramClient.ExpectedServerFinal(clientFirst, serverFirst, clientFinal, keys.ServerKey)); + } + + [Fact] + public void IncorrectProof_Fails() + { + const string password = "pencil"; + var salt = RandomNumberGenerator.GetBytes(16); + var keys = ScramClient.Derive(password, salt, 4096); + var credential = new ScramStoredCredential(salt, 4096, keys.StoredKey, keys.ServerKey); + const string clientFirst = "n,,n=user,r=abc123nonce"; + var (state, serverFirst) = ScramMechanism.Begin(clientFirst, credential); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + var proof = clientFinal[(clientFinal.LastIndexOf(",p=", StringComparison.Ordinal) + 3)..]; + var tampered = clientFinal.Replace($",p={proof}", ",p=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", StringComparison.Ordinal); + Assert.NotEqual(clientFinal, tampered); + Assert.Null(state.TryFinish(tampered)); + } + + [Fact] + public void MalformedStoredKeyLength_FailsWithoutThrowing() + { + const string password = "pencil"; + var salt = RandomNumberGenerator.GetBytes(16); + var keys = ScramClient.Derive(password, salt, 4096); + var credential = new ScramStoredCredential(salt, 4096, keys.StoredKey[..16], keys.ServerKey); + const string clientFirst = "n,,n=user,r=abc123nonce"; + var (state, serverFirst) = ScramMechanism.Begin(clientFirst, credential); + var clientFinal = ScramClient.CreateClientFinal(clientFirst, serverFirst, keys.ClientKey, keys.StoredKey); + Assert.Null(state.TryFinish(clientFinal)); + } + + [Fact] + public void MissingScramFields_AreNotUsable() + { + var record = MemoryNntpUserRecordStore.Create( + "alice", + "pw", + allowScram: true, + scramSalt: ReadOnlyMemory.Empty, + scramIterations: 65536); + Assert.False(record.HasScramMaterial); + } +} + +internal static class ScramClient +{ + public sealed record Keys(byte[] ClientKey, byte[] StoredKey, byte[] ServerKey); + + public static Keys Derive(string password, byte[] salt, int iterations) + { + var salted = Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(password), + salt, + iterations, + HashAlgorithmName.SHA256, + 32); + var clientKey = Hmac(salted, "Client Key"u8); + var storedKey = SHA256.HashData(clientKey); + var serverKey = Hmac(salted, "Server Key"u8); + return new Keys(clientKey, storedKey, serverKey); + } + + public static string CreateClientFinal(string clientFirst, string serverFirst, byte[] clientKey, byte[] storedKey) + { + var gs2 = clientFirst[..(clientFirst.IndexOf(",,", StringComparison.Ordinal) + 2)]; + var bare = clientFirst[gs2.Length..]; + var combinedNonce = Attribute(serverFirst, 'r'); + var withoutProof = $"c={Convert.ToBase64String(Encoding.ASCII.GetBytes(gs2))},r={combinedNonce}"; + var authMessage = $"{bare},{serverFirst},{withoutProof}"; + var signature = Hmac(storedKey, Encoding.UTF8.GetBytes(authMessage)); + var proof = Xor(clientKey, signature); + return $"{withoutProof},p={Convert.ToBase64String(proof)}"; + } + + public static string ExpectedServerFinal(string clientFirst, string serverFirst, string clientFinal, byte[] serverKey) + { + var gs2 = clientFirst[..(clientFirst.IndexOf(",,", StringComparison.Ordinal) + 2)]; + var bare = clientFirst[gs2.Length..]; + var proof = Attribute(clientFinal, 'p'); + var withoutProof = clientFinal.Replace($",p={proof}", string.Empty, StringComparison.Ordinal); + var authMessage = $"{bare},{serverFirst},{withoutProof}"; + return $"v={Convert.ToBase64String(Hmac(serverKey, Encoding.UTF8.GetBytes(authMessage)))}"; + } + + private static string Attribute(string message, char key) + { + foreach (var part in message.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (part.Length >= 2 && part[0] == key && part[1] == '=') + { + return part[2..]; + } + } + + throw new InvalidOperationException($"Missing SCRAM attribute {key}."); + } + + private static byte[] Hmac(byte[] key, ReadOnlySpan data) + { + using var hmac = IncrementalHash.CreateHMAC(HashAlgorithmName.SHA256, key); + hmac.AppendData(data); + return hmac.GetHashAndReset(); + } + + private static byte[] Xor(byte[] left, byte[] right) + { + var result = new byte[left.Length]; + for (var i = 0; i < left.Length; i++) + { + result[i] = (byte)(left[i] ^ right[i]); + } + + return result; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs new file mode 100644 index 00000000..0810364a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs @@ -0,0 +1,691 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Client; + +public sealed class CloudflareDnsClientTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task ListRecordsAsync_ValidSinglePage_ReturnsRecords() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Equal("1", records[0].Id); + Assert.Single(handler.Requests); + } + + [Fact] + public async Task ListRecordsAsync_FollowsPagination_AndFiltersExactName() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false }, + { "id": "x", "type": "A", "name": "other.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.3", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Equal(2, records.Count); + Assert.All(records, r => Assert.Equal(Fqdn, r.Name)); + Assert.Equal(2, handler.Requests.Count); + Assert.Contains("page=2", handler.Requests[1].RequestUri!.Query, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, handler.Requests[0].RequestUri!.ToString()); + } + + [Fact] + public async Task ListRecordsAsync_MissingResultInfo_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": [] }"""); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.False(ex.IsOutcomeUncertain); + Assert.Contains("result_info", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ListRecordsAsync_MissingTotalPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [], + "result_info": { "page": 1, "per_page": 100, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task ListRecordsAsync_NonPositiveTotalPages_IsPermanentFailure(int totalPages) + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": {{totalPages}}, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ListRecordsAsync_TotalPagesChangesBetweenPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 3, "count": 1, "total_count": 3 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages changed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task ListRecordsAsync_PageMetadataMismatch_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [], + "result_info": { "page": 9, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("page does not match", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ListRecordsAsync_ExceedsMaxPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + var declaredTotal = CloudflareDnsClient.MaxListPages + 1; + for (var page = 1; page <= CloudflareDnsClient.MaxListPages; page++) + { + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [ { "id": "{{page}}", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.{{page}}", "ttl": 1, "proxied": false } ], + "result_info": { "page": {{page}}, "per_page": 1, "total_pages": {{declaredTotal}}, "count": 1, "total_count": {{declaredTotal}} } + } + """); + } + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("exceeded", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(CloudflareDnsClient.MaxListPages.ToString(System.Globalization.CultureInfo.InvariantCulture), ex.Message, StringComparison.Ordinal); + Assert.Equal(CloudflareDnsClient.MaxListPages, handler.Requests.Count); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ListRecordsAsync_CancellationDuringPagination_Propagates() + { + using var cts = new CancellationTokenSource(); + var handler = new ScriptedHttpMessageHandler + { + OnRequest = (_, requestIndex) => + { + if (requestIndex >= 1) + { + cts.Cancel(); + } + }, + }; + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + await Assert.ThrowsAnyAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + } + + [Fact] + public async Task ListRecordsAsync_MatchingZoneId_IsAccepted() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": "zone-test" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var records = await CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Equal(ZoneId, records[0].ZoneId); + } + + [Fact] + public async Task ListRecordsAsync_OmittedZoneId_IsAcceptedPerApiContract() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var records = await CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Null(records[0].ZoneId); + } + + [Fact] + public async Task ListRecordsAsync_MismatchedZoneId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": "other-zone" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("other-zone", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\" \"")] + public async Task ListRecordsAsync_EmptyOrWhitespaceZoneId_IsPermanentFailure(string zoneJson) + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": {{zoneJson}} } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CreateRecordAsync_MismatchedZoneId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": { + "id": "new", + "type": "A", + "name": "nntpd01.usenet.ninja", + "content": "198.18.0.1", + "ttl": 1, + "proxied": false, + "zone_id": "other-zone" + } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.1", + }, + CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Reconcile_PaginationIntegrityFailure_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.99", "ttl": 1, "proxied": false } ] }"""); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Put); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task Reconcile_ZoneMismatch_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.99", "ttl": 1, "proxied": false, "zone_id": "other-zone" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post || r.Method == HttpMethod.Put || r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task RemoveAll_PaginationIntegrityFailure_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 100, "count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task Send_AddsBearerAuthorization_WithoutExposingKeyInExceptions() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.Unauthorized, + """ + { "success": false, "errors": [ { "code": 10000, "message": "Authentication error" } ], "result": null } + """); + + var client = CreateClient(handler); + var ex = await Assert.ThrowsAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Equal(401, ex.StatusCode); + Assert.True(ex.IsPermanentFailure); + Assert.Contains("Authentication error", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + Assert.Equal("Bearer", handler.Requests[0].Headers.Authorization!.Scheme); + Assert.Equal(TestHostFactory.TestCloudFlareApiKey, handler.Requests[0].Headers.Authorization!.Parameter); + } + + [Fact] + public async Task Send_UnsuccessfulSuccessFlag_IsFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { "success": false, "errors": [ { "code": 9109, "message": "Unauthorized to access requested resource" } ], "result": [] } + """); + + var client = CreateClient(handler); + var ex = await Assert.ThrowsAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Contains("Unauthorized to access requested resource", ex.Message, StringComparison.Ordinal); + Assert.Contains(9109, ex.CloudflareErrorCodes); + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task Send_RateLimit_RetriesThenSucceeds() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(1)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Empty(records); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task CreateUpdateDelete_UseExpectedPaths() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } }"""); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } }"""); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new" } }"""); + + var client = CreateClient(handler); + var created = await client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.1", + }, + CancellationToken.None); + Assert.Equal("new", created.Id); + + await client.UpdateRecordAsync( + ZoneId, + "new", + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.2", + }, + CancellationToken.None); + + await client.DeleteRecordAsync(ZoneId, "new", CancellationToken.None); + + Assert.Equal(HttpMethod.Post, handler.Requests[0].Method); + Assert.Equal(HttpMethod.Put, handler.Requests[1].Method); + Assert.Equal(HttpMethod.Delete, handler.Requests[2].Method); + Assert.EndsWith($"/zones/{ZoneId}/dns_records", handler.Requests[0].RequestUri!.AbsolutePath, StringComparison.Ordinal); + Assert.EndsWith($"/zones/{ZoneId}/dns_records/new", handler.Requests[2].RequestUri!.AbsolutePath, StringComparison.Ordinal); + } + + [Fact] + public async Task Send_Cancellation_Propagates() + { + var handler = new ScriptedHttpMessageHandler + { + Delay = TimeSpan.FromSeconds(30), + }; + handler.EnqueueJson(HttpStatusCode.OK, """{ "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1 } }"""); + + var client = CreateClient(handler); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(20)); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + } + + [Fact] + public void RequireTotalPages_RejectsNullAndInvalidMetadata() + { + Assert.Throws(() => CloudflareDnsClient.RequireTotalPages(null, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = null }, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 0 }, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 1, Page = 2 }, expectedPage: 1)); + + Assert.Equal(3, CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 3, Page = 1 }, expectedPage: 1)); + } + + [Fact] + public void EnsureRecordZoneMatches_AppliesOmissionAndMismatchPolicy() + { + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = null }, + ZoneId, + "List"); + + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = ZoneId }, + ZoneId, + "List"); + + var mismatch = Assert.Throws(() => + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = "other" }, + ZoneId, + "List")); + Assert.True(mismatch.IsPermanentFailure); + + var empty = Assert.Throws(() => + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = " " }, + ZoneId, + "List")); + Assert.True(empty.IsPermanentFailure); + } + + private static CloudflareDnsClient CreateClient(ScriptedHttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + }; + + var options = Options.Create(TestHostFactory.CreateValidOptions()); + return new CloudflareDnsClient(httpClient, options, NullLogger.Instance); + } + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public TimeSpan Delay { get; set; } + + public Action? OnRequest { get; set; } + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri); + if (request.Headers.Authorization is { } auth) + { + copy.Headers.Authorization = auth; + } + + Requests.Add(copy); + OnRequest?.Invoke(request, Requests.Count - 1); + + if (Delay > TimeSpan.Zero) + { + await Task.Delay(Delay, cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return _responses.Dequeue(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs new file mode 100644 index 00000000..9eb89650 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs @@ -0,0 +1,340 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Client; + +/// +/// Cancellation, shared operation budget, and per-request HTTP timeout regression coverage. +/// +public sealed class CloudflareDnsDeadlineTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task CallerCancellation_BeforeRequest_DoesNotStartHttpCall() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson(HttpStatusCode.OK, EmptyListJson()); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task RateLimit_CancelDuringDelay_PreventsSubsequentRequest() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + handler.EnqueueJson(HttpStatusCode.OK, EmptyListJson()); + + var client = CreateClient(handler); + using var cts = new CancellationTokenSource(); + client.DelayAsync = (_, ct) => + { + cts.Cancel(); + return Task.Delay(TimeSpan.FromSeconds(60), ct); + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMinutes(2), cts.Token, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + + Assert.Single(handler.Requests); + } + + [Fact] + public void CreateRequestTimeoutCts_CapsByRemainingBudget() + { + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(150), CancellationToken.None, out var opToken); + using var cts = CloudflareDnsClient.CreateRequestTimeoutCts(opToken, out var sendToken); + + Assert.False(sendToken.IsCancellationRequested); + Assert.True(SpinWait.SpinUntil(() => sendToken.IsCancellationRequested, TimeSpan.FromSeconds(2))); + } + + [Fact] + public void CreateRequestTimeoutCts_ThrowsWhenBudgetAlreadyExpired() + { + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(1), CancellationToken.None, out var opToken); + Assert.True(SpinWait.SpinUntil(() => budget.Remaining <= TimeSpan.Zero, TimeSpan.FromSeconds(2))); + + Assert.ThrowsAny(() => + CloudflareDnsClient.CreateRequestTimeoutCts(opToken, out _)); + } + + [Fact] + public async Task List_RequestTimeout_IsNotUncertainMutation() + { + var handler = new StallingHttpMessageHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = Timeout.InfiniteTimeSpan, + }; + var client = new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(40), CancellationToken.None, out var opToken); + + // Budget expires → OCE (operation cancel), not CloudflareDnsException uncertain. + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + } + + [Fact] + public async Task Mutation_PerRequestTimeout_WithoutOperationCancel_IsUncertain() + { + var handler = new StallingHttpMessageHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + // Legacy HttpClient timeout still used by some tests; client also CancelAfter(PerRequestTimeout). + Timeout = TimeSpan.FromMilliseconds(40), + }; + var client = new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }, + CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.False(ex.IsPermanentFailure); + } + + [Fact] + public async Task Reconciler_DoesNotResetDeadlineAcrossAttempts() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("transient") { FailedOperation = "List" }, + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMilliseconds(80); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance) + { + DelayAsync = async (_, ct) => + { + await Task.Delay(TimeSpan.FromMilliseconds(5), ct).ConfigureAwait(false); + }, + }; + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.True(client.ListCallCount >= 1); + } + + [Fact] + public async Task Reconcile_CancelDuringVerification_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + + using var cts = new CancellationTokenSource(); + var listCount = 0; + client.TransformListedRecords = (type, records, callCount) => + { + listCount = callCount; + // After planning lists (2) and creates (0), verification lists start — cancel then. + if (callCount >= 3) + { + cts.Cancel(); + } + + return records; + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMinutes(2); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + + Assert.True(listCount >= 3); + } + + [Fact] + public async Task FailedStartCleanup_UsesDedicatedBudget_DoesNotMaskOriginal() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("original-startup-failure") + { + IsPermanentFailure = true, + FailedOperation = "List", + }, + TransformListedAllRecords = (_, callCount) => + { + if (callCount > 1) + { + throw new CloudflareDnsException("cleanup-still-failing") + { + IsPermanentFailure = true, + FailedOperation = "ListAll", + }; + } + + return []; + }, + }; + + var options = TestHostFactory.CreateValidOptions(); + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance), + new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Equal("original-startup-failure", ex.Message); + Assert.Equal(TimeSpan.FromSeconds(15), CloudflareDnsReconciliationService.FailedStartCleanupTimeout); + } + + [Fact] + public async Task NormalCleanup_HonorsCallerToken() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + return Task.CompletedTask; + }; + + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + await Assert.ThrowsAnyAsync(() => + reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token)); + } + + private static CloudflareDnsClient CreateClient(HttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = Timeout.InfiniteTimeSpan, + }; + return new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + } + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static string EmptyListJson() => + """{ "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } }"""; + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(new HttpRequestMessage(request.Method, request.RequestUri)); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } + + private sealed class StallingHttpMessageHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(HttpStatusCode.OK); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs new file mode 100644 index 00000000..41f3f814 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs @@ -0,0 +1,394 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Client; + +/// +/// Hardening coverage: managed TTL policy budgets, 429/budget interaction, record validation, sanitized errors. +/// +public sealed class CloudflareDnsHardeningTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task RateLimit_RetryAfterExceedsBudget_CancelsWithoutSleepingPastDeadline() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMinutes(2)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + + var client = CreateClient(handler); + var delays = new List(); + client.DelayAsync = (delay, _) => + { + delays.Add(delay); + return Task.CompletedTask; + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromSeconds(5), CancellationToken.None, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + + Assert.Empty(delays); + } + + [Fact] + public async Task RateLimit_CallerCancellation_TakesPrecedence() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + + var client = CreateClient(handler); + using var callerCts = new CancellationTokenSource(); + client.DelayAsync = (_, ct) => + { + callerCts.Cancel(); + return Task.Delay(TimeSpan.FromSeconds(30), ct); + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMinutes(2), callerCts.Token, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + } + + [Fact] + public async Task Reconciler_SharedDeadline_CancelsBackoffWithoutSuccess() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("transient list failure") { FailedOperation = "List" }, + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMilliseconds(50); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance) + { + DelayAsync = (_, ct) => Task.Delay(TimeSpan.FromSeconds(30), ct), + }; + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + } + + [Fact] + public async Task FailureResponse_DoesNotEmbedRawBody() + { + const string marker = "RAW_BODY_SECRET_MARKER_SHOULD_NOT_APPEAR"; + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.BadRequest, + $$"""{ "success": false, "errors": [], "result": null, "extra": "{{marker}}" }"""); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Equal(400, ex.StatusCode); + Assert.DoesNotContain(marker, ex.Message, StringComparison.Ordinal); + Assert.Contains("no Cloudflare error details returned", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task List_MissingSuccess_IsFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure || ex.StatusCode == 200); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task List_NonBooleanSuccess_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": 1, + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task List_MissingRecordId_IsPermanentFailure_NoMutations() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + } + + [Fact] + public async Task List_MissingTtl_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("ttl", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task List_MissingProxied_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300 } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("proxied", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task List_MalformedProxied_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300, "proxied": "yes" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task ListAll_MissingId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "type": "TXT", "name": "nntpd01.usenet.ninja", "content": "x" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListAllRecordsForNameAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task Reconcile_CreateUsesManagedTtl() + { + var client = new FakeCloudflareDnsClient(); + await new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + var created = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, created.Ttl); + Assert.Equal(false, created.Proxied); + Assert.Equal(1, client.CreateCallCount); + } + + [Fact] + public async Task FailedStartCleanup_Timeout_DoesNotClaimSuccess_PreservesOriginal() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + client.OnMutate = async ct => + { + await Task.Delay(TimeSpan.FromSeconds(60), ct).ConfigureAwait(false); + }; + + var options = TestHostFactory.CreateValidOptions(); + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + + // Force reconcile to fail after ownership path by making list fail after first mutation attempt — + // simpler: inject permanent list failure then ensure cleanup uses 15s budget. + client.ListException = new CloudflareDnsException("boom") + { + IsPermanentFailure = true, + FailedOperation = "List", + }; + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Equal("boom", ex.Message); + Assert.Contains(client.Snapshot(), r => r.Id == "a1"); + } + + [Fact] + public void ManagedDnsPolicy_ConstantIs300() => + Assert.Equal(300, CloudflareManagedDnsPolicy.ManagedTtl); + + private static CloudflareDnsClient CreateClient(ScriptedHttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + }; + return new CloudflareDnsClient( + httpClient, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + } + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri) + { + Content = request.Content, + }; + Requests.Add(copy); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs new file mode 100644 index 00000000..6df88f4b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs @@ -0,0 +1,655 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Lifecycle; + +/// +/// Shutdown cleanup: remove every DNS record for the exact FQDN (all types), with lifecycle coverage. +/// +public sealed class CloudflareDnsCleanupTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task RemoveAll_DeletesAAndAaaa() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("aaaa", "AAAA", "2001:db8::10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RemoveAll_DeletesAllRecordTypesForExactFqdn() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("aaaa", "AAAA", "2001:db8::10"), + Record("cname", "CNAME", "target.example"), + Record("txt", "TXT", "v=spf1"), + Record("mx", "MX", "10 mail.example"), + Record("srv", "SRV", "1 1 443 target"), + Record("caa", "CAA", "0 issue \"letsencrypt.org\""), + Record("https", "HTTPS", "1 . alpn=h2"), + Record("svcb", "SVCB", "1 . alpn=h2")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + Assert.True(client.SuccessfulDeleteCount >= 9); + } + + [Fact] + public async Task RemoveAll_DeletesDuplicates() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("txt1", "TXT", "one"), + Record("txt2", "TXT", "two")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + Assert.Equal(4, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task ListAll_FollowsPagination_InHttpClient() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "errors": [], + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "proxied": false, "ttl": 1 } ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "errors": [], + "result": [ { "id": "2", "type": "TXT", "name": "nntpd01.usenet.ninja", "content": "x", "proxied": false, "ttl": 1 } ], + "result_info": { "page": 2, "per_page": 100, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + var client = CreateHttpClient(handler); + var records = await client.ListAllRecordsForNameAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Equal(2, records.Count); + Assert.Contains(records, r => r.Type == "TXT"); + Assert.Equal(2, handler.Requests.Count); + Assert.DoesNotContain("type=", handler.Requests[0].RequestUri!.Query, StringComparison.Ordinal); + } + + [Fact] + public async Task RemoveAll_PreservesParentChildAndSimilarSuffixHostnames() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("ours", "A", "198.18.0.10"), + new CloudflareDnsRecord { Id = "parent", Type = "A", Name = "usenet.ninja", Content = "198.18.0.1" }, + new CloudflareDnsRecord { Id = "child", Type = "A", Name = "other.nntpd01.usenet.ninja", Content = "198.18.0.2" }, + new CloudflareDnsRecord { Id = "suffix", Type = "A", Name = "xnntpd01.usenet.ninja", Content = "198.18.0.3" }, + new CloudflareDnsRecord { Id = "peer", Type = "TXT", Name = "nntpd02.usenet.ninja", Content = "keep" }); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + var snap = client.Snapshot(); + Assert.DoesNotContain(snap, r => r.Id == "ours"); + Assert.Contains(snap, r => r.Id == "parent"); + Assert.Contains(snap, r => r.Id == "child"); + Assert.Contains(snap, r => r.Id == "suffix"); + Assert.Contains(snap, r => r.Id == "peer"); + } + + [Fact] + public async Task RemoveAll_PartialDelete_ThenRetry_Converges() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("transient delete") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + }; + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("txt", "TXT", "hello")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RemoveAll_TimeoutAfterDeleteApplied_RetryVerifiesGone() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("timeout after delete") + { + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + ApplyDeleteThenFailUncertain = true, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Empty(client.Snapshot()); + Assert.True(client.ListAllCallCount >= 2); + } + + [Fact] + public async Task RemoveAll_TimeoutWhenDeleteNotApplied_RetryDeletes() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("timeout no apply") + { + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + ApplyDeleteThenFailUncertain = false, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Empty(client.Snapshot()); + Assert.True(client.SuccessfulDeleteCount >= 1); + } + + [Fact] + public async Task RemoveAll_Retry_ReReadsActualState() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("fail once") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + }; + client.Seed(Record("txt", "TXT", "x")); + + var listsBefore = 0; + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + Assert.True(client.ListAllCallCount > 2); + _ = listsBefore; + } + + [Fact] + public async Task RemoveAll_PermanentFailure_DoesNotRetryPointlessly() + { + var client = new FakeCloudflareDnsClient + { + ListAllException = new CloudflareDnsException("auth") + { + StatusCode = 401, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Equal(1, client.ListAllCallCount); + Assert.Equal(0, client.DeleteCallCount); + } + + [Fact] + public async Task RemoveAll_VerificationMismatch_FailsClosed() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10")); + client.TransformListedAllRecords = (records, callCount) => + { + // After deletes, verification lists still show a record. + if (client.SuccessfulDeleteCount > 0 && callCount > 1) + { + return + [ + Record("ghost", "TXT", "still-here"), + ]; + } + + return records; + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("cleanup completed", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task RemoveAll_RetryExhaustion_WithRemainingRecords_FailsClosed() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("always fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("txt", "TXT", "remain")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.NotEmpty(client.Snapshot()); + } + + [Fact] + public async Task RemoveAll_CancellationDuringDelete_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10"), Record("txt", "TXT", "x")); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + return Task.CompletedTask; + }; + + await Assert.ThrowsAnyAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token)); + } + + [Fact] + public async Task RemoveAll_CancellationDuringBackoff_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + using var cts = new CancellationTokenSource(); + var failedOnce = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.OnMutate = _ => + { + failedOnce.TrySetResult(); + return Task.CompletedTask; + }; + + var task = CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token); + await failedOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => task); + } + + [Fact] + public async Task RemoveAll_AlreadyAbsent_IsIdempotentSuccess() + { + var client = new FakeCloudflareDnsClient(); + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + Assert.Equal(0, client.DeleteCallCount); + Assert.True(client.ListAllCallCount >= 1); + } + + [Fact] + public async Task AmbiguousRecordName_FailsSafelyWithoutDeletingOthers() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("good", "A", "198.18.0.10")); + client.TransformListedAllRecords = (records, _) => + records.Append(new CloudflareDnsRecord + { + Id = "bad", + Type = "TXT", + Name = "nntpd01..usenet.ninja", + Content = "ambiguous", + }).ToArray(); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("ambiguous", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task Lifecycle_Stop_RemovesAllRecordsForExactFqdn() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + options.GracefulShutdownTimeout = TimeSpan.FromSeconds(30); + + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("txt", "TXT", "preexisting"), + new CloudflareDnsRecord { Id = "child", Type = "A", Name = "other." + Fqdn, Content = "198.18.0.9" }); + + var dns = CreateService(options, client); + var other = new FakeApplicationService("listener"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), other], options); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Contains(client.Snapshot(), r => r.Type == "A"); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + Assert.Contains(client.Snapshot(), r => r.Id == "child"); + Assert.Equal(1, other.StopCount); + } + + [Fact] + public async Task Lifecycle_StopOrder_DnsCleanupRunsAfterOtherServices() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + var order = new List(); + + var dns = CreateService(options, client); + var trackingDns = new OrderedStopService(TestHostFactory.WrapDns(dns), order, "dns"); + var other = new FakeApplicationService( + "listener", + onStop: _ => + { + order.Add("listener"); + return Task.CompletedTask; + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([trackingDns, other], options); + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(["listener", "dns"], order); + } + + [Fact] + public async Task Lifecycle_PartialStartup_DnsRollbackCleansFqdn() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + + var dns = CreateService(options, client); + var failing = new FakeApplicationService( + "failing", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), failing], options); + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + } + + [Fact] + public async Task StartFailure_DuringReconcile_AttemptsCleanup_DoesNotClaimSuccessOnCleanupFail() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("aaaa fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + // Cleanup delete also fails — must not report cleanup success; original start failure surfaces. + DeleteException = new CloudflareDnsException("cleanup fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + var service = CreateService(options, client); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + // Ownership was never marked active; Stop should no-op. + await service.StopAsync(CancellationToken.None); + Assert.True(client.ListAllCallCount >= 1); + } + + [Fact] + public async Task Stop_WhenOwnershipInactive_DoesNotClaimCleanup() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10")); + var service = CreateService(options, client); + + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, client.DeleteCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task ReconcileAndCleanup_SameInstance_AreSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var max = 0; + var concurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref max, Math.Max(Volatile.Read(ref max), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + var reconcile = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + + var cleanup = reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + await Task.Delay(30); + Assert.Equal(1, Volatile.Read(ref entered)); + + gate.TrySetResult(); + await Task.WhenAll(reconcile, cleanup); + Assert.Equal(1, max); + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RepeatedStop_IsSafe_AfterCleanup() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + var service = CreateService(options, client); + + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + } + + [Fact] + public async Task ShutdownDeadline_PropagatesCancellation_WithoutClaimingRemoval() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + options.GracefulShutdownTimeout = TimeSpan.FromMilliseconds(50); + + var client = new FakeCloudflareDnsClient(); + var dns = CreateService(options, client); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns)], options); + await lifecycle.StartAsync(CancellationToken.None); + + client.OnMutate = async ct => + { + await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); + }; + + // Stop should surface timeout/cancel from manager budget while cleanup is in progress. + await Assert.ThrowsAnyAsync(() => lifecycle.StopAsync(CancellationToken.None)); + } + + [Fact] + public void TryClassifyExactFqdn_RejectsAmbiguousAndNonExact() + { + Assert.True(CloudflareDnsReconciler.TryClassifyExactFqdn("NNTPD01.Usenet.Ninja.", "nntpd01.usenet.ninja", out var exact)); + Assert.True(exact); + + Assert.True(CloudflareDnsReconciler.TryClassifyExactFqdn("other.nntpd01.usenet.ninja", "nntpd01.usenet.ninja", out exact)); + Assert.False(exact); + + Assert.False(CloudflareDnsReconciler.TryClassifyExactFqdn("nntpd01..usenet.ninja", "nntpd01.usenet.ninja", out _)); + Assert.False(CloudflareDnsReconciler.TryClassifyExactFqdn(" ", "nntpd01.usenet.ninja", out _)); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static CloudflareDnsReconciliationService CreateService(NntpdOptions options, ICloudflareDnsClient client) + { + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + CreateReconciler(client), + NullLogger.Instance); + } + + private static CloudflareDnsClient CreateHttpClient(HttpMessageHandler handler) + { + var options = TestHostFactory.CreateValidOptions(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) { BaseAddress = new Uri("https://api.cloudflare.com/client/v4/") }; + return new CloudflareDnsClient( + http, + Options.Create(options), + NullLogger.Instance); + } + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static bool NamesEqual(string left, string right) => + string.Equals( + left.Trim().TrimEnd('.'), + right.Trim().TrimEnd('.'), + StringComparison.OrdinalIgnoreCase); + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } + + private sealed class OrderedStopService : IApplicationService + { + private readonly IApplicationService _inner; + private readonly List _order; + private readonly string _label; + + public OrderedStopService(IApplicationService inner, List order, string label) + { + _inner = inner; + _order = order; + _label = label; + } + + public string Name => _inner.Name; + public Task? Execution => _inner.Execution; + public Task StartAsync(CancellationToken cancellationToken) => _inner.StartAsync(cancellationToken); + + public async Task StopAsync(CancellationToken cancellationToken) + { + await _inner.StopAsync(cancellationToken).ConfigureAwait(false); + _order.Add(_label); + } + } + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + _responses.Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri); + Requests.Add(copy); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs new file mode 100644 index 00000000..29eea6fd --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs @@ -0,0 +1,254 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.ReaderService.Acme; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Lifecycle; + +[Collection(SerilogCollection.Name)] +public sealed class CloudflareDnsReconciliationServiceTests +{ + [Fact] + public async Task StartAsync_WithNoEligibleAddresses_Fails() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(IPAddress.Loopback), + new FakeCloudflareDnsClient()); + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Contains("No eligible IP addresses", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StartAsync_WhenReconcileFails_PropagatesFailure() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("boom"), + }; + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task Lifecycle_DnsReconciliationCompletesBeforeRunning() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient(); + var dnsService = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + client); + + var order = new List(); + var other = new FakeApplicationService("other") + { + SharedStartOrder = order, + }; + + var trackingDns = new TrackingApplicationService(TestHostFactory.WrapDns(dnsService), order, "dns"); + + await using var lifecycle = TestHostFactory.CreateLifecycle([trackingDns, other], options); + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Equal(["dns", "other"], order); + Assert.True(client.CreateCallCount >= 2); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Lifecycle_DnsFailure_PreventsRunning() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("list failed"), + }; + + var dnsService = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + var other = new FakeApplicationService("other"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dnsService), other], options); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(0, other.StartCount); + } + + [Fact] + public async Task Host_RegistersDnsReconciliation_BeforeOtherServices() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + var started = new List(); + builder.Services.AddSingleton( + new FakeApplicationService( + "app", + onStart: _ => + { + started.Add("app"); + return Task.CompletedTask; + })); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + // Shared configuration is published before Cloudflare so the zone id and FQDN exist. + Assert.Equal(typeof(NntpDbService), services[0].GetType()); + Assert.Equal(typeof(VectorNNTPD.ReaderService.NntpDb.NntpSharedConfigurationService), services[1].GetType()); + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.ReaderService.NntpDb.NntpSharedConfigurationService)); + AssertRegisteredBefore( + services, + typeof(VectorNNTPD.ReaderService.NntpDb.NntpSharedConfigurationService), + typeof(CloudflareDnsReconciliationApplicationService)); + Assert.Equal( + typeof(CloudflareDnsReconciliationService).Assembly, + services.Single(static service => service.GetType() == typeof(CloudflareDnsReconciliationApplicationService)).GetType().Assembly); + + AssertRegisteredBefore(services, typeof(CloudflareDnsReconciliationApplicationService), typeof(VectorNNTPD.ReaderService.Redis.RedisService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.Redis.RedisService), typeof(VectorNNTPD.Common.Messaging.RabbitMq.RabbitMqService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.Common.Messaging.RabbitMq.RabbitMqService), typeof(VectorNNTPD.ReaderService.RabbitMq.RabbitMqTopologyService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.RabbitMq.RabbitMqTopologyService), typeof(VectorNNTPD.ReaderService.RabbitMq.ArticleWork.ArticleWorkRpcService)); + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.ReaderService.Newsgroups.NewsgroupCatalogueService)); + // Reader role: no Transit peer/DNS services; SessionState lease renewal follows ninpaths. + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.Newsgroups.NewsgroupCatalogueService), typeof(VectorNNTPD.ReaderService.Moderation.ModeratorCatalogueService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.Moderation.ModeratorCatalogueService), typeof(VectorNNTPD.ReaderService.PostFilter.PostFilterPolicyService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.PostFilter.PostFilterPolicyService), typeof(VectorNNTPD.ReaderService.PostFilter.PostFilterRejectionEvidenceService)); + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.ReaderService.PostFilter.PostFilterRejectionEvidenceService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.PostFilter.PostFilterRejectionEvidenceService), typeof(VectorNNTPD.ReaderService.History.HistoryWriteService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.History.HistoryWriteService), typeof(VectorNNTPD.ReaderService.History.HistoryMaintenanceService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.History.HistoryMaintenanceService), typeof(IncomingSpoolWriterService)); + AssertRegisteredBefore(services, typeof(IncomingSpoolWriterService), typeof(VectorNNTPD.ReaderService.Email.EmailDeliveryService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.Email.EmailDeliveryService), typeof(VectorNNTPD.ReaderService.Ninpaths.NinpathsProcessingService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.ReaderService.Ninpaths.NinpathsProcessingService), typeof(SessionStateService)); + AssertRegisteredBefore(services, typeof(SessionStateService), typeof(NntpPlainListenerService)); + AssertRegisteredBefore(services, typeof(NntpPlainListenerService), typeof(AcmeCertificateApplicationService)); + AssertRegisteredBefore(services, typeof(AcmeCertificateApplicationService), typeof(NntpTlsListenerService)); + Assert.DoesNotContain(services, static s => s.GetType().Name == "AccountByteService"); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.ReaderService.Transit.TransitDnsRefreshService); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.ReaderService.Transit.TransitPeerStateService); + Assert.Equal(1, services.Count(static s => s is SessionStateService)); + + await host.StartAsync(); + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + Assert.Equal(["app"], started); + await host.StopAsync(); + } + + [Fact] + public async Task StartAsync_Cancellation_Propagates() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + await Assert.ThrowsAnyAsync(() => service.StartAsync(cts.Token)); + } + + private static void AssertRegisteredBefore(IApplicationService[] services, Type earlier, Type later) + { + var earlierIndex = -1; + var laterIndex = -1; + for (var i = 0; i < services.Length; i++) + { + var type = services[i].GetType(); + if (type == earlier && earlierIndex < 0) + { + earlierIndex = i; + } + else if (type == later && laterIndex < 0) + { + laterIndex = i; + } + } + + Assert.True(earlierIndex >= 0, $"{earlier.Name} must be registered."); + Assert.True(laterIndex >= 0, $"{later.Name} must be registered."); + Assert.True( + earlierIndex < laterIndex, + $"{earlier.Name} (index {earlierIndex}) must precede {later.Name} (index {laterIndex})."); + } + + private static CloudflareDnsReconciliationService CreateService( + NntpdOptions options, + ILocalIpAddressAssignee assignee, + ICloudflareDnsClient client) + { + var resolver = new BindAddressResolver(assignee, NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + } + + private sealed class TrackingApplicationService : IApplicationService + { + private readonly IApplicationService _inner; + private readonly List _order; + private readonly string _label; + + public TrackingApplicationService(IApplicationService inner, List order, string label) + { + _inner = inner; + _order = order; + _label = label; + } + + public string Name => _inner.Name; + + public Task? Execution => _inner.Execution; + + public async Task StartAsync(CancellationToken cancellationToken) + { + await _inner.StartAsync(cancellationToken).ConfigureAwait(false); + _order.Add(_label); + } + + public Task StopAsync(CancellationToken cancellationToken) => _inner.StopAsync(cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs new file mode 100644 index 00000000..6621cebe --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs @@ -0,0 +1,413 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Reconciliation; + +/// +/// Adversarial coverage for non-atomic Cloudflare mutations, uncertain outcomes, and fail-closed startup. +/// +public sealed class CloudflareDnsAdversarialTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task ASucceedsThenAaaaCreateFails_DoesNotReportSuccess_LeavesPartialRemoteState() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") { StatusCode = 500, IsOutcomeUncertain = true }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + Assert.True(client.SuccessfulCreateCount >= 1); + } + + [Fact] + public async Task AaaaSucceedsThenADeleteFails_DoesNotReportSuccess() + { + // Seed stale A that must be deleted after AAAA create path; fail deletes permanently. + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") { StatusCode = 500, IsOutcomeUncertain = true }, + }; + client.Seed( + Record("stale", "A", "198.18.0.99"), + Record("aaaa", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + // Desired A was created (create-before-delete) but stale may remain. + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task CreateSucceedsThenDeleteFails_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task DeleteWouldSucceedButCreateFailsFirst_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + // create-before-delete: create failed, so stale should still be present and desired absent. + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.DoesNotContain(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task UncertainCreateOutcome_AppliedRemotely_RetryConvergesAndVerifies() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + CreateFailTypeException = new CloudflareDnsException("timeout after apply") + { + IsOutcomeUncertain = true, + }, + ApplyCreateThenFailUncertain = true, + }; + + var reconciler = CreateReconciler(client); + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + Assert.Equal(2, snapshot.Count); + } + + [Fact] + public async Task UncertainOutcomeExhausted_FailsClosed_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeException = new CloudflareDnsException("transport timeout") + { + IsOutcomeUncertain = true, + }, + // Do not apply: simulates a timeout where we cannot know whether Cloudflare wrote — + // here we model the "never observed as present" exhausted path by not mutating. + ApplyCreateThenFailUncertain = false, + }; + + var reconciler = CreateReconciler(client); + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.Contains("must not proceed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(client.Snapshot(), r => r.Content == "198.18.0.10"); + } + + [Fact] + public async Task RetryAfterPartialMutation_ReReadsAndConverges() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA failed"), + }; + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.5", "fd12:3456:789a::1"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "10.0.0.5"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + + client.CreateFailType = null; + client.CreateFailTypeException = null; + + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.5", "fd12:3456:789a::1"), CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "10.0.0.5"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "fd12:3456:789a::1"); + Assert.Equal(2, snapshot.Count); + } + + [Fact] + public async Task PrivateIpv4AndIpv6_ArePublishedExactly() + { + var client = new FakeCloudflareDnsClient(); + var reconciler = CreateReconciler(client); + var desired = Desired("10.0.0.8", "192.168.1.8", "172.16.0.8", "fd00::8"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Equal(4, snapshot.Count); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "10.0.0.8"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "192.168.1.8"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "172.16.0.8"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "fd00::8"); + } + + [Fact] + public async Task VerificationMismatch_NeverReportsSuccess() + { + var client = new AlwaysMismatchAfterMutationsClient(); + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("must not proceed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CancellationDuringReconcile_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + cts.Token.ThrowIfCancellationRequested(); + return Task.CompletedTask; + }; + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + } + + [Fact] + public async Task Startup_AfterDnsPartialFailure_DoesNotReachRunning_AndRollbackStopsDnsService() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA failed"), + }; + + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(IPAddress.Parse("10.0.0.9"), IPAddress.Parse("fd00::9")), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var dns = new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + + var other = new FakeApplicationService("other"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), other], options); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(0, other.StartCount); + Assert.DoesNotContain( + client.Snapshot(), + r => string.Equals( + r.Name.Trim().TrimEnd('.'), + options.Fqdn.Trim().TrimEnd('.'), + StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task MutationTimeout_IsMarkedUncertain_ByHttpClient() + { + var handler = new ScriptedTimeoutHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = TimeSpan.FromMilliseconds(50), + }; + var client = new CloudflareDnsClient( + httpClient, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + }, + CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void BindAddressResolver_PublishesPrivateAddresses_ExcludesLinkLocal() + { + var privateV4 = IPAddress.Parse("10.1.2.3"); + var ula = IPAddress.Parse("fd12::1"); + var linkLocal = IPAddress.Parse("169.254.1.1"); + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(privateV4, ula, linkLocal), + NullLogger.Instance); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var resolved = resolver.Resolve(options); + + Assert.Contains(privateV4, resolved.IPv4); + Assert.Contains(ula, resolved.IPv6); + Assert.DoesNotContain(linkLocal, resolved.All); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + /// + /// Lists empty during mutation planning, then returns a mismatched set on verification lists. + /// + private sealed class AlwaysMismatchAfterMutationsClient : ICloudflareDnsClient + { + private int _creates; + + public Task> ListRecordsAsync( + string zoneId, + string fqdn, + string type, + CancellationToken cancellationToken) + { + if (_creates == 0) + { + return Task.FromResult>([]); + } + + // After any create, always report a mismatched address so verification cannot succeed. + return Task.FromResult>( + [ + new CloudflareDnsRecord + { + Id = "mismatch", + Type = type, + Name = fqdn, + Content = type == "AAAA" ? "2001:db8::99" : "198.18.0.99", + }, + ]); + } + + public Task> ListAllRecordsForNameAsync( + string zoneId, + string fqdn, + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public Task CreateRecordAsync( + string zoneId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + _creates++; + return Task.FromResult(new CloudflareDnsRecord + { + Id = $"c{_creates}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + }); + } + + public Task UpdateRecordAsync( + string zoneId, + string recordId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) => + Task.FromResult(new CloudflareDnsRecord + { + Id = recordId, + Type = request.Type, + Name = request.Name, + Content = request.Content, + }); + + public Task DeleteRecordAsync(string zoneId, string recordId, CancellationToken cancellationToken) => + Task.CompletedTask; + } + + private sealed class ScriptedTimeoutHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json"), + Headers = { }, + }; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs new file mode 100644 index 00000000..8cc4d587 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs @@ -0,0 +1,534 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Reconciliation; + +/// +/// Final regression coverage for staged create-all-then-delete, retry classification, and exact-set verify. +/// +public sealed class CloudflareDnsFinalRegressionTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task TransientFailure_ThenSuccessfulRetry_ReportsSuccessOnlyAfterVerify() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + CreateFailTypeException = new CloudflareDnsException("transient 502") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(2, client.Snapshot().Count); + Assert.True(client.CreateCallCount >= 2); + } + + [Fact] + public async Task PermanentAuthFailure_DoesNotRetryPointlessly() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("Authentication error") + { + StatusCode = 401, + IsPermanentFailure = true, + FailedOperation = "GET list", + }, + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Equal(401, ex.StatusCode); + Assert.Equal(1, client.ListCallCount); + Assert.Equal(0, client.CreateCallCount); + } + + [Fact] + public async Task TimeoutAfterMutationApplied_RetryConvergesWithoutDuplicateSuccessClaim() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = true, + CreateFailTypeException = new CloudflareDnsException("timeout after apply") + { + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.Equal("198.18.0.10", client.Snapshot()[0].Content); + } + + [Fact] + public async Task TimeoutAfterMutationNotApplied_RetryCreatesThenVerifies() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = false, + CreateFailTypeException = new CloudflareDnsException("timeout no apply") + { + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.True(client.SuccessfulCreateCount >= 1); + } + + [Fact] + public async Task Retry_ReReadsActualState_DoesNotAssumePriorMutationFailed() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = true, + CreateFailTypeException = new CloudflareDnsException("uncertain") { IsOutcomeUncertain = true }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.1", "fd00::1"), CancellationToken.None); + + // Attempt 1: list A+AAAA; attempt 2: list A+AAAA again (+ verify lists). Must exceed one attempt. + Assert.True(client.ListCallCount > 4); + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task RetryExhaustion_AfterPartialChanges_FailsClosed() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA always fails") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(client.Snapshot(), r => r.Type == "A"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + } + + [Fact] + public async Task CreateFailure_BeforeAnyDeletion_LeavesStaleIntact() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") + { + StatusCode = 500, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task DeleteFailure_AfterDesiredCreated_FailsClosed_KeepsSuperset() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") + { + StatusCode = 500, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task CrossFamily_CreatesAllMissingBeforeAnyDeletes() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(["Create:A", "Create:AAAA", "Delete", "Delete"], client.MutationOrder.Take(4).ToArray()); + Assert.DoesNotContain(client.Snapshot(), r => r.Content is "198.18.0.99" or "2001:db8::99"); + } + + [Fact] + public async Task AFailureWhileAaaaAlsoNeedsChanges_DoesNotDeleteYet() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeException = new CloudflareDnsException("A create failed") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Equal(0, client.SuccessfulDeleteCount); + Assert.DoesNotContain(client.MutationOrder, m => m.StartsWith("Create:AAAA", StringComparison.Ordinal)); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.Contains(client.Snapshot(), r => r.Content == "2001:db8::99"); + } + + [Fact] + public async Task AaaaFailureAfterACreated_BeforeDeletes_KeepsStale() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Equal(0, client.SuccessfulDeleteCount); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task Ipv6ExpandedAndCompressedForms_NormalizeToSameDesiredSet() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("aaaa", "AAAA", "2001:db8::1")); + + var desired = new ResolvedBindAddresses( + [ + IPAddress.Parse("2001:0db8:0000:0000:0000:0000:0000:0001"), + ]); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task ProxiedDesiredRecord_IsUpdatedBeforeSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "proxied", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + }); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + Assert.False(Assert.Single(client.Snapshot()).Proxied); + } + + [Fact] + public async Task Verify_RejectsProxiedRecordEvenIfContentMatches() + { + var client = new FakeCloudflareDnsClient(); + client.TransformListedRecords = (_, records, _) => + records + .Select(r => new CloudflareDnsRecord + { + Id = r.Id, + Type = r.Type, + Name = r.Name, + Content = r.Content, + Ttl = r.Ttl, + Proxied = true, + }) + .ToArray(); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("proxied", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CancellationDuringBackoff_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + using var cts = new CancellationTokenSource(); + var failedOnce = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var original = client.CreateException; + client.CreateException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }; + + // Signal after first create attempt by wrapping via OnMutate. + client.OnMutate = _ => + { + failedOnce.TrySetResult(); + return Task.CompletedTask; + }; + // CreateException throws after OnMutate — keep original fail. + client.CreateException = original; + + var task = CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token); + await failedOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => task); + } + + [Fact] + public async Task ConcurrentReconcile_SameInstance_IsSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var max = 0; + var concurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref max, Math.Max(Volatile.Read(ref max), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var first = reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + var second = reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + await Task.Delay(30); + Assert.Equal(1, Volatile.Read(ref entered)); + gate.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, max); + } + + [Fact] + public async Task ExternalModification_BetweenReadAndMutation_RetryConvergesOrFailsClosed() + { + var client = new FakeCloudflareDnsClient(); + var injected = 0; + client.OnMutate = _ => + { + // Between planned create and completion: external actor also publishes desired A. + if (Interlocked.Exchange(ref injected, 1) == 0) + { + client.Inject(Record("external", "A", "198.18.0.10")); + } + + return Task.CompletedTask; + }; + + // May succeed on retry after deduping, or exhaust if inject keeps happening — inject only once. + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.Equal("198.18.0.10", client.Snapshot()[0].Content); + } + + [Fact] + public async Task ExternalModification_DetectedOnFinalVerification_FailsClosed() + { + var client = new FakeCloudflareDnsClient(); + client.TransformListedRecords = (type, records, listCallCount) => + { + // After mutations (create call happened), inject an unexpected A on verification lists. + if (client.SuccessfulCreateCount > 0 + && string.Equals(type, "A", StringComparison.OrdinalIgnoreCase) + && listCallCount > 2) + { + return records + .Append(Record("external-stale", "A", "198.18.0.77")) + .ToArray(); + } + + return records; + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task PrivateAddresses_RemainPublished() + { + var client = new FakeCloudflareDnsClient(); + await CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("10.0.0.8", "192.168.0.8", "fd00::8"), + CancellationToken.None); + + Assert.Equal(3, client.Snapshot().Count); + } + + [Fact] + public async Task NoEligibleAddresses_StartupFails() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + new BindAddressResolver(new FakeLocalIpAddressAssignee(IPAddress.Loopback), NullLogger.Instance), + CreateReconciler(new FakeCloudflareDnsClient()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("No eligible IP addresses", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task DuplicateAndPagination_StillConverge() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10"), + Record("aaaa2", "AAAA", "2001:db8::10")); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task PermanentValidationFailure_OnDelete_DoesNotRetryIndefinitely() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("validation error") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + // Create once, delete once — no reconciler retry loop. + Assert.Equal(1, client.SuccessfulCreateCount); + Assert.Equal(1, client.DeleteCallCount); + Assert.DoesNotContain("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs new file mode 100644 index 00000000..4d23794b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs @@ -0,0 +1,383 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Cloudflare.Reconciliation; + +public sealed class CloudflareDnsReconcilerTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task Reconcile_WhenRecordsAlreadyMatch_MakesNoMutations() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.UpdateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task Reconcile_MissingRecords_CreatesThem() + { + var client = new FakeCloudflareDnsClient(); + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(2, client.CreateCallCount); + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10" && r.Proxied == false); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10" && r.Proxied == false); + } + + [Fact] + public async Task Reconcile_StaleRecords_AreRemoved() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99"), + Record("other", "A", "198.18.0.50", name: "other.usenet.ninja")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Content is "198.18.0.99" or "2001:db8::99"); + Assert.Contains(snapshot, r => r.Name == "other.usenet.ninja" && r.Content == "198.18.0.50"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + } + + [Fact] + public async Task Reconcile_MixedDesiredAndStale_RetainsDesiredRemovesStale() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("keep", "A", "198.18.0.10"), + Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Single(snapshot); + Assert.Equal("198.18.0.10", snapshot[0].Content); + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(1, client.DeleteCallCount); + } + + [Fact] + public async Task Reconcile_NoIpv4_RemovesAllARecords() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("2001:db8::10")]); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Type == "A"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + } + + [Fact] + public async Task Reconcile_NoIpv6_RemovesAllAaaaRecords() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Type == "AAAA"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + } + + [Fact] + public async Task Reconcile_EmptyDesired_ThrowsWithoutMutating() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a1", "A", "198.18.0.10")); + var reconciler = CreateReconciler(client); + + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, new ResolvedBindAddresses([]), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task Reconcile_DuplicateRecords_AreCollapsed() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10"), + Record("aaaa2", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Equal(2, snapshot.Count); + Assert.Equal(2, client.DeleteCallCount); + } + + [Fact] + public async Task Reconcile_UnrelatedTxtRecord_RemainsUntouched() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + new CloudflareDnsRecord + { + Id = "txt1", + Type = "TXT", + Name = Fqdn, + Content = "v=spf1 -all", + }); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Contains(client.Snapshot(), r => r.Type == "TXT" && r.Id == "txt1"); + } + + [Fact] + public async Task Reconcile_ApiFailure_SurfacesAndIsNotTreatedAsSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") { StatusCode = 500 }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.Ordinal); + Assert.Contains("create failed", ex.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task Reconcile_PartialFailureAfterAUpdates_Surfaces() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") { StatusCode = 502 }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.Ordinal); + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + } + + [Fact] + public async Task Reconcile_VerificationMismatch_Throws() + { + var client = new FakeCloudflareDnsClient(); + // Force verification lists to observe a mismatched set after mutations. + client.MutationsBeforeListFailure = 1; + client.FailListAfterMutationsException = new CloudflareDnsException( + "Cloudflare DNS verification failed for 'nntpd01.usenet.ninja': mismatched") + { + FailedOperation = "Verify", + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Reconcile_Cancellation_Propagates() + { + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var reconciler = CreateReconciler(client); + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + } + + [Fact] + public async Task Reconcile_ConcurrentCalls_AreSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var concurrent = 0; + var maxConcurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref maxConcurrent, Math.Max(Volatile.Read(ref maxConcurrent), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + var first = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + + var second = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await Task.Delay(50); + Assert.Equal(1, Volatile.Read(ref entered)); + + gate.TrySetResult(); + await Task.WhenAll(first, second); + + Assert.Equal(1, maxConcurrent); + } + + [Fact] + public async Task Reconcile_ProxiedRecord_IsUpdatedToDnsOnly() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "proxied", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + }); + + var reconciler = CreateReconciler(client); + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(false, updated.Proxied); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + } + + [Fact] + public async Task Reconcile_IncorrectTtl_IsUpdatedInPlace() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = false, + Ttl = 120, + }); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + Assert.Equal(false, updated.Proxied); + } + + [Fact] + public async Task Reconcile_IncorrectTtlAndProxied_UpdatesOnce() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + Ttl = 60, + }); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + Assert.Equal(false, updated.Proxied); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content, string? name = null) => + new() + { + Id = id, + Type = type, + Name = name ?? Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/ControlOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/ControlOptionsTests.cs new file mode 100644 index 00000000..170d2f6c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/ControlOptionsTests.cs @@ -0,0 +1,320 @@ +using System.Reflection; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +public sealed class ControlOptionsTests +{ + private const string AioeFingerprint = "22031AAC51E7C7FD664F1D8090DF6C712322A7F8"; + private const string AioeFingerprintGrouped = "2203 1AAC 51E7 C7FD 664F 1D80 90DF 6C71 2322 A7F8"; + private const string BigEightNewsgroups = "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*"; + private const int SourceAuthorityCount = 112; + private const int SourceAuthorizationCount = 317; + + [Fact] + public void Bind_ControlSection_Succeeds() + { + var options = Bind( + """ + { + "Control": { + "PgpAuthorities": { + "Source": { + "LastModified": "2023-08-05" + }, + "Authorities": [ + { "Name": "AIOE", "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8" } + ] + } + } + } + """); + + Assert.Equal("2023-08-05", options.PgpAuthorities.Source.LastModified); + Assert.Single(options.PgpAuthorities.Authorities); + Assert.Equal("AIOE", options.PgpAuthorities.Authorities[0].Name); + Assert.True(new ControlOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_PgpAuthorities_Succeeds() + { + var options = Bind( + """ + { + "Control": { + "PgpAuthorities": { + "Source": { + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl", + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl" + }, + "Authorities": [] + } + } + } + """); + + Assert.Equal("InnUrl", options.PgpAuthorities.Source.RetrievedFrom); + Assert.Equal( + "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl", + options.PgpAuthorities.Source.Url); + Assert.Equal( + "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + options.PgpAuthorities.Source.InnUrl); + Assert.Empty(options.PgpAuthorities.Authorities); + } + + [Fact] + public void Bind_KnownAuthority_HasExpectedFingerprint() + { + var options = BindProduction(); + var aioe = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "AIOE"); + Assert.Equal(AioeFingerprint, aioe.KeyFingerprint); + Assert.Equal("usenet@aioe.org", aioe.Contact); + Assert.Equal("http://news.aioe.org/hierarchy/aioe.txt", aioe.KeyUrl); + } + + [Fact] + public void FingerprintNormalization_PreservesHexBytes() + { + Assert.Equal(AioeFingerprint, PgpAuthorityFingerprint.Normalize(AioeFingerprintGrouped)); + Assert.Equal(AioeFingerprint, PgpAuthorityFingerprint.Normalize(AioeFingerprint.ToLowerInvariant())); + Assert.True(PgpAuthorityFingerprint.TryNormalize(AioeFingerprintGrouped, out var normalized)); + Assert.Equal(AioeFingerprint, normalized); + + var sourceDigits = new string(AioeFingerprintGrouped.Where(static c => !char.IsWhiteSpace(c)).ToArray()); + Assert.Equal(sourceDigits, normalized, ignoreCase: true); + Assert.False(PgpAuthorityFingerprint.TryNormalize("not-a-fingerprint", out _)); + Assert.False(PgpAuthorityFingerprint.TryNormalize("220", out _)); + Assert.False(PgpAuthorityFingerprint.TryNormalize(" ", out _)); + } + + [Fact] + public void Bind_PostConfigure_NormalizesGroupedFingerprint() + { + var configuration = LoadJson( + """ + { + "Control": { + "PgpAuthorities": { + "Authorities": [ + { "Name": "AIOE", "KeyFingerprint": "2203 1AAC 51E7 C7FD 664F 1D80 90DF 6C71 2322 A7F8" } + ] + } + } + } + """); + + var services = new ServiceCollection(); + services + .AddOptions() + .Bind(configuration.GetSection(ControlOptions.SectionName)) + .PostConfigure(static options => + { + // codeql[cs/linq/missed-where] TryNormalize out-params; foreach preserves mutate-in-place clarity. + foreach (var authority in options.PgpAuthorities.Authorities) + { + if (PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out var normalized)) + { + authority.KeyFingerprint = normalized; + } + } + }); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(AioeFingerprint, options.PgpAuthorities.Authorities[0].KeyFingerprint); + } + + [Fact] + public void Bind_MultipleHierarchyRules_RemainDistinct() + { + var options = BindProduction(); + var de = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "DE"); + var deAlt = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "DE.ALT"); + Assert.NotSame(de, deAlt); + Assert.Equal("de.*", de.Authorizations[0].Newsgroups); + Assert.Equal("de.alt.*", deAlt.Authorizations[0].Newsgroups); + + var bigEight = Assert.Single( + options.PgpAuthorities.Authorities, + a => a.Name == "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK"); + Assert.Equal(3, bigEight.Authorizations.Length); + Assert.All(bigEight.Authorizations, rule => Assert.Equal(BigEightNewsgroups, rule.Newsgroups)); + Assert.Equal( + ["checkgroups", "newgroup", "rmgroup"], + bigEight.Authorizations.Select(static r => r.Message).ToArray()); + + var maus = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "MAUS"); + Assert.Equal(6, maus.Authorizations.Length); + Assert.Equal(2, maus.Authorizations.Count(static r => r.Message == "checkgroups")); + Assert.Contains(maus.Authorizations, static r => r.From == "guenter@gst0hb.hb.provi.de"); + Assert.Contains(maus.Authorizations, static r => r.From == "guenter@gst0hb.north.de"); + } + + [Fact] + public void Validate_MissingOptionalMetadata_DoesNotFail() + { + var options = new ControlOptions + { + PgpAuthorities = new PgpAuthoritiesOptions + { + Authorities = + [ + new PgpAuthorityOptions { Name = "LOCAL-TEST" }, + ], + }, + }; + + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + } + + [Fact] + public void CatalogueTypesAndProductionJson_HaveNoPrivateKeyOrPassphraseConfiguration() + { + var secretFragments = new[] + { + "PrivateKey", + "Passphrase", + "Password", + "SecretKey", + "Armored", + "BEGIN PGP PRIVATE", + }; + + Type[] catalogueTypes = + [ + typeof(ControlOptions), + typeof(PgpAuthoritiesOptions), + typeof(PgpAuthorityOptions), + typeof(PgpAuthoritySourceOptions), + typeof(PgpAuthorityAuthorizationOptions), + ]; + + foreach (var type in catalogueTypes) + { + foreach (var property in type.GetProperties(BindingFlags.Instance | BindingFlags.Public)) + { + Assert.DoesNotContain( + secretFragments, + fragment => property.Name.Contains(fragment, StringComparison.OrdinalIgnoreCase)); + } + } + + using var document = JsonDocument.Parse(File.ReadAllText(FindProductionAppsettings())); + var controlJson = document.RootElement.GetProperty("Control").GetRawText(); + foreach (var fragment in secretFragments) + { + Assert.DoesNotContain(fragment, controlJson, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public void Validate_EmptyAuthorityConfiguration_Succeeds() + { + Assert.True(new ControlOptionsValidator().Validate(null, new ControlOptions()).Succeeded); + Assert.True( + new ControlOptionsValidator() + .Validate(null, new ControlOptions { PgpAuthorities = new PgpAuthoritiesOptions() }) + .Succeeded); + + var omitted = Bind("{}"); + Assert.Empty(omitted.PgpAuthorities.Authorities); + Assert.True(new ControlOptionsValidator().Validate(null, omitted).Succeeded); + + var emptySection = Bind("""{ "Control": { } }"""); + Assert.Empty(emptySection.PgpAuthorities.Authorities); + Assert.True(new ControlOptionsValidator().Validate(null, emptySection).Succeeded); + } + + [Fact] + public void ProductionAppsettings_BindsCompleteCatalogue() + { + var options = BindProduction(); + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + + Assert.Equal("2023-08-05", options.PgpAuthorities.Source.LastModified); + Assert.Equal("InnUrl", options.PgpAuthorities.Source.RetrievedFrom); + Assert.Equal(SourceAuthorityCount, options.PgpAuthorities.Authorities.Length); + Assert.Equal( + SourceAuthorizationCount, + options.PgpAuthorities.Authorities.Sum(static a => a.Authorizations.Length)); + Assert.Equal(55, options.PgpAuthorities.Authorities.Count(static a => a.KeyFingerprint is not null)); + Assert.All( + options.PgpAuthorities.Authorities.Where(static a => a.KeyFingerprint is not null), + static a => + { + Assert.True(PgpAuthorityFingerprint.TryNormalize(a.KeyFingerprint, out var normalized)); + Assert.Equal(normalized, a.KeyFingerprint); + }); + } + + [Fact] + public void Validate_RejectsNonHexadecimalFingerprint() + { + var options = new ControlOptions + { + PgpAuthorities = new PgpAuthoritiesOptions + { + Authorities = + [ + new PgpAuthorityOptions { Name = "AIOE", KeyFingerprint = "not-hex" }, + ], + }, + }; + + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + private static ControlOptions BindProduction() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(FindProductionAppsettings(), optional: false) + .Build(); + var options = new ControlOptions(); + configuration.GetSection(ControlOptions.SectionName).Bind(options); + return options; + } + + private static ControlOptions Bind(string json) + { + var options = new ControlOptions(); + LoadJson(json).GetSection(ControlOptions.SectionName).Bind(options); + return options; + } + + private static IConfigurationRoot LoadJson(string json) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + return new ConfigurationBuilder() + .AddJsonStream(stream) + .Build(); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/EmailOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/EmailOptionsTests.cs new file mode 100644 index 00000000..e877f566 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/EmailOptionsTests.cs @@ -0,0 +1,179 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +public sealed class EmailOptionsTests +{ + [Fact] + public void Disabled_EmptyHost_IsValid() + { + var options = Bind( + """ + { + "Email": { + "Enabled": false + } + } + """); + Assert.False(options.Enabled); + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void ProductionAppsettings_IsDisabledAndValid() + { + var path = FindProductionAppsettings(); + var configuration = new ConfigurationBuilder() + .AddJsonFile(path, optional: false) + .Build(); + var options = new EmailOptions(); + configuration.GetSection(EmailOptions.SectionName).Bind(options); + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + Assert.False(options.Enabled); + Assert.Equal("noreply@usenet.ninja", options.DefaultFrom); + Assert.Equal(SmtpSecurityMode.StartTls, options.Smtp.Security); + Assert.True(options.Smtp.RequireTlsForAuthentication); + Assert.True(string.IsNullOrEmpty(options.Smtp.Password)); + Assert.Equal(EmailSpoolOptions.DefaultDirectory, options.Spool.Directory); + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var email = document.RootElement.GetProperty("Email"); + var smtp = email.GetProperty("Smtp"); + Assert.False(smtp.TryGetProperty("EhloHostname", out _)); + Assert.False(smtp.TryGetProperty("DangerousAcceptAnyServerCertificate", out _)); + Assert.False(email.TryGetProperty("Queue", out _)); + Assert.True(email.TryGetProperty("Spool", out _)); + } + + [Fact] + public void Enabled_RequiresHostAndFrom() + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = string.Empty, + Smtp = new SmtpOptions { Host = string.Empty, Port = 587 }, + }; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + var failures = result.Failures ?? []; + Assert.Contains(failures, static f => f.Contains("Host", StringComparison.Ordinal)); + Assert.Contains(failures, static f => f.Contains("DefaultFrom", StringComparison.Ordinal)); + Assert.DoesNotContain(failures, static f => f.Contains("secret", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Enabled_AuthWithoutTls_FailsUnlessExplicit() + { + var options = ValidEnabled(); + options.Smtp.Username = "user"; + options.Smtp.Password = "password"; + options.Smtp.Security = SmtpSecurityMode.None; + options.Smtp.RequireTlsForAuthentication = true; + Assert.False(new EmailOptionsValidator().Validate(null, options).Succeeded); + + options.Smtp.RequireTlsForAuthentication = false; + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void UsernameWithoutPassword_FailsWithoutEchoingSecret() + { + var options = ValidEnabled(); + options.Smtp.Username = "user"; + options.Smtp.Password = string.Empty; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + Assert.DoesNotContain(result.Failures ?? [], static f => f.Contains("hunter2", StringComparison.Ordinal)); + } + + [Fact] + public void InvalidPortAndScanInterval_Fail() + { + var options = ValidEnabled(); + options.Smtp.Port = 0; + options.Spool.ScanInterval = TimeSpan.Zero; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + } + + [Fact] + public void Options_InvalidEnabledHost_FailsOnStart() + { + var services = new ServiceCollection(); + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes( + """ + { + "Email": { + "Enabled": true, + "DefaultFrom": "noreply@example.com", + "Smtp": { "Host": "", "Port": 587, "Security": "StartTls" } + } + } + """)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + services.AddSingleton(configuration); + services + .AddOptions() + .BindConfiguration(EmailOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, EmailOptionsValidator>(); + using var provider = services.BuildServiceProvider(); + Assert.Throws( + () => provider.GetRequiredService>().Value); + } + + private static EmailOptions ValidEnabled() => + new() + { + Enabled = true, + DefaultFrom = "noreply@example.com", + EnvelopeSender = "noreply@example.com", + Smtp = new SmtpOptions + { + Host = "smtp.example.com", + Port = 587, + Security = SmtpSecurityMode.StartTls, + }, + }; + + private static EmailOptions Bind(string json) + { + var options = new EmailOptions(); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + new ConfigurationBuilder() + .AddJsonStream(stream) + .Build() + .GetSection(EmailOptions.SectionName) + .Bind(options); + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + return options; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/ModerationOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/ModerationOptionsTests.cs new file mode 100644 index 00000000..c4dc6040 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/ModerationOptionsTests.cs @@ -0,0 +1,132 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +public sealed class ModerationOptionsTests +{ + [Fact] + public void Bind_EmptySection_IsValid() + { + var options = Bind( + """ + { + "Moderation": { + "Moderators": [] + } + } + """); + + Assert.Empty(options.Moderators); + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_OmittedSection_IsValid() + { + var options = Bind("""{ "Nntpd": { "ServerId": 1 } }"""); + Assert.Empty(options.Moderators); + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_LeftoverModerators_FailsValidation() + { + var options = Bind( + """ + { + "Moderation": { + "Moderators": [ + { + "Pattern": "comp.example.*", + "Address": "moderator@example.com", + "Username": "moderator-example" + } + ] + } + } + """); + + var result = new ModerationOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("nntpmoderators", result.FailureMessage, StringComparison.Ordinal); + } + + [Fact] + public void Bind_ProductionAppsettings_KeepsSourceProvenanceWithoutStaticList() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(FindProductionAppsettings(), optional: false) + .Build(); + var options = new ModerationOptions(); + configuration.GetSection(ModerationOptions.SectionName).Bind(options); + options.Moderators ??= []; + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + Assert.Equal("https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", options.Source.InnUrl); + Assert.Equal("InnUrl", options.Source.RetrievedFrom); + Assert.Equal("https://github.com/InterNetNews/inn/blob/main/samples/moderators", options.Source.Url); + Assert.Empty(options.Moderators); + } + + [Fact] + public void Options_LeftoverModerators_FailsOnStart() + { + var services = new ServiceCollection(); + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes( + """ + { + "Moderation": { + "Moderators": [ + { "Pattern": "comp.example.*", "Address": "mod@example.com", "Username": "mod" } + ] + } + } + """)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + services.AddSingleton(configuration); + services + .AddOptions() + .BindConfiguration(ModerationOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, ModerationOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var resolve = () => provider.GetRequiredService>().Value; + Assert.Throws(resolve); + } + + private static ModerationOptions Bind(string json) + { + var options = new ModerationOptions(); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + new ConfigurationBuilder() + .AddJsonStream(stream) + .Build() + .GetSection(ModerationOptions.SectionName) + .Bind(options); + options.Moderators ??= []; + return options; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs new file mode 100644 index 00000000..f632e720 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs @@ -0,0 +1,315 @@ +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +/// +/// Proves NNTPD-owned ACME state binds from Nntpd, that the ACME account +/// email and certificate password are not supplied by VECTOR__ACMEACCOUNT or +/// VECTOR__ACMECERTIFICATEPASSWORD, and that relative AcmeStateDir resolves +/// through Common +/// against . +/// +[Collection("WorkingDirectory")] +public sealed class NntpdAcmeConfigurationOwnershipTests +{ + private const string NntpdDirectoryUrl = "https://acme-staging-v02.api.letsencrypt.org/directory"; + private const string SharedAcmeAccount = "shared-acme@example.test"; + private const int NntpdRenewalDays = 14; + private const string NntpdStateDir = "certs/"; + private const string RootDirectoryUrl = "https://root-must-not-bind.example/directory"; + private const string RootEmail = "root-must-not-bind@example.test"; + private const string RootStateDir = "root-must-not-bind/"; + private const string TestCertificatePassword = "unit-test-nntpd-pfx-password"; + + [Fact] + public void Nntpd_json_does_not_own_the_acme_directory_url() + { + var json = File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"))); + Assert.DoesNotContain("\"AcmeDirectoryUrl\"", json, StringComparison.Ordinal); + } + + [Fact] + public void Shared_acme_account_email_is_not_supplied_by_vector_environment() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.NotEqual(SharedAcmeAccount, options.AcmeEmail); + Assert.Equal( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + "VECTOR__ACMEACCOUNT"); + } + + [Fact] + public void Nntpd_json_does_not_own_the_acme_renewal_threshold() + { + var json = File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"))); + Assert.DoesNotContain("\"AcmeRenewalThresholdDays\"", json, StringComparison.Ordinal); + } + + [Fact] + public void Nntpd_acme_state_dir_survives_configuration_binding() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + Assert.DoesNotContain("root-must-not-bind", options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Root_acme_values_cannot_overwrite_nntpd_owned_settings() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + var acme = host.Services.GetRequiredService>().Value; + + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + + Assert.NotEqual(RootEmail, options.AcmeEmail); + Assert.NotEqual(SharedAcmeAccount, options.AcmeEmail); + Assert.NotEqual(RootStateDir, options.AcmeStateDir); + + Assert.Same(options, acme); + } + + [Fact] + public void Vector_acme_certificate_password_is_not_supplied_by_the_environment_variable() + { + var previous = Environment.GetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + TestCertificatePassword); + + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.NotEqual(TestCertificatePassword, options.AcmeCertificatePassword); + } + finally + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + previous); + } + } + + [Fact] + public void Nntpd_does_not_require_an_application_specific_acme_password_variable() + { + Assert.Equal( + "VECTOR__ACMECERTIFICATEPASSWORD", + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + Assert.Equal( + AcmeCloudflareOptions.AcmeCertificatePasswordEnvironmentVariable, + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + Assert.Null( + typeof(NntpdOptions).GetField( + "AcmeCertificatePasswordEnvironmentVariable", + BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)); + Assert.DoesNotContain( + "NNTPD__", + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + StringComparison.OrdinalIgnoreCase); + + var previousShared = Environment.GetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + var previousNntpd = Environment.GetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD"); + try + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + TestCertificatePassword); + Environment.SetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD", "nntpd-specific-must-not-be-required"); + + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + var rabbit = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.NotEqual(TestCertificatePassword, options.AcmeCertificatePassword); + Assert.NotEqual("nntpd-specific-must-not-be-required", options.AcmeCertificatePassword); + Assert.Null(rabbit.Username); + Assert.Null(rabbit.Password); + } + finally + { + Environment.SetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, previousShared); + Environment.SetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD", previousNntpd); + } + } + + [Fact] + public void Relative_nntpd_acme_state_dir_resolves_under_appcontext_base_directory() + { + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + var expected = ExpectedResolvedStateDir(AppContext.BaseDirectory); + Assert.Equal(expected, options.AcmeStateDir); + Assert.StartsWith( + Path.GetFullPath(AppContext.BaseDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + options.AcmeStateDir, + StringComparison.OrdinalIgnoreCase); + Assert.Contains( + "ContentRootPath = AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "Program.cs"))), + StringComparison.Ordinal); + } + + [Fact] + public void Changing_process_working_directory_does_not_change_resolved_acme_state_directory() + { + var previous = Environment.CurrentDirectory; + var scratch = Directory.CreateTempSubdirectory("nntpd-acme-cwd-"); + try + { + Environment.CurrentDirectory = scratch.FullName; + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + var expected = ExpectedResolvedStateDir(AppContext.BaseDirectory); + var cwdResolved = Path.GetFullPath(NntpdStateDir.Trim()); + + Assert.Equal(expected, options.AcmeStateDir); + Assert.NotEqual(cwdResolved, options.AcmeStateDir); + Assert.DoesNotContain( + Path.GetFullPath(scratch.FullName).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + options.AcmeStateDir, + StringComparison.OrdinalIgnoreCase); + } + finally + { + Environment.CurrentDirectory = previous; + scratch.Delete(recursive: true); + } + } + + [Fact] + public void Nntpd_and_backfiller_use_the_same_common_acme_path_resolution_mechanism() + { + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + + var nntpdHosting = File.ReadAllText(FindRepoFile( + Path.Join("src", "VectorNNTPD.ReaderService", "Hosting", "NntpdServiceCollectionExtensions.cs"))); + var backFillerHosting = File.ReadAllText(FindRepoFile( + Path.Join("src", "VectorNNTPD.BackFillerService", "Hosting", "BackFillerServiceCollectionExtensions.cs"))); + + Assert.Contains("ApplicationLocalPath.ResolveApplicationLocalPath", nntpdHosting, StringComparison.Ordinal); + Assert.Contains("ApplicationLocalPath.ResolveApplicationLocalPath", backFillerHosting, StringComparison.Ordinal); + Assert.Contains("AppContext.BaseDirectory", nntpdHosting, StringComparison.Ordinal); + Assert.Contains("AppContext.BaseDirectory", backFillerHosting, StringComparison.Ordinal); + } + + private static string ExpectedResolvedStateDir(string contentRootPath) => + ApplicationLocalPath.ResolveApplicationLocalPath(NntpdStateDir, contentRootPath); + + private static IHost CreateHost(string? contentRootPath = null) + { + // VECTOR__ACMEACCOUNT overlays in-memory ACMEACCOUNT. Isolate from the + // operator environment so these tests prove the shared-account contract. + var previousAccount = Environment.GetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable); + Environment.SetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + SharedAcmeAccount); + try + { + return BuildHost(contentRootPath); + } + finally + { + Environment.SetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + previousAccount); + } + } + + private static IHost BuildHost(string? contentRootPath) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.ReaderService.Tests", + ContentRootPath = contentRootPath ?? AppContext.BaseDirectory, + EnvironmentName = Environments.Development, + }); + + builder.Configuration.AddInMemoryCollection(CreateConfigurationPairs()); + builder.Configuration.AddVectorEnvironmentVariables(); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + TestHostFactory.IsolateTransit(builder.Services); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static Dictionary CreateConfigurationPairs() + { + return new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:AcmeDirectoryUrl"] = NntpdDirectoryUrl, + [$"{NntpdOptions.SectionName}:AcmeEmail"] = "nntpd-section-must-not-bind@example.test", + [$"{NntpdOptions.SectionName}:AcmeRenewalThresholdDays"] = "14", + [$"{NntpdOptions.SectionName}:AcmeStateDir"] = NntpdStateDir, + [AcmeCloudflareOptions.AcmeAccountConfigurationKey] = SharedAcmeAccount, + ["AcmeDirectoryUrl"] = RootDirectoryUrl, + ["AcmeEmail"] = RootEmail, + ["AcmeRenewalThresholdDays"] = "30", + ["AcmeStateDir"] = RootStateDir, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d", + [$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja", + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }; + } + + private static string FindRepoFile(string relativePath) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var candidate = Path.Join(directory.FullName, relativePath); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException($"Could not locate {relativePath}."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdApplicationLocalPathTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdApplicationLocalPathTests.cs new file mode 100644 index 00000000..69c41036 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdApplicationLocalPathTests.cs @@ -0,0 +1,282 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +/// +/// Proves NNTPD application-local paths resolve against +/// , not the process working directory +/// or an IDE/project content root. +/// +[Collection("WorkingDirectory")] +public sealed class NntpdApplicationLocalPathTests +{ + [Fact] + public void Relative_certs_resolves_beneath_application_base() + { + using var host = CreateHost(decoyContentRoot: CreateTempDir("nntpd-decoy-content-")); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("certs/"), options.AcmeStateDir); + Assert.StartsWith(Normalized(AppContext.BaseDirectory), options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Resolved_certs_is_not_based_on_process_working_directory() + { + var previous = Environment.CurrentDirectory; + var cwd = CreateTempDir("nntpd-cwd-certs-"); + try + { + Environment.CurrentDirectory = cwd; + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("certs/"), options.AcmeStateDir); + Assert.NotEqual(Path.GetFullPath("certs/"), options.AcmeStateDir); + Assert.DoesNotContain(Normalized(cwd), options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(cwd); + } + } + + [Fact] + public void Changing_working_directory_does_not_change_resolved_certs_or_logs() + { + var previous = Environment.CurrentDirectory; + var first = CreateTempDir("nntpd-cwd-a-"); + var second = CreateTempDir("nntpd-cwd-b-"); + try + { + Environment.CurrentDirectory = first; + using var hostA = CreateHost(); + var certsA = hostA.Services.GetRequiredService>().Value.AcmeStateDir; + Environment.CurrentDirectory = second; + using var hostB = CreateHost(); + var certsB = hostB.Services.GetRequiredService>().Value.AcmeStateDir; + Assert.Equal(certsA, certsB); + Assert.Equal(Expected("certs/"), certsB); + Assert.Equal( + NntpdFileLogging.ResolveDirectory("logs/"), + ApplicationLocalPath.ResolveApplicationLocalPath("logs/", AppContext.BaseDirectory)); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(first); + TryDelete(second); + } + } + + [Fact] + public void Explicit_absolute_paths_remain_absolute() + { + var absoluteCerts = CreateTempDir("nntpd-abs-certs-"); + var absoluteLogs = CreateTempDir("nntpd-abs-logs-"); + try + { + using var host = CreateHost( + extra: + [ + new KeyValuePair($"{NntpdOptions.SectionName}:AcmeStateDir", absoluteCerts), + new KeyValuePair($"{NntpdOptions.SectionName}:LogDir", absoluteLogs), + ]); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected(absoluteCerts), options.AcmeStateDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath(absoluteLogs, AppContext.BaseDirectory), + NntpdFileLogging.ResolveDirectory(absoluteLogs)); + } + finally + { + TryDelete(absoluteCerts); + TryDelete(absoluteLogs); + } + } + + [Fact] + public void Explicit_relative_paths_resolve_beneath_application_base() + { + using var host = CreateHost( + extra: [new KeyValuePair($"{NntpdOptions.SectionName}:AcmeStateDir", "nested-certs/")]); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("nested-certs/"), options.AcmeStateDir); + Assert.Equal( + Expected("state-logs/"), + NntpdFileLogging.ResolveDirectory("state-logs/")); + } + + [Fact] + public void Nntpd_uses_the_same_common_path_resolution_as_backfiller() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal( + AcmeCloudflareOptionsValidator.ResolveAcmeStateDir("certs/", AppContext.BaseDirectory), + options.AcmeStateDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath("certs/", AppContext.BaseDirectory), + options.AcmeStateDir); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "Hosting", "NntpdServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "Logging", "NntpdFileLogging.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "Email", "FilesystemEmailSpool.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.ReaderService", "Hosting", "NntpdServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.BackFillerService", "Hosting", "BackFillerServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + } + + [Fact] + public void Log_directory_creation_uses_application_base_not_working_directory() + { + var previous = Environment.CurrentDirectory; + var cwd = CreateTempDir("nntpd-log-cwd-"); + var baseDir = CreateTempDir("nntpd-log-base-"); + try + { + Environment.CurrentDirectory = cwd; + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = "logs/", + [$"{NntpdOptions.SectionName}:ApplicationName"] = "VectorNNTPD.ReaderService", + }); + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + configuration[pair.Key] = pair.Value; + } + + NntpdFileLogging.BindResolvedFilePath(configuration, baseDir); + + var expected = ApplicationLocalPath.ResolveApplicationLocalPath("logs/", baseDir); + Assert.True(Directory.Exists(expected)); + Assert.False(Directory.Exists(Path.Join(cwd, "logs"))); + Assert.Equal(expected, NntpdFileLogging.ResolveDirectory("logs/", baseDir)); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(cwd); + TryDelete(baseDir); + } + } + + private static string Expected(string configured) => + ApplicationLocalPath.ResolveApplicationLocalPath(configured, AppContext.BaseDirectory); + + private static string Normalized(string path) => + Path.GetFullPath(path).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + + private static IHost CreateHost( + string? decoyContentRoot = null, + IEnumerable>? extra = null) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.ReaderService.Tests", + ContentRootPath = decoyContentRoot ?? AppContext.BaseDirectory, + EnvironmentName = Environments.Development, + }); + + var pairs = new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:AcmeStateDir"] = "certs/", + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d", + [$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja", + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }; + if (extra is not null) + { + foreach (var pair in extra) + { + pairs[pair.Key] = pair.Value; + } + } + + builder.Configuration.AddInMemoryCollection(pairs); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + TestHostFactory.IsolateTransit(builder.Services); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static string CreateTempDir(string prefix) => + Directory.CreateTempSubdirectory(prefix).FullName; + + private static void TryDelete(string path) + { + try + { + Directory.Delete(path, recursive: true); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private static string FindRepoFile(string relativePath) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var candidate = Path.Join(directory.FullName, relativePath); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException($"Could not locate {relativePath}."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdOptionsValidatorTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdOptionsValidatorTests.cs new file mode 100644 index 00000000..9938f8a8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Configuration/NntpdOptionsValidatorTests.cs @@ -0,0 +1,1907 @@ +using System.ComponentModel.DataAnnotations; +using System.Globalization; +using System.Net; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.ReaderService.Tests.Configuration; + +public sealed class NntpdOptionsValidatorTests +{ + private static NntpdOptionsValidator CreateValidator(ILocalIpAddressAssignee? assignee = null) => + new(assignee ?? new FakeLocalIpAddressAssignee(assignAll: true)); + + [Fact] + public void Validate_Rejects_OutstandingStoreLimitOutsideOneOrTwo() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(2, options.ArticleIngestion.MaxOutstandingStoresPerConnection); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.ArticleIngestion.MaxOutstandingStoresPerConnection = 1; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.ArticleIngestion.MaxOutstandingStoresPerConnection = 64; + var result = CreateValidator().Validate(null, options); + Assert.False(result.Succeeded); + Assert.Contains("MaxOutstandingStoresPerConnection", result.FailureMessage, StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_ForDefaultHistoryTime() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(TimeSpan.FromHours(2), options.HistoryTime); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForConfiguredHistoryTime() + { + var options = TestHostFactory.CreateValidOptions(); + options.HistoryTime = TimeSpan.FromHours(6); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void CheckPipelineDepth_IsNotAnOptionsProperty() + { + Assert.Null(typeof(NntpdOptions).GetProperty("CheckPipelineDepth")); + Assert.Equal(16, VectorNNTPD.ReaderService.Session.CheckPipeline.Depth); + } + + [Fact] + public void BindConfiguration_IgnoresCheckPipelineDepthKey() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:ServerId"] = "1", + ["Nntpd:BindAddress:0"] = "127.0.0.1", + ["Nntpd:CheckPipelineDepth"] = "64", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Null(typeof(NntpdOptions).GetProperty("CheckPipelineDepth")); + Assert.Equal(16, VectorNNTPD.ReaderService.Session.CheckPipeline.Depth); + } + + [Fact] + public void Validate_Succeeds_ForDefaultIdleTime() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultIdleTime, options.IdleTime); + Assert.Equal(300, new NntpdOptions().IdleTime); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForConfiguredIdleTime() + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = 120; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Validate_Fails_ForIdleTimeBelowOneSecond(int idleTime) + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = idleTime; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.IdleTime), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForIdleTimeAboveOneDay() + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = NntpdOptions.MaxIdleTime + 1; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.IdleTime), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursIdleTimeSeconds() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:IdleTime"] = "120", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(120, options.IdleTime); + } + + [Fact] + public void Validate_Succeeds_ForDefaultMaxArticleSize() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultMaxArticleSize, options.MaxArticleSize); + Assert.Equal(5_242_880, new NntpdOptions().MaxArticleSize); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Validate_Fails_ForNonPositiveMaxArticleSize(int size) + { + var options = TestHostFactory.CreateValidOptions(); + options.MaxArticleSize = size; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MaxArticleSize), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForMaxArticleSizeAboveCeiling() + { + var options = TestHostFactory.CreateValidOptions(); + options.MaxArticleSize = NntpdOptions.MaxMaxArticleSize + 1; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MaxArticleSize), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursMaxArticleSize() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:MaxArticleSize"] = "1048576", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(1_048_576, options.MaxArticleSize); + } + + [Fact] + public void Validate_Succeeds_ForDefaultMailComplaintsTo() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultMailComplaintsTo, options.MailComplaintsTo); + Assert.Equal("abuse@usenet.ninja", new NntpdOptions().MailComplaintsTo); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_WhenTop1000IsMissingOrEmpty() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Empty(options.Top1000); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.Top1000 = []; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.Top1000 = ["", " ", "\t"]; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForTop1000Mailboxes_IgnoringWhitespace() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["top1000@anthologeek.net", " ", "ops@usenet.ninja"]; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_ForInvalidTop1000Mailbox() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["not-a-mailbox"]; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.Top1000), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursTop1000() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:Top1000:0"] = "top1000@anthologeek.net", + ["Nntpd:Top1000:1"] = "ops@usenet.ninja", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(["top1000@anthologeek.net", "ops@usenet.ninja"], options.Top1000); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("not-a-mailbox")] + [InlineData("abuse@localhost")] + public void Validate_Fails_ForInvalidMailComplaintsTo(string value) + { + var options = TestHostFactory.CreateValidOptions(); + options.MailComplaintsTo = value; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MailComplaintsTo), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursMailComplaintsTo() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:MailComplaintsTo"] = "ops@usenet.ninja", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal("ops@usenet.ninja", options.MailComplaintsTo); + } + + [Fact] + public void Validate_Succeeds_ForDefaultXTraceKeyWhenConfigured() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + Assert.True(XTraceKeyParser.TryDecode(options.XTraceKey, out var key)); + Assert.Equal(32, key!.Length); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("not-a-key")] + [InlineData("0123456789abcdef")] + public void Validate_Fails_ForInvalidXTraceKey(string value) + { + var options = TestHostFactory.CreateValidOptions(); + options.XTraceKey = value; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.XTraceKeyConfigurationKey, StringComparison.Ordinal)); + var joined = NntpdOptionsValidator.JoinFailures(result); + if (!string.IsNullOrWhiteSpace(value)) + { + Assert.DoesNotContain(value.Trim(), joined, StringComparison.Ordinal); + } + } + + [Fact] + public void Validate_Fails_ForInvalidXTracePreviousKey() + { + var options = TestHostFactory.CreateValidOptions(); + options.XTracePreviousKey = "short"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.XTracePreviousKeyConfigurationKey, StringComparison.Ordinal)); + Assert.DoesNotContain("short", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_WhenNewsmasterCredentialsArePaired() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenOnlyNewsmasterUserIsSet() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.NewsmasterPasswordConfigurationKey, StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenOnlyNewsmasterPasswordIsSet() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + var joined = NntpdOptionsValidator.JoinFailures(result); + Assert.Contains(NntpdOptions.NewsmasterUserConfigurationKey, joined, StringComparison.Ordinal); + Assert.DoesNotContain("unit-test-newsmaster-password", joined, StringComparison.Ordinal); + } + + [Fact] + public void BindConfiguration_HonoursXTraceKey() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:XTraceKey"] = TestHostFactory.TestXTraceKey, + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + } + + [Fact] + public void Validate_Fails_ForHistoryTimeBelowOneSecond() + { + var options = TestHostFactory.CreateValidOptions(); + options.HistoryTime = TimeSpan.Zero; + Assert.True(CreateValidator().Validate(null, options).Failed); + } + + [Fact] + public void Validate_Succeeds_ForValidDefaults() + { + var result = CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()); + Assert.True(result.Succeeded); + } + + [Fact] + public void FeedDiagnostics_DefaultsAreOff() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.False(options.FeedDiagnostics.Enabled); + Assert.Equal(FeedDiagnosticsOptions.DefaultIntervalSeconds, options.FeedDiagnostics.IntervalSeconds); + Assert.True(options.FeedDiagnostics.IncludeSessions); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void SpeedTest_DefaultsAreDiagnosticSized() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(SpeedTestOptions.DefaultMaxDurationSeconds, options.SpeedTest.MaxDurationSeconds); + Assert.Equal(SpeedTestOptions.DefaultMaxBytes, options.SpeedTest.MaxBytes); + Assert.Equal(SpeedTestOptions.DefaultMaxConcurrent, options.SpeedTest.MaxConcurrent); + Assert.Equal(SpeedTestOptions.DefaultMaxConcurrentPerPeer, options.SpeedTest.MaxConcurrentPerPeer); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0, 4096, 2, 1)] + [InlineData(61, 4096, 2, 1)] + [InlineData(10, 512, 2, 1)] + [InlineData(10, 4096, 0, 1)] + [InlineData(10, 4096, 9, 1)] + [InlineData(10, 4096, 2, 0)] + [InlineData(10, 4096, 2, 5)] + public void SpeedTest_InvalidLimits_FailValidation( + int duration, + long maxBytes, + int concurrent, + int perPeer) + { + var options = TestHostFactory.CreateValidOptions(); + options.SpeedTest = new SpeedTestOptions + { + MaxDurationSeconds = duration, + MaxBytes = maxBytes, + MaxConcurrent = concurrent, + MaxConcurrentPerPeer = perPeer, + }; + Assert.True(CreateValidator().Validate(null, options).Failed); + } + + [Fact] + public void TransitQueueMemoryLimit_DefaultsToOneGibibyte() + { + Assert.Equal(1_073_741_824L, NntpdOptions.DefaultTransitQueueMemoryLimit); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, new NntpdOptions().TransitQueueMemoryLimit); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, TestHostFactory.CreateValidOptions().TransitQueueMemoryLimit); + Assert.True(CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()).Succeeded); + } + + [Fact] + public void BindConfiguration_HonoursTransitQueueMemoryLimit() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:TransitQueueMemoryLimit"] = "2097152", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(2_097_152L, options.TransitQueueMemoryLimit); + var valid = TestHostFactory.CreateValidOptions(); + valid.TransitQueueMemoryLimit = options.TransitQueueMemoryLimit; + Assert.True(CreateValidator().Validate(null, valid).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(long.MinValue)] + public void Validate_Fails_ForNonPositiveTransitQueueMemoryLimit(long limit) + { + var options = TestHostFactory.CreateValidOptions(); + options.TransitQueueMemoryLimit = limit; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("TransitQueueMemoryLimit", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_ForMinimumTransitQueueMemoryLimit() + { + var options = TestHostFactory.CreateValidOptions(); + options.TransitQueueMemoryLimit = 1; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForDefaultIngestionWorkerPoolSettings() + { + var options = TestHostFactory.CreateValidOptions(); + var ingestion = options.ArticleIngestion; + Assert.Equal(ArticleIngestionOptions.DefaultMinWorkers, ingestion.MinWorkers); + Assert.Equal(ArticleIngestionOptions.DefaultMaxWorkers, ingestion.MaxWorkers); + Assert.Equal(ArticleIngestionOptions.DefaultMaxPublishConcurrency, ingestion.MaxPublishConcurrency); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbWorkQueueMemoryLimit, + ingestion.OverviewDbWorkQueueMemoryLimit); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbMinPublisherWorkers, + ingestion.OverviewDbMinPublisherWorkers); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbMaxPublisherWorkers, + ingestion.OverviewDbMaxPublisherWorkers); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenOverviewDbMaxPublisherWorkersLessThanMin() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.OverviewDbMinPublisherWorkers = 4; + options.ArticleIngestion.OverviewDbMaxPublisherWorkers = 2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + "OverviewDbMaxPublisherWorkers", + NntpdOptionsValidator.JoinFailures(result), + StringComparison.Ordinal); + } + + [Fact] + public void Validate_Fails_WhenMaxWorkersLessThanMinWorkers() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.MinWorkers = 4; + options.ArticleIngestion.MaxWorkers = 2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("MaxWorkers", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Fails_WhenScaleUpThresholdNotGreaterThanScaleDown() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.ScaleUpPressureThreshold = 0.2; + options.ArticleIngestion.ScaleDownPressureThreshold = 0.2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("ScaleUpPressureThreshold", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(513)] + public void Validate_Fails_ForInvalidMinWorkers(int minWorkers) + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.MinWorkers = minWorkers; + options.ArticleIngestion.MaxWorkers = Math.Max(minWorkers, 1); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("MinWorkers", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void BindConfiguration_HonoursIngestionWorkerPoolSettings() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:ArticleIngestion:MinWorkers"] = "3", + ["Nntpd:ArticleIngestion:MaxWorkers"] = "12", + ["Nntpd:ArticleIngestion:MaxPublishConcurrency"] = "8", + ["Nntpd:ArticleIngestion:OverviewDbWorkQueueMemoryLimit"] = "536870912", + ["Nntpd:ArticleIngestion:OverviewDbMinPublisherWorkers"] = "3", + ["Nntpd:ArticleIngestion:OverviewDbMaxPublisherWorkers"] = "10", + ["Nntpd:ArticleIngestion:ScaleIntervalSeconds"] = "5", + ["Nntpd:ArticleIngestion:ScaleUpPressureThreshold"] = "0.55", + ["Nntpd:ArticleIngestion:ScaleDownPressureThreshold"] = "0.15", + ["Nntpd:ArticleIngestion:ScaleUpConsecutiveIntervals"] = "4", + ["Nntpd:ArticleIngestion:ScaleDownConsecutiveIntervals"] = "6", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(3, options.ArticleIngestion.MinWorkers); + Assert.Equal(12, options.ArticleIngestion.MaxWorkers); + Assert.Equal(8, options.ArticleIngestion.MaxPublishConcurrency); + Assert.Equal(536870912, options.ArticleIngestion.OverviewDbWorkQueueMemoryLimit); + Assert.Equal(3, options.ArticleIngestion.OverviewDbMinPublisherWorkers); + Assert.Equal(10, options.ArticleIngestion.OverviewDbMaxPublisherWorkers); + Assert.Equal(5, options.ArticleIngestion.ScaleIntervalSeconds); + Assert.Equal(0.55, options.ArticleIngestion.ScaleUpPressureThreshold); + Assert.Equal(0.15, options.ArticleIngestion.ScaleDownPressureThreshold); + Assert.Equal(4, options.ArticleIngestion.ScaleUpConsecutiveIntervals); + Assert.Equal(6, options.ArticleIngestion.ScaleDownConsecutiveIntervals); + + var valid = TestHostFactory.CreateValidOptions(); + valid.ArticleIngestion = options.ArticleIngestion; + Assert.True(CreateValidator().Validate(null, valid).Succeeded); + } + + [Fact] + public void LogDir_DefaultsToLogsSlash() + { + Assert.Equal("logs/", NntpdOptions.DefaultLogDir); + Assert.Equal(NntpdOptions.DefaultLogDir, new NntpdOptions().LogDir); + Assert.Equal(NntpdOptions.DefaultLogDir, TestHostFactory.CreateValidOptions().LogDir); + Assert.True(CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()).Succeeded); + } + + [Fact] + public void Validate_Fails_ForEmptyLogDir() + { + var options = TestHostFactory.CreateValidOptions(); + options.LogDir = " "; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains(nameof(NntpdOptions.LogDir), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForEmptyApplicationName() + { + var options = TestHostFactory.CreateValidOptions(); + options.ApplicationName = " "; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForTooShortShutdownTimeout() + { + var options = TestHostFactory.CreateValidOptions(); + options.GracefulShutdownTimeout = TimeSpan.FromMilliseconds(100); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForTooShortStartupTimeout() + { + var options = TestHostFactory.CreateValidOptions(); + options.StartupTimeout = TimeSpan.FromMilliseconds(10); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForInvalidWatchdogIntervalFraction() + { + var options = TestHostFactory.CreateValidOptions(); + options.Systemd = new SystemdOptions { WatchdogIntervalFraction = 1.5 }; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(4)] + [InlineData(8)] + [InlineData(16)] + public void Validate_Succeeds_ForValidStreamOutstandingArticleDepth(int depth) + { + var options = TestHostFactory.CreateValidOptions(); + options.Transit.StreamOutstandingArticleDepth = depth; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(3)] + [InlineData(17)] + [InlineData(0)] + [InlineData(64)] + public void Validate_Fails_ForInvalidStreamOutstandingArticleDepth(int depth) + { + var options = TestHostFactory.CreateValidOptions(); + options.Transit.StreamOutstandingArticleDepth = depth; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("StreamOutstandingArticleDepth", StringComparison.Ordinal)); + } + + [Fact] + public void Transit_WantTrashAndLogTrash_DefaultTrue() + { + var transit = new TransitOptions(); + Assert.True(transit.WantTrash); + Assert.True(transit.LogTrash); + var options = TestHostFactory.CreateValidOptions(); + Assert.True(options.Transit.WantTrash); + Assert.True(options.Transit.LogTrash); + } +} + +public sealed class NntpdConfigurationTests +{ + [Fact] + public void Bind_StandardizedPascalCaseNames_FromConfiguration() + { + // Isolated JSON (not the working-tree appsettings) so bind defaults and PascalCase names are asserted. + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(string.Empty, options.CloudFlareApiKey); + Assert.Equal("usenet.ninja", options.DnsSuffix); + Assert.Equal(1, options.ServerId); + Assert.Equal("reader01.usenet.ninja", options.Fqdn); + } + + [Fact] + public void Bind_ConsumesNntpdSectionBindSettings() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "127.0.0.1" ], + "BindPort": 1199, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_NntpdTransit_RejectsDatabaseOwnedWantTrashAndLogTrash() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "Transit": { + "LogTrash": false, + "WantTrash": true + } + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = Assert.Throws( + () => _ = host.Services.GetRequiredService>().Value); + Assert.Contains("nntptransitglobalrevision", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Bind_NntpdTransit_KeepsStreamOutstandingArticleDepth() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "Transit": { + "StreamOutstandingArticleDepth": 12 + } + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(12, options.Transit.StreamOutstandingArticleDepth); + Assert.True(options.Transit.WantTrash); + Assert.True(options.Transit.LogTrash); + } + + [Fact] + public void Bind_DoesNotConsumeRootLevelBindSettings() + { + var json = """ + { + "BindAddress": [ "203.0.113.10" ], + "BindPort": 2119, + "BindPortTls": 2563, + "Nntpd": { + "BindAddress": [ "127.0.0.1" ], + "BindPort": 1199, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json, forceTlsDisabled: false); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_RootLevelBindSettingsAlone_DoNotReplaceNntpdDefaults() + { + var json = """ + { + "BindAddress": [ "203.0.113.10" ], + "BindPort": 2119, + "BindPortTls": 2563, + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json, forceTlsDisabled: false); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + } + + [Fact] + public void Bind_OverlayPreservesNntpdBindPortTlsAgainstRoot() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "127.0.0.1", + [$"{NntpdOptions.SectionName}:BindPort"] = "1199", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "5633", + ["BindAddress:0"] = "203.0.113.10", + ["BindPort"] = "2119", + ["BindPortTls"] = "2563", + }) + .Build(); + + var options = new NntpdOptions(); + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(5633, options.BindPortTls); + Assert.True(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_HasNoDedicatedBindEnvironmentVariableConstants() + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy; + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindAddressEnvironmentVariable", flags)); + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindPortEnvironmentVariable", flags)); + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindPortTlsEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindAddressEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindPortEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindPortTlsEnvironmentVariable", flags)); + } + + [Fact] + public void Bind_HonoursConfiguredLogDir() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = "D:/nntp-logs/", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal("D:/nntp-logs/", options.LogDir); + var valid = TestHostFactory.CreateValidOptions(); + valid.LogDir = options.LogDir; + var validator = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)); + Assert.True(validator.Validate(null, valid).Succeeded); + } + + [Fact] + public void ProductionAppsettings_DeclaresIdleTimeDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal(NntpdOptions.DefaultIdleTime, doc.RootElement.GetProperty("Nntpd").GetProperty("IdleTime").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DeclaresMaxArticleSizeDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal( + NntpdOptions.DefaultMaxArticleSize, + doc.RootElement.GetProperty("Nntpd").GetProperty("MaxArticleSize").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DeclaresMailComplaintsToDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal( + NntpdOptions.DefaultMailComplaintsTo, + doc.RootElement.GetProperty("Nntpd").GetProperty("MailComplaintsTo").GetString()); + } + + [Fact] + public void ProductionAppsettings_DeclaresTop1000Recipients() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var top = doc.RootElement.GetProperty("Nntpd").GetProperty("Top1000"); + Assert.Equal(2, top.GetArrayLength()); + Assert.Equal("top1000@anthologeek.net", top[0].GetString()); + Assert.Equal("cknipe@opticnetworks.net", top[1].GetString()); + } + + [Fact] + public void ProductionAppsettings_DoesNotDeclareXTraceKey() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("XTraceKey", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("XTracePreviousKey", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("NewsmasterUser", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("NewsmasterPassword", out _)); + } + + [Fact] + public void ProductionAppsettings_DeclaresBindSettingsUnderNntpdOnly() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var root = doc.RootElement; + var nntpd = root.GetProperty("Nntpd"); + + Assert.False(root.TryGetProperty("BindAddress", out _)); + Assert.False(root.TryGetProperty("BindPort", out _)); + Assert.False(root.TryGetProperty("BindPortTls", out _)); + Assert.Equal("198.18.0.66", nntpd.GetProperty("BindAddress")[0].GetString()); + Assert.Equal("2c0f:f030:1442:501:198:18:0:66", nntpd.GetProperty("BindAddress")[1].GetString()); + Assert.Equal(1199, nntpd.GetProperty("BindPort").GetInt32()); + Assert.Equal(5633, nntpd.GetProperty("BindPortTls").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DoesNotDeclareSharedConfigColumns() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var root = doc.RootElement; + var nntpd = root.GetProperty("Nntpd"); + + Assert.False(root.TryGetProperty("CloudFlareZoneId", out _)); + Assert.False(root.TryGetProperty("DnsSuffix", out _)); + Assert.False(root.TryGetProperty("AcmeDirectoryUrl", out _)); + Assert.False(root.TryGetProperty("AcmeRenewalThresholdDays", out _)); + Assert.False(nntpd.TryGetProperty("CloudFlareZoneId", out _)); + Assert.False(nntpd.TryGetProperty("DnsSuffix", out _)); + Assert.False(nntpd.TryGetProperty("AcmeDirectoryUrl", out _)); + Assert.False(nntpd.TryGetProperty("AcmeRenewalThresholdDays", out _)); + } + + [Fact] + public void Bind_ConsumesNntpdZoneIdAndDnsSuffix() + { + var json = """ + { + "CloudFlareZoneId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "DnsSuffix": "root-must-not-bind.example", + "Nntpd": { + "BindAddress": [ "*" ], + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "example.test", + "ServerId": 7 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal("example.test", options.DnsSuffix); + Assert.Equal("reader07.example.test", options.Fqdn); + } + + [Fact] + public void ProductionAppsettings_DeclaresLogDirDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal("logs/", doc.RootElement.GetProperty("Nntpd").GetProperty("LogDir").GetString()); + Assert.Equal( + "Information", + doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[0] + .GetProperty("Args").GetProperty("restrictedToMinimumLevel").GetString()); + Assert.Equal( + "Debug", + doc.RootElement.GetProperty("Serilog").GetProperty("MinimumLevel") + .GetProperty("Override").GetProperty("VectorNNTPD.ReaderService").GetString()); + var async = doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[1]; + Assert.Equal("Async", async.GetProperty("Name").GetString()); + Assert.Equal(50000, async.GetProperty("Args").GetProperty("bufferSize").GetInt32()); + Assert.True(async.GetProperty("Args").GetProperty("blockWhenFull").GetBoolean()); + Assert.Equal( + "Debug", + async.GetProperty("Args").GetProperty("configure")[0].GetProperty("Args") + .GetProperty("restrictedToMinimumLevel").GetString()); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } + + [Fact] + public void OldUnderscoreSeparatedNames_AreNotBound() + { + // Underscore-separated legacy keys are not aliases. (DnsSuffix vs dnssuffix is the same + // case-insensitive configuration key and is therefore not covered here.) + var json = """ + { + "Nntpd": { + "bind_address": [ "198.18.0.66" ], + "bind_port": 1199, + "bind_port_tls": 5633, + "cloudflare_api_key": "should-not-bind", + "cloudflare_zone_id": "should-not-bind-zone", + "server_id": 42, + "ServerId": 7, + "DnsSuffix": "example.test", + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .PostConfigure(static o => + { + if (o.BindAddress is null || o.BindAddress.Length == 0) + { + o.BindAddress = ["*"]; + } + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.Equal(7, options.ServerId); + Assert.Equal("example.test", options.DnsSuffix); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(TestHostFactory.TestCloudFlareApiKey, options.CloudFlareApiKey); + Assert.NotEqual("should-not-bind", options.CloudFlareApiKey); + Assert.NotEqual("should-not-bind-zone", options.CloudFlareZoneId); + } + + [Fact] + public void EnvironmentVariables_UseExactCloudFlareNames() + { + const string envKey = "env-override-cloudflare-api-key"; + + var previousKey = Environment.GetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable); + + try + { + Environment.SetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, envKey); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:CloudFlareApiKey"] = "from-json-should-be-overridden", + [$"{NntpdOptions.SectionName}:CloudFlareZoneId"] = "5811a29d39a0732afb5f160c9b137c3d", + }) + .AddVectorEnvironmentVariables() + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .BindConfiguration(NntpdOptions.SectionName) + .Configure(static (options, config) => + { + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme(options, config); + }) + .PostConfigure(static options => + { + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + options.CloudFlareApiKey = string.Empty; + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.NotEqual(envKey, options.CloudFlareApiKey); + Assert.NotEqual("from-json-should-be-overridden", options.CloudFlareApiKey); + Assert.Equal(string.Empty, options.CloudFlareApiKey); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, "VECTOR__CLOUDFLAREAPIKEY"); + } + finally + { + Environment.SetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, previousKey); + } + } + + [Fact] + public void ValidationErrors_DoNotExposeApiKey() + { + const string secret = "super-secret-cloudflare-api-key-value"; + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = secret; + options.BindPort = 0; + + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + + Assert.True(result.Failed); + var joined = NntpdOptionsValidator.JoinFailures(result); + Assert.DoesNotContain(secret, joined, StringComparison.Ordinal); + Assert.False(NntpdOptionsValidator.ContainsSecret(joined, secret)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("\t")] + public void CloudFlareApiKey_BlankOrMissing_DoesNotFailOptionsValidation(string? apiKey) + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = apiKey!; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void CloudFlareApiKey_ValidNonblank_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = TestHostFactory.TestCloudFlareApiKey; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("\t")] + public void CloudFlareZoneId_BlankOrMissing_FailsSharedConfiguration(string? zoneId) + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + zoneId, + "usenet.ninja"), + ])); + Assert.Contains("cloudflarezoneid", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void CloudFlareZoneId_ValidNonblank_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareZoneId = "5811a29d39a0732afb5f160c9b137c3d"; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingCloudFlareApiKey_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = " "; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingCloudFlareZoneId_FailsSharedConfiguration() + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + string.Empty, + "usenet.ninja"), + ])); + Assert.Contains("cloudflarezoneid", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Defaults_ApplyWhenListenerSettingsOmitted() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .PostConfigure(static o => + { + if (o.BindAddress is null || o.BindAddress.Length == 0) + { + o.BindAddress = ["*"]; + } + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + Assert.Equal("usenet.ninja", options.DnsSuffix); + } + + [Fact] + public void MissingServerId_FailsValidationAndIsDistinctFromZero() + { + var missing = TestHostFactory.CreateValidOptions(); + missing.ServerId = null; + var missingResult = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, missing); + Assert.True(missingResult.Failed); + Assert.Contains("required", NntpdOptionsValidator.JoinFailures(missingResult), StringComparison.OrdinalIgnoreCase); + + var zero = TestHostFactory.CreateValidOptions(); + zero.ServerId = 0; + var zeroResult = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, zero); + Assert.True(zeroResult.Failed); + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(zeroResult), StringComparison.Ordinal); + + Assert.Null(missing.ServerId); + Assert.Equal(0, zero.ServerId); + } + + [Fact] + public void MissingServerId_FromConfiguration_FailsStartup() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "usenet.ninja" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .ValidateDataAnnotations() + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var ex = Assert.Throws( + () => _ = provider.GetRequiredService>().Value); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("198.18.0.66")] + [InlineData("2001:db8::1")] + public void BindAddress_AssignedAddresses_AreAccepted(string address) + { + var ip = IPAddress.Parse(address); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [address]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(ip)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void BindAddress_MultipleConfiguredAddresses_AreAllValidated() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["198.18.0.66", "2001:db8::1"]; + var assignee = new FakeLocalIpAddressAssignee( + IPAddress.Parse("198.18.0.66"), + IPAddress.Parse("2001:db8::1")); + var result = new NntpdOptionsValidator(assignee).Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void BindAddress_AssignedPrivateAndPublic_AreAccepted() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["10.0.0.5", "203.0.113.10"]; + var assignee = new FakeLocalIpAddressAssignee( + IPAddress.Parse("10.0.0.5"), + IPAddress.Parse("203.0.113.10")); + var result = new NntpdOptionsValidator(assignee).Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void BindAddress_UnassignedAddress_IsRejected() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["198.18.0.66"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(false)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("not assigned", NntpdOptionsValidator.JoinFailures(result), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void BindAddress_InvalidSyntax_IsRejected() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["not-an-ip"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData("*")] + [InlineData("0.0.0.0")] + [InlineData("::")] + public void BindAddress_Wildcards_AreAcceptedWithoutNicAssignment(string wildcard) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [wildcard]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(false)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Theory] + [InlineData(1)] + [InlineData(119)] + [InlineData(65535)] + public void BindPort_ValidRange_Succeeds(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPort = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(65536)] + public void BindPort_InvalidValues_Fail(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPort = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void BindPortTls_UnsetAndZero_DisableTls() + { + var unset = TestHostFactory.CreateValidOptions(); + unset.BindPortTls = default; + Assert.Equal(0, unset.BindPortTls); + Assert.False(unset.IsTlsListenerEnabled); + Assert.True( + new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, unset) + .Succeeded); + + var explicitZero = TestHostFactory.CreateValidOptions(); + explicitZero.BindPortTls = 0; + Assert.False(explicitZero.IsTlsListenerEnabled); + Assert.True( + new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, explicitZero) + .Succeeded); + } + + [Theory] + [InlineData(1)] + [InlineData(563)] + [InlineData(65535)] + public void BindPortTls_NonZero_EnablesTlsConfiguration(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = port; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = "unit-test-pfx-password"; + Assert.True(options.IsTlsListenerEnabled); + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(-1)] + [InlineData(65536)] + public void BindPortTls_OutOfRange_Fails(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(1, "nntpd01.usenet.ninja")] + [InlineData(8, "nntpd08.usenet.ninja")] + [InlineData(9, "nntpd09.usenet.ninja")] + [InlineData(10, "nntpd10.usenet.ninja")] + [InlineData(99, "nntpd99.usenet.ninja")] + public void Fqdn_UsesExactFormat_WithoutDotBeforeId(int serverId, string expected) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + options.DnsSuffix = "usenet.ninja"; + + Assert.Equal(expected, options.Fqdn); + Assert.Equal(NntpdOptions.ApplicationPrefix, "nntpd"); + Assert.Equal( + ApplicationFqdn.Build(NntpdOptions.ApplicationPrefix, serverId, "usenet.ninja"), + options.Fqdn); + Assert.Equal( + ApplicationFqdn.Build(NntpdOptions.ApplicationPrefix, serverId, "usenet.ninja"), + NntpdOptions.FormatFqdn(serverId, "usenet.ninja")); + Assert.NotEqual( + ApplicationFqdn.Build("backfiller", serverId, "usenet.ninja"), + options.Fqdn); + Assert.DoesNotContain("nntpd.", options.Fqdn, StringComparison.Ordinal); + Assert.StartsWith($"nntpd{serverId:00}.", options.Fqdn, StringComparison.Ordinal); + + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(1)] + [InlineData(99)] + [InlineData(100)] + [InlineData(255)] + public void ServerId_ValidBoundaries_Succeed(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(256)] + public void ServerId_OutOfRange_Fails(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Theory] + [InlineData(null, (int)ServerIdValidationStatus.Missing, false)] + [InlineData(-1, (int)ServerIdValidationStatus.OutOfRange, false)] + [InlineData(0, (int)ServerIdValidationStatus.OutOfRange, false)] + [InlineData(1, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(8, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(99, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(100, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(255, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(256, (int)ServerIdValidationStatus.OutOfRange, false)] + public void ServerId_DelegatesNumericRulesToCommon( + int? serverId, + int expectedStatus, + bool expectedValid) + { + Assert.Equal((ServerIdValidationStatus)expectedStatus, ServerIdRules.Classify(serverId)); + + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + + Assert.Equal(expectedValid, result.Succeeded); + if (!expectedValid) + { + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + } + + [Fact] + public void ServerId_RangeAttribute_UsesSharedCommonBounds() + { + var range = typeof(NntpdOptions) + .GetProperty(nameof(NntpdOptions.ServerId)) + ?.GetCustomAttribute(); + Assert.NotNull(range); + Assert.Equal(ServerIdRules.MinimumInclusive, Convert.ToInt32(range.Minimum, CultureInfo.InvariantCulture)); + Assert.Equal(ServerIdRules.MaximumInclusive, Convert.ToInt32(range.Maximum, CultureInfo.InvariantCulture)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("-bad.example")] + [InlineData("bad..example")] + [InlineData("bad_label.example")] + public void DnsSuffix_Invalid_FailsSharedConfiguration(string suffix) + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + "0123456789abcdef0123456789abcdef", + suffix), + ])); + Assert.Contains("dnssuffix", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fqdn_CannotBeIndependentlyConfiguredOrOverridden() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": 119, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "usenet.ninja", + "ServerId": 8, + "Fqdn": "evil.example.com" + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(8, options.ServerId); + Assert.Equal("reader08.usenet.ninja", options.Fqdn); + Assert.DoesNotContain("evil", options.Fqdn, StringComparison.Ordinal); + } + + [Fact] + public async Task HostStart_MissingCloudFlareApiKey_PublishesTheSharedSnapshot() + { + var port = TestHostFactory.GetFreeTcpPort(); + var json = $$""" + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": {{port}}, + "CloudFlareApiKey": "", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + await host.StartAsync(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, options.CloudFlareApiKey); + Assert.NotEqual(string.Empty, options.CloudFlareApiKey); + await host.StopAsync(); + } + + [Fact] + public void HostStart_MissingCloudFlareZoneId_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": " ", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(" ", options.CloudFlareZoneId); + } + + [Fact] + public async Task HostStart_MissingServerId_FailsBeforeRunning() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d" + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + Assert.Contains(NntpdOptions.ServerIdEnvironmentVariable, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task HostStart_ValidMandatorySettings_ReachesRunning() + { + var port = TestHostFactory.GetFreeTcpPort(); + var json = $$""" + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": {{port}}, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + await host.StartAsync(); + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + await host.StopAsync(); + } + + [Fact] + public void CloudFlareApiKey_NullFromConfiguration_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": null, + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.CloudFlareApiKey); + } + + [Fact] + public void CloudFlareZoneId_NullFromConfiguration_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": null, + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.True(string.IsNullOrWhiteSpace(options.CloudFlareZoneId)); + } + + [Fact] + public void ServerId_NullFromConfiguration_FailsStartup() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": null + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = Assert.Throws( + () => _ = host.Services.GetRequiredService>().Value); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void EnvironmentVariables_DocumentedNames_AreExact() + { + Assert.Equal("VECTOR__CLOUDFLAREAPIKEY", NntpdOptions.CloudFlareApiKeyEnvironmentVariable); + Assert.Equal("VECTOR__CLOUDFLAREZONEID", NntpdOptions.CloudFlareZoneIdEnvironmentVariable); + Assert.Equal("NNTPD__SERVERID", NntpdOptions.ServerIdEnvironmentVariable); + Assert.Equal("NNTPD__XTRACEKEY", NntpdOptions.XTraceKeyEnvironmentVariable); + Assert.Equal("NNTPD__XTRACEPREVIOUSKEY", NntpdOptions.XTracePreviousKeyEnvironmentVariable); + Assert.Equal("NNTPD__NEWSMASTERUSER", NntpdOptions.NewsmasterUserEnvironmentVariable); + Assert.Equal("NNTPD__NEWSMASTERPASSWORD", NntpdOptions.NewsmasterPasswordEnvironmentVariable); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.ServerIdEnvironmentVariable)); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.XTraceKeyEnvironmentVariable)); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.NewsmasterPasswordEnvironmentVariable)); + } + + [Fact] + public void ApplicationSpecificSettings_BindFromNntpdSection_NotVectorRootKeys() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Nntpd:ServerId"] = "12", + ["Nntpd:XTraceKey"] = TestHostFactory.TestXTraceKey, + ["ServerId"] = "99", + ["XTraceKey"] = new string('a', 64), + ["NewsmasterPassword"] = "vector-must-not-bind", + }) + .Build(); + + var options = new NntpdOptions(); + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + AcmeCloudflareOptions.OverlaySharedFromRoot(options, configuration); + + Assert.Equal(12, options.ServerId); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + Assert.True(string.IsNullOrEmpty(options.NewsmasterPassword)); + } + + private static IHost CreateEmptyNntpdHost(string json, bool forceTlsDisabled = true) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.ReaderService.Tests", + EnvironmentName = Environments.Development, + }); + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + builder.Configuration.AddJsonStream(jsonStream); + TestHostFactory.PromoteSharedKeysToRoot(builder.Configuration); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + }); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.PostConfigure(options => + { + if (forceTlsDisabled) + { + options.BindPortTls = 0; + } + + options.AcmeEmail = string.Empty; + }); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }); + builder.Services.AddSingleton(); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static IHost CreateConfiguredHost( + IReadOnlyDictionary? configuration = null, + ILocalIpAddressAssignee? assignee = null) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = + TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindAddress:1"] = null, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }); + + if (configuration is not null) + { + builder.Configuration.AddInMemoryCollection(configuration); + } + + builder.Services.AddSingleton( + assignee ?? new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.PostConfigure(static options => + { + options.BindAddress = ["*"]; + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + }); + TestHostFactory.IsolateTransit(builder.Services); + + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs new file mode 100644 index 00000000..cc595c3f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs @@ -0,0 +1,286 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Core.Lifecycle; + +[Collection(SerilogCollection.Name)] +public sealed class ApplicationLifecycleDisposalTests +{ + [Fact] + public async Task HostStopThenDispose_DoesNotThrowObjectDisposedException() + { + using var host = CreateHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await host.StopAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + // DI disposal previously reproduced ObjectDisposedException from DisposeAsync → StopAsync. + var dispose = Record.ExceptionAsync(async () => + { + if (host is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + host.Dispose(); + } + }); + + Assert.Null(await dispose); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartStopDispose_CompletesWithoutObjectDisposedException() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + var dispose = await Record.ExceptionAsync(async () => await lifecycle.DisposeAsync()); + Assert.Null(dispose); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_AfterNormalStop_IsIdempotent() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + await lifecycle.DisposeAsync(); + await lifecycle.DisposeAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_WhileRunning_StopsServicesOnce() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await lifecycle.DisposeAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_SurfacesShutdownFailure_WhenDisposeTriggersStop() + { + var stopAttempts = 0; + var service = new FakeApplicationService( + "bad", + onStop: _ => + { + Interlocked.Increment(ref stopAttempts); + throw new InvalidOperationException("dispose-stop-fail"); + }); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + async () => await lifecycle.DisposeAsync()); + Assert.Contains(ex.InnerExceptions, e => e.Message == "dispose-stop-fail"); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, stopAttempts); + + // Subsequent dispose is idempotent and does not rethrow or re-stop. + await lifecycle.DisposeAsync(); + Assert.Equal(1, stopAttempts); + } + + [Fact] + public async Task ConcurrentStopRequests_ShareSingleShutdown() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "svc", + onStop: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + var lifecycle = TestHostFactory.CreateLifecycle( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await lifecycle.StartAsync(CancellationToken.None); + + var stop1 = lifecycle.StopAsync(CancellationToken.None); + await entered.Task; + var stop2 = lifecycle.StopAsync(CancellationToken.None); + var stop3 = lifecycle.StopAsync(CancellationToken.None); + + release.TrySetResult(); + await Task.WhenAll(stop1, stop2, stop3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + + await lifecycle.DisposeAsync(); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StopAsync_AfterDispose_WhenAlreadyStopped_DoesNotThrow() + { + var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + await lifecycle.DisposeAsync(); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task ConcurrentStopAndDispose_DoNotThrowOrDoubleStopServices() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + await lifecycle.StartAsync(CancellationToken.None); + + var stop = lifecycle.StopAsync(CancellationToken.None); + var dispose = lifecycle.DisposeAsync().AsTask(); + + await Task.WhenAll(stop, dispose); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + + await lifecycle.DisposeAsync(); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task ConcurrentDisposeRequests_AreSafe() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + await lifecycle.StartAsync(CancellationToken.None); + + var d1 = lifecycle.DisposeAsync().AsTask(); + var d2 = lifecycle.DisposeAsync().AsTask(); + var d3 = lifecycle.DisposeAsync().AsTask(); + + await Task.WhenAll(d1, d2, d3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task ShutdownFailure_RemainsObservable_OnStopAsync() + { + var service = new FakeApplicationService( + "bad", + onStop: _ => throw new InvalidOperationException("stop-fail")); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StopAsync(CancellationToken.None)); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-fail"); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartupFailure_FollowedByDispose_Succeeds() + { + var service = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("start-fail")); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + await lifecycle.DisposeAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task ShutdownCancellation_Propagates() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "slow", + onStop: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await lifecycle.StartAsync(CancellationToken.None); + + using var cts = new CancellationTokenSource(); + var stop = lifecycle.StopAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => stop); + } + + [Fact] + public void HostWithSerilog_DoesNotRegisterMicrosoftConsoleProvider() + { + using var host = CreateHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.DoesNotContain(host.Services.GetLoggerProviders(), static p => p is ConsoleLoggerProvider); + Assert.Empty(host.Services.GetLoggerProviders()); + } + + private static IHost CreateHost(Action? configureServices = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs new file mode 100644 index 00000000..ff8413b1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs @@ -0,0 +1,189 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Tests.Core.Lifecycle; + +public sealed class ApplicationLifecycleTests +{ + [Fact] + public async Task StartAsync_TransitionsCreatedToRunning() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + Assert.Equal(ApplicationState.Created, lifecycle.State); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Equal(1, service.StartCount); + } + + [Fact] + public async Task StopAsync_TransitionsRunningToStopped() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StartAsync_WhenAlreadyRunning_ThrowsInvalidOperation() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Contains("Invalid application lifecycle transition", ex.Message, StringComparison.Ordinal); + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task ConcurrentStart_SecondCallerFailsPredictably() + { + var releaseStart = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var enteredStart = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + enteredStart.TrySetResult(); + await releaseStart.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var first = lifecycle.StartAsync(CancellationToken.None); + await enteredStart.Task; + + var second = lifecycle.StartAsync(CancellationToken.None); + + releaseStart.TrySetResult(); + await first; + + await Assert.ThrowsAsync(() => second); + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task RepeatedStop_IsIdempotent() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var stop1 = lifecycle.StopAsync(CancellationToken.None); + var stop2 = lifecycle.StopAsync(CancellationToken.None); + var stop3 = lifecycle.StopAsync(CancellationToken.None); + + await Task.WhenAll(stop1, stop2, stop3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StopAsync_BeforeStart_TransitionsToStopped() + { + await using var lifecycle = TestHostFactory.CreateLifecycle([]); + + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartupFailure_EndsInStopped_AndRethrows() + { + var ok = new FakeApplicationService("ok"); + var bad = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([ok, bad]); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Equal("boom", ex.Message); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, ok.StartCount); + Assert.Equal(1, ok.StopCount); + Assert.Equal(0, bad.StartCount); + } + + [Fact] + public async Task StartupCancellation_PropagatesAndStopsCleanly() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var cts = new CancellationTokenSource(); + + var start = lifecycle.StartAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task State_IsVisibleAcrossThreads() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var start = lifecycle.StartAsync(CancellationToken.None); + await entered.Task; + + var observed = await Task.Run(() => lifecycle.State); + Assert.Equal(ApplicationState.Starting, observed); + + release.TrySetResult(); + await start; + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task UnexpectedServiceTermination_IsObservable() + { + var service = new FakeApplicationService("a", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + service.CompleteExecution(new InvalidOperationException("service crashed")); + + await Assert.ThrowsAsync( + () => lifecycle.WaitAsync(CancellationToken.None)); + Assert.True(lifecycle.UnexpectedTermination.IsCompletedSuccessfully); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Core/Services/ApplicationServiceManagerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Core/Services/ApplicationServiceManagerTests.cs new file mode 100644 index 00000000..69d4c158 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Core/Services/ApplicationServiceManagerTests.cs @@ -0,0 +1,371 @@ +using System.Diagnostics; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Tests.Core.Services; + +public sealed class ApplicationServiceManagerTests +{ + [Fact] + public async Task StartAsync_StartsServicesInRegistrationOrder() + { + var order = new List(); + var a = new FakeApplicationService("a") { SharedStartOrder = order }; + var b = new FakeApplicationService("b") { SharedStartOrder = order }; + var c = new FakeApplicationService("c") { SharedStartOrder = order }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + await manager.StartAsync(CancellationToken.None); + + Assert.Equal(["a", "b", "c"], order); + Assert.Equal(3, manager.StartedServices.Count); + } + + [Fact] + public async Task StopAsync_StopsServicesInReverseOrder() + { + var startOrder = new List(); + var stopOrder = new List(); + var a = new FakeApplicationService("a") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var b = new FakeApplicationService("b") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var c = new FakeApplicationService("c") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + Assert.Equal(["a", "b", "c"], startOrder); + Assert.Equal(["c", "b", "a"], stopOrder); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task StartupFailure_RollsBackStartedServices() + { + var stopOrder = new List(); + var a = new FakeApplicationService("a") { SharedStopOrder = stopOrder }; + var b = new FakeApplicationService( + "b", + onStart: _ => throw new InvalidOperationException("fail-b")) + { + SharedStopOrder = stopOrder, + }; + var c = new FakeApplicationService("c") { SharedStopOrder = stopOrder }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + var ex = await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + + Assert.Equal("fail-b", ex.Message); + Assert.Equal(1, a.StartCount); + Assert.Equal(1, a.StopCount); + Assert.Equal(0, b.StartCount); + Assert.Equal(0, c.StartCount); + Assert.Equal(["a"], stopOrder); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task ShutdownFailure_IsAggregatedAndSurfaced() + { + var a = new FakeApplicationService( + "a", + onStop: _ => throw new InvalidOperationException("stop-a")); + var b = new FakeApplicationService( + "b", + onStop: _ => throw new InvalidOperationException("stop-b")); + var manager = TestHostFactory.CreateServiceManager([a, b]); + + await manager.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + Assert.Equal(2, ex.InnerExceptions.Count); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-b"); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-a"); + } + + [Fact] + public async Task CancellationDuringStartup_RollsBackAndThrows() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var a = new FakeApplicationService("a"); + var b = new FakeApplicationService( + "b", + onStart: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + var manager = TestHostFactory.CreateServiceManager([a, b]); + using var cts = new CancellationTokenSource(); + + var start = manager.StartAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, a.StartCount); + Assert.Equal(1, a.StopCount); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task CancellationDuringShutdown_Throws() + { + var enteredStop = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var a = new FakeApplicationService("a"); + var b = new FakeApplicationService( + "b", + onStop: async ct => + { + enteredStop.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + var manager = TestHostFactory.CreateServiceManager([a, b]); + + await manager.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(); + + var stop = manager.StopAsync(cts.Token); + await enteredStop.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => stop); + + // Reverse order: b canceled first; a must still receive a best-effort abort and leave tracking. + Assert.Empty(manager.StartedServices); + Assert.Equal(1, b.StopAttemptCount); + Assert.Equal(1, a.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_ThrowsTimeoutException() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50)); + var manager = TestHostFactory.CreateServiceManager([service], options); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task GracefulShutdownTimeout_ClearsTracking_AndStillAttemptsEarlierServices() + { + // Reverse stop order: slow (started second) consumes the overall budget first; cooperative + // (started first) must still be offered a stop attempt and leave StartedServices empty. + var cooperative = new FakeApplicationService("cooperative"); + var slow = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50)); + var manager = TestHostFactory.CreateServiceManager([cooperative, slow], options); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + + Assert.Empty(manager.StartedServices); + Assert.Equal(1, slow.StopAttemptCount); + Assert.Equal(1, cooperative.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_IsOverallBudget_NotPerService() + { + // Two non-cooperative stops must not receive a refreshed full timeout each. + var first = new FakeApplicationService("first") { HoldStopUntilReleased = true }; + var second = new FakeApplicationService("second") { HoldStopUntilReleased = true }; + var timeout = TimeSpan.FromMilliseconds(80); + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: timeout); + var manager = TestHostFactory.CreateServiceManager([first, second], options); + + await manager.StartAsync(CancellationToken.None); + + var sw = Stopwatch.StartNew(); + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + sw.Stop(); + + Assert.Empty(manager.StartedServices); + // Per-service refresh would take ~2× timeout; overall budget should stay near one window. + Assert.True( + sw.Elapsed < timeout + TimeSpan.FromMilliseconds(250), + $"Shutdown elapsed {sw.Elapsed} exceeded overall budget expectation for timeout {timeout}."); + Assert.Equal(1, second.StopAttemptCount); + Assert.Equal(1, first.StopAttemptCount); + } + + [Fact] + public async Task ShutdownException_RemovesServiceFromTracking() + { + var a = new FakeApplicationService( + "a", + onStop: _ => throw new InvalidOperationException("stop-a")); + var manager = TestHostFactory.CreateServiceManager([a]); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task DuplicateStart_ThrowsInvalidOperation() + { + var manager = TestHostFactory.CreateServiceManager([new FakeApplicationService("a")]); + + await manager.StartAsync(CancellationToken.None); + await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task ConcurrentLifecycleOperations_AreRejected() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + var manager = TestHostFactory.CreateServiceManager([service]); + + var start = manager.StartAsync(CancellationToken.None); + await entered.Task; + + await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + release.TrySetResult(); + await start; + } + + [Fact] + public async Task UnexpectedExecutionFault_RaisesEvent() + { + var service = new FakeApplicationService("a", withExecution: true); + var manager = TestHostFactory.CreateServiceManager([service]); + var observed = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + manager.UnexpectedServiceTermination += (_, args) => observed.TrySetResult(args); + + await manager.StartAsync(CancellationToken.None); + service.CompleteExecution(new InvalidOperationException("crash")); + + var args = await observed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal("a", args.ServiceName); + Assert.False(args.CompletedNormally); + Assert.IsType(args.Exception); + } + + [Fact] + public async Task ConcurrentStopAsync_IsRejected() + { + var service = new FakeApplicationService("a") + { + HoldStopUntilReleased = true, + }; + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await manager.StartAsync(CancellationToken.None); + + var stop1 = manager.StopAsync(CancellationToken.None); + await service.StopEntered.Task; + + await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + service.StopGate.TrySetResult(); + await stop1; + + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_NonCooperativeStop_KeepsAwaitingUntilReturn_ThenTimesOut() + { + var service = new FakeApplicationService("stubborn") + { + HoldStopUntilReleased = true, + IgnoreStopCancellation = true, + }; + var timeout = TimeSpan.FromMilliseconds(80); + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: timeout)); + + await manager.StartAsync(CancellationToken.None); + + var stop = manager.StopAsync(CancellationToken.None); + await service.StopEntered.Task; + + // Manager must not abandon the in-flight StopAsync when the budget elapses. + await Assert.ThrowsAsync(() => stop.WaitAsync(timeout + TimeSpan.FromMilliseconds(200))); + Assert.Single(manager.StartedServices); + + service.StopGate.TrySetResult(); + + await Assert.ThrowsAsync(() => stop); + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + Assert.Equal(1, service.StopCount); + + // Repeated stop must not re-invoke a service already removed from tracking. + await manager.StopAsync(CancellationToken.None); + Assert.Equal(1, service.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_DoesNotDoubleStop_AfterTimeoutClearsTracking() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50))); + + await manager.StartAsync(CancellationToken.None); + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + + await manager.StopAsync(CancellationToken.None); + Assert.Equal(1, service.StopAttemptCount); + } + + [Fact] + public async Task ServicesAreNotStartedTwice_AfterSuccessfulStartAndStop() + { + var service = new FakeApplicationService("a"); + var manager = TestHostFactory.CreateServiceManager([service]); + + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + // After full stop, started list is empty but duplicate start is blocked by design + // until we allow restart. Phase 0 treats restart as unsupported on the same manager + // only when started list is non-empty; empty allows start again. + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + Assert.Equal(2, service.StartCount); + Assert.Equal(2, service.StopCount); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/1 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/1 new file mode 100644 index 00000000..0a464332 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/1 @@ -0,0 +1,11 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Test article +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Organization: Leading space tab + folded header and trailing space. +Message-ID: +Xref: news.example.com example.test:1 + +This is a test message to example.test in native format. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/2 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/2 new file mode 100644 index 00000000..35e5862d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/2 @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.config +Subject: Test article #2 +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:54 -0800 +Message-ID: +Xref: news.example.com example.config:1 + +This is a test message to example.config in native format. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/3 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/3 new file mode 100644 index 00000000..a61360c5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/3 @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test,example.config +Subject: Test article #3 with a | | tab +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com example.test:2 example.config:2 + +This is an example crossposted message. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/4 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/4 new file mode 100644 index 00000000..136c0f48 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/4 @@ -0,0 +1,20 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: This + is an aggressively obnoxious article + for + testing + + overview +Date: Tue, 18 May 2004 01:43:21 -0700 +Expires: Wed, 19 May 2004 01:43:21 -0700 +Message-ID: +References: + +Xref: news.example.com example.test:3 +From: I + Am + User + +This is a header with complex headers for testing overview generation +using makehistory. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/5 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/5 new file mode 100644 index 00000000..ae6f8384 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/5 @@ -0,0 +1,26 @@ +T: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +The^Previous@Header#Is&Exactly*1000+Characters=Long: Whee! +Subject: Torture test for article parsing +TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT: +X-Test: The previous header is also exactly 1000 characters long and has + a colon-space but no content + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + and the previous continuation line is exactly 1000 characters long +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Expires: Wed, 19 May 2004 01:43:21 -0700 +Message-ID: +References: + +Xref: news.example.com example.test:4 +From: I + Am + User +X-Test: This header has the same name as a previous header + . + and has a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +. +nosense in it to torture the article parser. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/6 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/6 new file mode 100644 index 00000000..3cea95e7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/6 @@ -0,0 +1,137 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Long test article +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com example.test:5 + +This is an example article that's longer than 8192 bytes, used to test +large article rejection in article parsing. + +Each of these lines but the last is 60 characters, becoming 63 characters +with dot escaping. Everything before these lines is exactly 607 bytes. +Therefore, this article is exactly 8193 bytes long in wire format, but +less than that in native format. + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +.................... diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/7 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/7 new file mode 100644 index 00000000..3a976bf8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/7 @@ -0,0 +1,16 @@ +.Actually^Valid: Strangely enough +Subject: Wire format testing +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Message-ID: +X-Test: This header has + . + a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +. +nosense in it to torture the article parser. +.. +... +. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/LICENSE.ISC.txt b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/LICENSE.ISC.txt new file mode 100644 index 00000000..2fd0df6c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/LICENSE.ISC.txt @@ -0,0 +1,22 @@ +Copyright (c) 2004-2026 +by Internet Systems Consortium, Inc. ("ISC") +Copyright (c) 1991, 1994-2003 +by The Internet Software Consortium and Rich Salz + +This code is derived from software contributed to the Internet Software +Consortium by Rich Salz. + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Source: InterNetNews/inn LICENSE at commit 6843efceb8a15774d23e4e2892d08daf54574314 +https://github.com/InterNetNews/inn/blob/6843efceb8a15774d23e4e2892d08daf54574314/LICENSE diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/README.md b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/README.md new file mode 100644 index 00000000..80a81e94 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/README.md @@ -0,0 +1,45 @@ +# INN article test corpus (byte-exact fixtures) + +## Provenance + +| Field | Value | +|-------|-------| +| Repository | [InterNetNews/inn](https://github.com/InterNetNews/inn) | +| Source path | `tests/data/articles` | +| Upstream ref | `main` | +| Upstream commit | `6843efceb8a15774d23e4e2892d08daf54574314` | +| Retrieval date (UTC) | 2026-09-22 | +| Upstream commit date (UTC) | 2026-09-11 | + +Fixtures were imported from the GitHub archive of that exact commit: + +`https://github.com/InterNetNews/inn/archive/6843efceb8a15774d23e4e2892d08daf54574314.zip` + +Paths under `inn-6843efceb8a15774d23e4e2892d08daf54574314/tests/data/articles/` were copied **byte-for-byte**. No line-ending conversion, text decoding, or content normalization was applied. + +Machine-readable per-file hashes and byte characteristics: + +- `inn-articles.manifest.json` (same directory) + +## License / attribution + +INN as a whole (and unmarked code/data in the repository) is covered by the ISC-style license in the upstream `LICENSE` file: + +> Copyright (c) 2004-2026 by Internet Systems Consortium, Inc. ("ISC") +> Copyright (c) 1991, 1994-2003 by The Internet Software Consortium and Rich Salz +> +> Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. +> +> THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Full upstream license text: + +No separate license/notice file was present inside `tests/data/articles` itself at this commit. + +## Framing note (VectorNNTP TAKETHIS experiment) + +These fixtures are INN parser/spool test inputs. Most numeric and `bad-*` files are **LF-only spool articles** and do **not** include an NNTP multiline terminator (`.\r\n` / `\r\n.\r\n`). + +The `wire-*` files are wire-oriented. At this commit, only `wire-7` contains the five-byte `\r\n.\r\n` delimiter and ends with a `.\r\n` terminator suitable for a complete multiline framing benchmark. + +Do **not** treat filename categories as NNTP validity by themselves; use the manifest byte characteristics and framing classification. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-empty b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-empty new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-empty @@ -0,0 +1 @@ + diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-empty b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-empty new file mode 100644 index 00000000..99e9698d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-empty @@ -0,0 +1,4 @@ +Test: This article has a bad header +From: + +Empty header but with a colon-space. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-nospc b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-nospc new file mode 100644 index 00000000..0cea9203 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-nospc @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test:<-he: re + +No space before colon. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-space b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-space new file mode 100644 index 00000000..c0a66b15 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-space @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test : <-here + +Space before colon. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-trunc b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-trunc new file mode 100644 index 00000000..c2359d66 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-hdr-trunc @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test: + +Empty header with no colon-space. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-cont b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-cont new file mode 100644 index 00000000..e9e9b29e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-cont @@ -0,0 +1,12 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Article with too long of a header +T: Testing + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + Testing +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:1 + +This is an article with a header continuation line exceeding MAXARTLINELENGTH. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-hdr b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-hdr new file mode 100644 index 00000000..a133a639 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-long-hdr @@ -0,0 +1,10 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Article with too long of a header +T: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:1 + +This is an article with a header line exceeding MAXARTLINELENGTH. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-msgid b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-msgid new file mode 100644 index 00000000..a98e22ef --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-msgid @@ -0,0 +1,8 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Test article with stray CR | | and no message ID +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Xref: news.example.com example.test:4 + +This is a test message to example.test with a stray CR and no message ID. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-body b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-body new file mode 100644 index 00000000..8aba3720 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-body @@ -0,0 +1,6 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Bad article with no body +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-header b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-header new file mode 100644 index 00000000..b7dec89b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-no-header @@ -0,0 +1,2 @@ + +This is a test article that contains no header. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-body b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-body new file mode 100644 index 00000000..5f6da45b Binary files /dev/null and b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-body differ diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-header b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-header new file mode 100644 index 00000000..09782f23 Binary files /dev/null and b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-nul-header differ diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-subj b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-subj new file mode 100644 index 00000000..0f4e75a1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/bad-subj @@ -0,0 +1,8 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:3 + +This is a test message to example.test with no Subject header. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/inn-articles.manifest.json b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/inn-articles.manifest.json new file mode 100644 index 00000000..867bc24a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/inn-articles.manifest.json @@ -0,0 +1,436 @@ +{ + "repository": "InterNetNews/inn", + "sourcePath": "tests/data/articles", + "upstreamRef": "main", + "upstreamCommit": "6843efceb8a15774d23e4e2892d08daf54574314", + "retrievalDateUtc": "2026-09-22", + "fileCount": 25, + "totalBytes": 19544, + "files": [ + { + "relativePath": "1", + "length": 346, + "sha256": "a4ab547e5f1c1de730f4cf93a20a22e680c0ab2e1b7416cf71393133889bb0e8", + "upstreamFileName": "1", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "2", + "length": 285, + "sha256": "4d57fe8c5ba8193ebaf650e4a295a1af3738c311fb037c9e45784e38640f84aa", + "upstreamFileName": "2", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "3", + "length": 309, + "sha256": "88a3622ad30b41a131b7f0a85e28381dadb51dec803f65e351054a67f765e2e6", + "upstreamFileName": "3", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "4", + "length": 484, + "sha256": "e0c8efc04310d40053bdac08a637b7e9f9b3e0b80fb6c34fc53a1b80bc7d6bab", + "upstreamFileName": "4", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "5", + "length": 3810, + "sha256": "9f0ff1b216d75d25feb0d8e0d7cd181bed84d65447878e58b92c306c96634ad8", + "upstreamFileName": "5", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "6", + "length": 7932, + "sha256": "7db1d0ab9df4fa9318c4f3f2ad344969c034bcba83844c159f2cd0bcf022bbea", + "upstreamFileName": "6", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "7", + "length": 387, + "sha256": "723d2e133a19b3ea1d321b76728a1643d738d8252e952b9e397e29e7ec76a2ef", + "upstreamFileName": "7", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-empty", + "length": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "upstreamFileName": "bad-empty", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-empty", + "length": 81, + "sha256": "72b802aa141b9da402098d7752fb5df9937a69e87a0522bc2ba98620c94e0250", + "upstreamFileName": "bad-hdr-empty", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-nospc", + "length": 74, + "sha256": "5d0b224e4e96b2f7721216ee79b4b6a8384798b6cc21f93f079c33a8625ad42a", + "upstreamFileName": "bad-hdr-nospc", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-space", + "length": 71, + "sha256": "949c2a4e92d9776c3d341140a7b9d0dc5f129920c855b2e01b0d550f03e1c10b", + "upstreamFileName": "bad-hdr-space", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-trunc", + "length": 77, + "sha256": "2c774bced25643912ed6e76f170caf7ced866adb474839b359fb639a114be7d4", + "upstreamFileName": "bad-hdr-trunc", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": true + }, + { + "relativePath": "bad-long-cont", + "length": 1349, + "sha256": "08d6c7f7567826d9c21e6f984574f824f1d984554ddbd262616ea3afaaee73c1", + "upstreamFileName": "bad-long-cont", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-long-hdr", + "length": 1314, + "sha256": "ae69d5b5b0468df45cff370a9ec973bf15f6020431ad11ca5fb8f15dc34acfcc", + "upstreamFileName": "bad-long-hdr", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-msgid", + "length": 293, + "sha256": "d2197e7be69357add2b527fd91e014e5cfd74f31de257be15d94ffac30e51d87", + "upstreamFileName": "bad-msgid", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-no-body", + "length": 191, + "sha256": "f3503b1b140a4a5d962db445d2efcea6e89082b7b186553ca8c428cbe415ddbd", + "upstreamFileName": "bad-no-body", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-no-header", + "length": 49, + "sha256": "e06feff8243eeb2365e3fa0a1eaa4d37db3cd6c1cce26c96e3af45f2bca0295e", + "upstreamFileName": "bad-no-header", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-nul-body", + "length": 310, + "sha256": "6ebe35ae053f2b298cace44b8fc947b9f7a57a7fac0ea2a6b7b420afff738ee2", + "upstreamFileName": "bad-nul-body", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-nul-header", + "length": 303, + "sha256": "8c58d98396c666a09e8dabdbe6c90ec36a2f4100c570811f063159ba07b00ca6", + "upstreamFileName": "bad-nul-header", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-subj", + "length": 267, + "sha256": "db59026c63d0bf3ebc5a7e6d043e218cc0fc7e6a204c8459c1d4e98eb3936266", + "upstreamFileName": "bad-subj", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-7", + "length": 411, + "sha256": "3bb2fe13f24d179db4a79847f7c93cf98359a0ad9991bebe8c37b63b93a44a02", + "upstreamFileName": "wire-7", + "categoryByName": "wire", + "framingClassification": "complete_multiline_wire", + "containsCrlf": true, + "containsLfOnly": false, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": true, + "containsFiveByteCrlfDotCrlf": true, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-no-body", + "length": 191, + "sha256": "f3503b1b140a4a5d962db445d2efcea6e89082b7b186553ca8c428cbe415ddbd", + "upstreamFileName": "wire-no-body", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-strange", + "length": 477, + "sha256": "0bf53004be197496191b2fc25c3332d35aee1b59cd550c8b92f6f18a3d28ec8e", + "upstreamFileName": "wire-strange", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": true, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-truncated", + "length": 115, + "sha256": "1e37f7cf1606ee6bbd3424ebcd47ba9c994d57d2af7c9011829bd617cd2ed92f", + "upstreamFileName": "wire-truncated", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": true + }, + { + "relativePath": "xref", + "length": 417, + "sha256": "f4587f5fb11e872de275548347eeb7ebb5211ddc54913ca43d82d2f5207ea67a", + "upstreamFileName": "xref", + "categoryByName": "other", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + } + ] +} \ No newline at end of file diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-7 b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-7 new file mode 100644 index 00000000..6fe8c626 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-7 @@ -0,0 +1,17 @@ +..Actually^Valid: Strangely enough +Subject: Wire format testing +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Message-ID: +X-Test: This header has + . + a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +.. +nosense in it to torture the article parser. +... +.... +.. +. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-no-body b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-no-body new file mode 100644 index 00000000..8aba3720 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-no-body @@ -0,0 +1,6 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Bad article with no body +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-strange b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-strange new file mode 100644 index 00000000..3c0731a9 Binary files /dev/null and b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-strange differ diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-truncated b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-truncated new file mode 100644 index 00000000..c1667121 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/wire-truncated @@ -0,0 +1,5 @@ +Path: foo!bar +From: example@example.com +Subject: Test +Message-ID: +Date: Mon, 23 Dec 2002 16:00:04 \ No newline at end of file diff --git a/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/xref b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/xref new file mode 100644 index 00000000..6af82e6c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Data/InnArticles/xref @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test,example.config +Subject: Test article #3 with a | | tab +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com blah example:test1:2 example.test1:4 example.test2:-1 example.test3:2 example.test3 example.test2:1roger example.test2: + +This is a test of Xref parsing for message cancellation. diff --git a/tests/VectorNNTPD.ReaderService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs new file mode 100644 index 00000000..2b23d48b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs @@ -0,0 +1,56 @@ +using System.Diagnostics; +using VectorNNTPD.ReaderService.Diagnostics; + +namespace VectorNNTPD.ReaderService.Tests.Diagnostics; + +/// Bucketed ingestion-pipeline histogram contracts used by one-minute telemetry. +public sealed class IngestionPipelineMetricsTests +{ + [Fact] + public void CaptureInterval_ReportsCountAverageAndP95() + { + var metrics = new IngestionPipelineMetrics(); + for (var i = 0; i < 94; i++) + { + metrics.RecordNews(TimeSpan.FromMilliseconds(1)); + } + + for (var i = 0; i < 6; i++) + { + metrics.RecordNews(TimeSpan.FromMilliseconds(100)); + } + + var snapshot = metrics.CaptureInterval(); + Assert.Equal(100, snapshot.News.Count); + Assert.Equal(100, snapshot.News.P95Ms); + Assert.InRange(snapshot.News.AvgMs, 4, 12); + Assert.True(snapshot.News.MaxMs >= 90); + + var empty = metrics.CaptureInterval(); + Assert.Equal(0, empty.News.Count); + Assert.Equal(0, empty.News.P95Ms); + } + + [Fact] + public void ConfirmBusyPercent_IsConfirmSumOverBusyTime() + { + var metrics = new IngestionPipelineMetrics(); + var confirmTicks = (long)(0.040 * Stopwatch.Frequency); + RecordAgo(metrics.RecordPublishAndConfirm, TimeSpan.FromMilliseconds(40)); + metrics.AddBusyTicks((long)(0.100 * Stopwatch.Frequency)); + metrics.AddIdleTicks((long)(0.100 * Stopwatch.Frequency)); + metrics.RecordWorkerItem(Stopwatch.GetTimestamp() - confirmTicks); + + var snapshot = metrics.CaptureInterval(); + Assert.Equal(1, snapshot.WorkerItems); + Assert.InRange(snapshot.BusyPercent, 40, 60); + Assert.InRange(snapshot.ConfirmBusyPercent, 30, 50); + Assert.Equal(1.0, snapshot.ArticlesPerSecond(TimeSpan.FromSeconds(1)), precision: 6); + } + + private static void RecordAgo(Action record, TimeSpan ago) + { + var start = Stopwatch.GetTimestamp() - (long)(ago.TotalSeconds * Stopwatch.Frequency); + record(start); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/EmailDeliveryServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailDeliveryServiceTests.cs new file mode 100644 index 00000000..ae7bd6b1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailDeliveryServiceTests.cs @@ -0,0 +1,270 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Email.Smtp; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class EmailDeliveryServiceTests +{ + [Fact] + public async Task Disabled_DoesNotStartWorkerOrTouchSmtp() + { + using var harness = new EmailSpoolHarness(enabled: false); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Null(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, transport.Calls); + Assert.False(Directory.Exists(harness.Directory)); + } + + [Fact] + public async Task Worker_DeliversPreExistingEmlAndDeletesIt() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.Single(harness.EmlFiles()); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls == 1); + await service.StopAsync(CancellationToken.None); + Assert.Equal(1, transport.Calls); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + } + + [Fact] + public async Task Worker_DoesNotDeliverDeliveredOrFailedFiles() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var deliveredId = EmailSpoolFileName.NewId(); + var failedId = EmailSpoolFileName.NewId(); + await File.WriteAllBytesAsync( + Path.Join(harness.Directory, deliveredId + ".delivered"), + EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + await File.WriteAllBytesAsync( + Path.Join(harness.Directory, "failed", failedId + ".eml"), + EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.NotNull(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, transport.Calls); + Assert.True(File.Exists(Path.Join(harness.Directory, deliveredId + ".delivered"))); + Assert.True(File.Exists(Path.Join(harness.Directory, "failed", failedId + ".eml"))); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task TransientFailure_RetainsEml() + { + using var harness = new EmailSpoolHarness( + smtp: new SmtpOptions + { + Host = "127.0.0.1", + MaxAttempts = 1, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Transient, "4xx") { StatusCode = 451 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls >= 1 && harness.EmlFiles().Length == 1); + await service.StopAsync(CancellationToken.None); + Assert.Single(harness.EmlFiles()); + Assert.Empty(harness.FailedFiles()); + } + + [Fact] + public async Task PermanentFailure_MovesToFailed() + { + using var harness = new EmailSpoolHarness(); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Permanent, "5xx") { StatusCode = 550 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls == 1 && harness.FailedFiles().Length == 1); + await service.StopAsync(CancellationToken.None); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task Transient_IsRetriedThenSucceeds() + { + using var harness = new EmailSpoolHarness( + smtp: new SmtpOptions + { + Host = "127.0.0.1", + MaxAttempts = 3, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Transient, "4xx") { StatusCode = 451 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Accepted, (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + await WaitUntilAsync(() => transport.Calls == 2 && harness.EmlFiles().Length == 0); + await service.StopAsync(CancellationToken.None); + Assert.Equal(2, transport.Calls); + } + + [Fact] + public async Task Shutdown_StopsAcceptingAndRetainsUndelivered() + { + using var harness = new EmailSpoolHarness(); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.False(harness.Spool.IsAccepting); + Assert.Equal( + EmailEnqueueStatus.Unavailable, + (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task Cancellation_RetainsEml() + { + using var harness = new EmailSpoolHarness(); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = new ScriptedSmtpTransport + { + BeforeSend = async cancellationToken => + { + started.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.True(harness.EmlFiles().Length + harness.WorkFiles().Length == 1); + harness.Spool.RecoverClaims(); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task Worker_DeliversThroughFakeSmtp() + { + await using var smtp = new FakeSmtpServer(); + using var harness = new EmailSpoolHarness(smtp: smtp.ClientOptions(SmtpSecurityMode.None)); + var transport = SmtpTransportTests.Create(harness.Options.Smtp); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Accepted, (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + await WaitUntilAsync(() => smtp.LastMessage is not null); + await service.StopAsync(CancellationToken.None); + Assert.Contains("Subject: hello", System.Text.Encoding.ASCII.GetString(smtp.LastMessage!), StringComparison.Ordinal); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task TwoWorkers_DoNotDeliverTheSameFile() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slow = new ScriptedSmtpTransport + { + BeforeSend = async _ => + { + firstStarted.TrySetResult(); + await release.Task; + }, + }; + var fast = new ScriptedSmtpTransport(); + var a = harness.CreateDelivery(slow); + var b = harness.CreateDelivery(fast); + await a.StartAsync(CancellationToken.None); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await b.StartAsync(CancellationToken.None); + Assert.Equal(0, fast.Calls); + release.TrySetResult(); + await WaitUntilAsync(() => slow.Calls == 1); + await a.StopAsync(CancellationToken.None); + await b.StopAsync(CancellationToken.None); + Assert.Equal(1, slow.Calls + fast.Calls); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public void RetryDelay_IsBounded() + { + var smtp = new SmtpOptions + { + InitialRetryDelay = TimeSpan.FromSeconds(2), + MaximumRetryDelay = TimeSpan.FromSeconds(5), + }; + var delay = EmailRetryDelay.Compute(smtp, attempt: 8, new Random(1)); + Assert.True(delay <= TimeSpan.FromSeconds(5)); + Assert.True(delay >= TimeSpan.Zero); + } + + private static async Task WaitUntilAsync(Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (!predicate()) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException(); + } + + await Task.Delay(20); + } + } +} + +internal sealed class ScriptedSmtpTransport : ISmtpTransport +{ + public List Failures { get; init; } = []; + + public Func? BeforeSend { get; init; } + + public int Calls { get; private set; } + + public async Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + if (BeforeSend is not null) + { + await BeforeSend(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + if (Failures.Count > 0) + { + var failure = Failures[0]; + Failures.RemoveAt(0); + throw failure; + } + + return new SmtpSendResult + { + Succeeded = true, + Recipients = [new SmtpRecipientResult(item.Recipients[0], true, 250, "OK")], + DataStatusCode = 250, + }; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/EmailServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailServiceTests.cs new file mode 100644 index 00000000..9cd96885 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailServiceTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class EmailServiceTests +{ + [Fact] + public async Task SendAsync_Disabled_DoesNotCreateSpoolFile() + { + using var harness = new EmailSpoolHarness(enabled: false); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Disabled, result.Status); + Assert.False(Directory.Exists(harness.Directory)); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_Enabled_AcceptsWithoutSmtp() + { + using var harness = new EmailSpoolHarness(); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Accepted, result.Status); + Assert.Contains("spool", result.Detail, StringComparison.Ordinal); + Assert.Single(harness.EmlFiles()); + var bytes = await File.ReadAllBytesAsync(harness.EmlFiles()[0]); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("envelope@example.com", item!.EnvelopeSender.Address); + Assert.NotEqual(0, item.EncodedMessage.Length); + } + + [Fact] + public async Task SendAsync_Injection_IsRejectedWithoutSpoolFile() + { + using var harness = new EmailSpoolHarness(); + var result = await harness.CreateService().SendAsync(TextMessage(subject: "x\r\nBcc: evil@example.com")); + Assert.Equal(EmailEnqueueStatus.Rejected, result.Status); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_StoppedSpool_IsUnavailable() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.StopAccepting(); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Unavailable, result.Status); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_SpoolWriteFailure_ReturnsFailed() + { + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-email-failed-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = "from@example.com", + EnvelopeSender = "envelope@example.com", + Smtp = new SmtpOptions { Host = "127.0.0.1", Port = 25 }, + Spool = new EmailSpoolOptions { Directory = blocker }, + }; + var spool = new FilesystemEmailSpool(Options.Create(options), NullLogger.Instance); + var service = new EmailService( + spool, + new Rfc5322MessageEncoder(), + Options.Create(options), + NullLogger.Instance); + var result = await service.SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Failed, result.Status); + } + finally + { + File.Delete(blocker); + } + } + + [Fact] + public void ChannelEmailQueue_IsGone() + { + Assert.Null(typeof(EmailService).Assembly.GetType("VectorNNTPD.ReaderService.Email.ChannelEmailQueue")); + Assert.Null(typeof(EmailService).Assembly.GetType("VectorNNTPD.ReaderService.Email.IEmailQueue")); + } + + internal static EmailMessage TextMessage(string subject = "hello") => + new() + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Subject = subject, + Body = "body\r\n"u8.ToArray(), + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/EmailSpoolHarness.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailSpoolHarness.cs new file mode 100644 index 00000000..ea4fa7ba --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/EmailSpoolHarness.cs @@ -0,0 +1,101 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Email.Smtp; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +internal sealed class EmailSpoolHarness : IDisposable +{ + public EmailSpoolHarness(bool enabled = true, SmtpOptions? smtp = null) + { + Directory = Path.Join( + Path.GetTempPath(), + "vectornntp-email-spool", + Guid.NewGuid().ToString("N")); + Options = new EmailOptions + { + Enabled = enabled, + DefaultFrom = "from@example.com", + EnvelopeSender = "envelope@example.com", + Smtp = smtp ?? new SmtpOptions + { + Host = "127.0.0.1", + Port = 25, + Security = SmtpSecurityMode.None, + MaxAttempts = 1, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }, + Spool = new EmailSpoolOptions + { + Directory = Directory, + ScanInterval = TimeSpan.FromMilliseconds(20), + ShutdownTimeout = TimeSpan.FromSeconds(2), + }, + }; + Spool = new FilesystemEmailSpool(Microsoft.Extensions.Options.Options.Create(Options), NullLogger.Instance); + } + + public string Directory { get; } + + public EmailOptions Options { get; } + + public FilesystemEmailSpool Spool { get; } + + public EmailService CreateService() => + new( + Spool, + new Rfc5322MessageEncoder(), + Microsoft.Extensions.Options.Options.Create(Options), + NullLogger.Instance); + + public EmailDeliveryService CreateDelivery(ISmtpTransport transport) => + new( + Spool, + transport, + Microsoft.Extensions.Options.Options.Create(Options), + NullLogger.Instance); + + public string[] EmlFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.eml") + : []; + + public string[] WorkFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.wrk") + : []; + + public string[] TmpFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.tmp") + : []; + + public string[] FailedFiles() + { + var failed = Path.Join(Directory, "failed"); + return System.IO.Directory.Exists(failed) + ? System.IO.Directory.GetFiles(failed, "*.eml") + : []; + } + + public void Dispose() + { + try + { + if (System.IO.Directory.Exists(Directory)) + { + System.IO.Directory.Delete(Directory, recursive: true); + } + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (UnauthorizedAccessException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/FilesystemEmailSpoolTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/FilesystemEmailSpoolTests.cs new file mode 100644 index 00000000..0bc0ed5b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/FilesystemEmailSpoolTests.cs @@ -0,0 +1,220 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class FilesystemEmailSpoolTests +{ + [Fact] + public async Task WriteAsync_CreatesDirectoryAndAtomicEml() + { + using var harness = new EmailSpoolHarness(); + Assert.False(Directory.Exists(harness.Directory)); + var path = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(Directory.Exists(harness.Directory)); + Assert.True(File.Exists(path)); + Assert.EndsWith(".eml", path, StringComparison.OrdinalIgnoreCase); + Assert.Empty(harness.TmpFiles()); + var name = Path.GetFileNameWithoutExtension(path); + Assert.True(EmailSpoolFileName.IsId(name)); + Assert.DoesNotContain("hello", name, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("example.com", name, StringComparison.OrdinalIgnoreCase); + var bytes = await File.ReadAllBytesAsync(path); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("envelope@example.com", item!.EnvelopeSender.Address); + Assert.Contains("Subject: hello", Encoding.ASCII.GetString(item.EncodedMessage.Span), StringComparison.Ordinal); + Assert.StartsWith(EmailSpoolRecord.Magic, Encoding.ASCII.GetString(bytes), StringComparison.Ordinal); + } + + [Fact] + public async Task TmpFiles_AreNeverClaimed() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var tmp = Path.Join(harness.Directory, EmailSpoolFileName.NewId() + ".tmp"); + await File.WriteAllBytesAsync(tmp, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + Assert.True(File.Exists(tmp)); + } + + [Fact] + public void NewId_IsOpaqueAndUnique() + { + var ids = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < 64; i++) + { + var id = EmailSpoolFileName.NewId(); + Assert.True(EmailSpoolFileName.IsId(id)); + Assert.True(ids.Add(id)); + } + + Assert.False(EmailSpoolFileName.IsId("not-an-id")); + Assert.False(EmailSpoolFileName.IsId(new string('A', 32))); + } + + [Fact] + public async Task Collision_DoesNotOverwriteExistingEml() + { + using var harness = new EmailSpoolHarness(); + var first = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var original = await File.ReadAllBytesAsync(first); + var second = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.NotEqual(first, second); + Assert.Equal(original, await File.ReadAllBytesAsync(first)); + Assert.Equal(2, harness.EmlFiles().Length); + } + + [Fact] + public void RecoverClaims_RenamesWorkBackToEml() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var wrk = Path.Join(harness.Directory, id + ".wrk"); + File.WriteAllBytes(wrk, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + harness.Spool.RecoverClaims(); + Assert.False(File.Exists(wrk)); + Assert.True(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + } + + [Fact] + public async Task Restart_DiscoversPreExistingEml() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + var claim = restarted.TryClaimNext(); + Assert.NotNull(claim); + Assert.Equal("to@example.com", claim!.Item.Recipients[0].Address); + restarted.Release(claim); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task ConcurrentClaim_IsExclusive() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claims = new EmailSpoolClaim?[8]; + Parallel.For(0, claims.Length, i => + { + claims[i] = harness.Spool.TryClaimNext(); + }); + Assert.Equal(1, claims.Count(static claim => claim is not null)); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.WorkFiles()); + } + + [Fact] + public async Task MalformedFile_IsQuarantined() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var bad = Path.Join(harness.Directory, EmailSpoolFileName.NewId() + ".eml"); + await File.WriteAllTextAsync(bad, "not a spool file"); + Assert.Null(harness.Spool.TryClaimNext()); + Assert.False(File.Exists(bad)); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task DeliveredMarker_IsNeverClaimedRecoveredOrRetransmitted() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var delivered = Path.Join(harness.Directory, id + ".delivered"); + await File.WriteAllBytesAsync(delivered, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + harness.Spool.RecoverClaims(); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + Assert.Null(restarted.TryClaimNext()); + Assert.True(File.Exists(delivered)); + Assert.False(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + } + + [Fact] + public async Task FailedDirectory_IsNeverClaimedOrRequeued() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var failed = Path.Join(harness.Directory, "failed", id + ".eml"); + await File.WriteAllBytesAsync(failed, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + harness.Spool.RecoverClaims(); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + Assert.Null(restarted.TryClaimNext()); + Assert.True(File.Exists(failed)); + Assert.False(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task Complete_DeletesWorkFile_WithoutDeliveredMarker() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claim = harness.Spool.TryClaimNext(); + Assert.NotNull(claim); + harness.Spool.Complete(claim!); + Assert.False(File.Exists(claim!.WorkPath)); + Assert.False(File.Exists(Path.Join(harness.Directory, claim.Id + ".delivered"))); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + Assert.Null(harness.Spool.TryClaimNext()); + } + + [Fact] + public async Task Complete_DeleteFailure_LeavesDeliveredMarker() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claim = harness.Spool.TryClaimNext(); + Assert.NotNull(claim); + harness.Spool.CompleteWorkFileDelete = _ => throw new IOException("injected work-file delete failure"); + harness.Spool.Complete(claim!); + var delivered = Path.Join(harness.Directory, claim!.Id + ".delivered"); + Assert.True(File.Exists(delivered)); + Assert.False(File.Exists(claim.WorkPath)); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + Assert.Null(harness.Spool.TryClaimNext()); + } + + [Fact] + public async Task WriteAsync_WhenDirectoryIsAFile_Fails() + { + using var harness = new EmailSpoolHarness(); + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-email-spool-blocker-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + harness.Options.Spool.Directory = blocker; + var spool = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + await Assert.ThrowsAnyAsync( + () => spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None)); + } + finally + { + File.Delete(blocker); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/Rfc5322MessageEncoderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/Rfc5322MessageEncoderTests.cs new file mode 100644 index 00000000..66461583 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/Rfc5322MessageEncoderTests.cs @@ -0,0 +1,115 @@ +using System.Text; +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class Rfc5322MessageEncoderTests +{ + private readonly Rfc5322MessageEncoder _encoder = new(); + + [Fact] + public void Encode_UsesCrlfAndOmitsBcc() + { + var message = Create( + subject: "Hello", + body: "line1\nline2", + bcc: new EmailAddress("hidden@example.com")); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("From: from@example.com\r\n", text, StringComparison.Ordinal); + Assert.Contains("To: to@example.com\r\n", text, StringComparison.Ordinal); + Assert.Contains("Subject: Hello\r\n", text, StringComparison.Ordinal); + Assert.Contains("Cc: cc@example.com\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("Bcc:", text, StringComparison.Ordinal); + Assert.Contains("line1\r\nline2", text, StringComparison.Ordinal); + Assert.DoesNotContain("\nline1", text.Replace("\r\n", string.Empty, StringComparison.Ordinal), StringComparison.Ordinal); + } + + [Fact] + public void Encode_EncodesNonAsciiSubject() + { + var message = Create(subject: "café"); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("=?UTF-8?B?", text, StringComparison.Ordinal); + Assert.DoesNotContain("café", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_RejectsHeaderInjection() + { + Assert.Throws(() => + _encoder.Encode(Create(subject: "ok\r\nBcc: evil@example.com"))); + Assert.Throws(() => + new EmailHeader("X-Injected\r\nBcc", "x").Validate()); + Assert.Throws(() => + new EmailHeader("X-Good", "bad\r\nBcc: evil@example.com").Validate()); + Assert.Throws(() => + new EmailAddress("ok@example.com", "bad\r\nFrom: evil")); + } + + [Fact] + public void Encode_CustomHeadersAndNewsTransmission() + { + var message = new EmailMessage + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Subject = "group mid", + Body = "From: poster@example.com\r\n\r\nbody\r\n"u8.ToArray(), + ContentType = "application/news-transmission; usage=moderate", + Headers = [new EmailHeader("X-Moderated-Newsgroup", "comp.example")], + }; + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("Content-Type: application/news-transmission; usage=moderate\r\n", text, StringComparison.Ordinal); + Assert.Contains("X-Moderated-Newsgroup: comp.example\r\n", text, StringComparison.Ordinal); + Assert.Contains("usage=moderate", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_QuotedPrintableFor8BitBody() + { + var message = Create(bodyBytes: "naïve"u8.ToArray()); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("Content-Transfer-Encoding: quoted-printable", text, StringComparison.Ordinal); + Assert.Contains("=C3=AF", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_AttachmentIsMultipartBase64() + { + var message = Create(); + message = new EmailMessage + { + From = message.From, + To = message.To, + Subject = message.Subject, + Body = message.Body, + Attachments = [new EmailAttachment("note.txt", "hi"u8.ToArray(), "text/plain")], + }; + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("multipart/mixed", text, StringComparison.Ordinal); + Assert.Contains("filename=\"note.txt\"", text, StringComparison.Ordinal); + Assert.Contains("aGk=", text, StringComparison.Ordinal); + } + + [Fact] + public void EnvelopeRecipients_AreDistinct() + { + var message = Create(bcc: new EmailAddress("to@example.com")); + Assert.Equal(2, message.EnvelopeRecipients().Count); + } + + private static EmailMessage Create( + string subject = "Subject", + string body = "body\r\n", + byte[]? bodyBytes = null, + EmailAddress? bcc = null) => + new() + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Cc = [new EmailAddress("cc@example.com")], + Bcc = bcc is null ? [] : [bcc], + Subject = subject, + Body = bodyBytes ?? Encoding.ASCII.GetBytes(body), + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpAuthTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpAuthTests.cs new file mode 100644 index 00000000..d1e71c64 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpAuthTests.cs @@ -0,0 +1,130 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Email.Smtp; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class SmtpAuthTests +{ + [Fact] + public async Task AuthPlain_SucceedsAfterTls() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var logger = new MemoryLogger(); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create( + smtp, + FakeSmtpServer.TrustAnchors(server.Certificate!), + logger); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.Authenticated); + Assert.DoesNotContain(server.RawLines, static l => l.Contains("s3cret", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Messages, static m => m.Contains("s3cret", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Messages, static m => m.Contains("user", StringComparison.Ordinal)); + Assert.DoesNotContain(server.Commands, static c => c.Contains("PLAIN", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task AuthLogin_SucceedsWhenPlainMissing() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthLogin = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.Authenticated); + Assert.DoesNotContain(server.RawLines, static l => l.Contains("s3cret", StringComparison.Ordinal)); + } + + [Fact] + public async Task AuthFailure_IsNotRetriedByTransport() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "right", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "wrong"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Authentication, ex.Kind); + } + + [Fact] + public async Task Auth_NotAttemptedBeforeTlsWhenRequired() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + var commands = server.Commands.ToList(); + var startTls = commands.FindIndex(static c => c.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)); + var auth = commands.FindIndex(static c => c.Equals("AUTH", StringComparison.OrdinalIgnoreCase)); + Assert.True(auth > startTls); + } + + [Fact] + public async Task Credentials_AreNotCopiedIntoHeaders() + { + var message = EmailServiceTests.TextMessage(); + var encoded = System.Text.Encoding.ASCII.GetString(new Rfc5322MessageEncoder().Encode(message).Span); + Assert.DoesNotContain("AUTH", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Password", encoded, StringComparison.Ordinal); + } +} + +internal sealed class MemoryLogger : ILogger +{ + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpProtocolTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpProtocolTests.cs new file mode 100644 index 00000000..0177d5c7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpProtocolTests.cs @@ -0,0 +1,78 @@ +using System.Text; +using VectorNNTPD.ReaderService.Email.Smtp; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class SmtpProtocolTests +{ + [Theory] + [InlineData("250 OK", 250, false, "OK")] + [InlineData("250-PIPELINING", 250, true, "PIPELINING")] + [InlineData("354", 354, false, "")] + public void ParseReplyLine_Structured(string line, int code, bool continuation, string text) + { + Assert.True(SmtpResponseReader.TryParseReplyLine( + Encoding.ASCII.GetBytes(line), + out var parsed, + out var cont, + out var body)); + Assert.Equal(code, parsed); + Assert.Equal(continuation, cont); + Assert.Equal(text, body); + } + + [Theory] + [InlineData("OK")] + [InlineData("2OK")] + [InlineData("25X OK")] + [InlineData("250XOK")] + public void ParseReplyLine_RejectsMalformed(string line) + { + Assert.False(SmtpResponseReader.TryParseReplyLine(Encoding.ASCII.GetBytes(line), out _, out _, out _)); + } + + [Fact] + public void Capabilities_ParseExtensions() + { + var response = new SmtpResponse( + 250, + [ + "test.example", + "STARTTLS", + "AUTH PLAIN LOGIN", + "SIZE 1000", + "8BITMIME", + "SMTPUTF8", + ], + enhancedStatus: null, + text: "ehlo"); + var caps = SmtpCapabilities.Parse(response); + Assert.True(caps.StartTls); + Assert.True(caps.EightBitMime); + Assert.True(caps.SmtpUtf8); + Assert.Equal(1000, caps.MaxSize); + Assert.True(caps.SupportsAuth("PLAIN")); + Assert.True(caps.SupportsAuth("LOGIN")); + Assert.False(caps.SupportsAuth("CRAM-MD5")); + } + + [Fact] + public async Task DataEncoder_DotStuffsAndTerminates() + { + using var stream = new MemoryStream(); + await SmtpDataEncoder.WriteStuffedAsync( + stream, + ".hidden\r\nnormal\r\n"u8.ToArray(), + CancellationToken.None); + var text = Encoding.ASCII.GetString(stream.ToArray()); + Assert.Equal("..hidden\r\nnormal\r\n.\r\n", text); + } + + [Fact] + public void EnhancedStatus_IsSplit() + { + Assert.True(SmtpResponseReader.TrySplitEnhanced("2.0.0 OK", out var enhanced, out var rest)); + Assert.Equal("2.0.0", enhanced); + Assert.Equal("OK", rest); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTlsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTlsTests.cs new file mode 100644 index 00000000..2702e5a9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTlsTests.cs @@ -0,0 +1,116 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email.Smtp; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class SmtpTlsTests +{ + [Fact] + public async Task ImplicitTls_ValidCertificate_SucceedsWithoutPlaintext() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + Assert.True(certificate.HasPrivateKey); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create( + server.ClientOptions(SmtpSecurityMode.ImplicitTls), + FakeSmtpServer.TrustAnchors(certificate)); + var result = await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(result.Succeeded); + Assert.True(server.StartedTls); + Assert.False(server.SawPlaintextSmtpBeforeTls); + } + + [Fact] + public async Task ImplicitTls_UntrustedCertificate_Fails() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create(server.ClientOptions(SmtpSecurityMode.ImplicitTls)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + } + + [Fact] + public async Task ImplicitTls_HostnameMismatch_Fails() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate("wrong.example", includeLoopbackIp: false); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create( + server.ClientOptions(SmtpSecurityMode.ImplicitTls), + FakeSmtpServer.TrustAnchors(certificate)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + } + + [Fact] + public async Task StartTls_SecondEhloAndNoPlaintextAuth() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "secret", + Certificate = certificate, + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "secret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(certificate)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.StartedTls); + Assert.True(server.Authenticated); + var commands = server.Commands.ToList(); + var ehlo = commands.Count(static c => c.StartsWith("EHLO", StringComparison.OrdinalIgnoreCase)); + Assert.True(ehlo >= 2); + var startTlsIndex = commands.FindIndex(static c => c.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)); + var authIndex = commands.FindIndex(static c => c.Equals("AUTH", StringComparison.OrdinalIgnoreCase)); + Assert.True(startTlsIndex >= 0); + Assert.True(authIndex > startTlsIndex); + } + + [Fact] + public async Task StartTls_Required_FailsWhenNotAdvertised() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { AdvertiseStartTls = false }); + var transport = SmtpTransportTests.Create(server.ClientOptions(SmtpSecurityMode.StartTls)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + Assert.DoesNotContain(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + } + + [Fact] + public async Task StartTls_DoesNotDowngradeToPlaintext() + { + await using var server = new FakeSmtpServer(); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls); + var transport = SmtpTransportTests.Create(smtp); + await Assert.ThrowsAsync(() => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.False(server.StartedTls); + Assert.DoesNotContain(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + } + + [Fact] + public void SmtpOptions_HasNoCertificateBypassOrEhloHostname() + { + Assert.Null(typeof(SmtpOptions).GetProperty("DangerousAcceptAnyServerCertificate")); + Assert.Null(typeof(SmtpOptions).GetProperty("EhloHostname")); + Assert.Null(typeof(EmailOptions).GetProperty("Queue")); + Assert.NotNull(typeof(EmailOptions).GetProperty(nameof(EmailOptions.Spool))); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTransportTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTransportTests.cs new file mode 100644 index 00000000..19bc167f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Email/SmtpTransportTests.cs @@ -0,0 +1,186 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Email.Smtp; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Email; + +public sealed class SmtpTransportTests +{ + [Fact] + public async Task Plain_SendsMailRcptDataQuit() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var result = await transport.SendAsync(Item(), CancellationToken.None); + Assert.True(result.Succeeded); + Assert.Contains(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + Assert.Contains(server.Commands, static c => c.StartsWith("RCPT TO:", StringComparison.Ordinal)); + Assert.Contains(server.Commands, static c => c.Equals("DATA", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(server.Commands, static c => c.Equals("QUIT", StringComparison.OrdinalIgnoreCase)); + Assert.NotNull(server.LastMessage); + Assert.Contains("Subject: hello", System.Text.Encoding.ASCII.GetString(server.LastMessage), StringComparison.Ordinal); + } + + [Fact] + public async Task MultipleRecipients_OneConnection() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var item = Item(); + item = new EmailWorkItem + { + Message = item.Message, + EncodedMessage = item.EncodedMessage, + EnvelopeSender = item.EnvelopeSender, + Recipients = + [ + new EmailAddress("a@example.com"), + new EmailAddress("b@example.com"), + ], + }; + var result = await transport.SendAsync(item, CancellationToken.None); + Assert.True(result.Succeeded); + Assert.Equal(2, result.Recipients.Count); + Assert.Equal(1, server.Commands.Count(static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal))); + } + + [Fact] + public async Task Recipient5xx_IsPermanent() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { DefaultRcptCode = 550 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Permanent, ex.Kind); + } + + [Fact] + public async Task Recipient4xx_IsTransient() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { DefaultRcptCode = 450 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Transient, ex.Kind); + } + + [Fact] + public async Task MixedRecipients_ArePartial() + { + var options = new FakeSmtpServerOptions { DefaultRcptCode = 250 }; + options.RcptCodes["bad@example.com"] = 550; + await using var server = new FakeSmtpServer(options); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var item = Item(); + item = new EmailWorkItem + { + Message = item.Message, + EncodedMessage = item.EncodedMessage, + EnvelopeSender = item.EnvelopeSender, + Recipients = + [ + new EmailAddress("to@example.com"), + new EmailAddress("bad@example.com"), + ], + }; + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(item, CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Partial, ex.Kind); + } + + [Fact] + public async Task DataDotStuffing_IsApplied() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var encoded = "From: from@example.com\r\nTo: to@example.com\r\nSubject: x\r\n\r\n.secret\r\n"u8.ToArray(); + await transport.SendAsync(Item(encoded), CancellationToken.None); + var received = System.Text.Encoding.ASCII.GetString(server.LastMessage!); + Assert.Contains(".secret", received, StringComparison.Ordinal); + } + + [Fact] + public async Task ServerCloseAfterGreeting_IsNetworkFailure() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { CloseAfterGreeting = true }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Network, ex.Kind); + } + + [Fact] + public async Task OversizedMessage_UsesSizeLimit() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { Size = 16 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Message, ex.Kind); + } + + [Fact] + public async Task Cancellation_Propagates() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => transport.SendAsync(Item(), cts.Token)); + } + + [Fact] + public async Task MultilineEhlo_IsConsumed() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + await transport.SendAsync(Item(), CancellationToken.None); + Assert.Contains(server.Commands, static c => c.StartsWith("EHLO", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Ehlo_UsesApplicationFqdn() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + await transport.SendAsync(Item(), CancellationToken.None); + Assert.Contains( + server.Commands, + static c => c.Equals("EHLO nntpd01.usenet.ninja", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + server.Commands, + static c => c.Contains("localhost", StringComparison.OrdinalIgnoreCase)); + } + + internal static SmtpTransport Create( + SmtpOptions smtp, + IReadOnlyCollection? extraTrustedRoots = null, + ILogger? logger = null) + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = "from@example.com", + Smtp = smtp, + }; + return new SmtpTransport( + Options.Create(options), + Options.Create(new NntpdOptions { ServerId = 1, DnsSuffix = "usenet.ninja" }), + logger ?? NullLogger.Instance, + extraTrustedRoots); + } + + internal static EmailWorkItem Item(ReadOnlyMemory? encoded = null) + { + var message = EmailServiceTests.TextMessage(); + var bytes = encoded ?? new Rfc5322MessageEncoder().Encode(message); + return new EmailWorkItem + { + Message = message, + EncodedMessage = bytes, + EnvelopeSender = new EmailAddress("envelope@example.com"), + Recipients = [new EmailAddress("to@example.com")], + }; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/AccountBytesRedisIntegration.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/AccountBytesRedisIntegration.cs new file mode 100644 index 00000000..817f6fa7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/AccountBytesRedisIntegration.cs @@ -0,0 +1,197 @@ +using System.Collections.Concurrent; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Shared live Redis connection for AccountBytes Lua EVAL tests. Deletes only +/// nntpd:bytes: keys created for uniquely named test accounts. +/// +public sealed class AccountBytesRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _accounts = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + public string Endpoint { get; private set; } = string.Empty; + + public string ServerVersion { get; private set; } = string.Empty; + + internal RedisAccountByteStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = SessionStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + Endpoint = SessionStateRedisIntegration.DescribeTarget(options); + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + var endpoint = live.Multiplexer.GetEndPoints()[0]; + ServerVersion = live.Multiplexer.GetServer(endpoint).Version.ToString(); + Store = new RedisAccountByteStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var account in _accounts) + { + await DeleteKeyAsync(account); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "AccountBytes live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateAccountAsync() + { + EnsureConfigured(); + var account = "vnntp.bytes.lua." + Guid.NewGuid().ToString("N"); + _accounts.Add(account); + await DeleteKeyAsync(account); + return account; + } + + public async Task DeleteKeyAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + // codeql[cs/linq/missed-where] Async KeyExists filter; Where cannot await. + foreach (var account in _accounts.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)AccountByteKeys.Create(account))) + { + remaining.Add(Encoding.UTF8.GetString(AccountByteKeys.Create(account))); + } + } + + return remaining; + } + + public ValueTask ApplyAsync(string accountName, string batchId, long consumed, long mysqlRemainingAfter) => + Store.ApplyAsync(accountName, batchId, consumed, mysqlRemainingAfter); + + public ValueTask ApplyAsync(string accountName, long consumed, long mysqlRemainingAfter) => + Store.ApplyAsync(accountName, AccountByteBatchId.Create(), consumed, mysqlRemainingAfter); + + public ValueTask ObserveAsync(string accountName) => Store.ObserveAsync(accountName); + + public ValueTask DeleteAsync(string accountName) => Store.DeleteAsync(accountName); + + public async Task SeedAsync(string accountName, long remaining) + { + EnsureConfigured(); + _ = await Database.HashSetAsync( + (RedisKey)AccountByteKeys.Create(accountName), + "remaining", + remaining.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + public async Task BatchMarkExistsAsync(string accountName, string batchId) + { + EnsureConfigured(); + return await Database.HashExistsAsync( + (RedisKey)AccountByteKeys.Create(accountName), + AccountByteBatchId.FieldName(batchId)); + } + + public async Task KeyTypeAsync(string accountName) + { + EnsureConfigured(); + return await Database.KeyTypeAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + public async Task KeyTimeToLiveAsync(string accountName) + { + EnsureConfigured(); + return await Database.KeyTimeToLiveAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + public async Task DeleteWithoutFixtureTrackingAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(SessionStateRedisIntegration.SkipReason); + } + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/CanonicalArticleText.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/CanonicalArticleText.cs new file mode 100644 index 00000000..aa178b2a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/CanonicalArticleText.cs @@ -0,0 +1,85 @@ +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Networking.Proxy; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Destuffed article text that Common ArticleRecordFactory accepts. +/// +internal static class CanonicalArticleText +{ + /// Date header value accepted by NewsDateParser. + public const string Date = "Fri, 23 Aug 2024 07:30:10 +0000"; + + /// Builds destuffed headers + body without an NNTP terminator. + /// Message-ID value including angle brackets. + /// Destuffed body, typically CRLF-terminated. + /// Newsgroups header value. + /// Destuffed article text. + public static string Destuffed( + string messageId, + string body = "body\r\n", + string newsgroups = "alt.test", + string subject = "ingress-test", + string from = "user@example.test", + string? references = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentNullException.ThrowIfNull(body); + ArgumentException.ThrowIfNullOrWhiteSpace(newsgroups); + ArgumentNullException.ThrowIfNull(subject); + ArgumentException.ThrowIfNullOrWhiteSpace(from); + var referencesHeader = string.IsNullOrEmpty(references) + ? string.Empty + : "References: " + references + "\r\n"; + return + "Path: peer.example\r\n" + + "Date: " + Date + "\r\n" + + "Message-ID: " + messageId + "\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "From: " + from + "\r\n" + + "Subject: " + subject + "\r\n" + + referencesHeader + + "\r\n" + + body; + } + + /// NNTP stuffed wire for without the terminator. + public static string Stuffed( + string messageId, + string destuffedBody = "body\r\n", + string newsgroups = "alt.test") => + Destuffed(messageId, destuffedBody, newsgroups) + .Replace("\r\n.", "\r\n..", StringComparison.Ordinal); + + /// Builds a CanonicalV1 queue item for tests. + public static InboundArticle CreateQueued( + string messageId, + InboundArticleProducer producer, + string body = "body\r\n", + string newsgroups = "alt.test", + ReadOnlyMemory feed = default, + string subject = "ingress-test", + string from = "user@example.test", + string? references = null) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(Destuffed(messageId, body, newsgroups, subject, from, references))); + if (!created.IsAccepted) + { + throw new InvalidOperationException("Test article must be CanonicalV1."); + } + + return ArticleRecordIngress.CreateQueued( + messageId, + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + producer, + feed); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/ControllableTimeProvider.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/ControllableTimeProvider.cs new file mode 100644 index 00000000..c97b16d3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/ControllableTimeProvider.cs @@ -0,0 +1,189 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Test that advances monotonically and fires +/// callbacks so +/// completes without real waits. +/// +internal sealed class ControllableTimeProvider : TimeProvider +{ + private readonly object _gate = new(); + private readonly List _timers = []; + private long _ticks; + + /// + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + /// + public override long GetTimestamp() + { + lock (_gate) + { + return _ticks; + } + } + + /// + public override DateTimeOffset GetUtcNow() + { + lock (_gate) + { + return DateTimeOffset.UnixEpoch.AddTicks(_ticks); + } + } + + /// Gets whether any callback is currently scheduled. + public bool HasScheduledTimers + { + get + { + lock (_gate) + { + return _timers.Count > 0; + } + } + } + + /// + /// Gets whether a timer is waiting for its due instant. + /// A fired one-shot timer is not pending, so this becomes true again only when the next delay is registered. + /// + public bool HasPendingTimer + { + get + { + lock (_gate) + { + foreach (var timer in _timers) + { + if (timer.IsPending) + { + return true; + } + } + + return false; + } + } + } + + /// Advances virtual time and invokes timers that are now due. + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfLessThan(delta, TimeSpan.Zero); + lock (_gate) + { + _ticks += delta.Ticks; + } + + // Invoke due timers synchronously so tests do not depend on ThreadPool + // scheduling. Callbacks may start a new Delay; only timers already due + // at the current virtual time are collected on each pass. + for (var pass = 0; pass < 32; pass++) + { + List due = []; + lock (_gate) + { + var now = _ticks; + // codeql[cs/linq/missed-where] CollectIfDue has side effects under lock; no LINQ filter. + foreach (var timer in _timers) + { + if (timer.CollectIfDue(now)) + { + due.Add(timer); + } + } + } + + if (due.Count == 0) + { + return; + } + + foreach (var timer in due) + { + timer.Invoke(); + } + } + } + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ArgumentNullException.ThrowIfNull(callback); + var entry = new TimerEntry(this, callback, state); + lock (_gate) + { + _timers.Add(entry); + entry.Schedule(_ticks, dueTime, period); + } + + return entry; + } + + private void Remove(TimerEntry entry) + { + lock (_gate) + { + _timers.Remove(entry); + } + } + + private sealed class TimerEntry : ITimer + { + private readonly ControllableTimeProvider _owner; + private readonly TimerCallback _callback; + private readonly object? _state; + private long? _dueTicks; + private TimeSpan _period = Timeout.InfiniteTimeSpan; + + public TimerEntry(ControllableTimeProvider owner, TimerCallback callback, object? state) + { + _owner = owner; + _callback = callback; + _state = state; + } + + public void Schedule(long nowTicks, TimeSpan dueTime, TimeSpan period) + { + _period = period; + _dueTicks = dueTime == Timeout.InfiniteTimeSpan ? null : nowTicks + dueTime.Ticks; + } + + public bool IsPending => _dueTicks is not null; + + public bool CollectIfDue(long nowTicks) + { + if (_dueTicks is not { } due || due > nowTicks) + { + return false; + } + + _dueTicks = _period <= TimeSpan.Zero || _period == Timeout.InfiniteTimeSpan + ? null + : nowTicks + _period.Ticks; + + return true; + } + + public void Invoke() => _callback(_state); + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + lock (_owner._gate) + { + Schedule(_owner._ticks, dueTime, period); + } + + return true; + } + + public void Dispose() => _owner.Remove(this); + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs new file mode 100644 index 00000000..7324ebf9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs @@ -0,0 +1,183 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.NntpDb; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Seeds uniquely prefixed nntpmoderators rows on a live MySQL target and +/// deletes only those rows. Unconfigured runs do not open a connection. +/// +public sealed class MySqlModeratorIntegrationFixture : IAsyncLifetime +{ + private readonly List _insertedIds = []; + private NntpDbService? _nntpDb; + + public bool IsConfigured { get; private set; } + + public string RunId { get; } = Guid.NewGuid().ToString("N"); + + public string PatternPrefix => "test.vnntp.modcat." + RunId; + + public MySqlNntpModeratorRepository? Repository { get; private set; } + + public SeededModeratorRow Static { get; private set; } = null!; + + public SeededModeratorRow Template { get; private set; } = null!; + + public SeededModeratorRow Disabled { get; private set; } = null!; + + public SeededModeratorRow SharedFirst { get; private set; } = null!; + + public SeededModeratorRow SharedSecond { get; private set; } = null!; + + public SeededModeratorRow Boundary { get; private set; } = null!; + + public string SharedPattern => PatternPrefix + ".shared.*"; + + public string LongAccountName { get; } = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"; + + public string LongModeratorAddress { get; } = new string('a', 308) + "@example.org"; + + public async Task InitializeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is null) + { + return; + } + + IsConfigured = true; + var options = Options.Create(NntpDbIntegration.CreateOptions(connectionString)); + _nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + options, + NullLogger.Instance); + await _nntpDb.StartAsync(CancellationToken.None); + Repository = new MySqlNntpModeratorRepository( + _nntpDb, + NullLogger.Instance); + + Static = await InsertAsync( + connectionString, + PatternPrefix + ".static.*", + "moderator@example.org", + "TESTMOD01", + enabled: true); + Template = await InsertAsync( + connectionString, + PatternPrefix + ".template.*", + "moderator-%s@example.org", + "TESTMOD02", + enabled: true); + Disabled = await InsertAsync( + connectionString, + PatternPrefix + ".disabled.*", + "disabled@example.org", + "TESTMOD03", + enabled: false); + SharedFirst = await InsertAsync( + connectionString, + SharedPattern, + "shared-first@example.org", + "TESTMOD04", + enabled: true); + SharedSecond = await InsertAsync( + connectionString, + SharedPattern, + "shared-second@example.org", + "TESTMOD05", + enabled: true); + Boundary = await InsertAsync( + connectionString, + CreateBoundaryPattern(), + LongModeratorAddress, + LongAccountName, + enabled: true); + } + + public async Task DisposeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is not null && _insertedIds.Count > 0) + { + await CleanupAsync(connectionString).ConfigureAwait(false); + } + + if (_nntpDb is not null) + { + await _nntpDb.DisposeAsync().ConfigureAwait(false); + } + } + + public IReadOnlyList Ours(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + return rows.Where(row => row.GroupPattern.StartsWith(PatternPrefix, StringComparison.Ordinal)).ToArray(); + } + + private string CreateBoundaryPattern() + { + var prefix = PatternPrefix + ".long."; + var remaining = 255 - prefix.Length - 1; + return prefix + new string('x', remaining) + "*"; + } + + private async Task InsertAsync( + string connectionString, + string pattern, + string address, + string account, + bool enabled) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + "INSERT INTO nntpmoderators " + + "(group_pattern, moderator_address, account_name, is_enabled) " + + "VALUES (@pattern, @address, @account, @enabled); " + + "SELECT LAST_INSERT_ID();"; + command.Parameters.AddWithValue("@pattern", pattern); + command.Parameters.AddWithValue("@address", address); + command.Parameters.AddWithValue("@account", account); + command.Parameters.AddWithValue("@enabled", enabled ? "Y" : "N"); + var id = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + _insertedIds.Add(id); + return new SeededModeratorRow(id, pattern, address, account, enabled); + } + + private async Task CleanupAsync(string connectionString) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (_insertedIds.Count > 0) + { + await using var byId = connection.CreateCommand(); + var names = new string[_insertedIds.Count]; + for (var i = 0; i < _insertedIds.Count; i++) + { + names[i] = "@id" + i.ToString(CultureInfo.InvariantCulture); + byId.Parameters.AddWithValue(names[i], _insertedIds[i]); + } + + byId.CommandText = "DELETE FROM nntpmoderators WHERE moderator_id IN (" + string.Join(", ", names) + ")"; + await byId.ExecuteNonQueryAsync(); + } + + await using var byPrefix = connection.CreateCommand(); + byPrefix.CommandText = "DELETE FROM nntpmoderators WHERE group_pattern LIKE @prefix"; + byPrefix.Parameters.AddWithValue("@prefix", PatternPrefix + "%"); + await byPrefix.ExecuteNonQueryAsync(); + } +} + +/// One row inserted by . +public sealed record SeededModeratorRow( + long ModeratorId, + string GroupPattern, + string ModeratorAddress, + string AccountName, + bool Enabled); diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs new file mode 100644 index 00000000..7fc1e182 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs @@ -0,0 +1,664 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MySqlConnector; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Applies the official PostFilter DDL when missing and isolates test revisions +/// on a live MySQL target. Unconfigured runs do not open a connection. +/// +public sealed class MySqlPostFilterPolicyIntegrationFixture : IAsyncLifetime +{ + private NntpDbService? _nntpDb; + private long? _savedCurrentRevision; + private readonly List _revisions = []; + + public bool IsConfigured { get; private set; } + + public string? SkipReason { get; private set; } + + public long RevisionBase { get; private set; } = + 9_000_000_000L + (Environment.TickCount64 % 1_000_000L) * 100L; + + public MySqlPostFilterPolicyRepository? Repository { get; private set; } + + public async Task InitializeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is null) + { + SkipReason = NntpDbIntegration.SkipReason; + return; + } + + try + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (await HasColumnAsync(connection, "nntppostfilterpolicy", "policy_id") + && !await HasTableAsync(connection, "nntppostfiltercurrent")) + { + SkipReason = + "NntpDB still has the pre-revision PostFilter singleton. Apply docs/schema/postfilter.sql."; + return; + } + + if (!await HasTableAsync(connection, "nntppostfiltercurrent")) + { + foreach (var statement in PostFilterSchemaScript.ReadStatements()) + { + await using var command = connection.CreateCommand(); + command.CommandText = statement; + await command.ExecuteNonQueryAsync(); + } + } + else + { + try + { + await EnsureExtendedSchemaAsync(connection); + } + catch (MySqlException ex) + { + SkipReason = FormatPrepareFailure(ex); + return; + } + } + + _savedCurrentRevision = await ReadCurrentRevisionAsync(connection); + if (_savedCurrentRevision is { } saved && saved >= RevisionBase) + { + RevisionBase = saved + 1_000; + } + + IsConfigured = true; + } + catch (Exception ex) when (ex is not InvalidOperationException) + { + SkipReason = FormatPrepareFailure(ex); + return; + } + + var options = Options.Create(NntpDbIntegration.CreateOptions(connectionString)); + _nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + options, + NullLogger.Instance); + await _nntpDb.StartAsync(CancellationToken.None); + Repository = new MySqlPostFilterPolicyRepository( + _nntpDb, + NullLogger.Instance); + } + + public async Task DisposeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is not null && _revisions.Count > 0) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (_savedCurrentRevision is { } saved) + { + await using var restore = connection.CreateCommand(); + restore.CommandText = + "UPDATE nntppostfiltercurrent SET revision = @revision WHERE policy_id = 1"; + restore.Parameters.AddWithValue("@revision", saved); + try + { + await restore.ExecuteNonQueryAsync(); + } + catch (MySqlException) + { + // Restore can fail if the trigger forbids regression; drop current then reinsert. + await using var deleteCurrent = connection.CreateCommand(); + deleteCurrent.CommandText = "DELETE FROM nntppostfiltercurrent WHERE policy_id = 1"; + await deleteCurrent.ExecuteNonQueryAsync(); + await using var insertCurrent = connection.CreateCommand(); + insertCurrent.CommandText = + "INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, @revision)"; + insertCurrent.Parameters.AddWithValue("@revision", saved); + await insertCurrent.ExecuteNonQueryAsync(); + } + } + + foreach (var revision in _revisions.OrderByDescending(static value => value)) + { + await DeleteRevisionAsync(connection, revision); + } + } + + if (_nntpDb is not null) + { + await _nntpDb.DisposeAsync().ConfigureAwait(false); + } + } + + public long NextRevision() + { + var revision = RevisionBase + _revisions.Count + 1; + _revisions.Add(revision); + return revision; + } + + /// + /// Allocates a revision that remains after dispose so a live validation + /// publish can stay current. Must be higher than . + /// + public long NextPersistentRevision() => + RevisionBase + 90_000 + (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() % 1_000); + + /// + /// Remembers a published revision so dispose restores the pointer here + /// instead of the pre-test value. + /// + public void RememberPublishedRevision(long revision) => _savedCurrentRevision = revision; + + public async Task InsertDisabledRevisionAsync(long revision, Action? customize = null) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + @revision, UTC_TIMESTAMP(3), 'Disabled', + 86400000, 600000, + 0, 0, 0, 0, 0, 0, + 'N', NULL, 131072, 783, + '1.5', 4, 'RoundRobin', + 5000, 30000 + ) + """; + command.Parameters.AddWithValue("@revision", revision); + customize?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + public async Task InsertArtTypeAsync(long revision, string listKind, string artType) + { + await ExecuteAsync( + "INSERT INTO nntppostfilterarttypes (revision, list_kind, art_type) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue("@value", artType); + }); + } + + public async Task InsertAccountAsync(long revision, string listKind, string account) + { + await ExecuteAsync( + "INSERT INTO nntppostfilteraccounts (revision, list_kind, account_name) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue( + "@value", + PostFilterAccountIdentity.FromPolicyEntry(account)); + }); + } + + public async Task InsertCidrAsync(long revision, string listKind, string cidr) + { + await ExecuteAsync( + "INSERT INTO nntppostfiltercidrs (revision, list_kind, cidr) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue("@value", cidr); + }); + } + + public async Task InsertHostAsync(long revision, int order, string host) + { + await ExecuteAsync( + "INSERT INTO nntppostfiltersahosts (revision, host_order, host) VALUES (@revision, @order, @host)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@order", order); + command.Parameters.AddWithValue("@host", host); + }); + } + + public async Task PublishOptionsAsync(long revision, PostFilterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + await InsertPolicyScalarsAsync(revision, options); + foreach (var account in options.DeniedAccounts) + { + await InsertAccountAsync(revision, NntpPostFilterQueries.ListKindDeny, account); + } + + foreach (var account in options.AllowlistedAccounts) + { + await InsertAccountAsync(revision, NntpPostFilterQueries.ListKindAllow, account); + } + + foreach (var cidr in options.DeniedCidrs) + { + await InsertCidrAsync(revision, NntpPostFilterQueries.ListKindDeny, cidr); + } + + foreach (var cidr in options.AllowlistedCidrs) + { + await InsertCidrAsync(revision, NntpPostFilterQueries.ListKindAllow, cidr); + } + + foreach (var artType in options.RejectArtTypes) + { + await InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindReject, artType); + } + + foreach (var artType in options.SpamAssassin.ExcludeArtTypes) + { + await InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindSaExclude, artType); + } + + for (var i = 0; i < options.SpamAssassin.Hosts.Length; i++) + { + await InsertHostAsync(revision, i + 1, options.SpamAssassin.Hosts[i]); + } + + await PublishAsync(revision); + } + + public async Task InsertPolicyScalarsAsync(long revision, PostFilterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + @revision, UTC_TIMESTAMP(3), @gate, + @long_window_ms, @short_window_ms, + @max_messages_long, @max_bytes_long, @max_identical_long, + @max_messages_short, @max_bytes_short, @max_identical_short, + @sa_enabled, @sa_on_failure, @sa_max_article_size, @sa_port, + @sa_protocol_version, @sa_max_connections, @sa_host_selection, + @sa_connect_timeout_ms, @sa_operation_timeout_ms + ) + """; + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@gate", options.Gate.ToString()); + command.Parameters.AddWithValue( + "@long_window_ms", + checked((long)options.Quota.LongWindow.TotalMilliseconds)); + command.Parameters.AddWithValue( + "@short_window_ms", + checked((long)options.Quota.ShortWindow.TotalMilliseconds)); + command.Parameters.AddWithValue("@max_messages_long", options.Quota.MaxMessagesLong); + command.Parameters.AddWithValue("@max_bytes_long", options.Quota.MaxBytesLong); + command.Parameters.AddWithValue("@max_identical_long", options.Quota.MaxIdenticalLong); + command.Parameters.AddWithValue("@max_messages_short", options.Quota.MaxMessagesShort); + command.Parameters.AddWithValue("@max_bytes_short", options.Quota.MaxBytesShort); + command.Parameters.AddWithValue("@max_identical_short", options.Quota.MaxIdenticalShort); + command.Parameters.AddWithValue("@sa_enabled", options.SpamAssassin.Enabled ? "Y" : "N"); + command.Parameters.AddWithValue( + "@sa_on_failure", + options.SpamAssassin.OnFailure is { } onFailure + ? onFailure.ToString() + : DBNull.Value); + command.Parameters.AddWithValue("@sa_max_article_size", options.SpamAssassin.MaxArticleSize); + command.Parameters.AddWithValue("@sa_port", options.SpamAssassin.Port); + command.Parameters.AddWithValue("@sa_protocol_version", options.SpamAssassin.ProtocolVersion); + command.Parameters.AddWithValue("@sa_max_connections", options.SpamAssassin.MaxConnections); + command.Parameters.AddWithValue("@sa_host_selection", options.SpamAssassin.HostSelection.ToString()); + command.Parameters.AddWithValue( + "@sa_connect_timeout_ms", + checked((int)options.SpamAssassin.ConnectTimeout.TotalMilliseconds)); + command.Parameters.AddWithValue( + "@sa_operation_timeout_ms", + checked((int)options.SpamAssassin.OperationTimeout.TotalMilliseconds)); + await command.ExecuteNonQueryAsync(); + } + + public async Task PublishAsync(long revision) + { + await ExecuteAsync( + "UPDATE nntppostfiltercurrent SET revision = @revision WHERE policy_id = 1", + command => command.Parameters.AddWithValue("@revision", revision)); + } + + public async Task LoadAsync() + { + if (Repository is null) + { + throw new InvalidOperationException("PostFilter MySQL fixture is not configured."); + } + + return await Repository.LoadAsync(); + } + + public async Task ExecuteRawAsync(string sql, Action? bind = null) + { + await ExecuteAsync(sql, bind); + } + + public async Task ExecuteRawScalarAsync(string sql, Action? bind = null) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + bind?.Invoke(command); + return await command.ExecuteScalarAsync(); + } + + public async Task ShowCreateTableAsync(string table) + { + ArgumentException.ThrowIfNullOrWhiteSpace(table); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = "SHOW CREATE TABLE `" + table.Replace("`", string.Empty, StringComparison.Ordinal) + "`"; + await using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + throw new InvalidOperationException("SHOW CREATE TABLE returned no rows."); + } + + return reader.GetString(1); + } + + public async Task InsertRejectionAsync(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var db = new MySqlNntpDbConnection(connection); + await db.InsertPostFilterRejectionAsync(evidence, CancellationToken.None); + } + + public async Task BeginHeldWriteAsync() + { + var connection = MySqlNntpDbConnectionFactory.CreateConnection(RequireConnectionString()); + await connection.OpenAsync(); + var transaction = await connection.BeginTransactionAsync(); + return new HeldPostFilterWrite(connection, transaction); + } + + private async Task ExecuteAsync(string sql, Action? bind) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + bind?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + private string RequireConnectionString() => + NntpDbIntegration.TryGetConnectionString() + ?? throw new InvalidOperationException(NntpDbIntegration.SkipReason); + + private static string FormatPrepareFailure(Exception ex) + { + if (ex is MySqlException mysql) + { + return "PostFilter MySQL schema could not be prepared: MySqlException " + + mysql.Number.ToString(CultureInfo.InvariantCulture) + + " " + + (mysql.SqlState ?? "-") + + " " + + mysql.Message; + } + + return "PostFilter MySQL schema could not be prepared: " + + ex.GetType().Name + + " " + + ex.Message; + } + + private static async Task HasTableAsync(MySqlConnection connection, string table) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COUNT(*) FROM information_schema.tables " + + "WHERE table_schema = DATABASE() AND table_name = @name"; + command.Parameters.AddWithValue("@name", table); + var count = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + return count > 0; + } + + private static async Task EnsureExtendedSchemaAsync(MySqlConnection connection) + { + if (!await HasTableAsync(connection, "nntppostfilterrejections")) + { + var create = PostFilterSchemaScript.ReadStatements() + .Single(statement => statement.StartsWith( + "CREATE TABLE nntppostfilterrejections", + StringComparison.Ordinal)); + await using var command = connection.CreateCommand(); + command.CommandText = create; + await command.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntppostfilteraccounts") + && await ColumnTypeAsync(connection, "nntppostfilteraccounts", "account_name") is { } accountType + && !accountType.Contains("char(32)", StringComparison.OrdinalIgnoreCase)) + { + await using var hash = connection.CreateCommand(); + hash.CommandText = + """ + UPDATE nntppostfilteraccounts + SET account_name = LOWER(MD5(account_name)) + WHERE CHAR_LENGTH(account_name) <> 32 + """; + await hash.ExecuteNonQueryAsync(); + await using var alter = connection.CreateCommand(); + alter.CommandText = + "ALTER TABLE nntppostfilteraccounts MODIFY account_name CHAR(32) NOT NULL"; + await alter.ExecuteNonQueryAsync(); + try + { + await using var check = connection.CreateCommand(); + check.CommandText = + "ALTER TABLE nntppostfilteraccounts ADD CONSTRAINT chk_nntppostfilteraccounts_name " + + "CHECK (account_name REGEXP '^[0-9a-f]{32}$')"; + await check.ExecuteNonQueryAsync(); + } + catch (MySqlException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Constraint may already exist on a partially migrated schema. + } + } + + if (await HasTableAsync(connection, "nntppostfilterarttypes") + && await ColumnTypeAsync(connection, "nntppostfilterarttypes", "art_type") is { } artType + && !artType.Contains("enum", StringComparison.OrdinalIgnoreCase)) + { + await using var alter = connection.CreateCommand(); + alter.CommandText = + """ + ALTER TABLE nntppostfilterarttypes + MODIFY art_type ENUM( + 'Default','Control','Cancel','Mime','Binary','UuEncode','Base64','YEncoded', + 'BommaNews','UniData','Multipart','Html','PostScript','BinHex','Partial','PgpMessage' + ) NOT NULL + """; + await alter.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntppostfilterrejections") + && await ColumnTypeAsync(connection, "nntppostfilterrejections", "source_ip") is { } sourceIpType + && !sourceIpType.Contains("varbinary", StringComparison.OrdinalIgnoreCase)) + { + await using var convert = connection.CreateCommand(); + convert.CommandText = + """ + ALTER TABLE nntppostfilterrejections + ADD COLUMN source_ip_bin VARBINARY(16) NULL + """; + await convert.ExecuteNonQueryAsync(); + await using var copy = connection.CreateCommand(); + copy.CommandText = + """ + UPDATE nntppostfilterrejections + SET source_ip_bin = INET6_ATON(CAST(source_ip AS CHAR)) + """; + await copy.ExecuteNonQueryAsync(); + await using var drop = connection.CreateCommand(); + drop.CommandText = "ALTER TABLE nntppostfilterrejections DROP COLUMN source_ip"; + await drop.ExecuteNonQueryAsync(); + await using var rename = connection.CreateCommand(); + rename.CommandText = + """ + ALTER TABLE nntppostfilterrejections + CHANGE source_ip_bin source_ip VARBINARY(16) NOT NULL + """; + await rename.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntpusers") + && !await HasColumnAsync(connection, "nntpusers", "account_art_type")) + { + try + { + await using var alter = connection.CreateCommand(); + alter.CommandText = NntpUserQueries.AddAccountArtTypeColumn; + await alter.ExecuteNonQueryAsync(); + } + catch (MySqlException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Account capability is additive on nntpusers. PostFilter + // schema tests must still run when this ALTER is denied. + } + } + } + + private static async Task ColumnTypeAsync(MySqlConnection connection, string table, string column) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COLUMN_TYPE FROM information_schema.columns " + + "WHERE table_schema = DATABASE() AND table_name = @table AND column_name = @column"; + command.Parameters.AddWithValue("@table", table); + command.Parameters.AddWithValue("@column", column); + var value = await command.ExecuteScalarAsync(); + return value is null or DBNull ? null : Convert.ToString(value, CultureInfo.InvariantCulture); + } + + private static async Task HasColumnAsync(MySqlConnection connection, string table, string column) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COUNT(*) FROM information_schema.columns " + + "WHERE table_schema = DATABASE() AND table_name = @table AND column_name = @column"; + command.Parameters.AddWithValue("@table", table); + command.Parameters.AddWithValue("@column", column); + var count = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + return count > 0; + } + + private static async Task ReadCurrentRevisionAsync(MySqlConnection connection) + { + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"; + var scalar = await command.ExecuteScalarAsync(); + return scalar is null or DBNull + ? null + : Convert.ToInt64(scalar, CultureInfo.InvariantCulture); + } + + private static async Task DeleteRevisionAsync(MySqlConnection connection, long revision) + { + foreach (var table in new[] + { + "nntppostfilteraccounts", + "nntppostfiltercidrs", + "nntppostfilterarttypes", + "nntppostfiltersahosts", + }) + { + await using var command = connection.CreateCommand(); + command.CommandText = "DELETE FROM " + table + " WHERE revision = @revision"; + command.Parameters.AddWithValue("@revision", revision); + await command.ExecuteNonQueryAsync(); + } + + await using var policy = connection.CreateCommand(); + policy.CommandText = "DELETE FROM nntppostfilterpolicy WHERE revision = @revision"; + policy.Parameters.AddWithValue("@revision", revision); + await policy.ExecuteNonQueryAsync(); + } +} + +/// Open writer transaction used to prove uncommitted policy is invisible. +public sealed class HeldPostFilterWrite : IAsyncDisposable +{ + public HeldPostFilterWrite(MySqlConnection connection, MySqlTransaction transaction) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(transaction); + Connection = connection; + Transaction = transaction; + } + + public MySqlConnection Connection { get; } + + public MySqlTransaction Transaction { get; } + + public async Task ExecuteAsync(string sql, Action? bind = null) + { + await using var command = Connection.CreateCommand(); + command.Transaction = Transaction; + command.CommandText = sql; + bind?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + public Task CommitAsync() => Transaction.CommitAsync(); + + public Task RollbackAsync() => Transaction.RollbackAsync(); + + public async ValueTask DisposeAsync() + { + await Transaction.DisposeAsync(); + await Connection.DisposeAsync(); + } +} + +/// Serializes live PostFilter schema tests against one MySQL target. +[CollectionDefinition("NntpDbPostFilter")] +public sealed class NntpDbPostFilterCollection : ICollectionFixture +{ +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegration.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegration.cs new file mode 100644 index 00000000..39fb06d8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegration.cs @@ -0,0 +1,48 @@ +using MySqlConnector; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Opt-in live NntpDB seam. Ordinary tests never read ConnectionStrings:NntpDB +/// or the committed production appsettings target. +/// +internal static class NntpDbIntegration +{ + /// + /// Dedicated MySQL connection string for live nntpmoderators tests. + /// + public const string ConnectionStringEnvironmentVariable = "VECTORNNTP_NNTPDB_INTEGRATION"; + + /// Reason used when the live MySQL tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_NNTPDB_INTEGRATION to a dedicated MySQL connection string " + + "that contains nntpmoderators. Ordinary tests do not open MySQL and do not " + + "use ConnectionStrings:NntpDB."; + + /// Returns the opt-in connection string, or when unset. + public static string? TryGetConnectionString() + { + var value = Environment.GetEnvironmentVariable(ConnectionStringEnvironmentVariable); + return string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + } + + /// Describes the target without passwords or the full connection string. + public static string DescribeTarget(string connectionString) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + var builder = new MySqlConnectionStringBuilder(connectionString); + return "Server=" + builder.Server + + ";Database=" + builder.Database + + ";User ID=" + builder.UserID; + } + + /// Builds NntpDB options for the live factory. Never logs the string. + public static NntpDbOptions CreateOptions(string connectionString) => + new() + { + ConnectionString = connectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs new file mode 100644 index 00000000..4e87a8bb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_NNTPDB_INTEGRATION is set. +/// When the variable is set, the test runs and must fail if MySQL is unreachable. +/// +public sealed class NntpDbIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in string is absent. + public NntpDbIntegrationFactAttribute() + { + if (NntpDbIntegration.TryGetConnectionString() is null) + { + Skip = NntpDbIntegration.SkipReason; + } + } +} + +/// +/// xUnit 2 theory skip unless VECTORNNTP_NNTPDB_INTEGRATION is set. +/// +public sealed class NntpDbIntegrationTheoryAttribute : TheoryAttribute +{ + /// Initializes the attribute and skips when the opt-in string is absent. + public NntpDbIntegrationTheoryAttribute() + { + if (NntpDbIntegration.TryGetConnectionString() is null) + { + Skip = NntpDbIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationTests.cs new file mode 100644 index 00000000..538a7ae1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpDbIntegrationTests.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +public sealed class NntpDbIntegrationTests +{ + [Fact] + public void DescribeTarget_OmitsPasswordAndFullConnectionString() + { + const string connectionString = + "Server=198.18.0.70;Port=3306;Database=nntpdb;User ID=nntpd;Password=super-secret;"; + + var description = NntpDbIntegration.DescribeTarget(connectionString); + + Assert.Contains("Server=198.18.0.70", description, StringComparison.Ordinal); + Assert.Contains("Database=nntpdb", description, StringComparison.Ordinal); + Assert.Contains("User ID=nntpd", description, StringComparison.Ordinal); + Assert.DoesNotContain("super-secret", description, StringComparison.Ordinal); + Assert.DoesNotContain("Password", description, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(connectionString, description, StringComparison.Ordinal); + } + + [Fact] + public void TryGetConnectionString_ReadsOnlyOptInVariable() + { + Assert.Equal("VECTORNNTP_NNTPDB_INTEGRATION", NntpDbIntegration.ConnectionStringEnvironmentVariable); + Assert.Contains("do not use ConnectionStrings:NntpDB", NntpDbIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (NntpDbIntegration.TryGetConnectionString() is not null) + { + return; + } + + var attribute = new NntpDbIntegrationFactAttribute(); + Assert.Equal(NntpDbIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs new file mode 100644 index 00000000..d1bced0d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs @@ -0,0 +1,10 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// Loads docs/schema/nntpusers-account-art-type.sql. +internal static class NntpUsersArtTypeMigrationScript +{ + public static string FindPath() => PostFilterSchemaScript.FindDocsSql("nntpusers-account-art-type.sql"); + + public static IReadOnlyList ReadStatements() => + PostFilterSchemaScript.ReadStatements(FindPath()); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs new file mode 100644 index 00000000..d30bb65b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs @@ -0,0 +1,182 @@ +using System.Collections.Concurrent; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Shared live Redis connection for PostFilter quota Lua EVAL tests. Deletes only +/// keys created for uniquely named test accounts. Reuses +/// opt-in. +/// +public sealed class PostFilterQuotaRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _accounts = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + internal RedisPostFilterQuotaStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = SessionStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + Store = new RedisPostFilterQuotaStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var account in _accounts) + { + await DeleteKeysAsync(account); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "PostFilter quota live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateAccountAsync() + { + EnsureConfigured(); + var account = "vnntp.pf.q.lua." + Guid.NewGuid().ToString("N"); + _accounts.Add(account); + await DeleteKeysAsync(account); + return account; + } + + public async Task DeleteKeysAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(accountName)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + foreach (var account in _accounts.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(account))) + { + remaining.Add(Encoding.UTF8.GetString(PostFilterQuotaKeys.CreateQuota(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(account))) + { + remaining.Add(Encoding.UTF8.GetString(PostFilterQuotaKeys.CreateMultipost(account))); + } + } + + return remaining; + } + + public async Task SeedQuotaAsync(string accountName, string field, string value) + { + EnsureConfigured(); + _ = await Database.HashSetAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName), field, value); + } + + public async Task ReadQuotaFieldAsync(string accountName, string field) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName), field); + return value.IsNullOrEmpty ? null : (string)value!; + } + + public async Task ReadMultipostFieldAsync(string accountName, string field) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(accountName), field); + return value.IsNullOrEmpty ? null : (string)value!; + } + + public async Task QuotaFieldCountAsync(string accountName) + { + EnsureConfigured(); + return (int)await Database.HashLengthAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + } + + public async Task QuotaKeyTtlAsync(string accountName) + { + EnsureConfigured(); + return await Database.KeyTimeToLiveAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + } + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(SessionStateRedisIntegration.SkipReason); + } + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterSchemaScript.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterSchemaScript.cs new file mode 100644 index 00000000..648f9746 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/PostFilterSchemaScript.cs @@ -0,0 +1,125 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// Loads the operator DDL at docs/schema/postfilter.sql. +internal static class PostFilterSchemaScript +{ + public static string FindPath() => FindDocsSql("postfilter.sql"); + + /// Finds a file under repository docs/ or docs/schema/. + public static string FindDocsSql(string fileName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fileName); + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var schemaCandidate = Path.Join(dir.FullName, "docs", "schema", fileName); + if (File.Exists(schemaCandidate)) + { + return schemaCandidate; + } + + var docsCandidate = Path.Join(dir.FullName, "docs", fileName); + if (File.Exists(docsCandidate)) + { + return docsCandidate; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("docs/" + fileName + " was not found."); + } + + public static IReadOnlyList ReadStatements() => ReadStatements(FindPath()); + + /// Splits one operator SQL file using the same DELIMITER rules as postfilter.sql. + public static IReadOnlyList ReadStatements(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + var lines = File.ReadAllLines(path); + var statements = new List(); + var buffer = new List(); + var delimiter = ";"; + foreach (var line in lines) + { + var trimmed = line.Trim(); + if (TryReadDelimiter(trimmed, out var nextDelimiter)) + { + Flush(buffer, statements, delimiter); + delimiter = nextDelimiter; + continue; + } + + if (trimmed.StartsWith("-- VECTORNNTP_STMT", StringComparison.Ordinal) + || trimmed.StartsWith("-- BEGIN TRIGGER", StringComparison.Ordinal) + || trimmed.StartsWith("-- END TRIGGER", StringComparison.Ordinal)) + { + Flush(buffer, statements, delimiter); + continue; + } + + if (trimmed.StartsWith("--", StringComparison.Ordinal)) + { + continue; + } + + buffer.Add(line); + if (BufferEndsWith(buffer, delimiter)) + { + Flush(buffer, statements, delimiter); + } + } + + Flush(buffer, statements, delimiter); + return statements; + } + + private static bool TryReadDelimiter(string trimmed, out string delimiter) + { + delimiter = ";"; + const string prefix = "DELIMITER"; + if (!trimmed.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (trimmed.Length != prefix.Length + && !char.IsWhiteSpace(trimmed[prefix.Length])) + { + return false; + } + + var value = trimmed[prefix.Length..].Trim(); + delimiter = value.Length == 0 ? ";" : value; + return true; + } + + private static bool BufferEndsWith(List buffer, string delimiter) + { + var text = string.Join('\n', buffer).TrimEnd(); + return text.EndsWith(delimiter, StringComparison.Ordinal); + } + + private static void Flush(List buffer, List statements, string delimiter) + { + var text = string.Join('\n', buffer).Trim(); + buffer.Clear(); + if (text.Length == 0) + { + return; + } + + if (text.EndsWith(delimiter, StringComparison.Ordinal)) + { + text = text[..^delimiter.Length].TrimEnd(); + } + + text = text.TrimEnd(';').Trim(); + if (text.Length == 0) + { + return; + } + + statements.Add(text); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SerilogCollection.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SerilogCollection.cs new file mode 100644 index 00000000..58472837 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SerilogCollection.cs @@ -0,0 +1,10 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Serializes tests that mutate the process-wide Serilog.Log.Logger static. +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class SerilogCollection +{ + public const string Name = "Serilog"; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegration.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegration.cs new file mode 100644 index 00000000..47d21622 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegration.cs @@ -0,0 +1,373 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Net; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Opt-in live SessionState Redis seam. Ordinary tests never read Redis:Host +/// or open the test Redis. +/// +internal static class SessionStateRedisIntegration +{ + /// Dedicated Redis endpoint for live SessionState Lua tests (host:port). + public const string EndpointEnvironmentVariable = "VECTORNNTP_REDIS_INTEGRATION"; + + /// Authorized SessionState integration-test Redis. + public const string AuthorizedEndpoint = "198.18.0.70:6379"; + + /// Reason used when the live Redis tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_REDIS_INTEGRATION=198.18.0.70:6379 to run SessionState Lua " + + "against the dedicated test Redis. Ordinary tests do not open Redis and do " + + "not use Redis:Host."; + + /// Parses the opt-in endpoint, or when unset. + public static RedisOptions? TryGetOptions() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var endpoint = value.Trim(); + var port = RedisOptions.DefaultPort; + var host = endpoint; + var separator = endpoint.LastIndexOf(':'); + if (separator > 0 + && separator < endpoint.Length - 1 + && int.TryParse(endpoint.AsSpan(separator + 1), NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) + && parsed is > 0 and <= 65535) + { + host = endpoint[..separator]; + port = parsed; + } + + if (string.IsNullOrWhiteSpace(host)) + { + return null; + } + + return new RedisOptions { Host = [host], Port = port }; + } + + /// Describes the target without inventing extra connection material. + public static string DescribeTarget(RedisOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Host[0] + ":" + options.Port.ToString(CultureInfo.InvariantCulture); + } +} + +/// +/// Shared live Redis connection for SessionState Lua EVAL tests. Deletes only +/// keys created for uniquely named test accounts. +/// +public sealed class SessionStateRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _accounts = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + public string Endpoint { get; private set; } = string.Empty; + + public string ServerVersion { get; private set; } = string.Empty; + + internal RedisSessionStateStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = SessionStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + Endpoint = SessionStateRedisIntegration.DescribeTarget(options); + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + var endpoint = live.Multiplexer.GetEndPoints()[0]; + ServerVersion = live.Multiplexer.GetServer(endpoint).Version.ToString(); + Store = new RedisSessionStateStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var account in _accounts) + { + await DeleteKeysAsync(account); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "SessionState live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateAccountAsync() + { + EnsureConfigured(); + var account = "vnntp.sess.lua." + Guid.NewGuid().ToString("N"); + _accounts.Add(account); + await DeleteKeysAsync(account); + return account; + } + + public async Task DeleteKeysAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)SessionStateKeys.CreateSession(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)SessionStateKeys.CreateSource(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + foreach (var account in _accounts.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)SessionStateKeys.CreateSession(account))) + { + remaining.Add(Encoding.UTF8.GetString(SessionStateKeys.CreateSession(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)SessionStateKeys.CreateSource(account))) + { + remaining.Add(Encoding.UTF8.GetString(SessionStateKeys.CreateSource(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)AccountByteKeys.Create(account))) + { + remaining.Add(Encoding.UTF8.GetString(AccountByteKeys.Create(account))); + } + } + + return remaining; + } + + public ValueTask TryAdmitAsync( + string accountName, + string ip, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + long nowUnixMs, + long leaseMs, + bool trackSessions = false) => + Store.TryAdmitAsync( + accountName, + ip, + ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs), + trackSessions: trackSessions); + + public ValueTask ReleaseAsync( + string accountName, + string ip, + string ownerId, + long sessionGeneration, + long sourceGeneration) => + Store.ReleaseAsync(accountName, ip, ownerId, sessionGeneration, sourceGeneration); + + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + long nowUnixMs, + long leaseMs) => + Store.RenewAsync( + accountName, + ownerId, + sessionGeneration, + sources, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseOwnerAsync(string accountName, string ownerId) => + Store.ReleaseOwnerAsync(accountName, ownerId); + + public async Task SeedAsync(string accountName, string field, long expiryUnixMs, long generation, int count, bool session) + { + EnsureConfigured(); + var key = session ? SessionStateKeys.CreateSession(accountName) : SessionStateKeys.CreateSource(accountName); + _ = await Database.HashSetAsync( + (RedisKey)key, + field, + SessionStateKeys.Value(expiryUnixMs, generation, count)); + } + + public Task SeedSessionAsync(string accountName, string ownerId, long expiryUnixMs, long generation, int count) => + SeedAsync(accountName, ownerId, expiryUnixMs, generation, count, session: true); + + public Task SeedSourceAsync(string accountName, string ip, string ownerId, long expiryUnixMs, long generation, int count) => + SeedAsync(accountName, SessionStateKeys.SourceField(ip, ownerId), expiryUnixMs, generation, count, session: false); + + public async Task ReadSessionAsync(string accountName, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)SessionStateKeys.CreateSession(accountName), ownerId); + return Parse(value); + } + + public async Task ReadSourceAsync(string accountName, string ip, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync( + (RedisKey)SessionStateKeys.CreateSource(accountName), + SessionStateKeys.SourceField(ip, ownerId)); + return Parse(value); + } + + public async Task SessionCountAsync(string accountName) + { + var fields = await HashValuesAsync(SessionStateKeys.CreateSession(accountName)); + var total = 0; + // codeql[cs/linq/missed-where] Parse pattern-match; avoid double Parse from Where+body. + foreach (var value in fields) + { + if (Parse(value) is { } ownership) + { + total += ownership.Count; + } + } + + return total; + } + + public async Task DistinctSourceCountAsync(string accountName) + { + var names = await HashFieldNamesAsync(SessionStateKeys.CreateSource(accountName)); + var ips = new HashSet(StringComparer.Ordinal); + // codeql[cs/linq/missed-where] TrySplitField out-params; foreach preserves parse-and-collect. + foreach (var field in names) + { + if (SessionStateKeys.TrySplitField(field, out var ip, out _)) + { + ips.Add(ip); + } + } + + return ips.Count; + } + + public async Task KeyTimeToLiveAsync(string accountName, bool session) + { + EnsureConfigured(); + var key = session ? SessionStateKeys.CreateSession(accountName) : SessionStateKeys.CreateSource(accountName); + return await Database.KeyTimeToLiveAsync((RedisKey)key); + } + + public static string Format(string ip) => SourceAddressIdentity.Format(IPAddress.Parse(ip)); + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(SessionStateRedisIntegration.SkipReason); + } + } + + private async Task HashValuesAsync(byte[] key) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)key); + return [.. entries.Select(static entry => entry.Value)]; + } + + private async Task HashFieldNamesAsync(byte[] key) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)key); + return [.. entries.Select(static entry => (string)entry.Name!)]; + } + + private static Ownership? Parse(RedisValue value) + { + if (value.IsNullOrEmpty) + { + return null; + } + + return SessionStateKeys.TrySplitValue((string)value!, out var expiry, out var generation, out var count) + ? new Ownership(expiry, generation, count) + : null; + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} + +/// Parsed SessionState HASH value. +public readonly record struct Ownership(long ExpiryUnixMs, long Generation, int Count); diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs new file mode 100644 index 00000000..f61bc2e2 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs @@ -0,0 +1,34 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +public sealed class SessionStateRedisIntegrationConventionTests +{ + [Fact] + public void DescribeTarget_UsesHostAndPortOnly() + { + var description = SessionStateRedisIntegration.DescribeTarget( + new RedisOptions { Host = ["198.18.0.70"], Port = 6379 }); + Assert.Equal(SessionStateRedisIntegration.AuthorizedEndpoint, description); + } + + [Fact] + public void TryGetOptions_ReadsOnlyOptInVariable() + { + Assert.Equal("VECTORNNTP_REDIS_INTEGRATION", SessionStateRedisIntegration.EndpointEnvironmentVariable); + Assert.Contains("do not use Redis:Host", SessionStateRedisIntegration.SkipReason, StringComparison.Ordinal); + Assert.Contains(SessionStateRedisIntegration.AuthorizedEndpoint, SessionStateRedisIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (SessionStateRedisIntegration.TryGetOptions() is not null) + { + return; + } + + var attribute = new SessionStateRedisIntegrationFactAttribute(); + Assert.Equal(SessionStateRedisIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs new file mode 100644 index 00000000..7fe96452 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_REDIS_INTEGRATION is set. +/// When the variable is set, the test runs and must fail if Redis is unreachable. +/// +public sealed class SessionStateRedisIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public SessionStateRedisIntegrationFactAttribute() + { + if (SessionStateRedisIntegration.TryGetOptions() is null) + { + Skip = SessionStateRedisIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs new file mode 100644 index 00000000..6f814a8d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Skips at discovery unless VECTORNNTP_SPAMD_INTEGRATION is set to host:port. +/// Ordinary CI does not set the variable, so these tests never dial a default address. +/// When the variable is set, the test runs and fails if that endpoint is unreachable or malformed. +/// +public sealed class SpamdIntegrationFactAttribute : FactAttribute +{ + /// Environment variable that opts in to a real SPAMD endpoint. + public const string EndpointEnvironmentVariable = "VECTORNNTP_SPAMD_INTEGRATION"; + + /// Skip text used when the opt-in variable is absent. + public const string SkipReason = + "Set VECTORNNTP_SPAMD_INTEGRATION=host:port to run opt-in SPAMD compatibility tests. " + + "Ordinary CI does not set this and must not contact an external scanner."; + + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public SpamdIntegrationFactAttribute() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + Skip = SkipReason; + } + } + + /// Parses host:port. An unset variable is not an endpoint. + /// The variable is set but is not host:port. + public static (string Host, int Port) RequireEndpoint() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException(SkipReason); + } + + var separator = value.LastIndexOf(':'); + if (separator <= 0 + || separator == value.Length - 1 + || !int.TryParse(value[(separator + 1)..], out var port) + || port is < 1 or > 65535) + { + throw new InvalidOperationException( + $"{EndpointEnvironmentVariable} must be host:port. The value '{value}' is not usable."); + } + + return (value[..separator].Trim(), port); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TestHostFactory.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TestHostFactory.cs new file mode 100644 index 00000000..35d29050 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TestHostFactory.cs @@ -0,0 +1,216 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +internal static class TestHostFactory +{ + internal static IApplicationService WrapDns(CloudflareDnsReconciliationService service) => + new CloudflareDnsReconciliationApplicationService(service); + + public const string TestCloudFlareApiKey = "unit-test-cloudflare-api-key"; + + /// 64-hex AES-256 test key for POST X-Trace (not a production secret). + public const string TestXTraceKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + /// Offline placeholder for ConnectionStrings:NntpDB (not a live server). + public const string TestNntpDbConnectionString = "Server=127.0.0.1;Database=nntpdb;User ID=test;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=32;ConnectionIdleTimeout=300;"; + + /// + /// Deliberately malformed NntpDB string similar to the observed parse failure + /// (unclosed quote; rejected by MySqlConnectionStringBuilder). + /// + public const string MalformedNntpDbConnectionString = + "Server=127.0.0.1;Port=3306;Database=nntpdb;User ID=nntpd;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=128;ConnectionIdleTimeout=300;Password=\"unclosed"; + + /// Fake password used only in malformed-string secret-redaction tests. + public const string FakeNntpDbPassword = "fake-nntpdb-password-not-real"; + + /// Malformed NntpDB string that contains . + public const string MalformedNntpDbConnectionStringWithPassword = + "Server=127.0.0.1;Database=nntpdb;User ID=nntpd;Password=\"" + FakeNntpDbPassword + ";"; + + /// Shared parent for test file logs so hosts do not write under the repo logs/. + public static readonly string SharedTestLogDir = Path.Join(Path.GetTempPath(), "vectornntp-nntpd-testhost-logs"); + + /// Unique Nntpd:LogDir so parallel hosts do not share a Serilog file lock. + public static string NewTestLogDir() + { + var dir = Path.Join(SharedTestLogDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + public static readonly IPAddress TestIpv4 = IPAddress.Parse("198.18.0.10"); + public static readonly IPAddress TestIpv6 = IPAddress.Parse("2001:db8::10"); + + public static NntpdOptions CreateOptions( + TimeSpan? gracefulShutdownTimeout = null, + TimeSpan? startupTimeout = null) + { + return CreateValidOptions(gracefulShutdownTimeout, startupTimeout); + } + + public static NntpdOptions CreateValidOptions( + TimeSpan? gracefulShutdownTimeout = null, + TimeSpan? startupTimeout = null) + { + return new NntpdOptions + { + ApplicationName = "VectorNNTPD.ReaderService.Tests", + GracefulShutdownTimeout = gracefulShutdownTimeout ?? TimeSpan.FromSeconds(5), + StartupTimeout = startupTimeout, + StopHostOnUnexpectedServiceTermination = true, + BindAddress = ["*"], + BindPort = GetFreeTcpPort(), + BindPortTls = 0, + CloudFlareApiKey = TestCloudFlareApiKey, + CloudFlareZoneId = "5811a29d39a0732afb5f160c9b137c3d", + XTraceKey = TestXTraceKey, + DnsSuffix = "usenet.ninja", + ServerId = 1, + }; + } + + /// Reserves an ephemeral TCP port on loopback for offline listener tests. + public static int GetFreeTcpPort() + { + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return ((IPEndPoint)socket.LocalEndPoint!).Port; + } + + internal static ApplicationServiceManager CreateServiceManager( + IEnumerable services, + NntpdOptions? options = null) + { + return new ApplicationServiceManager( + services, + options ?? CreateOptions(), + NullLogger.Instance); + } + + public static ApplicationLifecycle CreateLifecycle( + IEnumerable services, + NntpdOptions? options = null) + { + options ??= CreateOptions(); + var manager = CreateServiceManager(services, options); + return new ApplicationLifecycle( + manager, + options, + NullLogger.Instance); + } + + /// + /// Registers deterministic networking and Cloudflare test configuration before hosting services. + /// + public static void ConfigureNntpdTestHost(HostApplicationBuilder builder, ILocalIpAddressAssignee? assignee = null) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ServerId)}"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = TestXTraceKey, + [NntpdOptions.XTraceKeyConfigurationKey] = TestXTraceKey, + // Replace appsettings BindAddress entirely (in-memory must clear leftover indices). + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindAddress:1"] = null, + ["Redis:Host:0"] = "127.0.0.1", + ["Redis:Port"] = "6379", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = TestNntpDbConnectionString, + [$"{NntpdOptions.SectionName}:LogDir"] = NewTestLogDir(), + }); + + var localAssignee = assignee ?? new FakeLocalIpAddressAssignee(TestIpv4, TestIpv6); + builder.Services.AddSingleton(localAssignee); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + PromoteSharedKeysToRoot(builder.Configuration); + IsolateTransit(builder.Services); + + // Ensure machine-specific appsettings bind entries cannot leak into host tests. + // Force TLS off so ACME never contacts Let's Encrypt during offline host tests + // (committed appsettings may set BindPortTls > 0 for local operator use). + // Use an ephemeral plain port so NntpPlainListenerService can bind without conflicts. + builder.Services.PostConfigure(static options => + { + options.BindAddress = ["*"]; + options.BindPort = GetFreeTcpPort(); + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + }); + } + + /// + /// Copies leftover Nntpd: shared keys to the configuration root so older + /// test fixtures keep working after the Common-owned root contract. + /// + public static void PromoteSharedKeysToRoot(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration is not IConfigurationBuilder builder) + { + return; + } + + var extras = new Dictionary(StringComparer.OrdinalIgnoreCase); + string[] keys = + [ + NntpdOptions.CloudFlareApiKeyConfigurationKey, + AcmeCloudflareOptions.AcmeAccountConfigurationKey, + nameof(NntpdOptions.AcmeCertificatePassword), + nameof(NntpdOptions.AcmeStateDir), + nameof(NntpdOptions.AcmeDirectoryUrl), + nameof(NntpdOptions.AcmeRenewalThresholdDays), + ]; + + // codeql[cs/linq/missed-where] Dual configuration lookup; foreach preserves short-circuit clarity. + foreach (var key in keys) + { + if (configuration[key] is null && configuration[$"{NntpdOptions.SectionName}:{key}"] is { } value) + { + extras[key] = value; + } + } + + if (extras.Count > 0) + { + builder.AddInMemoryCollection(extras); + } + } + + /// + /// Drops operator Transit peers inherited from VectorNNTPD.ReaderService.json. + /// Host tests use deny-by-default Transit so local operator peers do not leak in. + /// + public static void IsolateTransit(IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.PostConfigure(static options => options.Clear()); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs new file mode 100644 index 00000000..1cf13558 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs @@ -0,0 +1,280 @@ +using System.Collections.Concurrent; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +/// +/// Opt-in live TransitPeerState Redis seam. Reuses +/// VECTORNNTP_REDIS_INTEGRATION and never reads Redis:Host. +/// +internal static class TransitPeerStateRedisIntegration +{ + /// Dedicated Redis endpoint for live Transit Lua tests (host:port). + public const string EndpointEnvironmentVariable = SessionStateRedisIntegration.EndpointEnvironmentVariable; + + /// Authorized TransitPeerState integration-test Redis. + public const string AuthorizedEndpoint = SessionStateRedisIntegration.AuthorizedEndpoint; + + /// Reason used when the live Redis tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_REDIS_INTEGRATION=198.18.0.70:6379 to run TransitPeerState Lua " + + "against the dedicated test Redis. Ordinary tests do not open Redis and do " + + "not use Redis:Host."; + + /// Parses the opt-in endpoint, or when unset. + public static VectorNNTPD.ReaderService.Configuration.RedisOptions? TryGetOptions() => + SessionStateRedisIntegration.TryGetOptions(); +} + +/// +/// Shared live Redis connection for TransitPeerState Lua EVAL tests. Deletes only +/// keys created for uniquely named test peer identifiers. +/// +public sealed class TransitPeerStateRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _identifiers = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + public string Endpoint { get; private set; } = string.Empty; + + public string ServerVersion { get; private set; } = string.Empty; + + internal RedisTransitPeerStateStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = TransitPeerStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + Endpoint = SessionStateRedisIntegration.DescribeTarget(options); + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + var endpoint = live.Multiplexer.GetEndPoints()[0]; + ServerVersion = live.Multiplexer.GetServer(endpoint).Version.ToString(); + Store = new RedisTransitPeerStateStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var identifier in _identifiers) + { + await DeleteKeyAsync(identifier); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "TransitPeerState live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateIdentifierAsync() + { + EnsureConfigured(); + var identifier = "vnntp.tconn.lua." + Guid.NewGuid().ToString("N"); + _identifiers.Add(identifier); + await DeleteKeyAsync(identifier); + return identifier; + } + + public async Task DeleteKeyAsync(string identifier) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + // codeql[cs/linq/missed-where] Async KeyExists filter; Where cannot await. + foreach (var identifier in _identifiers.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)TransitPeerStateKeys.Create(identifier))) + { + remaining.Add(Encoding.UTF8.GetString(TransitPeerStateKeys.Create(identifier))); + } + } + + return remaining; + } + + public ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + long nowUnixMs, + long leaseMs) => + Store.TryAdmitAsync( + identifier, + ownerId, + maxIncoming, + generation, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseAsync(string identifier, string ownerId, long generation) => + Store.ReleaseAsync(identifier, ownerId, generation); + + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + long nowUnixMs, + long leaseMs) => + Store.RenewAsync( + identifier, + ownerId, + generation, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseOwnerAsync(string identifier, string ownerId) => + Store.ReleaseOwnerAsync(identifier, ownerId); + + public async Task SeedAsync(string identifier, string ownerId, long expiryUnixMs, long generation, int count) + { + EnsureConfigured(); + _ = await Database.HashSetAsync( + (RedisKey)TransitPeerStateKeys.Create(identifier), + ownerId, + TransitPeerStateKeys.Value(expiryUnixMs, generation, count)); + } + + public async Task ReadAsync(string identifier, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)TransitPeerStateKeys.Create(identifier), ownerId); + return Parse(value); + } + + public async Task CountAsync(string identifier) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + var total = 0; + // codeql[cs/linq/missed-where] Parse pattern-match; avoid double Parse from Where+body. + foreach (var entry in entries) + { + if (Parse(entry.Value) is { } ownership) + { + total += ownership.Count; + } + } + + return total; + } + + public async Task KeyTimeToLiveAsync(string identifier) + { + EnsureConfigured(); + return await Database.KeyTimeToLiveAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + public async Task FieldCountAsync(string identifier) + { + EnsureConfigured(); + return (int)await Database.HashLengthAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(TransitPeerStateRedisIntegration.SkipReason); + } + } + + private static Ownership? Parse(RedisValue value) + { + if (value.IsNullOrEmpty) + { + return null; + } + + return TransitPeerStateKeys.TrySplitValue((string)value!, out var expiry, out var generation, out var count) + ? new Ownership(expiry, generation, count) + : null; + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_REDIS_INTEGRATION is set. +/// +public sealed class TransitPeerStateRedisIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public TransitPeerStateRedisIntegrationFactAttribute() + { + if (TransitPeerStateRedisIntegration.TryGetOptions() is null) + { + Skip = TransitPeerStateRedisIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs new file mode 100644 index 00000000..c76df75e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.ReaderService.Tests.Fixtures; + +public sealed class TransitPeerStateRedisIntegrationConventionTests +{ + [Fact] + public void OptInVariable_MatchesSessionStateDedicatedTestRedis() + { + Assert.Equal( + SessionStateRedisIntegration.EndpointEnvironmentVariable, + TransitPeerStateRedisIntegration.EndpointEnvironmentVariable); + Assert.Equal( + SessionStateRedisIntegration.AuthorizedEndpoint, + TransitPeerStateRedisIntegration.AuthorizedEndpoint); + Assert.Contains("do not use Redis:Host", TransitPeerStateRedisIntegration.SkipReason, StringComparison.Ordinal); + Assert.Contains(TransitPeerStateRedisIntegration.AuthorizedEndpoint, TransitPeerStateRedisIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (TransitPeerStateRedisIntegration.TryGetOptions() is not null) + { + return; + } + + var attribute = new TransitPeerStateRedisIntegrationFactAttribute(); + Assert.Equal(TransitPeerStateRedisIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json new file mode 100644 index 00000000..c7741e55 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Fixtures/startup-smoke-reader.json @@ -0,0 +1,2987 @@ +{ + "Moderation": { + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", + "RetrievedFrom": "InnUrl", + "Url": "https://github.com/InterNetNews/inn/blob/main/samples/moderators" + } + }, + "Email": { + "Enabled": false, + "DefaultFrom": "noreply@usenet.ninja", + "EnvelopeSender": "noreply@usenet.ninja", + "Smtp": { + "Host": "", + "Port": 587, + "Security": "StartTls", + "Username": "", + "Password": "", + "RequireTlsForAuthentication": true, + "ConnectTimeout": "00:00:15", + "CommandTimeout": "00:00:30", + "MaxAttempts": 3, + "InitialRetryDelay": "00:00:02", + "MaximumRetryDelay": "00:01:00" + }, + "Spool": { + "Directory": "spool/smtp", + "ShutdownTimeout": "00:00:15", + "ScanInterval": "00:00:01" + } + }, + "Control": { + "PgpAuthorities": { + "Authorities": [ + { + "AdminGroup": "aioe.system", + "Authorizations": [ + { + "From": "usenet@aioe.org", + "Message": "checkgroups", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "newgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "rmgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + } + ], + "Contact": "usenet@aioe.org", + "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8", + "KeyUrl": "http://news.aioe.org/hierarchy/aioe.txt", + "Name": "AIOE", + "SyncableServer": "nntp.aioe.org", + "Url": "http://news.aioe.org/hierarchy/" + }, + { + "AdminGroup": "alabama.config", + "Authorizations": [ + { + "From": "jpc@suespammers.org", + "Message": "checkgroups", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "newgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "rmgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + } + ], + "Contact": "jpc@suespammers.org", + "Name": "ALABAMA & HSV" + }, + { + "AdminGroup": "at.usenet.gruppen", + "Authorizations": [ + { + "From": "control@usenet.backbone.at", + "Message": "checkgroups", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "newgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "rmgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + } + ], + "KeyUrl": "http://www.usenet.at/pgpkey.asc", + "Name": "AT", + "Url": "http://www.usenet.at/" + }, + { + "AdminGroup": "aus.net.news", + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "AUS", + "Url": "http://aus.news-admin.org/" + }, + { + "AdminGroup": "be.announce", + "Authorizations": [ + { + "From": "group-admin@usenet.be", + "Message": "checkgroups", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "newgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "rmgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + } + ], + "Contact": "be-hierarchy-admin@usenet.be", + "KeyFingerprint": "302A459470DE1FD5818C5864D2F70871", + "KeyUrl": "http://usenet.be/be.announce.newgroups.asc", + "Name": "BE", + "Url": "http://usenet.be/" + }, + { + "AdminGroup": "belwue.infos", + "Authorizations": [ + { + "From": "news@news.belwue.de", + "Message": "checkgroups", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "newgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "rmgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + } + ], + "Contact": "news@belwue.de", + "KeyFingerprint": "2EFF2B8A891FE5AA6F89672450B542D9", + "KeyUrl": "http://www.belwue.de/fileadmin/belwue/Dokumente/Newsserver/belwue-hir-control.asc", + "Name": "BELWUE", + "Url": "http://www.belwue.de/produkte/dienste/dienste-usenet1.html" + }, + { + "AdminGroup": "bionet.general", + "Authorizations": [ + { + "From": "Biosci-control-key@net.bio.net", + "Message": "checkgroups", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "newgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "rmgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + } + ], + "KeyFingerprint": "EBC0F1BA260BC6D6FB8DEDC4AE5D1054", + "Name": "BIONET", + "Url": "http://www.bio.net/" + }, + { + "AdminGroup": "bit.admin", + "Authorizations": [ + { + "From": "bit@newsadmin.com", + "Message": "checkgroups", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "newgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "rmgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + } + ], + "Name": "BIT", + "Url": "http://www.newsadmin.com/bit/bit.htm" + }, + { + "AdminGroup": "bln.misc", + "Authorizations": [ + { + "From": "news@fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + } + ], + "Contact": "news@fu-berlin.de", + "KeyFingerprint": "2B24B5C78A4861407BCD5A88281D90763F368EA0", + "KeyUrl": "https://ftp.fu-berlin.de/doc/news/bln/bln-key.txt", + "Name": "BLN", + "Url": "https://ftp.fu-berlin.de/doc/news/bln/bln-info.txt" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "BNE", + "Url": "http://bne.news-admin.org/" + }, + { + "Contact": "bofh-control@lists.killfile.org", + "KeyFingerprint": "40B58A561E006152083E38B3CEF569807DC1A266", + "Name": "BOFH" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "CANB", + "Url": "http://canb.news-admin.org/" + }, + { + "Authorizations": [ + { + "From": "felix.rauch@nice.ch", + "Message": "checkgroups", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "newgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "rmgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + } + ], + "Contact": "ch-news-admin@use-net.ch", + "KeyFingerprint": "7180D68CA7DE2C70624A486ED99602DF", + "KeyUrl": "http://www.use-net.ch/Usenet/adminkey.html", + "Name": "CH", + "Url": "http://www.use-net.ch/Usenet/" + }, + { + "Contact": "koordination@cl-netz.de", + "KeyUrl": "http://www.cl-netz.de/control.txt", + "Name": "CL", + "Url": "http://www.cl-netz.de/" + }, + { + "AdminGroup": "clari.net.admin", + "Authorizations": [ + { + "From": "cl*@clarinet.com", + "Message": "rmgroup", + "Newsgroups": "clari.*", + "VerificationIdentity": "ClariNet.Group" + } + ], + "Contact": "support@clari.net", + "KeyUrl": "http://www.clari.net/tech/clarikey.txt", + "Name": "CLARI" + }, + { + "AdminGroup": "cn.announce", + "Authorizations": [ + { + "From": "control@bentium.com", + "Message": "checkgroups", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "newgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "rmgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + } + ], + "KeyFingerprint": "6297EE33F71625C1A49E47BAC53E5E9E", + "Name": "CN", + "Url": "http://news.yaako.com/" + }, + { + "AdminGroup": "cn.bbs.admin.announce", + "Authorizations": [ + { + "From": "control@cn-bbs.org", + "Message": "checkgroups", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "newgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "rmgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + } + ], + "Name": "CN.BBS", + "Url": "http://bbs.cn.news-admin.org/" + }, + { + "AdminGroup": "news.announce.newgroups", + "Authorizations": [ + { + "From": "group-admin@isc.org", + "Message": "checkgroups", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "newgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "rmgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + } + ], + "Contact": "board@big-8.org", + "KeyFingerprint": "8E181BEF0BE638962C1A5C98FAFE7B550C18C8B7", + "Name": "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK", + "Url": "https://www.big-8.org/" + }, + { + "AdminGroup": "de.admin.news.announce", + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "checkgroups", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "newgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Contact": "moderator@dana.de", + "KeyFingerprint": "5BB05288BF55194F667DC2AE16262825", + "KeyUrl": "http://www.dana.de/pgp/dana.txt", + "Name": "DE", + "Url": "http://www.dana.de/" + }, + { + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.alt.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Name": "DE.ALT" + }, + { + "AdminGroup": "demon.news", + "Authorizations": [ + { + "From": "newsmaster@demon.net", + "Message": "checkgroups", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "newgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "rmgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + } + ], + "Contact": "newsmaster@demon.net", + "KeyUrl": "ftp://ftp.demon.co.uk/pub/news/doc/demon.news.txt", + "Name": "DEMON" + }, + { + "Authorizations": [ + { + "From": "news@news.dknet.dk", + "Message": "checkgroups", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "newgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "rmgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + } + ], + "KeyFingerprint": "7CB2C750F37D5D738CEE2E3F558072FF", + "KeyUrl": "http://www.usenet.dk/dk-admin/pubkey.html", + "Name": "DK", + "Url": "http://www.usenet.dk/dk-admin/" + }, + { + "AdminGroup": "easynet.support", + "Authorizations": [ + { + "From": "newsmaster@easynet.net", + "Message": "checkgroups", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "newgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "rmgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + } + ], + "Contact": "Christiaan Keet ", + "Name": "EASYNET" + }, + { + "Authorizations": [ + { + "From": "news@news.ut.ee", + "Message": "checkgroups", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "newgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "rmgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + } + ], + "Contact": "usenet@news.ut.ee", + "KeyUrl": "http://news.ut.ee/pubkey.asc", + "Name": "EE", + "Url": "http://news.ut.ee/" + }, + { + "AdminGroup": "eug.config", + "Authorizations": [ + { + "From": "newsadmin@efn.org", + "Message": "checkgroups", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "newgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "rmgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + } + ], + "Name": "EFN & EUG" + }, + { + "AdminGroup": "england.news.policy", + "Authorizations": [ + { + "From": "admin@england.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "newgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + } + ], + "Contact": "admin@england.news-admin.org", + "KeyFingerprint": "DA3EC20146E561CBA24309CA136D311F", + "Name": "ENGLAND" + }, + { + "AdminGroup": "es.news.anuncios", + "Authorizations": [ + { + "From": "moderador@corus-es.org", + "Message": "checkgroups", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "newgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "rmgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + } + ], + "Contact": "moderador@corus-es.org", + "KeyUrl": "http://www.corus-es.org/docs/esnews.asc", + "Name": "ES", + "Url": "http://www.corus-es.org/docs/es_newsadmins_faq.txt" + }, + { + "Authorizations": [ + { + "From": "mod-ena@ennui.org", + "Message": "checkgroups", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "newgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "rmgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + } + ], + "Contact": "", + "KeyUrl": "http://ennui.org/esp/mod-ena.asc", + "Name": "ESP", + "Url": "http://ennui.org/esp/" + }, + { + "Authorizations": [ + { + "From": "news@eternal-september.org", + "Message": "checkgroups", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "newgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "rmgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + } + ], + "Contact": "news@eternal-september.org", + "KeyFingerprint": "B79A2BC67D5AFF7918E2AC914BC125F1", + "KeyUrl": "http://www.eternal-september.org/control/pgpkey.txt", + "Name": "ETERNAL-SEPTEMBER", + "Url": "http://www.eternal-september.org/" + }, + { + "AdminGroup": "europa.usenet.admin", + "Authorizations": [ + { + "From": "group-admin@usenet.eu.org", + "Message": "checkgroups", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "newgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "rmgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + } + ], + "KeyFingerprint": "3A05A849FB16292575E3DEBB69E01DB4", + "KeyUrl": "http://www.europa.usenet.eu.org/pgp/index.html", + "Name": "EUROPA", + "Url": "http://www.europa.usenet.eu.org/" + }, + { + "AdminGroup": "ffm.admin", + "Authorizations": [ + { + "From": "ffm.admin@arcor.de", + "Message": "checkgroups", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "newgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "rmgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + } + ], + "KeyFingerprint": "53A082626FC781C9CF53AB00A3F8C211", + "KeyUrl": "ftp://ftp.arcor-online.net/pub/news/PGPKEY.FFM", + "Name": "FFM", + "Url": "http://ffm.arcornews.de/" + }, + { + "AdminGroup": "fido.belg.news", + "Authorizations": [ + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "checkgroups", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "newgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "rmgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + } + ], + "Name": "FIDO.BELG" + }, + { + "Authorizations": [ + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + } + ], + "KeyUrl": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/PGP-Key", + "Name": "FIDO.GER", + "Url": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/fido.ger-info.english" + }, + { + "AdminGroup": "fido7.postmasters", + "Authorizations": [ + { + "From": "newgroups-request@fido7.org", + "Message": "checkgroups", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "newgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "rmgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + } + ], + "KeyUrl": "http://www.fido7.org/pgpcontrol.html", + "Name": "FIDO7", + "Url": "http://www.fido7.org/" + }, + { + "AdminGroup": "fj.news.announce", + "Authorizations": [ + { + "From": "committee@fj-news.org", + "Message": "checkgroups", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "newgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "rmgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + } + ], + "Contact": "committee@fj-news.org", + "KeyUrl": "http://www.is.tsukuba.ac.jp/~yas/fj/fj.asc", + "Name": "FJ", + "Url": "http://www.fj-news.org/index.html.en" + }, + { + "AdminGroup": "flora.general", + "Authorizations": [ + { + "From": "news@flora.ottawa.on.ca", + "Message": "checkgroups", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "newgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "rmgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + } + ], + "Contact": "russell@flora.org", + "Name": "FLORA" + }, + { + "AdminGroup": "fr.usenet.forums.annonces", + "Authorizations": [ + { + "From": "control@usenet-fr.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "newgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + } + ], + "KeyFingerprint": "D5F369B297573622015354E7FA423E89970EB10B", + "KeyUrl": "https://www.usenet-fr.net/pgp-fr-2020.txt", + "Name": "FR", + "Url": "https://www.usenet-fr.net/" + }, + { + "AdminGroup": "france.admin.evolutions", + "Authorizations": [ + { + "From": "control@usenet-france.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "newgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + } + ], + "Contact": "control@usenet-france.news.eu.org", + "Name": "FRANCE" + }, + { + "AdminGroup": "gnu.gnusenet.config", + "Authorizations": [ + { + "From": "usenet@gnu.org", + "Message": "checkgroups", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "newgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "rmgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + } + ], + "KeyUrl": "http://www.gnu.org/usenet/usenet-pgp-key.txt", + "Name": "GNU", + "Url": "http://www.gnu.org/usenet/usenet.html" + }, + { + "Authorizations": [ + { + "From": "news@gnuu.de", + "Message": "rmgroup", + "Newsgroups": "gnuu.*", + "VerificationIdentity": "news@gnuu.de" + } + ], + "KeyUrl": "http://www.gnuu.de/config/PGPKEY.GNUU", + "Name": "GNUU", + "Url": "http://www.gnuu.de/" + }, + { + "AdminGroup": "grisbi.admin", + "Authorizations": [ + { + "From": "grisbi-control@grisbi.org", + "Message": "checkgroups", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "newgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "rmgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + } + ], + "Contact": "newsmaster@grisbi.org", + "KeyFingerprint": "EB350C030080BD2A7E0CA4C9F2C62A3DC86CC6E1", + "KeyUrl": "http://news.grisbi.org/public-key.asc", + "Name": "GRISBI", + "Url": "http://news.grisbi.org/" + }, + { + "Authorizations": [ + { + "From": "news@zhaum.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "newgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + } + ], + "Name": "HACKTIC" + }, + { + "AdminGroup": "hamburg.koordination", + "Authorizations": [ + { + "From": "hamburg@steering-group.net", + "Message": "checkgroups", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "newgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "rmgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + } + ], + "Contact": "hamburg@steering-group.net", + "KeyFingerprint": "3EE70CBB6E0194EE456FC557F4B9548E", + "KeyUrl": "http://www.steering-group.net/hamburg/hamburg.koordination.txt", + "Name": "HAMBURG", + "Url": "http://www.steering-group.net/hamburg/" + }, + { + "AdminGroup": "hamster.de.config", + "Authorizations": [ + { + "From": "hamster-control@snafu.de", + "Message": "checkgroups", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "newgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "rmgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + } + ], + "Contact": "hamster-contact@snafu.de", + "KeyFingerprint": "1275A9428AD61F776ACFB40C79155F93", + "Name": "HAMSTER" + }, + { + "AdminGroup": "han.news.admin", + "Authorizations": [ + { + "From": "newgroups-request@usenet.or.kr", + "Message": "checkgroups", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "newgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "rmgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + } + ], + "Contact": "newgroups-request@usenet.or.kr", + "KeyUrl": "ftp://ftp.usenet.or.kr/pub/korea/usenet/pgp/PGPKEY.han", + "Name": "HAN" + }, + { + "AdminGroup": "hiv.config", + "Authorizations": [ + { + "From": "news@hivnet.org", + "Message": "checkgroups", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "newgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "rmgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + } + ], + "Contact": "news@hivnet.org", + "KeyFingerprint": "5DD60EDC1E2DEA0BB0564DD65253D7A4", + "Name": "HIV" + }, + { + "AdminGroup": "houston.usenet.config", + "Authorizations": [ + { + "From": "news@academ.com", + "Message": "checkgroups", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "newgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "rmgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + } + ], + "Name": "HOUSTON" + }, + { + "AdminGroup": "hr.news.admin", + "Authorizations": [ + { + "From": "newsmaster@carnet.hr", + "Message": "checkgroups", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "newgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "rmgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + } + ], + "Contact": "newsmaster@carnet.hr", + "KeyFingerprint": "0EE574FB1C407ADB0AACA52F71921BA3ED63AD9A", + "KeyUrl": "http://newsfeed.carnet.hr/control/key.txt", + "Name": "HR", + "SyncableServer": "news.carnet.hr", + "Url": "http://newsfeed.carnet.hr/control/" + }, + { + "Authorizations": [ + { + "From": "news-admin@humanityquest.com", + "Message": "checkgroups", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "newgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "rmgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + } + ], + "Contact": "news-admin@humanityquest.com", + "KeyFingerprint": "BA3DB306B6F552AABA8F32F08C4F504016F9C046", + "KeyUrl": "http://www.humanityquest.com/projects/newsgroups/PGP.htm", + "Name": "HUMANITYQUEST", + "Url": "http://www.humanityquest.com/projects/newsgroups/" + }, + { + "AdminGroup": "hun.admin.news", + "Authorizations": [ + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "checkgroups", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "newgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "rmgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + } + ], + "KeyUrl": "http://gatling.ikk.sztaki.hu/~kissg/news/hun.admin.news.asc", + "Name": "HUN", + "Url": "http://gatling.ikk.sztaki.hu/~kissg/news/" + }, + { + "AdminGroup": "ie.news.group", + "Authorizations": [ + { + "From": "control@usenet.ie", + "Message": "checkgroups", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "newgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "rmgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + } + ], + "Contact": "control@usenet.ie", + "Name": "IE" + }, + { + "AdminGroup": "is.isnet", + "Authorizations": [ + { + "From": "group-admin@usenet.is", + "Message": "checkgroups", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "newgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "rmgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + } + ], + "Contact": "IS Group Admins ", + "KeyFingerprint": "33328D461E5E1C7F48608E72E53ECAEA", + "KeyUrl": "http://www.usenet.is/group-admin.asc", + "Name": "IS", + "Url": "http://www.usenet.is/" + }, + { + "AdminGroup": "isu.newgroups", + "Authorizations": [ + { + "From": "news@news.isu.edu.tw", + "Message": "checkgroups", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "newgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "rmgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + } + ], + "Contact": "news@news.isu.edu.tw", + "KeyUrl": "http://news.isu.edu.tw/isu.asc", + "Name": "ISU", + "Url": "http://news.isu.edu.tw/" + }, + { + "AdminGroup": "it.news.annunci", + "Authorizations": [ + { + "From": "gcn@news.nic.it", + "Message": "checkgroups", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "newgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "rmgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + } + ], + "Contact": "gcn@news.nic.it", + "KeyFingerprint": "94A4F7B54696D6C7A673F298C48CD0E0", + "KeyUrl": "http://www.news.nic.it/pgp.txt", + "Name": "IT", + "Url": "http://www.news.nic.it/" + }, + { + "AdminGroup": "italia.announce.newgroups", + "Authorizations": [ + { + "From": "news@news.cineca.it", + "Message": "checkgroups", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "newgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "rmgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + } + ], + "Contact": "news@news.cineca.it", + "KeyFingerprint": "0FBB7162DA5D5DB8D586FC2802671A6B", + "KeyUrl": "http://news.cineca.it/italia/italia-pgp.txt", + "Name": "ITALIA", + "Url": "http://news.cineca.it/italia/" + }, + { + "AdminGroup": "japan.admin.announce", + "Authorizations": [ + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "checkgroups", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "newgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "rmgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + } + ], + "Contact": "Tsuneo Tanaka ", + "KeyFingerprint": "6AFA1947691B107438534B1BD8BA3E85", + "KeyUrl": "http://grex.cyberspace.org/~tt/japan.admin.announce.asc", + "Name": "JAPAN", + "Url": "http://www.asahi-net.or.jp/~AE5T-KSN/japan-e.html" + }, + { + "AdminGroup": "jlug.config", + "Authorizations": [ + { + "From": "news@linux.or.jp", + "Message": "checkgroups", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "newgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "rmgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + } + ], + "Contact": "news@linux.or.jp", + "KeyUrl": "http://www.linux.or.jp/pgpkey/news", + "Name": "JLUG", + "Url": "http://www.linux.or.jp/community/news/index.html" + }, + { + "AdminGroup": "ka.admin", + "Contact": "usenet@karlsruhe.org", + "KeyFingerprint": "DE19BB2576198117F067D223E8C87C90", + "KeyUrl": "http://www.karlsruhe.org/pubkey-news.karlsruhe.org.asc", + "Name": "KA", + "Url": "http://www.karlsruhe.org/" + }, + { + "Authorizations": [ + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "checkgroups", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + }, + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "rmgroup", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + } + ], + "Name": "KANTO" + }, + { + "AdminGroup": "kassel.admin", + "Authorizations": [ + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + } + ], + "Name": "KASSEL" + }, + { + "AdminGroup": "linux.admin.news", + "Authorizations": [ + { + "From": "linux-admin@bofh.it", + "Message": "checkgroups", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "newgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "rmgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + } + ], + "Contact": "Marco d'Itri ", + "KeyFingerprint": "81B327994FCE32D11BC9010DBBB32E41", + "Name": "LINUX" + }, + { + "AdminGroup": "malta.config", + "Authorizations": [ + { + "From": "cmeli@cis.um.edu.mt", + "Message": "checkgroups", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "newgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "rmgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + } + ], + "Contact": "cmeli@cis.um.edu.mt", + "KeyFingerprint": "2017015CF0D01A42E41330580B1448A6", + "KeyUrl": "http://www.cis.um.edu.mt/news-malta/PGP.PUBLICKEY", + "Name": "MALTA", + "Url": "http://www.malta.news-admin.org/" + }, + { + "AdminGroup": "maus.info", + "Authorizations": [ + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + } + ], + "KeyFingerprint": "8252C77026B972A1379855983F26623E", + "Name": "MAUS" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "MELB", + "Url": "http://melb.news-admin.org/" + }, + { + "AdminGroup": "mensa.config", + "Authorizations": [ + { + "From": "usenet@newsgate.mensa.org", + "Message": "checkgroups", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "newgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "rmgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + } + ], + "Contact": "usenet@newsgate.mensa.org", + "KeyFingerprint": "52B9396385D908068E197344973C5005DC7DB7A7", + "Name": "MENSA" + }, + { + "AdminGroup": "microsoft.public.news.server", + "Authorizations": [ + { + "From": "control-microsoft@trigofacile.com", + "Message": "checkgroups", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "newgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "rmgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + } + ], + "Contact": "control-microsoft@trigofacile.com", + "KeyFingerprint": "DF705FC9F615D52E02DBA3CB63A98D13E60E2FAA", + "KeyUrl": "http://www.trigofacile.com/divers/usenet/clefs/pgpkey-microsoft.asc", + "Name": "MICROSOFT", + "SyncableServer": "msnews.microsoft.com", + "Url": "http://www.trigofacile.com/divers/usenet/clefs/index.htm" + }, + { + "AdminGroup": "milw.config", + "Authorizations": [ + { + "From": "milw@usenet.mil.wi.us", + "Message": "checkgroups", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "newgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "rmgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + } + ], + "Contact": "milw@usenet.mil.wi.us", + "KeyFingerprint": "6E9B9F7098AB9CE5C3C00582215BF49E", + "KeyUrl": "http://usenet.mil.wi.us/pgpkey", + "Name": "MILW", + "Url": "http://usenet.mil.wi.us/" + }, + { + "AdminGroup": "muc.admin", + "Authorizations": [ + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "checkgroups", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "newgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "rmgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + } + ], + "KeyFingerprint": "43C70E7C45C706E0BD6F76CE07395E66", + "KeyUrl": "http://home.arcor.de/andreas-barth/muc-admin.html", + "Name": "MUC" + }, + { + "AdminGroup": "netscape.public.admin", + "Authorizations": [ + { + "From": "news@netscape.com", + "Message": "checkgroups", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "newgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "rmgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + } + ], + "Contact": "news@netscape.com", + "KeyFingerprint": "B78055121F9C170B8666AD3BDB6835EC", + "Name": "NETSCAPE", + "Url": "http://www.mozilla.org/community.html" + }, + { + "AdminGroup": "nippon.news.group", + "Authorizations": [ + { + "From": "news@gcd.org", + "Message": "rmgroup", + "Newsgroups": "nippon.*", + "VerificationIdentity": "nippon.news.group" + } + ], + "KeyFingerprint": "BCCF15CDB13CDFB3C3DE356F2FF746DB", + "KeyUrl": "http://www.gcd.org/news/nippon/", + "Name": "NIPPON", + "Url": "http://www.gcd.org/news/nippon/" + }, + { + "AdminGroup": "nl.newsgroups", + "Authorizations": [ + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "checkgroups", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "newgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "rmgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + } + ], + "Contact": "nl-admin@nic.surfnet.nl", + "KeyFingerprint": "45200BD5A121EA7CEFB2956C25754D27", + "Name": "NL", + "Url": "http://nl.news-admin.org/info/nladmin.html" + }, + { + "Authorizations": [ + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "news@kink.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + } + ], + "KeyFingerprint": "6B62EB534D5D2F9635D9C89CB0650E4C", + "Name": "NL-ALT" + }, + { + "Authorizations": [ + { + "From": "news@nl.linux.org", + "Message": "checkgroups", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "newgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "rmgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + } + ], + "KeyFingerprint": "63DCB2510AF3DD72C2BDC6FDC1C544CF", + "Name": "NLO", + "SyncableServer": "news.nl.linux.org", + "Url": "http://mail.nl.linux.org/" + }, + { + "AdminGroup": "nntpworld.config", + "Authorizations": [ + { + "From": "usenet@nntpworld.net", + "Message": "checkgroups", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "newgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "rmgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + } + ], + "Contact": "usenet@nntpworld.net", + "KeyFingerprint": "C7BBE0567A8A5A8609B5E329E1BB8CE1", + "KeyUrl": "http://www.nntpworld.net/nntpworld.asc", + "Name": "NNTPWORLD", + "Url": "http://www.nntpworld.net/" + }, + { + "AdminGroup": "no.usenet.admin", + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "checkgroups", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "newgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + } + ], + "KeyUrl": "http://www.usenet.no/pgp-key.txt", + "Name": "NO", + "Url": "http://www.usenet.no/" + }, + { + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.alt.*", + "VerificationIdentity": "no-hir-control" + } + ], + "Name": "NO.ALT" + }, + { + "AdminGroup": "nrw.admin.announce", + "Authorizations": [ + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "checkgroups", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "newgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "rmgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + } + ], + "Contact": "moderator@nrw.usenetverwaltung.de", + "KeyFingerprint": "134A80FED634B464AF32083F620EB1E2", + "KeyUrl": "http://nrw.usenetverwaltung.de/pgp/nrw.asc", + "Name": "NRW", + "Url": "http://nrw.usenetverwaltung.de/" + }, + { + "AdminGroup": "nz.net.announce", + "Authorizations": [ + { + "From": "root@usenet.net.nz", + "Message": "checkgroups", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "newgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "rmgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + } + ], + "Contact": "root@usenet.net.nz", + "KeyFingerprint": "07DF48AAD0EDAA881670C591653D1A28", + "Name": "NZ", + "Url": "http://www.faqs.org/faqs/usenet/nz-news-hierarchy" + }, + { + "Authorizations": [ + { + "From": "news@news2.open-news-network.org", + "Message": "checkgroups", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "newgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "rmgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + } + ], + "Name": "OPENNEWS", + "Url": "http://www.open-news-network.org/" + }, + { + "AdminGroup": "openwatcom.contributors", + "Authorizations": [ + { + "From": "admin@openwatcom.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "newgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + } + ], + "Contact": "admin@openwatcom.news-admin.org", + "KeyUrl": "http://cmeerw.org/files/openwatcom/pgp-openwatcom.asc", + "Name": "OPENWATCOM", + "SyncableServer": "news.openwatcom.org", + "Url": "http://www.openwatcom.org/" + }, + { + "Authorizations": [ + { + "From": "*@opera.com", + "Message": "checkgroups", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "newgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "rmgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + } + ], + "Contact": "usenet@opera.com", + "Name": "OPERA", + "SyncableServer": "news.opera.com" + }, + { + "Authorizations": [ + { + "From": "postmaster@upb.de", + "Message": "checkgroups", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "newgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "rmgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + } + ], + "Contact": "news@uni-paderborn.de", + "Name": "PBINFO" + }, + { + "Authorizations": [ + { + "From": "newsadmin@perl.org", + "Message": "checkgroups", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "newgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "rmgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + } + ], + "Contact": "newsadmin@perl.org", + "KeyFingerprint": "438FD1BA4DCC3B1ABED82BCC32988A7D8B2ACFBB", + "KeyUrl": "http://www.nntp.perl.org/about/newsadmin@perl.org.pgp", + "Name": "PERL", + "Url": "http://www.nntp.perl.org/about/" + }, + { + "AdminGroup": "pgh.config", + "Authorizations": [ + { + "From": "pgh-config@psc.edu", + "Message": "checkgroups", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "newgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "rmgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + } + ], + "Name": "PGH" + }, + { + "Authorizations": [ + { + "From": "news@postgresql.org", + "Message": "checkgroups", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "newgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "rmgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + } + ], + "Contact": "news@postgresql.org", + "KeyUrl": "http://news.hub.org/gpg_public_keys.html", + "Name": "PGSQL", + "Url": "http://news.hub.org/gpg_public_keys.html" + }, + { + "AdminGroup": "pl.news.admin", + "Authorizations": [ + { + "From": "michalj@*fuw.edu.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + } + ], + "KeyUrl": "http://www.usenet.pl/doc/news-pl-new-site-faq.html#pgp", + "Name": "PL", + "Url": "http://www.usenet.pl/doc/news-pl-new-site-faq.html" + }, + { + "AdminGroup": "pt.internet.usenet", + "Authorizations": [ + { + "From": "pmelo@*.inescc.pt", + "Message": "checkgroups", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "newgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "rmgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + } + ], + "KeyUrl": "http://www.usenet-pt.org/control@usenet-pt.org.asc", + "Name": "PT", + "Url": "http://www.usenet-pt.org/" + }, + { + "AdminGroup": "relcom.netnews", + "Authorizations": [ + { + "From": "coord@*.relcom.ru", + "Message": "checkgroups", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "newgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "rmgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + } + ], + "KeyUrl": "ftp://ftp.relcom.ru/pub/relcom/netinfo/coordpubkey.txt", + "Name": "RELCOM", + "Url": "ftp://ftp.relcom.ru/pub/relcom/netinfo/" + }, + { + "AdminGroup": "rocksolid.shared.rocksolid", + "Authorizations": [ + { + "From": "usenet@novabbs.com", + "Message": "checkgroups", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "newgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "rmgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + } + ], + "Contact": "usenet@novabbs.com", + "KeyFingerprint": "FF846D730B72CC0CB69B64802A1C655BF6D10B13", + "KeyUrl": "http://news.novabbs.com/hierarchy/rocksolid.txt", + "Name": "ROCKSOLID", + "Url": "http://news.novabbs.com/hierarchy/" + }, + { + "AdminGroup": "saar.admin.news", + "Authorizations": [ + { + "From": "control@saar-admin-news.de", + "Message": "checkgroups", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "newgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "rmgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + } + ], + "KeyUrl": "http://www.saar-admin-news.de/saar-control.asc", + "Name": "SAAR", + "Url": "http://www.saar-admin-news.de/" + }, + { + "AdminGroup": "sat.usenet.config", + "Authorizations": [ + { + "From": "satgroup@endicor.com", + "Message": "checkgroups", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "newgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "rmgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + } + ], + "Contact": "satgroup@endicor.com", + "Name": "SAT" + }, + { + "AdminGroup": "schule.admin", + "Authorizations": [ + { + "From": "newsctrl@schule.de", + "Message": "checkgroups", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "newgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "rmgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + } + ], + "Contact": "schule-admin@roxel.ms.sub.org", + "KeyFingerprint": "6406F0AEE146850CBDCA0E538B1E73D2", + "KeyUrl": "http://www.afaik.de/usenet/admin/schule/control/schule.asc", + "Name": "SCHULE", + "Url": "http://home.pages.de/~schule-admin/" + }, + { + "AdminGroup": "scot.newsgroups.discuss", + "Authorizations": [ + { + "From": "control@scot.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "newgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + } + ], + "KeyUrl": "http://scot.news-admin.org/signature.html", + "Name": "SCOT", + "Url": "http://scot.news-admin.org/" + }, + { + "AdminGroup": "scout.admin", + "Authorizations": [ + { + "From": "control@news.scoutnet.org", + "Message": "checkgroups", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "newgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "rmgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + } + ], + "KeyUrl": "http://news.scoutnet.org/scout-pgpkey.asc", + "Name": "SCOUT", + "Url": "http://news.scoutnet.org/" + }, + { + "AdminGroup": "se.internet.news.meddelanden", + "Authorizations": [ + { + "From": "usenet@usenet-se.net", + "Message": "checkgroups", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "newgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "rmgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + } + ], + "Contact": "usenet@usenet-se.net", + "KeyFingerprint": "6803F0FD0CC34E696F0D0C603C586396", + "Name": "SE" + }, + { + "AdminGroup": "sfnet.ryhmat+listat", + "Authorizations": [ + { + "From": "sfnet@*cs.tut.fi", + "Message": "checkgroups", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "newgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "rmgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + } + ], + "Contact": "sfnet@cs.tut.fi", + "KeyFingerprint": "DE7933C2D359D12844E56A0CB6E30E536933A636", + "Name": "SFNET", + "Url": "http://www.cs.tut.fi/sfnet/" + }, + { + "AdminGroup": "si.news.announce.newsgroups", + "Authorizations": [ + { + "From": "news-admin@arnes.si", + "Message": "checkgroups", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "newgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "rmgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + } + ], + "KeyUrl": "http://www.arnes.si/news/config/", + "Name": "SI", + "Url": "http://www.arnes.si/news/config/" + }, + { + "AdminGroup": "staroffice.admin", + "Authorizations": [ + { + "From": "news@stardivision.de", + "Message": "checkgroups", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "newgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "rmgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + } + ], + "Contact": "news@starnews.sun.com", + "KeyFingerprint": "C63E816F2A19D3847251F91BE3B9B2C9", + "Name": "STAROFFICE", + "SyncableServer": "starnews.sun.com" + }, + { + "AdminGroup": "stgt.net", + "Authorizations": [ + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "checkgroups", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "newgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "rmgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + } + ], + "Contact": "news@news.uni-stuttgart.de", + "KeyFingerprint": "BAA40A548FF5F51E724851AE0951DE44", + "KeyUrl": "http://news.uni-stuttgart.de/hierarchie/stgt/stgt-control.txt", + "Name": "STGT", + "Url": "http://news.uni-stuttgart.de/hierarchie/stgt/" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "SYD", + "Url": "http://syd.news-admin.org/" + }, + { + "AdminGroup": "szaf.admin", + "Authorizations": [ + { + "From": "hirtenrat@szaf.org", + "Message": "rmgroup", + "Newsgroups": "szaf.*", + "VerificationIdentity": "Hirtenrat" + } + ], + "Contact": "hirtenrat@szaf.org", + "KeyUrl": "http://news.szaf.org/szaf/szaf-key.txt", + "Name": "SZAF" + }, + { + "AdminGroup": "thur.net.news.groups", + "Authorizations": [ + { + "From": "usenet@thur.de", + "Message": "checkgroups", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "newgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "rmgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + } + ], + "Contact": "usenet@thur.de", + "KeyFingerprint": "7E3D731393D4CA7839DE3CE737EE22F1", + "Name": "THUR", + "SyncableServer": "news.thur.de", + "Url": "http://www.thur.de/thurnet/old/thurnews.html" + }, + { + "Authorizations": [ + { + "From": "usenet@agate.berkeley.edu", + "Message": "checkgroups", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "newgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "rmgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + } + ], + "Contact": "Chris van den Berg ", + "KeyFingerprint": "96B88E9A9809377D0EEC8188DB9029BF", + "KeyUrl": "http://www.net.berkeley.edu/usenet/usenet.asc", + "Name": "UCB", + "Url": "http://www.net.berkeley.edu/usenet/" + }, + { + "AdminGroup": "ufra.admin", + "KeyFingerprint": "F7AD96D87A3F7E84020C839ADB8FEBB8", + "Name": "UFRA" + }, + { + "AdminGroup": "uk.net.news.announce", + "Authorizations": [ + { + "From": "control@usenet.org.uk", + "Message": "checkgroups", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "newgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "rmgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + } + ], + "KeyUrl": "http://www.usenet.org.uk/newsadmins.html", + "Name": "UK", + "Url": "http://www.usenet.org.uk/" + }, + { + "AdminGroup": "un.public.usenet.admin", + "Authorizations": [ + { + "From": "news@news.itu.int", + "Message": "checkgroups", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "newgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "rmgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + } + ], + "Name": "UN" + }, + { + "AdminGroup": "us.config", + "Authorizations": [ + { + "From": "us-control@lists.killfile.org", + "Message": "checkgroups", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "newgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "rmgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + } + ], + "Contact": "us-control@lists.killfile.org", + "KeyFingerprint": "BB96EB2CCFD075C8E9DEC2C21DA29D87B73CAF1B", + "Name": "US" + }, + { + "AdminGroup": "uw.newsgroups", + "Authorizations": [ + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "checkgroups", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "newgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "rmgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + } + ], + "Name": "UW", + "SyncableServer": "news.uwaterloo.ca" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "WA", + "Url": "http://wa.news-admin.org/" + }, + { + "AdminGroup": "wales.usenet.config", + "Authorizations": [ + { + "From": "control@wales-usenet.org", + "Message": "checkgroups", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "newgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "rmgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + } + ], + "Contact": "committee@wales-usenet.org", + "KeyFingerprint": "2D9EDEDF12DA345C49E1EE28E3AB0DAD", + "KeyUrl": "http://www.wales-usenet.org/english/newsadmin.txt", + "Name": "WALES", + "Url": "http://www.wales-usenet.org/" + }, + { + "AdminGroup": "x-privat.info", + "Authorizations": [ + { + "From": "dmitry@x-privat.org", + "Message": "checkgroups", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "newgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "rmgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + } + ], + "Contact": "dmitry@x-privat.org", + "KeyFingerprint": "9B0A7E682780C796476B039051056843", + "KeyUrl": "http://www.x-privat.org/dmitry.asc", + "Name": "X-PRIVAT", + "Url": "http://www.x-privat.org/" + }, + { + "AdminGroup": "z-netz.koordination.user+sysops", + "Authorizations": [ + { + "From": "teko@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + } + ], + "Contact": "teko@dinoex.sub.org", + "KeyUrl": "ftp://ftp.dinoex.de/pub/keys/z-netz.koordination.user+sysops.asc", + "Name": "Z-NETZ" + } + ], + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl", + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl" + } + } + }, + "Nntpd": { + "AcmeStateDir": "certs/", + "AllowCleartextAuth": true, + "BindAddress": [ + "127.0.0.1" + ], + "BindPort": 11990, + "BindPortTls": 15633, + "GracefulShutdownTimeout": "00:01:00", + "HistoryTime": "02:00:00", + "IdleTime": 300, + "LogDir": "logs/", + "MailComplaintsTo": "abuse@usenet.ninja", + "MaxArticleSize": 5242880, + "ProxyHosts": [], + "ServerId": 1, + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "StartupTimeout": null, + "StopHostOnUnexpectedServiceTermination": true, + "Top1000": [ + "top1000@anthologeek.net", + "cknipe@opticnetworks.net" + ], + "Transit": { + "StreamOutstandingArticleDepth": 8 + }, + "Systemd": { + "EnableWatchdog": true, + "NotifyReadyOnApplicationRunning": true, + "NotifyStoppingOnApplicationShutdown": true, + "ReportLifecycleStatus": true, + "WatchdogIntervalFraction": 0.5 + }, + "ArticleIngestion": { + "MaxArticleBytes": 4194304, + "MinWorkers": 2, + "MaxWorkers": 32, + "MaxPublishConcurrency": 32, + "OverviewDbWorkQueueMemoryLimit": 1073741824, + "OverviewDbMinPublisherWorkers": 2, + "OverviewDbMaxPublisherWorkers": 32, + "OverviewDbPublisherBatchSize": 100, + "OverviewDbPublisherShutdownSeconds": 2, + "ScaleIntervalSeconds": 2, + "ScaleUpPressureThreshold": 0.4, + "ScaleDownPressureThreshold": 0.1, + "ScaleUpConsecutiveIntervals": 2, + "ScaleDownConsecutiveIntervals": 3 + }, + "TransitQueueMemoryLimit": 4294967296 + }, + "NntpDb": { + "StartupTimeout": "00:00:15" + }, + "Redis": { + "Host": [ "198.18.0.70" ], + "Port": 6379 + }, + "Serilog": { + "Enrich": [ "FromLogContext" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information", + "System": "Warning", + "VectorNNTPD.ReaderService": "Debug" + } + }, + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File", "Serilog.Sinks.Async" ], + "WriteTo": [ + { + "Args": { + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "restrictedToMinimumLevel": "Information" + }, + "Name": "Console" + }, + { + "Args": { + "blockWhenFull": true, + "bufferSize": 50000, + "configure": [ + { + "Args": { + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "path": "logs/pending-.log", + "restrictedToMinimumLevel": "Debug", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Name": "File" + } + ] + }, + "Name": "Async" + } + ], + "News": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/news-.log", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Inpaths": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/inpaths-.log", + "retainedFileCountLimit": 1, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/GlobalUsings.cs b/tests/VectorNNTPD.ReaderService.Tests/GlobalUsings.cs new file mode 100644 index 00000000..31762881 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/GlobalUsings.cs @@ -0,0 +1,4 @@ +global using Xunit; +global using VectorNNTPD.Common.NntpDb; +global using VectorNNTPD.ReaderService.Tests.Fixtures; +global using VectorNNTPD.ReaderService.Tests.TestDoubles; diff --git a/tests/VectorNNTPD.ReaderService.Tests/History/HistoryDbTests.cs b/tests/VectorNNTPD.ReaderService.Tests/History/HistoryDbTests.cs new file mode 100644 index 00000000..64a6d142 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/History/HistoryDbTests.cs @@ -0,0 +1,328 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.History; + +public sealed class HistoryDbTests +{ + private static readonly byte[] MessageId = ""u8.ToArray(); + private static readonly byte[] OtherId = ""u8.ToArray(); + + [Fact] + public void HistoryTime_Default_IsTwoHours() + { + Assert.Equal(TimeSpan.FromHours(2), new VectorNNTPD.ReaderService.Configuration.NntpdOptions().HistoryTime); + } + + [Fact] + public void IdenticalMessageIds_ProduceIdenticalKeys() + { + var left = HistoryDigest.FromMessageId(MessageId); + var right = HistoryDigest.FromMessageId(MessageId); + Assert.Equal(left, right); + Assert.Equal(HistoryRedisKeys.Create(left), HistoryRedisKeys.Create(right)); + } + + [Fact] + public void HistoryDigest_MatchesCommonArticleIdBytes() + { + var history = HistoryDigest.FromMessageId(MessageId); + var articleId = VectorNNTPD.Common.Articles.ArticleId.FromMessageId(MessageId); + Span historyBytes = stackalloc byte[HistoryDigest.Length]; + Span articleIdBytes = stackalloc byte[VectorNNTPD.Common.Articles.ArticleId.Length]; + history.CopyTo(historyBytes); + articleId.CopyTo(articleIdBytes); + Assert.True(historyBytes.SequenceEqual(articleIdBytes)); + } + + [Fact] + public void DifferentMessageIds_ProduceDifferentKeys() + { + var left = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)); + var right = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(OtherId)); + Assert.False(left.AsSpan().SequenceEqual(right)); + } + + [Fact] + public void RedisKey_IsBinaryNamespacePlusDigest() + { + var digest = HistoryDigest.FromMessageId(MessageId); + var key = HistoryRedisKeys.Create(digest); + Assert.Equal(HistoryRedisKeys.KeyLength, key.Length); + Assert.True(key.AsSpan().StartsWith(HistoryRedisKeys.NamespacePrefix)); + Assert.Equal("nntpd:hist:"u8.Length + HistoryDigest.Length, key.Length); + + var suffix = key.AsSpan(HistoryRedisKeys.NamespacePrefix.Length); + var reconstructed = HistoryDigest.FromSpan(suffix); + Assert.Equal(digest, reconstructed); + Assert.False(suffix.StartsWith("nntpd:hist:"u8)); + } + + [Fact] + public void Namespace_IsolatesHistoryKeys() + { + var key = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)); + Assert.StartsWith("nntpd:hist:", Encoding.ASCII.GetString(key[..HistoryRedisKeys.NamespacePrefix.Length])); + } + + [Fact] + public async Task MemoryMiss_RedisMiss_ReturnsUnseen_AndRecordsLocal() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unseen, result); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(1, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task MemoryHit_DoesNotQueryRedis() + { + var redis = new FakeRedisService(); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + redis.Database.KeyExistsCount = 0; + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Seen, result); + Assert.Equal(0, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task MemoryMiss_RedisHit_ReturnsSeen_AndWarmsMemory() + { + var redis = new FakeRedisService(); + var digest = HistoryDigest.FromMessageId(MessageId); + redis.Database.Seed(HistoryRedisKeys.Create(digest)); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Seen, result); + Assert.True(history.ContainsLocal(digest)); + Assert.Equal(0, redis.Database.SetCount); + } + + [Fact] + public async Task DoubleMiss_EnqueuesAsyncRedisWrite_WithoutWaiting() + { + var redis = new FakeRedisService(); + redis.Database.BlockSet = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = Create(redis); + + var lookup = history.LookupAsync(MessageId); + var completed = await Task.WhenAny(lookup.AsTask(), Task.Delay(TimeSpan.FromSeconds(1))); + Assert.Same(lookup.AsTask(), completed); + Assert.Equal(HistoryLookupResult.Unseen, await lookup); + Assert.Equal(0, redis.Database.SetCount); + + redis.Database.BlockSet.SetResult(); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.True(redis.Database.Contains(HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)))); + await writer.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task RedisLookupFailure_IsUnavailable_NotUnseen() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unavailable, result); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + } + + [Fact] + public async Task RedisLookupFailure_DoesNotPopulateLocal_AndDoesNotHammerRedis() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(MessageId)); + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.True(redis.IsUnavailable); + + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(OtherId)); + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.Equal(1, redis.BeginRejectCount); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(OtherId))); + } + + [Fact] + public async Task RedisRecovery_RestoresMissAndHit() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(MessageId)); + + redis.Recover(); + redis.Database.ExistsException = null; + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + redis.Database.Seed(HistoryRedisKeys.Create(HistoryDigest.FromMessageId(OtherId))); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(OtherId)); + } + + [Fact] + public async Task Writer_SetFailure_DoesNotChangeAlreadyIssuedUnseen() + { + var redis = new FakeRedisService(); + redis.Database.SetException = new RedisUnavailableException("write-fail"); + var history = Create(redis); + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await writer.StopAsync(CancellationToken.None); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(MessageId)); + } + + [Fact] + public async Task LocalExpiry_MatchesConfiguredRetention() + { + var clock = new ManualTimeProvider(); + var redis = new FakeRedisService(); + var queue = new HistoryWriteQueue(); + var history = new HistoryDb(redis, TimeSpan.FromMinutes(5), NullLogger.Instance, clock, queue); + _ = await history.LookupAsync(MessageId); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + clock.Advance(TimeSpan.FromMinutes(5).Add(TimeSpan.FromSeconds(1))); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + } + + [Fact] + public void WriteQueue_DropsWhenFull() + { + var queue = new HistoryWriteQueue(capacity: 1); + var first = HistoryDigest.FromMessageId(MessageId); + var second = HistoryDigest.FromMessageId(OtherId); + Assert.True(queue.TryEnqueue(first)); + Assert.False(queue.TryEnqueue(second)); + } + + [Fact] + public async Task Writer_Stop_DrainsOrCancelsWithoutThrowing() + { + var redis = new FakeRedisService(); + redis.Database.BlockSet = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await writer.StopAsync(cts.Token); + redis.Database.BlockSet.TrySetResult(); + } + + [Fact] + public async Task Writer_RedisWriteFailure_IsObserved() + { + var redis = new FakeRedisService(); + redis.Database.SetException = new RedisUnavailableException("write-fail"); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await writer.StopAsync(CancellationToken.None); + Assert.Equal(1, redis.Database.SetCount); + } + + [Fact] + public async Task Peek_Miss_DoesNotRecordLocal() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + var result = await history.PeekAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unseen, result); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(1, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task Capture_CountsLookupsHitsMissesAndErrors() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(MessageId)); + redis.IsUnavailable = true; + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(OtherId)); + + var snapshot = history.Capture(); + Assert.Equal(3, snapshot.Lookups); + Assert.Equal(1, snapshot.Hits); + Assert.Equal(1, snapshot.Misses); + Assert.Equal(1, snapshot.Errors); + Assert.True(snapshot.WaitTicks >= 0); + } + + [Fact] + public async Task Remember_RecordsLocalWithoutExists() + { + var redis = new FakeRedisService(); + var history = Create(redis); + redis.Database.KeyExistsCount = 0; + + history.Remember(MessageId); + + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(0, redis.Database.KeyExistsCount); + var peek = await history.PeekAsync(MessageId); + Assert.Equal(HistoryLookupResult.Seen, peek); + Assert.Equal(0, redis.Database.KeyExistsCount); + } + + private static HistoryDb Create(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/History/HistoryStoreCapacityTests.cs b/tests/VectorNNTPD.ReaderService.Tests/History/HistoryStoreCapacityTests.cs new file mode 100644 index 00000000..fcb43bce --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/History/HistoryStoreCapacityTests.cs @@ -0,0 +1,135 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.History; + +public sealed class HistoryStoreCapacityTests +{ + [Fact] + public void Add_DoesNotScan_AndHonorsHardCap() + { + const int cap = 2048; + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromHours(2), clock, cap); + + for (var i = 0; i < cap + 256; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + Assert.Equal(cap, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.False(store.Contains(HistoryDigest.FromMessageId(Id(cap + 10)))); + } + + [Fact] + public void Maintain_RemovesExpired_AndFreesSlots_OffCheckPath() + { + const int cap = 64; + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromMinutes(5), clock, cap); + for (var i = 0; i < cap; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + clock.Advance(TimeSpan.FromMinutes(5).Add(TimeSpan.FromSeconds(1))); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.False(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.Equal(cap - 1, store.ApproximateCount); + + var removed = store.Maintain(); + Assert.Equal(1, store.MaintenanceScanCount); + Assert.Equal(cap - 1, removed); + Assert.Equal(0, store.ApproximateCount); + + store.Add(HistoryDigest.FromMessageId(Id(10_000))); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(10_000)))); + Assert.Equal(1, store.MaintenanceScanCount); + } + + [Theory] + [InlineData(10_000)] + [InlineData(100_000)] + public void UniqueInserts_StayBounded_WithoutMaintenanceScan(int count) + { + var store = new LocalHistoryStore(TimeSpan.FromHours(2), TimeProvider.System); + for (var i = 0; i < count; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + Assert.Equal(count, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(count - 1)))); + } + + [Fact] + public void Maintain_DoesNotRemoveRefreshedLiveEntry() + { + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromMinutes(5), clock); + var digest = HistoryDigest.FromMessageId(Id(1)); + var observedExpiry = clock.GetUtcNow().Add(TimeSpan.FromMinutes(5)).UtcTicks; + store.Add(digest); + + clock.Advance(TimeSpan.FromMinutes(6)); + store.Add(digest); + + Assert.False(store.TryRemoveObservedExpiry(digest, observedExpiry)); + Assert.Equal(0, store.Maintain()); + Assert.True(store.Contains(digest)); + Assert.Equal(1, store.EntryCountForTests); + Assert.Equal(1, store.ApproximateCount); + } + + [Fact] + public void ConcurrentInserts_NeverExceedHardCap() + { + const int cap = 128; + var store = new LocalHistoryStore(TimeSpan.FromHours(2), TimeProvider.System, cap); + Parallel.For(0, 2_000, i => store.Add(HistoryDigest.FromMessageId(Id(i)))); + + Assert.True(store.EntryCountForTests <= cap); + Assert.True(store.ApproximateCount <= cap); + Assert.Equal(store.EntryCountForTests, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + } + + [Fact] + public async Task MaintenanceService_Stop_CompletesWithoutThrowing() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var service = new HistoryMaintenanceService( + history, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await service.StartAsync(CancellationToken.None); + Assert.NotNull(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, history.Local.MaintenanceScanCount); + } + + private static byte[] Id(int i) => Encoding.ASCII.GetBytes($""); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/HostIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/HostIntegrationTests.cs new file mode 100644 index 00000000..42bd7871 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/HostIntegrationTests.cs @@ -0,0 +1,185 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Hosting.Systemd; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting; + +[Collection(SerilogCollection.Name)] +public sealed class HostIntegrationTests +{ + [Fact] + public async Task Host_StartsAndStopsCleanly() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await host.StopAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_RepeatedStop_IsIdempotent() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + await host.StopAsync(); + + Assert.Equal(ApplicationState.Stopped, host.Services.GetRequiredService().State); + } + + [Fact] + public async Task Host_WithSystemdNotifierFakes_ReportsReadyOnlyAfterRunning() + { + var notify = new FakeSystemdNotifyBridge(); + var runtime = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(); + + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + services.AddSingleton(notify); + services.AddSingleton(runtime); + }); + + Assert.Equal(0, notify.ReadyCount); + await host.StartAsync(); + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + Assert.Equal(1, notify.ReadyCount); + + await host.StopAsync(); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Host_StartupFailure_Propagates_AndLeavesApplicationStopped() + { + using var host = CreateTestHost(services => + { + services.AddSingleton( + new FakeApplicationService("bad", onStart: _ => throw new InvalidOperationException("startup-fail"))); + }); + + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Equal("startup-fail", ex.Message); + + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_StopApplication_TriggersGracefulShutdown() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + + var lifetime = host.Services.GetRequiredService(); + var lifecycle = host.Services.GetRequiredService(); + + lifetime.StopApplication(); + await host.WaitForShutdownAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_RespectsGracefulShutdownTimeoutConfiguration() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + + using var host = CreateTestHost( + services => + { + services.AddSingleton(service); + }, + configure: options => + { + options.GracefulShutdownTimeout = TimeSpan.FromSeconds(1); + }, + hostShutdownTimeout: TimeSpan.FromSeconds(5)); + + await host.StartAsync(); + + // Host StopAsync should surface the application shutdown timeout from the service manager. + await Assert.ThrowsAsync(() => host.StopAsync()); + Assert.Equal(ApplicationState.Stopped, host.Services.GetRequiredService().State); + } + + [Fact] + public async Task Host_UnexpectedServiceTermination_StopsHost() + { + var service = new FakeApplicationService("fragile", withExecution: true); + + using var host = CreateTestHost(services => + { + services.AddSingleton(service); + }); + + await host.StartAsync(); + + var lifecycle = host.Services.GetRequiredService(); + var shutdown = host.WaitForShutdownAsync(); + + service.CompleteExecution(new InvalidOperationException("boom")); + + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(5)); + var shutdownError = await Assert.ThrowsAsync( + () => shutdown.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.Equal( + "An application service terminated unexpectedly while the application was Running.", + shutdownError.Message); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + private static IHost CreateTestHost( + Action? configureServices = null, + Action? configure = null, + TimeSpan? hostShutdownTimeout = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => + { + // Tests avoid noisy console output; exclusivity is asserted separately. + lc.MinimumLevel.Fatal(); + }); + + builder.Services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + if (hostShutdownTimeout is { } timeout) + { + options.ShutdownTimeout = timeout; + } + }); + + builder.Services.AddNntpReaderHosting(configure, includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/NntpServerRoleIsolationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/NntpServerRoleIsolationTests.cs new file mode 100644 index 00000000..0f02d83d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/NntpServerRoleIsolationTests.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Hosting +{ + public sealed class NntpServerRoleIsolationTests + { + [Fact] + public void Reader_host_registers_reader_markers_without_transit_catalogue() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddNntpReaderHosting(); + + Assert.Contains( + services, + static d => d.ServiceType == typeof(NntpServerRole) + && Equals(d.ImplementationInstance, NntpServerRole.Reader)); + Assert.Contains(services, static d => d.ImplementationType == typeof(SessionStateService)); + Assert.Contains(services, static d => d.ImplementationType == typeof(PostFilterPolicyService)); + Assert.DoesNotContain(services, static d => d.ImplementationType == typeof(TransitCatalogueService)); + Assert.DoesNotContain(services, static d => d.ImplementationType == typeof(TransitPeerAuthorization)); + } + + [Fact] + public void Transit_hosting_on_reader_executable_throws() + { + var services = new ServiceCollection(); + services.AddLogging(); + var ex = Assert.Throws(() => services.AddNntpTransitHosting()); + Assert.Contains("Transit", ex.Message, StringComparison.Ordinal); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs new file mode 100644 index 00000000..a6a13a88 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Hosting; + +[Collection(SerilogCollection.Name)] +public sealed class ObsoleteConsoleFormatterCleanupTests +{ + [Fact] + public void RemoveObsoleteMicrosoftConsoleFormatterConfiguration_StripsConsoleLoggerOptionsBindings() + { + var services = new ServiceCollection(); + services.Configure(options => + { + options.FormatterName = ConsoleFormatterNames.Systemd; + }); + + Assert.Contains(services, NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + + NntpdServiceCollectionExtensions.RemoveObsoleteMicrosoftConsoleFormatterConfiguration(services); + + Assert.DoesNotContain(services, NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + } + + [Fact] + public void PlatformHosting_AfterAddSystemd_DoesNotRetainMicrosoftConsoleFormatterConfiguration() + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Nntpd:LogDir"] = TestHostFactory.NewTestLogDir(), + }); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + + Assert.DoesNotContain( + builder.Services, + NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + + using var host = builder.Build(); + + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.Empty(host.Services.GetLoggerProviders()); + + // ConsoleLoggerOptions may still resolve via Options infrastructure defaults, but must not + // carry the systemd MEL formatter name that AddSystemd would otherwise leave behind. + var consoleOptions = host.Services.GetService>(); + Assert.True( + consoleOptions is null + || string.IsNullOrEmpty(consoleOptions.Value.FormatterName) + || !string.Equals( + consoleOptions.Value.FormatterName, + ConsoleFormatterNames.Systemd, + StringComparison.Ordinal)); + } + + [Fact] + public void PlatformHosting_PreservesSerilogConsoleSinkConfiguration() + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Nntpd:LogDir"] = TestHostFactory.NewTestLogDir(), + }); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + + using var host = builder.Build(); + var logger = host.Services.GetRequiredService>(); + + // Behavioral check: logging remains functional through Serilog after cleanup. + logger.LogInformation("Serilog console pipeline remains active after formatter cleanup"); + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs new file mode 100644 index 00000000..1d6a8ecc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs @@ -0,0 +1,134 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +public sealed class SystemdLifecycleNotifierTests +{ + [Fact] + public async Task Readiness_FollowsSuccessfulInitialization_AndIsNotDuplicated() + { + var notify = new FakeSystemdNotifyBridge(); + var service = new FakeApplicationService("svc"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + + await notifier.StartAsync(CancellationToken.None); + Assert.Equal(0, notify.ReadyCount); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(1, notify.ReadyCount); + Assert.Contains(notify.Notifications, n => n.StartsWith("STATUS=", StringComparison.Ordinal)); + + // Force another Running notification attempt via duplicate event is not possible; + // stopping and ensuring ready count stays at 1 after shutdown start. + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Readiness_IsNotSentDuringStartup() + { + var notify = new FakeSystemdNotifyBridge(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "slow", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + var start = lifecycle.StartAsync(CancellationToken.None); + await entered.Task; + + Assert.Equal(ApplicationState.Starting, lifecycle.State); + Assert.Equal(0, notify.ReadyCount); + Assert.Contains(notify.Notifications, n => n.Contains("Starting", StringComparison.Ordinal)); + + release.TrySetResult(); + await start; + + Assert.Equal(1, notify.ReadyCount); + } + + [Fact] + public async Task Readiness_IsNotSentAfterFailedStartup() + { + var notify = new FakeSystemdNotifyBridge(); + var service = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Equal(0, notify.ReadyCount); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Shutdown_PreventsSubsequentReadinessReporting() + { + var notify = new FakeSystemdNotifyBridge(); + await using var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.StoppingCount); + Assert.True(lifecycle.ShutdownRequested); + + // A second stop is idempotent and must not emit another readiness notification. + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Notifications_AreNoOpsWhenBridgeDisabled() + { + var notify = new FakeSystemdNotifyBridge { IsEnabled = false }; + await using var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Empty(notify.Notifications); + } + + private static SystemdLifecycleNotifier CreateNotifier( + ApplicationLifecycle lifecycle, + FakeSystemdNotifyBridge notify, + Action? configure = null) + { + var options = TestHostFactory.CreateOptions(); + configure?.Invoke(options); + + return new SystemdLifecycleNotifier( + lifecycle, + notify, + RecordingSystemdRuntimeFactory.LinuxUnderSystemd(), + Options.Create(options), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs new file mode 100644 index 00000000..38f103ca --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +public sealed class SystemdRuntimeDetectionTests +{ + [Fact] + public void NonLinux_DisablesWatchdogEvenIfEnvironmentLooksConfigured() + { + using var scope = new EnvironmentVariableScope( + (SystemdRuntime.WatchdogUsecVariable, "5000000"), + (SystemdRuntime.WatchdogPidVariable, Environment.ProcessId.ToString())); + + var runtime = new SystemdRuntime([], NullLogger.Instance); + + if (runtime.IsLinux) + { + // On a real Linux CI agent this assertion is skipped; detection is covered by fakes. + return; + } + + Assert.False(runtime.IsLinux); + Assert.False(runtime.IsSystemdService); + Assert.False(runtime.IsWatchdogConfigured); + Assert.Null(runtime.WatchdogTimeout); + } + + [Fact] + public void MissingWatchdogConfiguration_IsSafe() + { + using var scope = new EnvironmentVariableScope( + (SystemdRuntime.WatchdogUsecVariable, null), + (SystemdRuntime.WatchdogPidVariable, null)); + + var fake = RecordingSystemdRuntimeFactory.LinuxConsole(); + Assert.False(fake.IsWatchdogConfigured); + Assert.Null(fake.WatchdogTimeout); + } + + [Fact] + public void InvalidWatchdogUsec_IsIgnored_OnLinuxPath() + { + // Parse behavior is exercised via interval helper + fake runtimes offline. + // Malformed WATCHDOG_USEC must never invent an interval. + var fake = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(watchdog: null); + Assert.False(fake.IsWatchdogConfigured); + } + + [Fact] + public void FakeDetection_DistinguishesSystemdFromConsoleAndNonLinux() + { + var underSystemd = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromSeconds(5)); + var console = RecordingSystemdRuntimeFactory.LinuxConsole(); + var windows = RecordingSystemdRuntimeFactory.NonLinux(); + + Assert.True(underSystemd.IsLinux && underSystemd.IsSystemdService && underSystemd.IsWatchdogConfigured); + Assert.True(console.IsLinux && !console.IsSystemdService && !console.IsWatchdogConfigured); + Assert.True(!windows.IsLinux && !windows.IsSystemdService && !windows.IsWatchdogConfigured); + } +} + +internal sealed class EnvironmentVariableScope : IDisposable +{ + private readonly (string Name, string? Previous)[] _previous; + + public EnvironmentVariableScope(params (string Name, string? Value)[] values) + { + _previous = values + .Select(v => (v.Name, Environment.GetEnvironmentVariable(v.Name))) + .ToArray(); + + foreach (var (name, value) in values) + { + Environment.SetEnvironmentVariable(name, value); + } + } + + public void Dispose() + { + foreach (var (name, previous) in _previous) + { + Environment.SetEnvironmentVariable(name, previous); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdTestDoubles.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdTestDoubles.cs new file mode 100644 index 00000000..16fc3fed --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdTestDoubles.cs @@ -0,0 +1,140 @@ +using VectorNNTPD.ReaderService.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +internal sealed class FakeSystemdNotifyBridge : ISystemdNotifyBridge +{ + private readonly object _sync = new(); + private readonly List _notifications = []; + private int _readyCount; + private int _stoppingCount; + private int _watchdogCount; + + public bool IsEnabled { get; set; } = true; + + public Exception? ThrowOnNotify { get; set; } + + public IReadOnlyList Notifications + { + get + { + lock (_sync) + { + return _notifications.ToArray(); + } + } + } + + public int ReadyCount => Volatile.Read(ref _readyCount); + + public int StoppingCount => Volatile.Read(ref _stoppingCount); + + public int WatchdogCount => Volatile.Read(ref _watchdogCount); + + public TaskCompletionSource ReadySent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StoppingSent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource WatchdogSent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void NotifyReady() + { + Record("READY=1", ref _readyCount); + ReadySent.TrySetResult(); + } + + public void NotifyStopping() + { + Record("STOPPING=1", ref _stoppingCount); + StoppingSent.TrySetResult(); + } + + public void NotifyStatus(string status) + { + if (ThrowOnNotify is not null) + { + throw ThrowOnNotify; + } + + if (!IsEnabled) + { + return; + } + + lock (_sync) + { + _notifications.Add("STATUS=" + status); + } + } + + public void NotifyWatchdog() + { + Record("WATCHDOG=1", ref _watchdogCount); + WatchdogSent.TrySetResult(); + } + + private void Record(string value, ref int counter) + { + if (ThrowOnNotify is not null) + { + throw ThrowOnNotify; + } + + if (!IsEnabled) + { + return; + } + + lock (_sync) + { + _notifications.Add(value); + } + + Interlocked.Increment(ref counter); + } +} + +internal sealed class FakeSystemdRuntime : ISystemdRuntime +{ + public bool IsLinux { get; set; } + + public bool IsSystemdService { get; set; } + + public bool IsNotifyEnabled { get; set; } + + public TimeSpan? WatchdogTimeout { get; set; } + + public bool IsWatchdogConfigured => WatchdogTimeout is not null; +} + +internal sealed class FakeApplicationHealth : IApplicationHealth +{ + public bool IsHealthyForWatchdog { get; set; } +} + +internal sealed class RecordingSystemdRuntimeFactory +{ + public static FakeSystemdRuntime LinuxUnderSystemd(TimeSpan? watchdog = null) => new() + { + IsLinux = true, + IsSystemdService = true, + IsNotifyEnabled = true, + WatchdogTimeout = watchdog, + }; + + public static FakeSystemdRuntime LinuxConsole() => new() + { + IsLinux = true, + IsSystemdService = false, + IsNotifyEnabled = false, + WatchdogTimeout = null, + }; + + public static FakeSystemdRuntime NonLinux() => new() + { + IsLinux = false, + IsSystemdService = false, + IsNotifyEnabled = false, + WatchdogTimeout = null, + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs new file mode 100644 index 00000000..ab489b5f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs @@ -0,0 +1,34 @@ +using VectorNNTPD.ReaderService.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +public sealed class SystemdWatchdogIntervalTests +{ + [Fact] + public void Calculate_UsesConfiguredFraction_AndLeavesMargin() + { + var interval = SystemdWatchdogInterval.Calculate(TimeSpan.FromSeconds(10), fraction: 0.5); + + Assert.Equal(TimeSpan.FromSeconds(5), interval); + Assert.True(interval < TimeSpan.FromSeconds(10)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1)] + [InlineData(1.5)] + [InlineData(double.NaN)] + public void Calculate_RejectsInvalidFraction(double fraction) + { + Assert.Throws( + () => SystemdWatchdogInterval.Calculate(TimeSpan.FromSeconds(10), fraction)); + } + + [Fact] + public void Calculate_RejectsNonPositiveTimeout() + { + Assert.Throws( + () => SystemdWatchdogInterval.Calculate(TimeSpan.Zero)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs new file mode 100644 index 00000000..382c570b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs @@ -0,0 +1,240 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting.Systemd; + +namespace VectorNNTPD.ReaderService.Tests.Hosting.Systemd; + +public sealed class SystemdWatchdogServiceTests +{ + [Fact] + public async Task Watchdog_DisabledWhenNotConfigured() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(watchdog: null), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + Assert.Equal(0, notify.WatchdogCount); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_DisabledWhenOptionFalse() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromSeconds(1)), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }, + configure: o => o.Systemd.EnableWatchdog = false); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_DisabledOnNonLinux() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.NonLinux(), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_SendsHeartbeatsWhileHealthy_AndStopsOnShutdown() + { + var notify = new FakeSystemdNotifyBridge(); + var health = new FakeApplicationHealth { IsHealthyForWatchdog = true }; + var lifetime = new TestHostApplicationLifetime(); + + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(100)), + health: health, + hostLifetime: lifetime, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsActive); + Assert.True(service.HeartbeatInterval > TimeSpan.Zero); + Assert.True(service.HeartbeatInterval < TimeSpan.FromMilliseconds(100)); + + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(notify.WatchdogCount >= 1); + + await service.StopAsync(CancellationToken.None); + var countAfterStop = notify.WatchdogCount; + Assert.Equal(countAfterStop, notify.WatchdogCount); + Assert.False(service.IsActive); + } + + [Fact] + public async Task Watchdog_DoesNotReportHealthyDuringStartupOrUnhealthyPeriods() + { + var notify = new FakeSystemdNotifyBridge(); + var health = new FakeApplicationHealth { IsHealthyForWatchdog = false }; + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(80)), + health: health, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsActive); + + var unhealthyGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = Task.Run(async () => + { + await Task.Delay(service.HeartbeatInterval + TimeSpan.FromMilliseconds(30)); + unhealthyGate.TrySetResult(); + }); + await unhealthyGate.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, notify.WatchdogCount); + + health.IsHealthyForWatchdog = true; + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(notify.WatchdogCount >= 1); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_StopsAfterFatalSupervisedServiceFailure() + { + var appService = new FakeApplicationService("fragile", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([appService]); + await lifecycle.StartAsync(CancellationToken.None); + + var health = new ApplicationHealth(lifecycle); + Assert.True(health.IsHealthyForWatchdog); + + var notify = new FakeSystemdNotifyBridge(); + using var watchdog = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(80)), + health: health, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await watchdog.StartAsync(CancellationToken.None); + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + appService.CompleteExecution(new InvalidOperationException("fatal")); + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(health.IsHealthyForWatchdog); + + var count = notify.WatchdogCount; + var settle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = Task.Run(async () => + { + await Task.Delay(watchdog.HeartbeatInterval + TimeSpan.FromMilliseconds(40)); + settle.TrySetResult(); + }); + await settle.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(count, notify.WatchdogCount); + + await watchdog.StopAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_NotificationFailure_IsSurfacedAndStopsHost() + { + var notify = new FakeSystemdNotifyBridge + { + ThrowOnNotify = new InvalidOperationException("notify-failed"), + }; + var health = new FakeApplicationHealth { IsHealthyForWatchdog = true }; + var lifetime = new TestHostApplicationLifetime(); + + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(60)), + health: health, + hostLifetime: lifetime, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + await lifetime.StoppingRequested.Task.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.True(lifetime.StopRequested); + + await service.StopAsync(CancellationToken.None); + Assert.False(service.IsActive); + } + + [Fact] + public async Task ApplicationHealth_TracksLifecycleAndUnexpectedTermination() + { + var appService = new FakeApplicationService("svc", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([appService]); + var health = new ApplicationHealth(lifecycle); + + Assert.False(health.IsHealthyForWatchdog); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.True(health.IsHealthyForWatchdog); + + appService.CompleteExecution(new InvalidOperationException("fatal")); + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(health.IsHealthyForWatchdog); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.False(health.IsHealthyForWatchdog); + Assert.True(lifecycle.ShutdownRequested); + } + + private static SystemdWatchdogService CreateWatchdog( + FakeSystemdNotifyBridge notify, + ISystemdRuntime runtime, + IApplicationHealth health, + IHostApplicationLifetime? hostLifetime = null, + Action? configure = null) + { + var options = TestHostFactory.CreateOptions(); + configure?.Invoke(options); + + return new SystemdWatchdogService( + runtime, + notify, + health, + hostLifetime ?? new TestHostApplicationLifetime(), + Options.Create(options), + NullLogger.Instance); + } +} + +internal sealed class TestHostApplicationLifetime : IHostApplicationLifetime +{ + private readonly CancellationTokenSource _started = new(); + private readonly CancellationTokenSource _stopping = new(); + private readonly CancellationTokenSource _stopped = new(); + + public CancellationToken ApplicationStarted => _started.Token; + + public CancellationToken ApplicationStopping => _stopping.Token; + + public CancellationToken ApplicationStopped => _stopped.Token; + + public bool StopRequested { get; private set; } + + public TaskCompletionSource StoppingRequested { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void StopApplication() + { + StopRequested = true; + StoppingRequested.TrySetResult(); + _stopping.Cancel(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/FileLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/FileLoggingTests.cs new file mode 100644 index 00000000..80d48cc8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/FileLoggingTests.cs @@ -0,0 +1,587 @@ +using System.Collections; +using System.Globalization; +using System.IO.Compression; +using System.Reflection; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Serilog; +using Serilog.Core; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class FileLoggingTests +{ + [Fact] + public void ProductionAppsettings_DeclaresAsyncFileSinkContract() + { + using var doc = JsonDocument.Parse(File.ReadAllText(FindProductionAppsettings())); + var async = doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[1]; + Assert.Equal("Async", async.GetProperty("Name").GetString()); + Assert.Equal(50000, async.GetProperty("Args").GetProperty("bufferSize").GetInt32()); + Assert.True(async.GetProperty("Args").GetProperty("blockWhenFull").GetBoolean()); + var file = async.GetProperty("Args").GetProperty("configure")[0]; + Assert.Equal("File", file.GetProperty("Name").GetString()); + var args = file.GetProperty("Args"); + Assert.Equal("Debug", args.GetProperty("restrictedToMinimumLevel").GetString()); + Assert.Equal("Day", args.GetProperty("rollingInterval").GetString()); + // 14 daily files including the active file. .log and .log.gz for one day count once. + Assert.Equal(14, args.GetProperty("retainedFileCountLimit").GetInt32()); + Assert.True(args.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", args.GetProperty("flushToDiskInterval").GetString()); + Assert.False(args.GetProperty("rollOnFileSizeLimit").GetBoolean()); + Assert.Equal(JsonValueKind.Null, args.GetProperty("fileSizeLimitBytes").ValueKind); + Assert.False(args.TryGetProperty("hooks", out _)); + Assert.Equal( + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + args.GetProperty("outputTemplate").GetString()); + Assert.Equal("logs/pending-.log", args.GetProperty("path").GetString()); + var news = doc.RootElement.GetProperty("Serilog").GetProperty("News"); + Assert.True(news.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", news.GetProperty("flushToDiskInterval").GetString()); + Assert.False(news.TryGetProperty("hooks", out _)); + var inpaths = doc.RootElement.GetProperty("Serilog").GetProperty("Inpaths"); + Assert.True(inpaths.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", inpaths.GetProperty("flushToDiskInterval").GetString()); + Assert.False(inpaths.TryGetProperty("hooks", out _)); + var usingNames = doc.RootElement.GetProperty("Serilog").GetProperty("Using") + .EnumerateArray().Select(static e => e.GetString()).ToArray(); + Assert.Contains("Serilog.Sinks.File", usingNames); + Assert.Contains("Serilog.Sinks.Async", usingNames); + Assert.DoesNotContain("Serilog.Sinks.File.Archive", usingNames); + Assert.Equal("-.log", NntpdFileLogging.RollingPathSuffix); + Assert.Equal(".gz", NntpdFileLogging.GzipArchiveSuffix); + } + + [Fact] + public void BindResolvedFilePath_OverwritesProductionJsonPlaceholderFromLogDir() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.ReaderService", + }); + + NntpdFileLogging.BindResolvedFilePath(configuration); + + var expected = NntpdFileLogging.RollingFilePath( + logDir, + ApplicationJsonConfiguration.EntryAssemblyName); + Assert.Equal(expected, configuration["Serilog:WriteTo:1:Args:configure:0:Args:path"]); + Assert.DoesNotContain("VectorNNTPD.ReaderService-", configuration["Serilog:WriteTo:1:Args:configure:0:Args:path"], StringComparison.Ordinal); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(logDir), configuration["Serilog:News:path"]); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(logDir), configuration["Serilog:Inpaths:path"]); + Assert.True(Directory.Exists(logDir)); + } + finally + { + TryDelete(logDir); + } + } + + [Fact] + public void ProductionJson_BindsUnlimitedDailyAsyncGzipFileSink() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.ReaderService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + var loggerConfiguration = new LoggerConfiguration() + .ReadFrom.Configuration(NntpdConfiguredSinks.WithoutWriteTo(configuration)); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + + var sinks = WalkLogEventSinks(logger).ToArray(); + var fileSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.Null(ReadInstanceField(fileSink, "_fileSizeLimitBytes")); + Assert.False(Assert.IsType(ReadInstanceField(fileSink, "_rollOnFileSizeLimit")!)); + Assert.Equal(14, ReadInstanceField(fileSink, "_retainedFileCountLimit")); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_buffered")!)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + + var asyncSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); + var queue = ReadInstanceField(asyncSink, "_queue") + ?? throw new InvalidOperationException("BackgroundWorkerSink._queue was not found."); + var boundedCapacity = queue.GetType().GetProperty("BoundedCapacity")?.GetValue(queue) + ?? throw new InvalidOperationException("BoundedCapacity was not found."); + Assert.Equal(50000, Convert.ToInt32(boundedCapacity, CultureInfo.InvariantCulture)); + AssertFlushesOncePerSecond(sinks); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void DirectRegistration_MatchesProductionAsyncGzipFileSink() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.ReaderService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + var levels = NntpdConfiguredSinks.WithoutWriteTo(configuration); + Assert.Empty(levels.GetSection("Serilog:WriteTo").GetChildren()); + Assert.Equal("Information", levels["Serilog:MinimumLevel:Default"]); + Assert.Equal("Debug", levels["Serilog:MinimumLevel:Override:VectorNNTPD.ReaderService"]); + + var loggerConfiguration = new LoggerConfiguration() + .ReadFrom.Configuration(levels); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + + var sinks = WalkLogEventSinks(logger).ToArray(); + var fileSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.Null(ReadInstanceField(fileSink, "_fileSizeLimitBytes")); + Assert.False(Assert.IsType(ReadInstanceField(fileSink, "_rollOnFileSizeLimit")!)); + Assert.Equal(14, ReadInstanceField(fileSink, "_retainedFileCountLimit")); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_buffered")!)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + + var asyncSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); + var queue = ReadInstanceField(asyncSink, "_queue") + ?? throw new InvalidOperationException("BackgroundWorkerSink._queue was not found."); + var boundedCapacity = queue.GetType().GetProperty("BoundedCapacity")?.GetValue(queue) + ?? throw new InvalidOperationException("BoundedCapacity was not found."); + Assert.Equal(50000, Convert.ToInt32(boundedCapacity, CultureInfo.InvariantCulture)); + AssertFlushesOncePerSecond(sinks); + Assert.Single(sinks, static n => n.GetType().Name.Equals("ConsoleSink", StringComparison.Ordinal)); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void DirectRegistration_OmitsFileWhenOnlyConsoleIsConfigured() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Warning", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + }) + .Build(); + + var loggerConfiguration = new LoggerConfiguration(); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + var sinks = WalkLogEventSinks(logger).ToArray(); + Assert.Single(sinks, static n => n.GetType().Name.Equals("ConsoleSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + } + + [Fact] + public void DirectRegistration_LeavesUnsupportedFileArgumentsOnConfigurationBinding() + { + var logDir = CreateTempLogDir(); + try + { + var path = NntpdFileLogging.RollingFilePath(logDir, "VectorNNTPD.ReaderService"); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Serilog:Using:0"] = "Serilog.Sinks.File", + ["Serilog:WriteTo:0:Name"] = "File", + ["Serilog:WriteTo:0:Args:path"] = path, + ["Serilog:WriteTo:0:Args:shared"] = "true", + ["Serilog:WriteTo:0:Args:rollingInterval"] = "Day", + }) + .Build(); + + var loggerConfiguration = new LoggerConfiguration(); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + var fileSink = Assert.Single( + WalkLogEventSinks(logger), + static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_shared")!)); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void ProductionNewsAndInpaths_FlushCurrentFileOncePerSecond() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.ReaderService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + Assert.Equal(TimeSpan.FromSeconds(1), NntpdNewsLogging.ReadSettings(configuration).FlushToDiskInterval); + Assert.Equal(TimeSpan.FromSeconds(1), NntpdPathSurveyLogging.ReadSettings(configuration).FlushToDiskInterval); + + using var news = NntpdNewsLogging.CreateLogger(configuration); + using var inpaths = NntpdPathSurveyLogging.CreateLogger( + configuration, + new IgnoringCompletedPathSurveyFileHandler(), + NullLogger.Instance); + var newsSinks = WalkLogEventSinks(news).ToArray(); + var inpathsSinks = WalkLogEventSinks(inpaths).ToArray(); + Assert.Equal(14, ReadInstanceField(Assert.Single(newsSinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)), "_retainedFileCountLimit")); + Assert.Equal(1, ReadInstanceField(Assert.Single(inpathsSinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)), "_retainedFileCountLimit")); + AssertFlushesOncePerSecond(newsSinks); + AssertFlushesOncePerSecond(inpathsSinks); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void ResolveDirectory_UsesCommonHelperAgainstApplicationBase() + { + var relative = NntpdFileLogging.ResolveDirectory(NntpdOptions.DefaultLogDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath(NntpdOptions.DefaultLogDir, AppContext.BaseDirectory), + relative); + + var absolute = Path.Join(Path.GetTempPath(), "vectornntp-logdir-abs"); + Assert.Equal(Path.GetFullPath(absolute), NntpdFileLogging.ResolveDirectory(absolute)); + Assert.DoesNotContain("logs/", NntpdFileLogging.RollingFilePath(absolute, "VectorNNTPD.ReaderService"), StringComparison.Ordinal); + } + + [Fact] + public void RollingFilePath_UsesApplicationName() + { + var path = NntpdFileLogging.RollingFilePath("C:\\logs", "VectorNNTPD.ReaderService"); + Assert.Equal(Path.Join("C:\\logs", "VectorNNTPD.ReaderService-.log"), path); + } + + [Fact] + public void CompletedApplicationLog_IsReplacedByGzip_WithTheOriginalBytes() + { + var dir = CreateTempLogDir(); + try + { + var rolled = Path.Join(dir, "VectorNNTPD.ReaderService-20260101.log"); + var original = "completed-day"u8.ToArray(); + File.WriteAllBytes(rolled, original); + var expectedName = NntpdFileLogging.GzipArchiveFileName(rolled); + Assert.Equal("VectorNNTPD.ReaderService-20260101.log.gz", expectedName); + + Assert.True(new GzipLogFileCompressor().TryCompressAndReplace(rolled)); + + var archive = Path.Join(dir, expectedName); + Assert.True(File.Exists(archive)); + Assert.False(File.Exists(rolled)); + using var gz = new GZipStream(File.OpenRead(archive), CompressionMode.Decompress); + using var reader = new MemoryStream(); + gz.CopyTo(reader); + Assert.Equal(original, reader.ToArray()); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void FailedCompression_DoesNotDeleteTheOriginalLog() + { + var dir = CreateTempLogDir(); + try + { + var rolled = Path.Join(dir, "VectorNNTPD.ReaderService-20260102.log"); + File.WriteAllText(rolled, "newer-day"); + using (new FileStream(rolled, FileMode.Open, FileAccess.Read, FileShare.None)) + { + Assert.False(new GzipLogFileCompressor().TryCompressAndReplace(rolled)); + } + + Assert.True(File.Exists(rolled)); + Assert.False(File.Exists(rolled + ".gz")); + Assert.Equal("newer-day", File.ReadAllText(rolled)); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void ConsoleReceivesInformation_ButNotDebug_FileReceivesDebugAndTakeThis() + { + var dir = CreateTempLogDir(); + using var captured = new StringWriter(); + var previous = Console.Out; + Console.SetOut(captured); + try + { + using var host = CreateFileLoggingHost(dir); + var factory = host.Services.GetRequiredService(); + var logger = factory.CreateLogger("VectorNNTPD.ReaderService.Session.NntpSession"); + + Assert.True(logger.IsEnabled(LogLevel.Information)); + Assert.True(logger.IsEnabled(LogLevel.Debug)); + Assert.True(logger.IsEnabled(LogLevel.Trace)); + + logger.LogInformation("file-logging-info-marker"); + logger.LogDebug("file-logging-debug-marker"); + CommandLogMessages.CommandRx(logger, "198.51.100.20:9", "TAKETHIS "); + + Log.CloseAndFlush(); + } + finally + { + Console.SetOut(previous); + } + + var console = captured.ToString(); + Assert.Contains("file-logging-info-marker", console, StringComparison.Ordinal); + Assert.DoesNotContain("file-logging-debug-marker", console, StringComparison.Ordinal); + Assert.DoesNotContain("TAKETHIS", console, StringComparison.Ordinal); + + var applicationName = ApplicationJsonConfiguration.EntryAssemblyName; + var daily = Assert.Single( + Directory.GetFiles(dir, applicationName + "-*.log"), + path => Path.GetFileName(path).StartsWith(applicationName + "-", StringComparison.Ordinal) + && path.EndsWith(".log", StringComparison.Ordinal) + && !path.EndsWith(".log.gz", StringComparison.Ordinal)); + Assert.Matches("^" + System.Text.RegularExpressions.Regex.Escape(applicationName) + @"-\d{8}\.log$", Path.GetFileName(daily)); + Assert.DoesNotContain("VectorNNTPD.ReaderService-", Path.GetFileName(daily), StringComparison.Ordinal); + Assert.Empty(Directory.GetFiles(dir, "*.gz")); + + var fileText = File.ReadAllText(daily); + Assert.Contains("file-logging-info-marker", fileText, StringComparison.Ordinal); + Assert.Contains("file-logging-debug-marker", fileText, StringComparison.Ordinal); + Assert.Contains("TAKETHIS ", fileText, StringComparison.Ordinal); + + TryDelete(dir); + } + + [Fact] + public void ConfiguredLogDir_IsHonoured_AndCreatedWhenMissing() + { + var parent = Path.Join(Path.GetTempPath(), "vectornntp-logdir-" + Guid.NewGuid().ToString("N")); + var dir = Path.Join(parent, "nested-logs"); + Assert.False(Directory.Exists(dir)); + try + { + using var host = CreateFileLoggingHost(dir); + var logger = host.Services.GetRequiredService() + .CreateLogger("VectorNNTPD.ReaderService"); + logger.LogInformation("created-dir-marker"); + Log.CloseAndFlush(); + + Assert.True(Directory.Exists(dir)); + Assert.Contains( + Directory.GetFiles(dir, "*.log"), + path => File.ReadAllText(path).Contains("created-dir-marker", StringComparison.Ordinal)); + } + finally + { + TryDelete(parent); + } + } + + private static IHost CreateFileLoggingHost(string logDir) + { + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + ContentRootPath = AppContext.BaseDirectory, + }); + TestHostFactory.ConfigureNntpdTestHost(builder); + var settings = new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = logDir, + [$"{NntpdOptions.SectionName}:ApplicationName"] = "VectorNNTPD.ReaderService", + ["Serilog:Using:0"] = "Serilog.Sinks.Console", + ["Serilog:Using:1"] = "Serilog.Sinks.File", + ["Serilog:Using:2"] = "Serilog.Sinks.Async", + ["Serilog:MinimumLevel:Default"] = "Information", + ["Serilog:MinimumLevel:Override:Microsoft"] = "Warning", + ["Serilog:MinimumLevel:Override:Microsoft.Hosting.Lifetime"] = "Information", + ["Serilog:MinimumLevel:Override:System"] = "Warning", + ["Serilog:MinimumLevel:Override:VectorNNTPD.ReaderService"] = "Verbose", + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Information", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + }; + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + settings[pair.Key] = pair.Value; + } + + builder.Configuration.AddInMemoryCollection(settings); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static string CreateTempLogDir() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-filelog-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } + + private static void TryDelete(string path) + { + try + { + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private static object? ReadInstanceField(object instance, string name) + { + for (var type = instance.GetType(); type is not null; type = type.BaseType) + { + var field = type.GetField(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (field is not null) + { + return field.GetValue(instance); + } + } + + return null; + } + + private static void AssertFlushesOncePerSecond(IEnumerable sinks) + { + var flush = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("PeriodicFileFlushSink", StringComparison.Ordinal)); + var timer = ReadInstanceField(flush, "_timer") + ?? throw new InvalidOperationException("PeriodicFileFlushSink._timer was not found."); + var holder = timer.GetType().GetField("_timer", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(timer) + ?? throw new InvalidOperationException("Timer._timer was not found."); + var queueTimer = holder.GetType().GetField("_timer", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(holder) + ?? throw new InvalidOperationException("TimerHolder._timer was not found."); + var period = queueTimer.GetType().GetField("_period", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(queueTimer) + ?? throw new InvalidOperationException($"{queueTimer.GetType().FullName}._period was not found."); + Assert.Equal(1000u, Assert.IsType(period)); + } + + private sealed class IgnoringCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler + { + public void OnCompletedFile(string completedFilePath) + { + } + } + + private static IEnumerable WalkLogEventSinks(object root) + { + var seen = new HashSet(ReferenceEqualityComparer.Instance); + var pending = new Stack(); + pending.Push(root); + while (pending.Count > 0) + { + var current = pending.Pop(); + if (!seen.Add(current)) + { + continue; + } + + yield return current; + foreach (var value in current.GetType() + .GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Select(field => field.GetValue(current))) + { + if (value is ILogEventSink sink) + { + pending.Push(sink); + continue; + } + + if (value is IEnumerable sinks) + { + foreach (var inner in sinks) + { + pending.Push(inner); + } + } + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/InnNewsTextFormatterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/InnNewsTextFormatterTests.cs new file mode 100644 index 00000000..fee30d47 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/InnNewsTextFormatterTests.cs @@ -0,0 +1,172 @@ +using System.Text; +using Serilog.Events; +using Serilog.Parsing; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +public sealed class InnNewsTextFormatterTests +{ + [Fact] + public void Format_IgnoresMessageTemplate_AndEmitsInnLine() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {NewsDisposition} {Message}"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.SitesProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Aug 25 13:37:41.839 + ? 0 ?\n", writer.ToString()); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_Junk_UsesJ_SpacePaddedDay_AndOmitsSite() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('j')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Jan 5 00:00:00.000 j ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("newsgroup not carried", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + } + + [Fact] + public void Format_Junk_UsesSuppliedReason_NotDerived() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('j')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.ReasonProperty, new ScalarValue("peer-only: junk.local")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 peer-only: junk.local\n", writer.ToString()); + } + + [Fact] + public void Format_Rejected_UsesSuppliedCodeAndReason() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {Message}"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('-')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("<23k82@bar.net>")), + new LogEventProperty(NntpdNewsLogging.ResponseCodeProperty, new ScalarValue(437)), + new LogEventProperty(NntpdNewsLogging.ReasonProperty, new ScalarValue("Poison newsgroup")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Aug 25 13:37:54.638 - ? <23k82@bar.net> 0 Poison newsgroup\n", writer.ToString()); + Assert.DoesNotContain("437", writer.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_Moderated_UsesM_NotPlusOrQuestion() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('m')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Aug 25 13:37:41.839 m ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain(" + ", line, StringComparison.Ordinal); + Assert.NotEqual('?', (char)NewsLogDisposition.Moderated); + } + + [Fact] + public void Format_PreservesMessageIdBytesAsAscii() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var id = ""; + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue(id)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Contains(id, writer.ToString(), StringComparison.Ordinal); + Assert.EndsWith(id + " 0 ?\n", writer.ToString(), StringComparison.Ordinal); + Assert.Equal(id, Encoding.ASCII.GetString(Encoding.ASCII.GetBytes(id))); + } + + [Fact] + public void Format_NamedPeer_RoundTripsSizeAndOutboundPlaceholder_WithoutSession() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("BlueWorldHosting")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.SizeProperty, new ScalarValue(1584)), + new LogEventProperty(NntpdNewsLogging.SitesProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + writer.ToString()); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/InnPathSurveyTextFormatterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/InnPathSurveyTextFormatterTests.cs new file mode 100644 index 00000000..b4a40dd1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/InnPathSurveyTextFormatterTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using Serilog.Events; +using Serilog.Parsing; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +public sealed class InnPathSurveyTextFormatterTests +{ + [Fact] + public void Format_IgnoresMessageTemplate_AndEmitsPathLine() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {Message} {PathSurveyPath}"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue("peer.example!not-for-mail")), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: peer.example!not-for-mail\n", writer.ToString()); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_EmptyPath_EmitsPrefixAndNewline() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: \n", writer.ToString()); + } + + [Fact] + public void Format_PreservesExactPathBytesAsLatin1() + { + var path = "AbC!Example.COM!not-for-mail"; + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue(path)), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: " + path + "\n", writer.ToString()); + Assert.Equal(path, Encoding.Latin1.GetString(Encoding.Latin1.GetBytes(path))); + } + + [Fact] + public void Format_DoesNotEmitMessageIdNewsgroupsOrDisposition() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue("only.path")), + new LogEventProperty("NewsMessageId", new ScalarValue("")), + new LogEventProperty("NewsDisposition", new ScalarValue('+')), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Path: only.path\n", line); + Assert.DoesNotContain("", line, StringComparison.Ordinal); + Assert.DoesNotContain("+", line, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/LoggingPipelineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/LoggingPipelineTests.cs new file mode 100644 index 00000000..754ab30f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/LoggingPipelineTests.cs @@ -0,0 +1,424 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Logging.Debug; +using Microsoft.Extensions.Options; +using Serilog; +using Serilog.Extensions.Logging; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Diagnostics; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class LoggingPipelineTests +{ + [Fact] + public void ApplicationInformationAndDebug_AreEnabled_ForVectorNntpCategory() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var root = factory.CreateLogger("VectorNNTPD.ReaderService"); + var listener = factory.CreateLogger("VectorNNTPD.ReaderService.Networking.Listeners.NntpPlainListenerService"); + + Assert.True(root.IsEnabled(LogLevel.Information)); + Assert.True(root.IsEnabled(LogLevel.Debug)); + Assert.True(listener.IsEnabled(LogLevel.Information)); + Assert.True(listener.IsEnabled(LogLevel.Debug)); + } + + [Fact] + public void MicrosoftAndSystem_RemainWarning_WhileHostingLifetimeIsInformation() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var microsoft = factory.CreateLogger("Microsoft.Extensions.Options"); + var system = factory.CreateLogger("System.Net.Http"); + var lifetime = factory.CreateLogger("Microsoft.Hosting.Lifetime"); + + Assert.False(microsoft.IsEnabled(LogLevel.Information)); + Assert.True(microsoft.IsEnabled(LogLevel.Warning)); + Assert.False(system.IsEnabled(LogLevel.Information)); + Assert.True(system.IsEnabled(LogLevel.Warning)); + Assert.True(lifetime.IsEnabled(LogLevel.Information)); + } + + [Fact] + public void SerilogFactory_IsExclusive_NoMicrosoftProviders() + { + using var host = CreateProductionConfiguredHost(); + var factories = host.Services.GetServices().ToArray(); + Assert.Single(factories); + Assert.Equal("SerilogLoggerFactory", factories[0].GetType().Name); + var providers = host.Services.GetLoggerProviders(); + Assert.Empty(providers); + Assert.DoesNotContain(providers, static p => p is ConsoleLoggerProvider); + Assert.DoesNotContain(providers, static p => p is DebugLoggerProvider); + } + + [Fact] + public void ConnectionAcceptance_ReachesSerilogSink() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + var logger = host.Services.GetRequiredService>(); + + Assert.True(logger.IsEnabled(LogLevel.Information)); + NetworkingLogMessages.PlainConnectionAccepted(logger, "192.0.2.10:40000"); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("Plain connection accepted", StringComparison.Ordinal)); + Assert.Equal(Serilog.Events.LogEventLevel.Information, evt.Level); + Assert.Equal( + "VectorNNTPD.ReaderService.Networking.Listeners.NntpPlainListenerService", + evt.Properties["SourceContext"].ToString().Trim('"')); + Assert.DoesNotContain("password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void FeedDiagnosticsSnapshot_ReachesSerilogSink() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + var logger = host.Services.GetRequiredService>(); + var text = FeedDiagnosticsFormatter.Format( + new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = TimeSpan.FromSeconds(10), + Peers = [], + Sessions = [], + IntervalArticles = 12, + IntervalBytes = 9_000_000, + }, + includeSessions: false); + + FeedDiagnosticsLogMessages.Snapshot(logger, text); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("FEED interval=", StringComparison.Ordinal)); + Assert.Contains("rate=", evt.RenderMessage(), StringComparison.Ordinal); + Assert.Equal( + "VectorNNTPD.ReaderService.Diagnostics.FeedDiagnosticsService", + evt.Properties["SourceContext"].ToString().Trim('"')); + Assert.DoesNotContain("@", evt.RenderMessage(), StringComparison.Ordinal); + Assert.DoesNotContain("password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void LoggingInitialized_ReachesSerilogSink_ThroughILogger() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + + NntpdLoggingExtensions.WriteLoggingInitialized( + host.Services, + "Production", + AppContext.BaseDirectory); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("Application logging initialized", StringComparison.Ordinal)); + Assert.Equal(Serilog.Events.LogEventLevel.Information, evt.Level); + Assert.Contains("SerilogLoggerFactory", evt.RenderMessage(), StringComparison.Ordinal); + Assert.Contains(NntpdLogCategories.Hosting, evt.RenderMessage(), StringComparison.Ordinal); + Assert.True(evt.Properties.TryGetValue("Application", out var application)); + Assert.Equal( + "\"" + ApplicationJsonConfiguration.EntryAssemblyName + "\"", + application.ToString()); + Assert.DoesNotContain("not-the-entry-assembly", application.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("CloudFlare", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ConfiguredConsoleSink_ReceivesInformationEvent() + { + using var captured = new StringWriter(); + var previous = Console.Out; + Console.SetOut(captured); + try + { + using var host = CreateProductionConfiguredHost(); + var logger = host.Services.GetRequiredService>(); + NetworkingLogMessages.PlainConnectionAccepted(logger, "198.51.100.10:119"); + } + finally + { + Console.SetOut(previous); + } + + var output = captured.ToString(); + Assert.Contains("Plain connection accepted", output, StringComparison.Ordinal); + Assert.Contains("198.51.100.10:119", output, StringComparison.Ordinal); + Assert.DoesNotContain("password", output, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void DateAndTakeThis_AreNotHotPathSuppressed() + { + Assert.False(NntpCommandLogFormat.SuppressHotPathCommand(NntpVerb.Date)); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommandLog("DATE")); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommand(NntpVerb.TakeThis)); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommandLog("TAKETHIS ")); + } + + [Fact] + public async Task LivePlainAccept_AndDate_EmitThroughSerilog() + { + var sink = new CollectingSink(); + using var serilog = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.FromLogContext() + .WriteTo.Sink(sink) + .CreateLogger(); + using var factory = new SerilogLoggerFactory(serilog); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + options.ProxyHosts = []; + + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + factory, + factory.CreateLogger()); + + await service.StartAsync(CancellationToken.None); + var endpoint = service.LocalEndPoints[0]; + using var client = new TcpClient(); + await client.ConnectAsync(endpoint); + await using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + await using var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true) + { + NewLine = "\r\n", + AutoFlush = true, + }; + + var greeting = await reader.ReadLineAsync(); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + + await writer.WriteLineAsync("DATE"); + var date = await reader.ReadLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + await writer.WriteLineAsync("QUIT"); + _ = await reader.ReadLineAsync(); + await service.StopAsync(CancellationToken.None); + + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("Plain connection accepted from", StringComparison.Ordinal) + && e.Properties.ContainsKey("TcpPeer")); + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("RX:", StringComparison.Ordinal) + && e.Properties.TryGetValue("Command", out var rx) + && rx.ToString().Contains("DATE", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("TX:", StringComparison.Ordinal) + && e.Properties.TryGetValue("Command", out var tx) + && tx.ToString().Contains("DATE", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.Properties.TryGetValue("SourceContext", out var sessionCtx) + && sessionCtx.ToString().Contains("VectorNNTPD.ReaderService.Session.NntpSession", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.Properties.TryGetValue("SourceContext", out var dateCtx) + && dateCtx.ToString().Contains("VectorNNTPD.ReaderService.Session.Commands.Date", StringComparison.Ordinal)); + } + + [Fact] + public void ProductionAssembly_ReferencesFileSinksAndCommon() + { + var names = typeof(NntpdLoggingExtensions).Assembly + .GetReferencedAssemblies() + .Select(static a => a.Name) + .ToHashSet(StringComparer.Ordinal); + Assert.Contains("Serilog.Sinks.Console", names); + Assert.Contains("Serilog.Sinks.File", names); + Assert.Contains("VectorNNTPD.Common", names); + Assert.DoesNotContain("Serilog.Sinks.File.Archive", names); + } + + [Fact] + public void ProductionSerilogSection_ConsoleIsInformationOnly() + { + var section = ProductionSerilogSection(); + Assert.Equal("Serilog.Sinks.Console", section["Serilog:Using:0"]); + Assert.Equal("Console", section["Serilog:WriteTo:0:Name"]); + Assert.Equal("Information", section["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"]); + Assert.Equal("Debug", section["Serilog:MinimumLevel:Override:VectorNNTPD.ReaderService"]); + Assert.Equal("Async", section["Serilog:WriteTo:1:Name"]); + Assert.Equal("50000", section["Serilog:WriteTo:1:Args:bufferSize"]); + Assert.Equal("true", section["Serilog:WriteTo:1:Args:blockWhenFull"]); + Assert.Equal("File", section["Serilog:WriteTo:1:Args:configure:0:Name"]); + Assert.Equal("Debug", section["Serilog:WriteTo:1:Args:configure:0:Args:restrictedToMinimumLevel"]); + Assert.Equal("Day", section["Serilog:WriteTo:1:Args:configure:0:Args:rollingInterval"]); + Assert.Equal("14", section["Serilog:WriteTo:1:Args:configure:0:Args:retainedFileCountLimit"]); + Assert.Null(section["Serilog:WriteTo:1:Args:configure:0:Args:fileSizeLimitBytes"]); + } + + [Fact] + public void CommandLoggers_UseHostFactory_NotNullLogger() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + _ = new NntpCommandDispatcher(factory); + var dateLogger = NntpCommandLoggers.For(typeof(Date)); + Assert.True(dateLogger.IsEnabled(LogLevel.Information)); + Assert.NotEqual(NullLogger.Instance.GetType(), dateLogger.GetType()); + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + } + + [Fact] + public async Task LivePlainAccept_AndDate_AppearOnConfiguredConsole() + { + using var captured = new StringWriter(); + var previous = Console.Out; + NntpPlainListenerService? service = null; + Console.SetOut(captured); + try + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var logger = host.Services.GetRequiredService>(); + var sessionLogger = factory.CreateLogger(); + var dateLogger = factory.CreateLogger("VectorNNTPD.ReaderService.Session.Commands.Date"); + + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + Assert.True(logger.IsEnabled(LogLevel.Information)); + Assert.True(sessionLogger.IsEnabled(LogLevel.Information)); + Assert.True(dateLogger.IsEnabled(LogLevel.Information)); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + options.ProxyHosts = []; + + service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + factory, + logger); + + await service.StartAsync(CancellationToken.None); + var endpoint = service.LocalEndPoints[0]; + using var client = new TcpClient(); + await client.ConnectAsync(endpoint); + await using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + await using var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true) + { + NewLine = "\r\n", + AutoFlush = true, + }; + + var greeting = await reader.ReadLineAsync(); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + await writer.WriteLineAsync("DATE"); + var date = await reader.ReadLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + await writer.WriteLineAsync("QUIT"); + _ = await reader.ReadLineAsync(); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + service = null; + } + finally + { + Console.SetOut(previous); + if (service is not null) + { + await service.DisposeAsync(); + } + } + + var output = captured.ToString(); + Assert.Contains("Plain connection accepted", output, StringComparison.Ordinal); + Assert.DoesNotContain("RX: DATE", output, StringComparison.Ordinal); + Assert.DoesNotContain("TX: DATE executed in", output, StringComparison.Ordinal); + Assert.DoesNotContain("TX: DATE [", output, StringComparison.Ordinal); + Assert.DoesNotContain("TAKETHIS", output, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("password", output, StringComparison.OrdinalIgnoreCase); + } + + private static IHost CreateProductionConfiguredHost(CollectingSink? sink = null) + { + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + ContentRootPath = AppContext.BaseDirectory, + }); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.Configuration.AddInMemoryCollection(ProductionSerilogSection()); + builder.ConfigureNntpdLogging(sink is null + ? null + : lc => + { + lc.WriteTo.Sink(sink); + }); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static Dictionary ProductionSerilogSection() + { + var section = new Dictionary + { + ["Serilog:Using:0"] = "Serilog.Sinks.Console", + ["Serilog:Using:1"] = "Serilog.Sinks.File", + ["Serilog:Using:2"] = "Serilog.Sinks.Async", + ["Serilog:MinimumLevel:Default"] = "Information", + ["Serilog:MinimumLevel:Override:Microsoft"] = "Warning", + ["Serilog:MinimumLevel:Override:Microsoft.Hosting.Lifetime"] = "Information", + ["Serilog:MinimumLevel:Override:System"] = "Warning", + ["Serilog:MinimumLevel:Override:VectorNNTPD.ReaderService"] = "Debug", + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Information", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + ["Serilog:Enrich:0"] = "FromLogContext", + ["Serilog:Properties:Application"] = "not-the-entry-assembly", + }; + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + section[pair.Key] = pair.Value; + } + + return section; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/PathSurveyCompletedFileTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/PathSurveyCompletedFileTests.cs new file mode 100644 index 00000000..b77f11f0 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/PathSurveyCompletedFileTests.cs @@ -0,0 +1,148 @@ +using System.IO.Compression; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +public sealed class PathSurveyCompletedFileTests +{ + [Fact] + public void CompletedFile_IsExposedToHandler_BeforeGzip_AndOriginalBytesAreInTheArchive() + { + var dir = CreateTempLogDir(); + try + { + var handler = new RecordingCompletedPathSurveyFileHandler(); + using (var writer = new SerilogPathSurveyWriter( + Configuration(dir, retention: 5, sizeLimit: 80), + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 8; i++) + { + writer.Write("aaaaaaaaaaaaaaaaaaaaaaaa"u8); + } + } + + Assert.NotEmpty(handler.Completed); + Assert.All(handler.GzipExistedAtHandoff, static existed => Assert.False(existed)); + Assert.All(handler.Completed, static path => Assert.False(File.Exists(path))); + var archives = Directory.GetFiles(dir, "inpaths*.log.gz"); + Assert.NotEmpty(archives); + Assert.NotEmpty(Directory.GetFiles(dir, "inpaths*.log")); + using var gz = new GZipStream(File.OpenRead(archives[0]), CompressionMode.Decompress); + using var reader = new StreamReader(gz); + Assert.Contains("Path: ", reader.ReadToEnd(), StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void HandlerFailure_DoesNotSkipGzip_AndDoesNotDropTheCompletedBytes() + { + var dir = CreateTempLogDir(); + try + { + var handler = new ThrowingCompletedPathSurveyFileHandler(); + using (var writer = new SerilogPathSurveyWriter( + Configuration(dir, retention: 5, sizeLimit: 80), + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 8; i++) + { + writer.Write("bbbbbbbbbbbbbbbbbbbbbbbb"u8); + } + } + + Assert.False(string.IsNullOrEmpty(handler.Path)); + Assert.False(File.Exists(handler.Path)); + var archive = handler.Path + ".gz"; + Assert.True(File.Exists(archive)); + using var gz = new GZipStream(File.OpenRead(archive), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + Assert.Contains("Path: ", reader.ReadToEnd(), StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void NewsRoll_CompressesTheCompletedFile_WithoutAPathSurveyHandler() + { + var dir = CreateTempLogDir(); + try + { + using (var writer = new SerilogNewsLogWriter(NewsConfiguration(dir))) + { + writer.Write(new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray())); + writer.Write(new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray())); + } + + var archives = Directory.GetFiles(dir, "news*.log.gz"); + Assert.NotEmpty(archives); + Assert.All(archives, static archive => Assert.False(File.Exists(archive[..^3]))); + Assert.Contains(Directory.GetFiles(dir), static path => path.EndsWith(".log", StringComparison.OrdinalIgnoreCase) + && !path.EndsWith(".log.gz", StringComparison.OrdinalIgnoreCase)); + using var gz = new GZipStream(File.OpenRead(archives[0]), CompressionMode.Decompress); + using var reader = new StreamReader(gz); + var text = reader.ReadToEnd(); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.DoesNotContain("", text, StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + private static IConfiguration Configuration(string dir, int retention, long sizeLimit) => + new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Serilog:Inpaths:path"] = Path.Join(dir, "inpaths-.log"), + ["Serilog:Inpaths:rollingInterval"] = "Infinite", + ["Serilog:Inpaths:retainedFileCountLimit"] = retention.ToString(), + ["Serilog:Inpaths:buffered"] = "false", + ["Serilog:Inpaths:rollOnFileSizeLimit"] = "true", + ["Serilog:Inpaths:fileSizeLimitBytes"] = sizeLimit.ToString(), + }).Build(); + + private static IConfiguration NewsConfiguration(string dir) => + new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Serilog:News:path"] = Path.Join(dir, "news-.log"), + ["Serilog:News:rollingInterval"] = "Infinite", + ["Serilog:News:retainedFileCountLimit"] = "5", + ["Serilog:News:buffered"] = "false", + ["Serilog:News:rollOnFileSizeLimit"] = "true", + ["Serilog:News:fileSizeLimitBytes"] = "1", + }).Build(); + + private static string CreateTempLogDir() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-inpaths-completed-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + private static void TryDelete(string dir) + { + try + { + Directory.Delete(dir, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/SerilogLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/SerilogLoggingTests.cs new file mode 100644 index 00000000..ff288bb5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/SerilogLoggingTests.cs @@ -0,0 +1,277 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Logging.Debug; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class SerilogLoggingTests +{ + [Fact] + public void ConfigureNntpdLogging_RegistersSerilog_AndRemovesDefaultProviders() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + builder.Services.AddSingleton(new FakeApplicationService("svc")); + + using var host = builder.Build(); + + var factory = host.Services.GetRequiredService(); + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + + var providers = host.Services.GetLoggerProviders(); + Assert.DoesNotContain(providers, static p => p is ConsoleLoggerProvider); + Assert.DoesNotContain(providers, static p => p is DebugLoggerProvider); + Assert.DoesNotContain( + providers, + static p => p.GetType().Name.Contains("EventLog", StringComparison.Ordinal)); + + // AddSerilog replaces ILoggerFactory rather than stacking MEL providers. + Assert.Empty(providers); + } + + [Fact] + public void ConfigureNntpdLogging_DoesNotRegisterDuplicateSerilogFactories() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var factories = host.Services.GetServices().ToArray(); + Assert.Single(factories); + Assert.Equal("SerilogLoggerFactory", factories[0].GetType().Name); + } + + [Fact] + public void ApplicationILogger_ResolvesThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink); + + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Structured probe {ProbeId} for {Component}", 42, "lifecycle"); + + var evt = Assert.Single( + sink.Events, + e => e.MessageTemplate.Text.Contains("Structured probe", StringComparison.Ordinal)); + Assert.Equal(LogEventLevel.Information, evt.Level); + Assert.True(evt.Properties.TryGetValue("ProbeId", out var probeId)); + Assert.True(evt.Properties.ContainsKey("Component")); + Assert.Equal("42", probeId.ToString().Trim('"')); + } + + [Fact] + public void StructuredLogging_PreservesExceptions() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink); + var logger = host.Services.GetRequiredService>(); + + var ex = new InvalidOperationException("boom"); + logger.LogError(ex, "Failure in {Phase}", "test"); + + var evt = Assert.Single(sink.Events, e => e.Exception is not null); + Assert.Same(ex, evt.Exception); + Assert.True(evt.Properties.ContainsKey("Phase")); + } + + [Fact] + public async Task LifecycleEvents_FlowThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("lifecycle state transition", StringComparison.OrdinalIgnoreCase) + || e.RenderMessage().Contains("Running", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.RenderMessage().Contains("Stopped", StringComparison.Ordinal) + || e.MessageTemplate.Text.Contains("shutdown", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Logger_RemainsAvailableDuringShutdown_AndHostDisposesCleanly() + { + var sink = new CollectingSink(); + var stopLogged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var service = new FakeApplicationService( + "svc", + onStop: ct => + { + // Resolve logger during stop to prove it is still usable. + return Task.CompletedTask; + }); + + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(service); + }); + + await host.StartAsync(); + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Pre-stop marker"); + + await host.StopAsync(); + logger.LogInformation("Post-stop marker still reachable from resolved instance"); + + Assert.Contains(sink.Events, e => e.RenderMessage().Contains("Pre-stop marker", StringComparison.Ordinal)); + Assert.True(service.StopCount >= 1); + stopLogged.TrySetResult(); + } + + [Fact] + public async Task StartupFailure_IsLogged_AndDoesNotLeakHost() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton( + new FakeApplicationService("bad", onStart: _ => throw new InvalidOperationException("startup-fail"))); + }); + + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Equal("startup-fail", ex.Message); + + Assert.Contains( + sink.Events, + e => e.Exception is InvalidOperationException + || e.RenderMessage().Contains("startup", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void BootstrapLogger_CapturesFatalBeforeHostBuild() + { + var previous = Log.Logger; + try + { + var sink = new CollectingSink(); + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(sink) + .CreateLogger(); + + var boom = new InvalidOperationException("bootstrap-fail"); + Log.Fatal(boom, "Host construction failed for {ApplicationName}", "VectorNNTPD.ReaderService"); + + var evt = Assert.Single(sink.Events); + Assert.Equal(LogEventLevel.Fatal, evt.Level); + Assert.Same(boom, evt.Exception); + Assert.True(evt.Properties.ContainsKey("ApplicationName")); + } + finally + { + Log.CloseAndFlush(); + Log.Logger = previous; + } + } + + [Fact] + public void CloseAndFlush_CanBeCalledExactlyOnceWithoutThrowing() + { + var previous = Log.Logger; + try + { + Log.Logger = new LoggerConfiguration() + .WriteTo.Sink(new CollectingSink()) + .CreateLogger(); + + Log.Information("flush-probe"); + Log.CloseAndFlush(); + Log.CloseAndFlush(); + } + finally + { + Log.Logger = previous; + } + } + + [Fact] + public void PlatformHosting_StillRegistersWithSerilogExclusiveLogging() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.Empty(host.Services.GetLoggerProviders()); + Assert.NotNull(host.Services.GetService()); + } + + [Fact] + public async Task HostedServiceStop_EmitsShutdownLogsThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + + Assert.Contains( + sink.Events, + e => e.RenderMessage().Contains("stopping", StringComparison.OrdinalIgnoreCase) + || e.RenderMessage().Contains("shutdown", StringComparison.OrdinalIgnoreCase) + || e.MessageTemplate.Text.Contains("Host stopping", StringComparison.OrdinalIgnoreCase)); + } + + private static IHost CreateSerilogHost( + CollectingSink sink, + Action? configureServices = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.Configuration[$"{NntpdOptions.SectionName}:ServerId"] = "1"; + builder.Configuration[$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d"; + builder.Configuration[$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja"; + builder.ConfigureNntpdLogging(lc => + { + lc.MinimumLevel.Verbose(); + lc.WriteTo.Sink(sink); + }); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + return builder.Build(); + } +} + +internal sealed class CollectingSink : ILogEventSink +{ + private readonly ConcurrentQueue _events = new(); + + public IReadOnlyCollection Events => _events.ToArray(); + + public void Emit(LogEvent logEvent) + { + ArgumentNullException.ThrowIfNull(logEvent); + _events.Enqueue(logEvent); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Logging/SourceGeneratedLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Logging/SourceGeneratedLoggingTests.cs new file mode 100644 index 00000000..88242924 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Logging/SourceGeneratedLoggingTests.cs @@ -0,0 +1,317 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Serilog.Events; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.Logging; + +namespace VectorNNTPD.ReaderService.Tests.Logging; + +/// +/// Observable contracts for source-generated logging: templates, structured properties, +/// EventIds, levels, and exception preservation. +/// +[Collection(SerilogCollection.Name)] +public sealed class SourceGeneratedLoggingTests +{ + [Fact] + public void CommandRx_PreservesStructuredTemplateAndEventId() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandRx(logger, "198.18.0.70:49860", "HELP"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1600, logger.EventId.Id); + Assert.Equal("[198.18.0.70:49860] RX: HELP", logger.Formatted); + Assert.Equal("198.18.0.70:49860", logger.GetString("Client")); + Assert.Equal("HELP", logger.GetString("Command")); + Assert.DoesNotContain("{Command}", logger.Formatted, StringComparison.Ordinal); + Assert.DoesNotContain("$\"", logger.Formatted, StringComparison.Ordinal); + } + + [Fact] + public void CommandRejected_PreservesStructuredProtocolFields() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandRejected( + logger, + "198.18.0.70:49860", + NntpVerb.Unknown, + NntpVerb.None, + NntpParseStatus.UnknownVerb, + "empty command"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1601, logger.EventId.Id); + Assert.Contains("RX rejected:", logger.Formatted, StringComparison.Ordinal); + Assert.Equal("198.18.0.70:49860", logger.GetString("Client")); + Assert.Equal(NntpVerb.Unknown, logger.GetValue("Verb")); + Assert.Equal(NntpVerb.None, logger.GetValue("Qualifier")); + Assert.Equal(NntpParseStatus.UnknownVerb, logger.GetValue("Status")); + Assert.Equal("empty command", logger.GetString("Detail")); + } + + [Fact] + public void CommandFailed_PreservesException() + { + var logger = new CapturingLogger(); + var boom = new InvalidOperationException("handler-fail"); + + CommandLogMessages.CommandFailed(logger, boom, "198.18.0.70:49860", "DATE"); + + Assert.Equal(LogLevel.Error, logger.Level); + Assert.Equal(1604, logger.EventId.Id); + Assert.Same(boom, logger.Exception); + Assert.Equal("[198.18.0.70:49860] DATE failed", logger.Formatted); + Assert.Equal("DATE", logger.GetString("Command")); + } + + [Fact] + public void CommandTx_PreservesTimingShape() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTx(logger, "198.18.0.70:49860", "HELP", 0.012); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1602, logger.EventId.Id); + Assert.Equal("[198.18.0.70:49860] TX: HELP executed in 0.012s", logger.Formatted); + Assert.Equal(0.012, logger.GetValue("ElapsedSeconds")); + } + + [Fact] + public void CommandTxWithStatus_PreservesStatusLineAndTimingShape() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTxWithStatus( + logger, + "198.18.0.70:49860", + "CHECK", + "238 send article to be transferred", + 0.012); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1615, logger.EventId.Id); + Assert.Equal( + "[198.18.0.70:49860] TX: CHECK [238 send article to be transferred] executed in 0.012s", + logger.Formatted); + Assert.Equal("CHECK", logger.GetString("Command")); + Assert.Equal( + "238 send article to be transferred", + logger.GetString("StatusLine")); + Assert.Equal(0.012, logger.GetValue("ElapsedSeconds")); + } + + [Fact] + public void CommandTxWithStatusAndDetail_PreservesBothStatusAndDetail() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTxWithStatusAndDetail( + logger, + "198.18.0.70:49860", + "STARTTLS", + "382 Continue with TLS negotiation", + 0.001, + "TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1616, logger.EventId.Id); + Assert.Equal( + "[198.18.0.70:49860] TX: STARTTLS [382 Continue with TLS negotiation] executed in 0.001s [TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384]", + logger.Formatted); + Assert.Equal("382 Continue with TLS negotiation", logger.GetString("StatusLine")); + Assert.Equal("TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384", logger.GetString("Detail")); + } + + [Fact] + public void RedactRxCommand_IsExplicitLoggingStringBoundary() + { + var command = NntpCommandParser.Parse("HELP"u8); + var text = NntpCommandLogFormat.RedactRxCommand(command, "HELP"u8); + + Assert.Equal("HELP", text); + + var pass = NntpCommandParser.Parse("AUTHINFO PASS secret"u8); + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxCommand(pass, "AUTHINFO PASS secret"u8)); + } + + [Fact] + public void DisabledInformation_DoesNotInvokeFormatter() + { + var logger = new CapturingLogger(enabled: false); + + CommandLogMessages.CommandRx(logger, "198.18.0.70:49860", "HELP"); + + Assert.Null(logger.Formatted); + Assert.Empty(logger.Properties); + } + + [Fact] + public void LifecycleTransition_UsesReservedEventId1000() + { + var logger = new CapturingLogger(); + + LifecycleLogMessages.LifecycleTransition( + logger, + ApplicationStateLog.Starting, + ApplicationStateLog.Running); + + Assert.Equal(LogLevel.Information, logger.Level); + Assert.Equal(1000, logger.EventId.Id); + Assert.Equal( + "Application lifecycle state transition: Starting -> Running", + logger.Formatted); + } + + [Fact] + public void SessionEndedWithError_PreservesClientAndException() + { + var logger = new CapturingLogger(); + var boom = new IOException("peer reset"); + var client = IPAddress.Parse("198.18.0.70"); + + SessionLogMessages.SessionEndedWithError(logger, boom, client); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1611, logger.EventId.Id); + Assert.Same(boom, logger.Exception); + Assert.Equal(client, logger.GetValue("Client")); + } + + [Fact] + public void PlainConnectionAccepted_PreservesEndpointProperty() + { + var logger = new CapturingLogger(); + + NetworkingLogMessages.PlainConnectionAccepted(logger, "198.18.0.70:42122"); + + Assert.Equal(LogLevel.Information, logger.Level); + Assert.Equal(1405, logger.EventId.Id); + Assert.Equal("Plain connection accepted from 198.18.0.70:42122", logger.Formatted); + Assert.Equal("198.18.0.70:42122", logger.GetString("TcpPeer")); + } + + [Fact] + public void GeneratedMessages_FlowThroughSerilogStructuredProperties() + { + var sink = new CollectingSink(); + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(lc => + { + lc.MinimumLevel.Verbose(); + lc.WriteTo.Sink(sink); + }); + + using var host = builder.Build(); + var logger = host.Services.GetRequiredService() + .CreateLogger("VectorNNTPD.ReaderService.Tests.SourceGenerated"); + + var boom = new SocketException((int)SocketError.ConnectionReset); + CommandLogMessages.QuitUncaught(logger, boom, "198.18.0.70:49860", boom.GetType().Name, boom.SocketErrorCode); + + var evt = Assert.Single(sink.Events, e => e.Exception is SocketException); + Assert.Equal(LogEventLevel.Error, evt.Level); + Assert.Same(boom, evt.Exception); + Assert.Equal( + "[{Client}] QUIT uncaught during termination: {Type} socket={Socket}", + evt.MessageTemplate.Text); + Assert.True(evt.Properties.ContainsKey("Client")); + Assert.True(evt.Properties.ContainsKey("Type")); + Assert.True(evt.Properties.ContainsKey("Socket")); + Assert.Equal("198.18.0.70:49860", Scalar(evt, "Client")); + } + + [Fact] + public void EventIds_AreUniqueAcrossGeneratedMessageFiles() + { + var ids = new[] + { + 1000, 1001, 1002, 1003, 1004, 1005, + 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, + 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, + 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, + 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, + 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, + 1314, 1315, 1316, 1317, 1318, + 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, + 1500, 1501, 1502, 1503, + 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, + 1615, 1616, + 1700, 1701, 1702, 1703, 1704, 1705, 1706, + 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, + 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, + 1828, 1829, + 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, + 1914, 1915, 1916, 1917, 1918, + 2000, 2001, 2002, 2003, 2004, + }; + + Assert.Equal(ids.Length, ids.Distinct().Count()); + } + + private static string Scalar(LogEvent evt, string name) => + evt.Properties[name].ToString().Trim('"'); + + private sealed class CapturingLogger : ILogger + { + private readonly bool _enabled; + + public CapturingLogger(bool enabled = true) => _enabled = enabled; + + public LogLevel? Level { get; private set; } + + public EventId EventId { get; private set; } + + public Exception? Exception { get; private set; } + + public string? Formatted { get; private set; } + + public Dictionary Properties { get; } = new(StringComparer.Ordinal); + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => _enabled; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Level = logLevel; + EventId = eventId; + Exception = exception; + Formatted = formatter(state, exception); + if (state is IEnumerable> pairs) + { + foreach (var pair in pairs) + { + Properties[pair.Key] = pair.Value; + } + } + } + + public string GetString(string name) => + Convert.ToString(Properties[name], CultureInfo.InvariantCulture) ?? string.Empty; + + public T GetValue(string name) => + Assert.IsType(Properties[name]); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs new file mode 100644 index 00000000..72756e62 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs @@ -0,0 +1,125 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Tests.Email; + +namespace VectorNNTPD.ReaderService.Tests.Moderation; + +public sealed class EmailModerationSubmissionServiceTests +{ + [Fact] + public async Task DisabledEmail_IsUnavailableWithoutSpoolFile() + { + using var harness = new EmailSpoolHarness(enabled: false); + var service = Create(harness, enabled: false); + var result = await service.SubmitAsync(Submission()); + Assert.Equal(ModerationSubmissionStatus.Unavailable, result.Status); + Assert.False(Directory.Exists(harness.Directory)); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task EnabledEmail_SpoolIsAccepted() + { + using var harness = new EmailSpoolHarness(); + var service = Create(harness, enabled: true); + var result = await service.SubmitAsync(Submission()); + Assert.Equal(ModerationSubmissionStatus.Accepted, result.Status); + Assert.Contains("spool", result.Detail, StringComparison.Ordinal); + Assert.Single(harness.EmlFiles()); + var bytes = await File.ReadAllBytesAsync(harness.EmlFiles()[0]); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("comp-example@moderators.isc.org", item!.Recipients[0].Address); + Assert.Equal("noreply@example.com", item.EnvelopeSender.Address); + var encoded = System.Text.Encoding.ASCII.GetString(item.EncodedMessage.Span); + Assert.Contains("application/news-transmission; usage=moderate", encoded, StringComparison.Ordinal); + Assert.Contains("X-Moderated-Newsgroup: comp.example", encoded, StringComparison.Ordinal); + Assert.Contains("Newsgroups: comp.example", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Date:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Path:", encoded, StringComparison.Ordinal); + } + + [Fact] + public async Task SpoolWriteFailure_IsFailed() + { + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-moderation-blocker-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + var options = EnabledOptions(); + options.Spool.Directory = blocker; + var spool = new FilesystemEmailSpool(Options.Create(options), NullLogger.Instance); + var service = new EmailModerationSubmissionService( + new EmailService(spool, new Rfc5322MessageEncoder(), Options.Create(options), NullLogger.Instance), + new ModerationEmailComposer(Options.Create(options)), + Options.Create(options)); + Assert.Equal(ModerationSubmissionStatus.Failed, (await service.SubmitAsync(Submission())).Status); + } + finally + { + File.Delete(blocker); + } + } + + [Fact] + public void PercentS_Address_ExpandsInComposerDestination() + { + var composer = new ModerationEmailComposer(Options.Create(EnabledOptions())); + var message = composer.Compose(Submission(moderator: "comp-example@moderators.isc.org")); + Assert.Equal("comp-example@moderators.isc.org", message.To[0].Address); + Assert.Equal("comp.example ", message.Subject); + } + + [Fact] + public void Destuff_RemovesStuffingDots() + { + var destuffed = ModerationEmailComposer.DestuffProtoArticle("..hidden\r\nbody\r\n"u8); + Assert.Equal(".hidden\r\nbody\r\n", System.Text.Encoding.ASCII.GetString(destuffed)); + } + + internal static EmailModerationSubmissionService Create(EmailSpoolHarness harness, bool enabled) + { + var options = enabled ? harness.Options : new EmailOptions(); + if (enabled) + { + options.DefaultFrom = "noreply@example.com"; + options.EnvelopeSender = "noreply@example.com"; + } + + var boxed = Options.Create(options); + return new EmailModerationSubmissionService( + new EmailService(harness.Spool, new Rfc5322MessageEncoder(), boxed, NullLogger.Instance), + new ModerationEmailComposer(boxed), + boxed); + } + + internal static EmailOptions EnabledOptions() => + new() + { + Enabled = true, + DefaultFrom = "noreply@example.com", + EnvelopeSender = "noreply@example.com", + Smtp = new SmtpOptions { Host = "127.0.0.1", Port = 25 }, + Spool = new EmailSpoolOptions + { + Directory = Path.Join(Path.GetTempPath(), "vectornntp-email-unused", Guid.NewGuid().ToString("N")), + }, + }; + + internal static ModerationSubmission Submission(string moderator = "comp-example@moderators.isc.org") => + new() + { + ProtoArticle = "From: poster@example.com\r\nNewsgroups: comp.example\r\nSubject: hi\r\nMessage-ID: \r\nDate: Fri, 25 Sep 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(), + MessageId = "", + Newsgroups = ["comp.example"], + TargetModeratedGroup = "comp.example", + ModeratorAddress = moderator, + AuthenticatedUsername = "alice", + Sender = ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorCatalogueServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorCatalogueServiceTests.cs new file mode 100644 index 00000000..00e9b76c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorCatalogueServiceTests.cs @@ -0,0 +1,184 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Moderation; + +public sealed class ModeratorCatalogueServiceTests +{ + [Fact] + public async Task StartAsync_LoadsEnabledRows_InModeratorIdOrder() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 2, + GroupPattern = "later.*", + ModeratorAddress = "later@example.com", + AccountName = "later", + }); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "first.*", + ModeratorAddress = "first@example.com", + AccountName = "first", + }); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 3, + GroupPattern = "disabled.*", + ModeratorAddress = "disabled@example.com", + AccountName = "disabled", + Enabled = false, + }); + + await using var catalogue = Create(repository); + await catalogue.StartAsync(CancellationToken.None); + + Assert.Equal(1, repository.QueryCount); + Assert.True(catalogue.HasSnapshot); + Assert.Equal(2, catalogue.Current.Count); + Assert.True(catalogue.Current.TryResolve("first.group"u8, out var first)); + Assert.Equal("first@example.com", first.Address); + Assert.False(catalogue.Current.TryResolve("disabled.group"u8, out _)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenRepositoryFails() + { + var repository = new MemoryNntpModeratorRepository + { + Exception = new NntpDbUnavailableException("down"), + }; + await using var catalogue = Create(repository); + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task Refresh_Failure_RetainsLastKnownGood() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "group.a", + ModeratorAddress = "a@example.com", + AccountName = "mod-a", + }); + var clock = new ControllableTimeProvider(); + await using var catalogue = Create(repository, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var original = catalogue.Current; + + repository.Exception = new NntpDbUnavailableException("down"); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => repository.QueryCount >= 2); + + Assert.Same(original, catalogue.Current); + Assert.True(catalogue.Current.TryResolve("group.a"u8, out _)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_Success_AtomicallyReplacesCurrent() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "old.*", + ModeratorAddress = "old@example.com", + AccountName = "old", + }); + var clock = new ControllableTimeProvider(); + await using var catalogue = Create(repository, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var original = catalogue.Current; + + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 2, + GroupPattern = "new.*", + ModeratorAddress = "new@example.com", + AccountName = "new", + }); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => !ReferenceEquals(original, catalogue.Current)); + + Assert.True(catalogue.Current.TryResolve("new.group"u8, out var identity)); + Assert.Equal("new@example.com", identity.Address); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task ConcurrentPublish_IsSafe() + { + await using var catalogue = Create(new MemoryNntpModeratorRepository()); + var first = ModeratorTestSnapshot.Create(("a.*", "a@example.com", "a")); + var second = ModeratorTestSnapshot.Create(("b.*", "b@example.com", "b")); + Parallel.For(0, 32, i => catalogue.PublishForTests(i % 2 == 0 ? first : second)); + Assert.True(catalogue.HasSnapshot); + Assert.True( + catalogue.Current.TryResolve("a.group"u8, out _) + || catalogue.Current.TryResolve("b.group"u8, out _)); + } + + [Fact] + public void QueryConstant_SelectsEnabledRowsByModeratorId() + { + Assert.Contains("FROM nntpmoderators", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("is_enabled = 'Y'", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("ORDER BY moderator_id", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("group_pattern", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("moderator_address", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("account_name", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + } + + [Fact] + public async Task FakeNntpDb_MapsEnabledRowsAndBackendFailure() + { + var factory = new FakeNntpDbConnectionFactory + { + Moderators = + [ + new NntpModeratorRow(1, "comp.foo.*", "moderator@example.org", "MODERATOR01"), + ], + }; + await using var connection = (FakeNntpDbConnection)await factory.OpenAsync("Server=test;", CancellationToken.None); + var rows = await connection.QueryEnabledModeratorsAsync(CancellationToken.None); + var row = Assert.Single(rows); + Assert.Equal(1, row.ModeratorId); + Assert.Equal("comp.foo.*", row.GroupPattern); + Assert.Equal("moderator@example.org", row.ModeratorAddress); + Assert.Equal("MODERATOR01", row.AccountName); + + connection.QueryModeratorsException = new NntpDbUnavailableException("down"); + await Assert.ThrowsAsync( + async () => await connection.QueryEnabledModeratorsAsync(CancellationToken.None)); + } + + private static ModeratorCatalogueService Create( + INntpModeratorRepository repository, + TimeProvider? clock = null, + TimeSpan? interval = null) => + new( + repository, + NullLogger.Instance, + clock ?? TimeProvider.System, + interval ?? ModeratorCatalogueService.RefreshInterval); + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorSnapshotTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorSnapshotTests.cs new file mode 100644 index 00000000..5ad65add --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Moderation/ModeratorSnapshotTests.cs @@ -0,0 +1,206 @@ +using System.Text; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Moderation; + +public sealed class ModeratorSnapshotTests +{ + [Fact] + public void FirstMatch_WinsOverLaterGeneralPattern() + { + var auth = ModeratorTestSnapshot.Create( + ("comp.example.moderated", "specific@example.com", "mod-specific"), + ("comp.example.*", "general@example.com", "mod-general")); + + Assert.True(auth.TryResolve(Bytes("comp.example.moderated"), out var identity)); + Assert.Equal("specific@example.com", identity.Address); + Assert.Equal("mod-specific", identity.Username); + } + + [Fact] + public void Resolve_UnknownGroup_Fails() + { + var auth = ModeratorTestSnapshot.Create(("group.a", "a@example.com", "moderator-a")); + Assert.False(auth.TryResolve(Bytes("group.b"), out _)); + } + + [Fact] + public void CanApprove_RequiresMatchingUsernameAndAddress() + { + var auth = Standard(); + Assert.True(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("moderator-b@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorB, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.NormalUser, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(null, Bytes("moderator-a@example.com"), Bytes("group.a"))); + } + + [Fact] + public void CanApprove_AddressComparison_IsCaseInsensitive() + { + var auth = Standard(); + Assert.True(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("Moderator-A@Example.COM"), Bytes("group.a"))); + } + + [Fact] + public void UsernameComparison_IsOrdinal() + { + var auth = Standard(); + Assert.False(auth.CanApprove("Moderator-A", Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.True(auth.IsAuthenticatedModerator(MapNntpAuthenticationProvider.ModeratorA)); + Assert.False(auth.IsAuthenticatedModerator("Moderator-A")); + Assert.False(auth.IsAuthenticatedModerator(MapNntpAuthenticationProvider.NormalUser)); + } + + [Fact] + public void TryAuthorizeApproval_AllModeratedTargetsMustBeCovered() + { + var auth = Standard(); + Assert.True(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com"], + ["group.a"], + out _)); + Assert.False(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com"], + ["group.a", "group.b"], + out var detail)); + Assert.Equal("unauthorized moderator principal", detail); + } + + [Fact] + public void TryAuthorizeApproval_RejectsExtraUnrelatedIdentity() + { + var auth = Standard(); + Assert.False(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com", "other@example.com"], + ["group.a"], + out var detail)); + Assert.Equal("conflicting Approved", detail); + } + + [Fact] + public void TryAuthorizeApproval_SameModeratorOnTwoPatterns_Succeeds() + { + var auth = ModeratorTestSnapshot.Create( + ("group.a", "shared@example.com", "moderator-a"), + ("group.b", "shared@example.com", "moderator-a")); + Assert.True(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["shared@example.com"], + ["group.a", "group.b"], + out _)); + } + + [Fact] + public void EmptyCatalogue_ResolvesNothing() + { + var auth = ModeratorSnapshot.Empty; + Assert.False(auth.TryResolve(Bytes("group.a"), out _)); + Assert.False(auth.TryAuthorizeApproval("moderator-a", ["a@example.com"], ["group.a"], out var detail)); + Assert.Equal("unresolved moderator", detail); + } + + [Theory] + [InlineData("fido7.some.group", "fido7-some-group@fido7.org")] + [InlineData("perl.foo.bar", "news-moderator-perl-foo-bar@perl.org")] + [InlineData("comp.test", "comp-test@moderators.isc.org")] + public void InnTemplate_ExpandsPercentSWithDotsToDashes(string group, string expected) + { + Assert.Equal(expected, ModeratorAddressTemplate.Expand(AddressFor(group), Bytes(group))); + } + + [Fact] + public void InnRoutingCatalogue_FirstMatchAndCatchAll() + { + var auth = InnRouting(); + Assert.True(auth.TryResolve(Bytes("fido7.some.group"), out var fido7)); + Assert.Equal("fido7-some-group@fido7.org", fido7.Address); + Assert.Equal(string.Empty, fido7.Username); + Assert.True(auth.TryResolve(Bytes("perl.foo.bar"), out var perl)); + Assert.Equal("news-moderator-perl-foo-bar@perl.org", perl.Address); + Assert.True(auth.TryResolve(Bytes("comp.test"), out var catchAll)); + Assert.Equal("comp-test@moderators.isc.org", catchAll.Address); + Assert.Equal("*", catchAll.Pattern); + } + + [Fact] + public void InnRoutingCatalogue_DoesNotAuthorizeInjection() + { + var auth = InnRouting(); + Assert.False(auth.IsAuthenticatedModerator("moderator-a")); + Assert.False(auth.CanApprove("moderator-a", Bytes("fido7-some-group@fido7.org"), Bytes("fido7.some.group"))); + Assert.False(auth.TryAuthorizeApproval( + "moderator-a", + ["fido7-some-group@fido7.org"], + ["fido7.some.group"], + out var detail)); + Assert.Equal("unauthorized moderator principal", detail); + } + + [Fact] + public void RoutingOnlyEntry_CompileDoesNotDropMissingUsername() + { + var auth = ModeratorTestSnapshot.Create(("fido7.*", "%s@fido7.org", string.Empty)); + Assert.True(auth.TryResolve(Bytes("fido7.announce"), out var identity)); + Assert.Equal("fido7-announce@fido7.org", identity.Address); + Assert.Equal(string.Empty, identity.Username); + } + + [Fact] + public void StaticMailbox_IsUsedLiterally() + { + var auth = ModeratorTestSnapshot.Create(("comp.foo.*", "moderator@example.org", "MODERATOR01")); + Assert.True(auth.TryResolve(Bytes("comp.foo.test"), out var identity)); + Assert.Equal("moderator@example.org", identity.Address); + Assert.True(auth.CanApprove("MODERATOR01", Bytes("moderator@example.org"), Bytes("comp.foo.test"))); + } + + [Fact] + public void Create_PreservesModeratorIdOrder_NotAlphabetical() + { + var auth = ModeratorSnapshot.Create( + [ + new NntpModeratorRow(2, "zzz.*", "later@example.com", "later"), + new NntpModeratorRow(1, "aaa.*", "first@example.com", "first"), + ]); + Assert.True(auth.TryResolve(Bytes("zzz.group"), out var identity)); + Assert.Equal("later@example.com", identity.Address); + } + + [Fact] + public void DatabaseFailureDoesNotAuthorize_EmptySnapshot() + { + Assert.False(ModeratorSnapshot.Empty.TryAuthorizeApproval( + "moderator-a", + ["a@example.com"], + ["group.a"], + out _)); + } + + private static ModeratorSnapshot InnRouting() => + ModeratorTestSnapshot.Create( + ("fido7.*", "%s@fido7.org", string.Empty), + ("fj.*", "%s@moderators.fj-news.org", string.Empty), + ("medlux.*", "%s@news.medlux.ru", string.Empty), + ("nl.*", "%s@nl.news-admin.org", string.Empty), + ("perl.*", "news-moderator-%s@perl.org", string.Empty), + ("relcom.*", "%s@moderators.relcom.ru", string.Empty), + ("si.*", "%s@arnes.si", string.Empty), + ("*", "%s@moderators.isc.org", string.Empty)); + + private static string AddressFor(string group) => + group.StartsWith("fido7.", StringComparison.Ordinal) ? "%s@fido7.org" + : group.StartsWith("perl.", StringComparison.Ordinal) ? "news-moderator-%s@perl.org" + : "%s@moderators.isc.org"; + + private static ModeratorSnapshot Standard() => + ModeratorTestSnapshot.Create( + ("group.a", "moderator-a@example.com", MapNntpAuthenticationProvider.ModeratorA), + ("group.b", "moderator-b@example.com", MapNntpAuthenticationProvider.ModeratorB)); + + private static byte[] Bytes(string value) => Encoding.ASCII.GetBytes(value); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs new file mode 100644 index 00000000..3c321856 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs @@ -0,0 +1,205 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Moderation; + +/// +/// Live MySQL coverage for . +/// Skips unless VECTORNNTP_NNTPDB_INTEGRATION is set. Fails when that +/// variable is set and MySQL is unreachable. +/// +public sealed class MySqlNntpModeratorRepositoryIntegrationTests + : IClassFixture +{ + private readonly MySqlModeratorIntegrationFixture _fixture; + + public MySqlNntpModeratorRepositoryIntegrationTests(MySqlModeratorIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [NntpDbIntegrationFact] + public async Task LoadsEnabledModeratorsFromRealMySql() + { + var ours = await LoadOursAsync(); + + AssertMapped(ours, _fixture.Static); + AssertMapped(ours, _fixture.Template); + AssertMapped(ours, _fixture.SharedFirst); + AssertMapped(ours, _fixture.SharedSecond); + AssertMapped(ours, _fixture.Boundary); + Assert.Equal(5, ours.Count); + } + + [NntpDbIntegrationFact] + public async Task ExcludesDisabledModeratorsFromRealMySql() + { + var ours = await LoadOursAsync(); + + Assert.DoesNotContain(ours, row => row.ModeratorId == _fixture.Disabled.ModeratorId); + Assert.DoesNotContain( + ours, + row => string.Equals(row.AccountName, _fixture.Disabled.AccountName, StringComparison.Ordinal) + && string.Equals(row.GroupPattern, _fixture.Disabled.GroupPattern, StringComparison.Ordinal)); + } + + [NntpDbIntegrationFact] + public async Task PreservesModeratorIdOrdering() + { + var ours = await LoadOursAsync(); + + for (var i = 1; i < ours.Count; i++) + { + Assert.True( + ours[i - 1].ModeratorId < ours[i].ModeratorId, + "Enabled nntpmoderators rows must be returned in moderator_id ASC."); + } + + var shared = ours + .Where(row => string.Equals(row.GroupPattern, _fixture.SharedPattern, StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(2, shared.Length); + Assert.Equal(Math.Min(_fixture.SharedFirst.ModeratorId, _fixture.SharedSecond.ModeratorId), shared[0].ModeratorId); + Assert.Equal(Math.Max(_fixture.SharedFirst.ModeratorId, _fixture.SharedSecond.ModeratorId), shared[1].ModeratorId); + Assert.NotEqual(shared[0].ModeratorAddress, shared[1].ModeratorAddress); + } + + [NntpDbIntegrationFact] + public async Task MapsCharAccountNameCorrectly() + { + var ours = await LoadOursAsync(); + + var shortName = Find(ours, _fixture.Static); + Assert.Equal("TESTMOD01", shortName.AccountName); + Assert.Equal(9, shortName.AccountName.Length); + Assert.False(shortName.AccountName.EndsWith(' ')); + + var fullWidth = Find(ours, _fixture.Boundary); + Assert.Equal(_fixture.LongAccountName, fullWidth.AccountName); + Assert.Equal(32, fullWidth.AccountName.Length); + Assert.False(fullWidth.AccountName.EndsWith(' ')); + } + + [NntpDbIntegrationFact] + public async Task MapsTemplateModeratorAddressCorrectly() + { + var ours = await LoadOursAsync(); + var template = Find(ours, _fixture.Template); + + Assert.Equal("moderator-%s@example.org", template.ModeratorAddress); + Assert.Contains("%s", template.ModeratorAddress, StringComparison.Ordinal); + Assert.Equal("TESTMOD02", template.AccountName); + } + + [NntpDbIntegrationFact] + public async Task MapsBoundaryLengthsFromRealMySql() + { + var ours = await LoadOursAsync(); + var boundary = Find(ours, _fixture.Boundary); + + Assert.Equal(255, boundary.GroupPattern.Length); + Assert.Equal(_fixture.Boundary.GroupPattern, boundary.GroupPattern); + Assert.Equal(320, boundary.ModeratorAddress.Length); + Assert.Equal(_fixture.LongModeratorAddress, boundary.ModeratorAddress); + Assert.Equal(_fixture.LongAccountName, boundary.AccountName); + } + + [NntpDbIntegrationFact] + public async Task CataloguePublishesMappedRowsFromRealMySql() + { + RequireConfigured(); + var repository = _fixture.Repository!; + var rows = await repository.GetEnabledAsync(CancellationToken.None); + var ours = _fixture.Ours(rows); + var snapshot = ModeratorSnapshot.Create(ours); + + Assert.Equal(5, snapshot.Count); + Assert.True(snapshot.IsAuthenticatedModerator("TESTMOD01")); + Assert.True(snapshot.IsAuthenticatedModerator("TESTMOD02")); + Assert.True(snapshot.IsAuthenticatedModerator(_fixture.LongAccountName)); + Assert.False(snapshot.IsAuthenticatedModerator("TESTMOD03")); + + Assert.True(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".static.child"), out var staticIdentity)); + Assert.Equal("moderator@example.org", staticIdentity.Address); + Assert.Equal("TESTMOD01", staticIdentity.Username); + + Assert.True(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".template.child"), out var templateIdentity)); + var expectedTemplate = "moderator-" + + (_fixture.PatternPrefix + ".template.child").Replace('.', '-') + + "@example.org"; + Assert.Equal(expectedTemplate, templateIdentity.Address); + Assert.Equal("TESTMOD02", templateIdentity.Username); + + var sharedGroup = _fixture.PatternPrefix + ".shared.child"; + Assert.True(snapshot.TryResolve(GroupBytes(sharedGroup), out var sharedIdentity)); + var firstShared = _fixture.SharedFirst.ModeratorId < _fixture.SharedSecond.ModeratorId + ? _fixture.SharedFirst + : _fixture.SharedSecond; + Assert.Equal(firstShared.ModeratorAddress, sharedIdentity.Address); + Assert.Equal(firstShared.AccountName, sharedIdentity.Username); + + Assert.False(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".disabled.child"), out _)); + + await using var catalogue = new ModeratorCatalogueService( + repository, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + await catalogue.StartAsync(CancellationToken.None); + try + { + Assert.True(catalogue.Current.Count >= 5); + Assert.True(catalogue.IsAuthenticatedModerator("TESTMOD01")); + Assert.True(catalogue.IsAuthenticatedModerator(_fixture.LongAccountName)); + Assert.False(catalogue.IsAuthenticatedModerator("TESTMOD03")); + } + finally + { + await catalogue.StopAsync(CancellationToken.None); + } + } + + [NntpDbIntegrationFact] + public async Task GetEnabledAsync_HonorsCancellation() + { + RequireConfigured(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync( + () => _fixture.Repository!.GetEnabledAsync(cts.Token).AsTask()); + } + + private async Task> LoadOursAsync() + { + RequireConfigured(); + var rows = await _fixture.Repository!.GetEnabledAsync(CancellationToken.None); + return _fixture.Ours(rows); + } + + private void RequireConfigured() + { + Assert.True(_fixture.IsConfigured, NntpDbIntegration.SkipReason); + Assert.NotNull(_fixture.Repository); + } + + private static void AssertMapped(IReadOnlyList rows, SeededModeratorRow expected) + { + var actual = Find(rows, expected); + Assert.Equal(expected.ModeratorId, actual.ModeratorId); + Assert.Equal(expected.GroupPattern, actual.GroupPattern); + Assert.Equal(expected.ModeratorAddress, actual.ModeratorAddress); + Assert.Equal(expected.AccountName, actual.AccountName); + } + + private static NntpModeratorRow Find(IReadOnlyList rows, SeededModeratorRow expected) + { + var match = rows.SingleOrDefault(row => row.ModeratorId == expected.ModeratorId); + Assert.NotNull(match); + return match; + } + + private static byte[] GroupBytes(string group) => Encoding.ASCII.GetBytes(group); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/BindAddressResolverTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/BindAddressResolverTests.cs new file mode 100644 index 00000000..a25fe812 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/BindAddressResolverTests.cs @@ -0,0 +1,149 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking; + +namespace VectorNNTPD.ReaderService.Tests.Networking; + +public sealed class BindAddressResolverTests +{ + [Fact] + public void Resolve_ExplicitIpv4AndIpv6_ReturnsBoth() + { + var v4 = IPAddress.Parse("198.18.0.66"); + var v6 = IPAddress.Parse("2001:db8::66"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [v4.ToString(), v6.ToString()]; + + var resolved = resolver.Resolve(options); + + Assert.Equal(2, resolved.All.Count); + Assert.Single(resolved.IPv4); + Assert.Single(resolved.IPv6); + Assert.Contains(v4, resolved.IPv4); + Assert.Contains(v6, resolved.IPv6); + } + + [Fact] + public void Resolve_WildcardStar_ExpandsEligibleInterfaceAddresses() + { + var publicV4 = IPAddress.Parse("203.0.113.10"); + var privateV4 = IPAddress.Parse("10.0.0.5"); + var v6 = IPAddress.Parse("2001:db8::5"); + var loopback = IPAddress.Loopback; + var linkLocal = IPAddress.Parse("169.254.10.10"); + var multicast = IPAddress.Parse("224.0.0.1"); + + var resolver = CreateResolver(publicV4, privateV4, v6, loopback, linkLocal, multicast); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var resolved = resolver.Resolve(options); + + Assert.Equal(3, resolved.All.Count); + Assert.Contains(publicV4, resolved.IPv4); + Assert.Contains(privateV4, resolved.IPv4); + Assert.Contains(v6, resolved.IPv6); + Assert.DoesNotContain(loopback, resolved.All); + Assert.DoesNotContain(linkLocal, resolved.All); + Assert.DoesNotContain(multicast, resolved.All); + } + + [Fact] + public void Resolve_Ipv4AnyWildcard_ExpandsOnlyIpv4() + { + var v4 = IPAddress.Parse("198.18.0.20"); + var v6 = IPAddress.Parse("2001:db8::20"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["0.0.0.0"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v4, resolved.IPv4[0]); + Assert.Empty(resolved.IPv6); + } + + [Fact] + public void Resolve_Ipv6AnyWildcard_ExpandsOnlyIpv6() + { + var v4 = IPAddress.Parse("198.18.0.21"); + var v6 = IPAddress.Parse("2001:db8::21"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["::"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v6, resolved.IPv6[0]); + Assert.Empty(resolved.IPv4); + } + + [Fact] + public void Resolve_Duplicates_AreEliminated() + { + var v4 = IPAddress.Parse("198.18.0.30"); + var resolver = CreateResolver(v4); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [v4.ToString(), v4.ToString(), "*"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v4, resolved.IPv4[0]); + } + + [Fact] + public void Resolve_ExplicitLoopbackOnly_YieldsEmptySet() + { + var resolver = CreateResolver(IPAddress.Loopback); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + + var resolved = resolver.Resolve(options); + + Assert.False(resolved.HasAny); + Assert.Empty(resolved.All); + } + + [Fact] + public void Resolve_NoEligibleAddressesOnWildcard_YieldsEmptySet() + { + var resolver = CreateResolver(IPAddress.Loopback, IPAddress.IPv6Loopback, IPAddress.Parse("169.254.1.1")); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var resolved = resolver.Resolve(options); + + Assert.False(resolved.HasAny); + } + + [Theory] + [InlineData("127.0.0.1", false)] + [InlineData("::1", false)] + [InlineData("0.0.0.0", false)] + [InlineData("::", false)] + [InlineData("169.254.1.1", false)] + [InlineData("fe80::1", false)] + [InlineData("224.0.0.1", false)] + [InlineData("ff02::1", false)] + [InlineData("10.0.0.1", true)] + [InlineData("192.168.1.1", true)] + [InlineData("203.0.113.1", true)] + [InlineData("2001:db8::1", true)] + public void IsEligibleForDns_ClassifiesAddresses(string text, bool expected) + { + var address = IPAddress.Parse(text); + Assert.Equal(expected, IpAddressEligibility.IsEligibleForDns(address)); + } + + private static BindAddressResolver CreateResolver(params IPAddress[] local) + { + return new BindAddressResolver( + new FakeLocalIpAddressAssignee(local), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs new file mode 100644 index 00000000..01f78ac7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs @@ -0,0 +1,163 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Listeners; + +/// Connection-acceptance log format contract (TCP peer vs PROXY effective client; TLS params). +public sealed class ConnectionAcceptanceLoggingTests +{ + [Fact] + public void LogPlainAccepted_Direct_OmitsClientSuffix() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.18.0.70"), 42122)); + ConnectionAcceptanceLogging.LogPlainAccepted(logger, identity); + + Assert.Single(logger.Messages); + var message = logger.Messages.Single(); + Assert.Equal("Plain connection accepted from 198.18.0.70:42122", message); + Assert.DoesNotContain("client ", message, StringComparison.Ordinal); + Assert.DoesNotContain("proxy ", message, StringComparison.Ordinal); + } + + [Fact] + public void LogPlainAccepted_Proxy_DistinguishesTcpPeerAndEffectiveClient() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Parse("198.18.0.10"), 40000), + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 42122), + proxyProtocolVersion: 2); + ConnectionAcceptanceLogging.LogPlainAccepted(logger, identity); + + Assert.Single(logger.Messages); + Assert.Equal( + "Proxy connection accepted from 198.18.0.10:40000; proxy 198.18.0.70:42122", + logger.Messages.Single()); + } + + [Fact] + public void LogTlsAccepted_Direct_IncludesNegotiatedParameters() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.18.0.70"), 40106)); + ConnectionAcceptanceLogging.LogTlsAccepted(logger, identity, "TLSv1.3", "TLS_AES_256_GCM_SHA384"); + + Assert.Single(logger.Messages); + Assert.Equal( + "TLS connection accepted from 198.18.0.70:40106 (TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384)", + logger.Messages.Single()); + } + + [Fact] + public void LogTlsAccepted_Proxy_IncludesNegotiatedParameters() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Parse("198.18.0.10"), 40000), + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 40106), + proxyProtocolVersion: 2); + ConnectionAcceptanceLogging.LogTlsAccepted(logger, identity, "TLSv1.3", "TLS_AES_256_GCM_SHA384"); + + Assert.Single(logger.Messages); + Assert.Equal( + "TLS/Proxy connection accepted from 198.18.0.10:40000; proxy 198.18.0.70:40106 (TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384)", + logger.Messages.Single()); + } + + [Fact] + public void FormatSslProtocol_UsesOperationalLabels() + { + Assert.Equal("TLSv1.3", TlsNegotiationLogging.FormatSslProtocol(SslProtocols.Tls13)); + Assert.Equal("TLSv1.2", TlsNegotiationLogging.FormatSslProtocol(SslProtocols.Tls12)); + } + + [Fact] + public async Task PlainListener_DirectAccept_LogsRequiredFormat() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.ProxyHosts = []; + + var recording = new RecordingLogger(); + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + recording); + + await service.StartAsync(CancellationToken.None); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(service.LocalEndPoints[0]); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!recording.Messages.Any(m => m.StartsWith("Plain connection accepted from ", StringComparison.Ordinal)) + && !cts.IsCancellationRequested) + { + await Task.Delay(10, cts.Token); + } + + var accept = Assert.Single( + recording.Messages, + m => m.StartsWith("Plain connection accepted from ", StringComparison.Ordinal)); + Assert.DoesNotContain("; client ", accept, StringComparison.Ordinal); + Assert.DoesNotContain("proxy ", accept, StringComparison.Ordinal); + await service.StopAsync(CancellationToken.None); + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ListenerBindStartupTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ListenerBindStartupTests.cs new file mode 100644 index 00000000..82050b67 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Listeners/ListenerBindStartupTests.cs @@ -0,0 +1,554 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Hosting.Systemd; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Listeners; + +/// +/// Bind-failure startup contract: all-or-nothing listener start, cleanup, and lifecycle propagation. +/// +public sealed class ListenerBindStartupTests +{ + [Fact] + public async Task PlainStartAsync_FirstBindSucceedsSecondFails_DisposesFirstAndFails() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var logger = new CapturingLogger(); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder, logger); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(2, binder.BindAttempts); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + Assert.Contains( + logger.Records, + r => r.Level == LogLevel.Error + && r.Exception == bindError + && r.Message.Contains("Failed to bind Plain NNTP listener", StringComparison.Ordinal) + && r.Message.Contains("listener startup cannot continue", StringComparison.Ordinal) + && r.Message.Contains(options.BindPort.ToString(), StringComparison.Ordinal)); + } + + [Fact] + public async Task PlainStartAsync_FirstBindFailsImmediately_FailsWithoutAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var service = CreatePlain(MultiEndpointPlainOptions(), binder); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(1, binder.BindAttempts); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + } + + [Fact] + public async Task TlsStartAsync_FirstBindSucceedsSecondFails_DisposesFirstAndFails() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var logger = new CapturingLogger(); + var options = MultiEndpointTlsOptions(); + await using var service = CreateTls(options, binder, logger); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(2, binder.BindAttempts); + Assert.False(service.ListenersBound); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + Assert.Contains( + logger.Records, + r => r.Level == LogLevel.Error + && r.Exception == bindError + && r.Message.Contains("Failed to bind TLS NNTP listener", StringComparison.Ordinal) + && r.Message.Contains("listener startup cannot continue", StringComparison.Ordinal) + && r.Message.Contains(options.BindPortTls.ToString(), StringComparison.Ordinal)); + } + + [Fact] + public async Task TlsStartAsync_FirstBindFailsImmediately_FailsWithoutAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var service = CreateTls(MultiEndpointTlsOptions(), binder); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(1, binder.BindAttempts); + Assert.False(service.ListenersBound); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + } + + [Fact] + public async Task PlainBindFailure_PropagatesToApplicationServiceManager() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder); + var managerLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + + var ex = await Assert.ThrowsAsync(() => manager.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Empty(manager.StartedServices); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("Application service NntpPlainListener started", StringComparison.Ordinal)); + Assert.Contains( + managerLogger.Records, + r => r.Level == LogLevel.Error + && r.Message.Contains("NntpPlainListener failed during startup", StringComparison.Ordinal)); + } + + [Fact] + public async Task TlsBindFailure_PropagatesToApplicationServiceManager() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + var options = MultiEndpointTlsOptions(); + await using var service = CreateTls(options, binder); + var managerLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + + var ex = await Assert.ThrowsAsync(() => manager.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Empty(manager.StartedServices); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.Contains( + managerLogger.Records, + r => r.Level == LogLevel.Error + && r.Message.Contains("NntpTlsListener failed during startup", StringComparison.Ordinal)); + } + + [Fact] + public async Task ListenerBindFailure_DoesNotReachRunning_AndOmitsSuccessLifecycleLogs() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder); + var managerLogger = new CapturingLogger(); + var lifecycleLogger = new CapturingLogger(); + var hostedLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + await using var lifecycle = new ApplicationLifecycle( + manager, + options, + lifecycleLogger); + var hosted = new NntpdHostedService( + lifecycle, + new NntpdHostLifetime( + lifecycle, + new TestHostApplicationLifetime(), + Options.Create(options), + NullLogger.Instance), + Options.Create(options), + hostedLogger); + + var ex = await Assert.ThrowsAsync(() => hosted.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.NotEqual(ApplicationState.Running, lifecycle.State); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Empty(manager.StartedServices); + Assert.False(service.HasActiveAcceptLoops); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.DoesNotContain( + lifecycleLogger.Records, + r => r.Message.Contains("Starting -> Running", StringComparison.Ordinal)); + Assert.DoesNotContain( + lifecycleLogger.Records, + r => r.Message.Contains("Application initialization completed", StringComparison.Ordinal)); + Assert.DoesNotContain( + hostedLogger.Records, + r => r.Message.Contains("Application entered Running", StringComparison.Ordinal) + || r.Message.Contains("Application started", StringComparison.Ordinal)); + } + + [Fact] + public async Task PlainStartAsync_SuccessfulBind_StartsAllEndpoints() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + await using var service = CreatePlain(options, listenBinder: null); + + await service.StartAsync(CancellationToken.None); + + Assert.NotEmpty(service.LocalEndPoints); + Assert.Equal(options.BindPort, service.LocalEndPoints[0].Port); + Assert.NotNull(service.Execution); + Assert.False(service.Execution!.IsCompleted); + Assert.True(service.HasActiveAcceptLoops); + await service.StopAsync(CancellationToken.None); + Assert.False(service.HasActiveAcceptLoops); + } + + [Fact] + public async Task PlainStartAsync_SuccessfulBind_UsesNntpdSectionBindSettings() + { + var port = TestHostFactory.GetFreeTcpPort(); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "127.0.0.1", + [$"{NntpdOptions.SectionName}:BindPort"] = port.ToString(), + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + ["BindAddress:0"] = "203.0.113.10", + ["BindPort"] = "2119", + ["BindPortTls"] = "2563", + }) + .Build(); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = []; + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + + await using var service = CreatePlain(options, listenBinder: null); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(port, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.NotEmpty(service.LocalEndPoints); + Assert.Equal(port, service.LocalEndPoints[0].Port); + Assert.Equal(IPAddress.Loopback, service.LocalEndPoints[0].Address); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task SuccessfulStartup_ReachesRunning_AndKeepsSuccessLogs() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + await using var service = CreatePlain(options, listenBinder: null); + var managerLogger = new CapturingLogger(); + var lifecycleLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + await using var lifecycle = new ApplicationLifecycle( + manager, + options, + lifecycleLogger); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Contains(manager.StartedServices, s => ReferenceEquals(s, service)); + Assert.Contains( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.Contains( + lifecycleLogger.Records, + r => r.Message.Contains("Starting -> Running", StringComparison.Ordinal)); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_DoesNotFail_WhenAcceptFailsAfterSuccessfulBind() + { + var binder = new ScriptedListenSocketBinder([null]); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = 23456; + await using var service = CreatePlain(options, binder); + + await service.StartAsync(CancellationToken.None); + + Assert.NotNull(service.Execution); + Assert.False(service.Execution!.IsFaulted); + Assert.False(service.Execution.IsCompleted); + Assert.NotEmpty(service.LocalEndPoints); + Assert.True(service.HasActiveAcceptLoops); + + foreach (var socket in binder.Sockets) + { + socket.Dispose(); + } + + Assert.False(service.Execution.IsFaulted); + Assert.False(service.Execution.IsCompleted); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task SocketAcceptListener_Start_PropagatesBindFailure_BeforeAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var listener = new SocketAcceptListener( + new ListenBinding(IPAddress.Loopback, 23456, DualMode: false), + static (_, _) => ValueTask.CompletedTask, + NullLogger.Instance, + binder); + + var ex = Assert.Throws(() => listener.Start()); + + Assert.Same(bindError, ex); + Assert.False(listener.AcceptLoopActive); + } + + private static SocketException AddressInUse() => + new((int)SocketError.AddressAlreadyInUse); + + private static NntpdOptions MultiEndpointPlainOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1", "::1"]; + options.BindPort = 23456; + options.BindPortTls = 0; + return options; + } + + private static NntpdOptions MultiEndpointTlsOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1", "::1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 23457; + return options; + } + + [Fact] + public async Task Plain_bind_failure_with_tls_disabled_does_not_publish() + { + var options = MultiEndpointPlainOptions(); + options.BindPortTls = 0; + options.BindAddress = [TestHostFactory.TestIpv4.ToString(), TestHostFactory.TestIpv6.ToString()]; + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "healthy-a", + Type = "A", + Name = options.Fqdn, + Content = "198.18.0.10", + Ttl = 300, + Proxied = false, + }); + var publication = CreatePublication(options, client); + await publication.StartAsync(CancellationToken.None); + await using var service = CreatePlain(options, new ScriptedListenSocketBinder(AddressInUse()), publication: publication); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await publication.StopAsync(CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Contains(client.Snapshot(), static record => record.Id == "healthy-a"); + } + + [Fact] + public async Task Tls_bind_failure_does_not_publish() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [TestHostFactory.TestIpv4.ToString()]; + options.BindPortTls = 563; + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "healthy-a", + Type = "A", + Name = options.Fqdn, + Content = "198.18.0.10", + Ttl = 300, + Proxied = false, + }); + var publication = CreatePublication(options, client); + await publication.StartAsync(CancellationToken.None); + await using var service = CreateTls(options, new ScriptedListenSocketBinder(AddressInUse()), publication: publication); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await publication.StopAsync(CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Contains(client.Snapshot(), static record => record.Id == "healthy-a"); + } + + private static NntpPlainListenerService CreatePlain( + NntpdOptions options, + IListenSocketBinder? listenBinder, + ILogger? logger = null, + CloudflareDnsReconciliationService? publication = null) + { + return new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + logger ?? NullLogger.Instance, + listenBinder: listenBinder, + publication: publication); + } + + private static NntpTlsListenerService CreateTls( + NntpdOptions options, + IListenSocketBinder listenBinder, + ILogger? logger = null, + CloudflareDnsReconciliationService? publication = null) + { + return new NntpTlsListenerService( + Options.Create(options), + new AvailableTlsCertificateProvider(), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + logger ?? NullLogger.Instance, + listenBinder: listenBinder, + publication: publication); + } + + private static CloudflareDnsReconciliationService CreatePublication(NntpdOptions options, FakeCloudflareDnsClient client) + { + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance, + new ListenerBoundDnsPublicationLease()); + } + + private sealed class ScriptedListenSocketBinder : IListenSocketBinder + { + private readonly Exception?[] _outcomes; + + public ScriptedListenSocketBinder(params Exception?[] outcomes) + { + _outcomes = outcomes; + } + + public List Sockets { get; } = []; + + public int BindAttempts { get; private set; } + + public void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog) + { + Sockets.Add(socket); + var index = BindAttempts++; + var outcome = index < _outcomes.Length ? _outcomes[index] : null; + if (outcome is not null) + { + throw outcome; + } + + // Bind an ephemeral loopback port so AcceptAsync parks instead of + // throwing synchronously (which would spin Start() on the caller). + var address = endpoint.AddressFamily == AddressFamily.InterNetwork + ? IPAddress.Loopback + : IPAddress.IPv6Loopback; + socket.Bind(new IPEndPoint(address, 0)); + socket.Listen(backlog); + } + } + + private sealed class AvailableTlsCertificateProvider : ITlsCertificateContextProvider + { + public bool IsAvailable => true; + + public TlsCertificateLease Acquire() => + throw new InvalidOperationException("Bind-startup tests do not perform TLS handshakes."); + + public void PublishFromPfx(ReadOnlySpan pfxBytes, string password) + { + } + } + + private sealed record CapturedLog(LogLevel Level, EventId EventId, string Message, Exception? Exception); + + private sealed class CapturingLogger : ILogger + { + public ConcurrentBag Records { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Records.Add(new CapturedLog(logLevel, eventId, formatter(state, exception), exception)); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Proxy/ProxyProtocolTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Proxy/ProxyProtocolTests.cs new file mode 100644 index 00000000..71bcbe2c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Proxy/ProxyProtocolTests.cs @@ -0,0 +1,1049 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Proxy; + +public sealed class TrustedProxyHostsTests +{ + [Fact] + public void EmptyProxyHosts_IsDisabled() + { + var hosts = new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + Assert.False(hosts.IsEnabled); + Assert.False(hosts.IsTrusted(IPAddress.Loopback)); + } + + [Fact] + public void MissingProxyHosts_IsDisabled() + { + var hosts = new TrustedProxyHosts(Options.Create(new NntpdOptions())); + Assert.False(hosts.IsEnabled); + } + + [Fact] + public void SingleIpv4_IsTrusted() + { + var hosts = Create(["198.51.100.10"]); + Assert.True(hosts.IsEnabled); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("198.51.100.11"))); + } + + [Fact] + public void MultipleIpv4_MatchExactly() + { + var hosts = Create(["198.51.100.10", "198.51.100.11"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.11"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("198.51.100.12"))); + } + + [Fact] + public void Ipv6_IsTrusted() + { + var hosts = Create(["2001:db8::10"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("2001:db8::10"))); + Assert.True(hosts.IsTrusted(IPAddress.Parse("2001:db8:0:0:0:0:0:10"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("2001:db8::11"))); + } + + [Fact] + public void Ipv4MappedPeer_MatchesConfiguredIpv4() + { + var hosts = Create(["198.51.100.10"]); + var mapped = IPAddress.Parse("::ffff:198.51.100.10"); + Assert.True(hosts.IsTrusted(mapped)); + } + + [Fact] + public void DuplicateAddresses_AreDeduped() + { + var hosts = Create(["198.51.100.10", "198.51.100.10", "::ffff:198.51.100.10"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + } + + private static TrustedProxyHosts Create(string[] entries) => + new(Options.Create(new NntpdOptions { ProxyHosts = entries })); +} + +public sealed class ProxyHostsConfigurationTests +{ + [Fact] + public void Validate_RejectsInvalidProxyHost() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["not-an-ip"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("ProxyHosts", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsCidr() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["198.51.100.0/24"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_RejectsAnyAddressWildcard() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["0.0.0.0"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_AcceptsIpv4AndIpv6() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["198.51.100.10", "2001:db8::1"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } +} + +public sealed class ProxyProtocolParserTests +{ + private static readonly IPEndPoint TcpPeer = new(IPAddress.Parse("198.51.100.1"), 45000); + + [Fact] + public void V1_Tcp4_ParsesClientEndpoint() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 41234 119\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out var error); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(string.Empty, error); + Assert.Equal(IPAddress.Parse("203.0.113.10"), result.Identity.ClientAddress); + Assert.Equal(41234, result.Identity.ClientPort); + Assert.Equal(TcpPeer, result.Identity.TcpPeer); + Assert.Equal(header.Length, result.HeaderLength); + } + + [Fact] + public void V1_Tcp6_ParsesClientEndpoint() + { + var header = Encoding.ASCII.GetBytes( + "PROXY TCP6 2001:db8::2 2001:db8::1 41234 119\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(IPAddress.Parse("2001:db8::2"), result.Identity.ClientAddress); + Assert.Equal(41234, result.Identity.ClientPort); + } + + [Fact] + public void V1_Unknown_UsesTcpPeer() + { + var header = Encoding.ASCII.GetBytes("PROXY UNKNOWN\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(TcpPeer, result.Identity.Client); + } + + [Fact] + public void V1_PartialDelivery_NeedsMore() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 41234 119\r\n"); + for (var i = 1; i < header.Length; i++) + { + var status = ProxyProtocolParser.TryParse(header.AsSpan(0, i), TcpPeer, out _, out _); + Assert.Equal(ProxyParseStatus.NeedMore, status); + } + + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V1_Malformed_Fails() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 not-an-ip 198.51.100.20 41234 119\r\n"); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("invalid address", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V1_ExceedsMaxLine_Fails() + { + var line = "PROXY UNKNOWN " + new string('x', 200) + "\r\n"; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(Encoding.ASCII.GetBytes(line), TcpPeer, out _, out _)); + } + + [Fact] + public void V1_LeadingZeroPort_Fails() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 01234 119\r\n"); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_Tcp4_ParsesClientEndpoint() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + srcPort: 40000, + dstPort: 119); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out var error); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(string.Empty, error); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + Assert.Equal(40000, result.Identity.ClientPort); + Assert.Equal(2, result.Version); + } + + [Fact] + public void V2_Tcp6_ParsesClientEndpoint() + { + var header = BuildV2Tcp6( + IPAddress.Parse("2001:db8::aa"), + IPAddress.Parse("2001:db8::bb"), + srcPort: 40001, + dstPort: 119); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(IPAddress.Parse("2001:db8::aa"), result.Identity.ClientAddress); + Assert.Equal(40001, result.Identity.ClientPort); + } + + [Fact] + public void V2_LocalCommand_UsesTcpPeer() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; // version 2, LOCAL + header[13] = 0x00; + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.True(result.LocalCommand); + Assert.Equal(TcpPeer, result.Identity.Client); + } + + [Fact] + public void V2_PartialDelivery_NeedsMore() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + for (var i = 1; i < header.Length; i++) + { + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, i), TcpPeer, out _, out _)); + } + } + + [Fact] + public void V2_InvalidSignature_Fails() + { + var header = new byte[16]; + header.AsSpan().Fill(0x41); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void UnrecognizedFirstByte_Fails() + { + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse([(byte)'G'], TcpPeer, out _, out var error)); + Assert.Contains("PROXY header required", error, StringComparison.Ordinal); + } + + [Fact] + public void V2_UnexpectedCommand_Fails() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + header[12] = 0x22; // version 2, unsupported command 2 + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("unsupported command", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_UnlistedFamilyProtocol_FallsBackToTcpPeer() + { + // Valid nibbles (family=1, protocol=0) but not a listed combination → UNSPEC fallback. + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x10; + header[14] = 0x00; + header[15] = 0x00; + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(TcpPeer.Address, result.Identity.ClientAddress); + Assert.Equal(TcpPeer.Port, result.Identity.ClientPort); + } + + [Fact] + public void V2_InvalidProtocolNibble_Fails() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x13; // AF_INET + invalid protocol 3 + header[14] = 0x00; + header[15] = 0x00; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_AbsentCrc_IsAccepted() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + } + + [Fact] + public void V2_ValidCrc32C_IsAccepted() + { + var header = BuildV2Tcp4WithCrc( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119, + corruptCrc: false); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + Assert.Equal(40000, result.Identity.ClientPort); + } + + [Fact] + public void V2_InvalidCrc32C_IsRejected() + { + var header = BuildV2Tcp4WithCrc( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119, + corruptCrc: true); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("CRC32C", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_UnsupportedFamilyNibble_Fails() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x41; // family 4 (unspecified) + STREAM + header[14] = 0x00; + header[15] = 0x00; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("unsupported address family/protocol", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_MaxAddressLength_LocalCommand_Parses() + { + // Length field is uint16; MaxV2AddressOctets (65535) is the wire maximum — no larger value exists. + var header = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; // LOCAL + header[13] = 0x00; + BinaryPrimitives.WriteUInt16BigEndian(header.AsSpan(14, 2), ProxyProtocolParser.MaxV2AddressOctets); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.True(result.LocalCommand); + Assert.Equal(ProxyProtocolParser.MaxV2HeaderOctets, result.HeaderLength); + } + + [Fact] + public void V2_OneByteBelowMaxAddressLength_NeedsMore() + { + var header = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; + header[13] = 0x00; + BinaryPrimitives.WriteUInt16BigEndian(header.AsSpan(14, 2), ProxyProtocolParser.MaxV2AddressOctets); + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, header.Length - 1), TcpPeer, out _, out _)); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_OneByteBelowComplete_NeedsMore() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, header.Length - 1), TcpPeer, out _, out _)); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + internal static byte[] BuildV2Tcp4(IPAddress src, IPAddress dst, ushort srcPort, ushort dstPort) + { + var header = new byte[28]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; // ver 2 PROXY + header[13] = 0x11; // TCP4 + header[14] = 0x00; + header[15] = 12; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 20); + header[24] = (byte)(srcPort >> 8); + header[25] = (byte)srcPort; + header[26] = (byte)(dstPort >> 8); + header[27] = (byte)dstPort; + return header; + } + + internal static byte[] BuildV2Tcp6(IPAddress src, IPAddress dst, ushort srcPort, ushort dstPort) + { + var header = new byte[52]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x21; // TCP6 + header[14] = 0x00; + header[15] = 36; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 32); + header[48] = (byte)(srcPort >> 8); + header[49] = (byte)srcPort; + header[50] = (byte)(dstPort >> 8); + header[51] = (byte)dstPort; + return header; + } + + internal static byte[] BuildV2Tcp4WithCrc( + IPAddress src, + IPAddress dst, + ushort srcPort, + ushort dstPort, + bool corruptCrc) + { + // Address block (12) + CRC TLV (3 + 4) = 19. + var header = new byte[16 + 19]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x11; + header[14] = 0x00; + header[15] = 19; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 20); + header[24] = (byte)(srcPort >> 8); + header[25] = (byte)srcPort; + header[26] = (byte)(dstPort >> 8); + header[27] = (byte)dstPort; + header[28] = 0x03; // PP2_TYPE_CRC32C + header[29] = 0x00; + header[30] = 0x04; + header[31] = 0x00; + header[32] = 0x00; + header[33] = 0x00; + header[34] = 0x00; + var crc = Crc32C.Compute(header); + if (corruptCrc) + { + crc ^= 0xFFFFFFFFu; + } + + BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(31, 4), crc); + return header; + } +} + +public sealed class ConnectionClientIdentityTests +{ + [Fact] + public void Direct_DoesNotAliasMutableEndpoints() + { + var original = new IPEndPoint(IPAddress.Parse("198.51.100.8"), 3333); + var identity = ConnectionClientIdentity.Direct(original); + + Assert.False(ReferenceEquals(identity.TcpPeer, identity.Client)); + Assert.False(ReferenceEquals(identity.TcpPeer, original)); + Assert.False(ReferenceEquals(identity.Client, original)); + + original.Port = 9999; + Assert.Equal(3333, identity.ClientPort); + Assert.Equal(3333, identity.TcpPeer.Port); + + identity.Client.Port = 1; + Assert.Equal(3333, identity.TcpPeer.Port); + Assert.Equal(IPAddress.Parse("198.51.100.8"), identity.TcpPeer.Address); + } + + [Fact] + public void FromTrustedProxy_ClonesBothEndpoints() + { + var peer = new IPEndPoint(IPAddress.Loopback, 40000); + var client = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 51515); + var identity = ConnectionClientIdentity.FromTrustedProxy(peer, client, proxyProtocolVersion: 1); + + peer.Port = 1; + client.Port = 2; + Assert.Equal(40000, identity.TcpPeer.Port); + Assert.Equal(51515, identity.ClientPort); + Assert.False(ReferenceEquals(identity.TcpPeer, peer)); + Assert.False(ReferenceEquals(identity.Client, client)); + } +} + +public sealed class ProxyPreambleLiveTests +{ + [Fact] + public async Task TrustedPeer_FragmentedV1Header_EstablishesClientIdentity() + { + var proxyIp = IPAddress.Loopback; + var trusted = new TrustedProxyHosts([proxyIp]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + Assert.True(trusted.IsTrusted(tcpPeer.Address)); + + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.77 127.0.0.1 51515 119\r\nHELLO"); + await client.SendAsync(header.AsMemory(0, 10)); + await client.SendAsync(header.AsMemory(10)); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(IPAddress.Parse("203.0.113.77"), resolution.Identity.ClientAddress); + Assert.Equal(51515, resolution.Identity.ClientPort); + Assert.True(resolution.Identity.IsTrustedProxy); + Assert.Equal("HELLO"u8.ToArray(), resolution.Leftover.ToArray()); + } + + [Fact] + public async Task TrustedPeer_SendsNothing_TimesOutWithoutFallbackIdentity() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var ex = await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + Assert.Contains("timed out", ex.Message, StringComparison.OrdinalIgnoreCase); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_IncompleteV1ThenStall_TimesOut() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("PROXY TCP4 203.0.113.1 "u8.ToArray()); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_IncompleteV2ThenStall_TimesOut() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("\r\n\r\n\0\r\n"u8.ToArray()); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_NonProxyBytes_FailsWithoutTcpFallback() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("CAPABILITIES\r\n"u8.ToArray()); + + var ex = await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromSeconds(2)).AsTask()); + Assert.Contains("PROXY header required", ex.Message, StringComparison.Ordinal); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_ClosesWithoutHeader_Fails() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + client.Dispose(); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromSeconds(2)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_MissingProxyOnTlsListener_DoesNotPublishSession() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + // First peer: trusted TCP but ordinary application bytes — must not become a session. + using (var bad = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await bad.ConnectAsync(host.EndPoint); + await bad.SendAsync("CAPABILITIES\r\n"u8.ToArray()); + await WaitForRemoteCloseAsync(bad); + } + + // Second peer: valid PROXY then TLS — must be the only published connection. + using var good = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await good.ConnectAsync(host.EndPoint); + await good.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.66 127.0.0.1 45000 563\r\n")); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(good, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.Equal(IPAddress.Parse("203.0.113.66"), server.ClientIdentity.ClientAddress); + Assert.Equal(45000, server.ClientIdentity.ClientPort); + Assert.True(server.ClientIdentity.IsTrustedProxy); + } + + [Fact] + public async Task UntrustedPeer_CannotReplaceIdentityWithProxyHeader() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + Assert.False(trusted.IsTrusted(tcpPeer.Address)); + + var forged = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.1 127.0.0.1 1 119\r\n"); + await client.SendAsync(forged); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.False(resolution.Identity.IsTrustedProxy); + Assert.False(resolution.Identity.UsedProxyHeader); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.Equal(tcpPeer.Port, resolution.Identity.ClientPort); + Assert.True(resolution.Leftover.IsEmpty); + + var leftover = new byte[forged.Length]; + var read = await accepted.ReceiveAsync(leftover.AsMemory(), SocketFlags.None); + Assert.Equal(forged.Length, read); + Assert.Equal(forged, leftover); + } + + [Fact] + public async Task UntrustedPeer_V2SignatureBytesRemainOnSocket() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var forged = ProxyProtocolParserTests.BuildV2Tcp4( + IPAddress.Parse("203.0.113.9"), + IPAddress.Loopback, + 1, + 119); + await client.SendAsync(forged); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.False(resolution.Identity.UsedProxyHeader); + + var leftover = new byte[forged.Length]; + var read = await accepted.ReceiveAsync(leftover.AsMemory(), SocketFlags.None); + Assert.Equal(forged.Length, read); + Assert.Equal(forged, leftover); + } + + [Fact] + public async Task UntrustedPeer_EstablishesDirectIdentityWithoutWaitingForProxy() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + cts.Token); + Assert.False(resolution.Identity.UsedProxyHeader); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.False(cts.IsCancellationRequested); + } + + [Fact] + public async Task EmptyTrustedSet_UsesTcpPeerWithoutReading() + { + var trusted = new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.Equal(tcpPeer.Port, resolution.Identity.ClientPort); + Assert.False(resolution.Identity.UsedProxyHeader); + } + + [Fact] + public async Task TlsWithTrustedProxy_HeaderThenHandshake_UsesProxyClientIdentity() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(host.EndPoint); + + var header = Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.44 127.0.0.1 42424 563\r\n"); + await socket.SendAsync(header); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(socket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.True(server.IsTls); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.44"), server.ClientIdentity.ClientAddress); + Assert.Equal(42424, server.ClientIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, server.ClientIdentity.TcpPeer.Address); + + var payload = new byte[] { 0x10, 0x20, 0x30 }; + await ssl.WriteAsync(payload); + await ssl.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task TlsWithTrustedProxy_HeaderAndHandshakePipelined_Succeeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(host.EndPoint); + + // PROXY header first; TLS ClientHello follows immediately on the same connection. + var header = ProxyProtocolParserTests.BuildV2Tcp4( + IPAddress.Parse("203.0.113.55"), + IPAddress.Loopback, + 43000, + 563); + await socket.SendAsync(header); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(socket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.Equal(IPAddress.Parse("203.0.113.55"), server.ClientIdentity.ClientAddress); + Assert.Equal(43000, server.ClientIdentity.ClientPort); + Assert.Equal(2, server.ClientIdentity.ProxyProtocolVersion); + + var payload = new byte[] { 0xAA, 0xBB }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public void Session_ExposesEffectiveClientWithoutInspectingTransport() + { + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Loopback, 40000), + new IPEndPoint(IPAddress.Parse("203.0.113.9"), 51515), + proxyProtocolVersion: 1); + + var sessionFromIdentity = new NntpSession( + new IdentityOnlyConnection(identity), + NullLogger.Instance); + Assert.Equal(IPAddress.Parse("203.0.113.9"), sessionFromIdentity.ClientAddress); + Assert.Equal(51515, sessionFromIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, sessionFromIdentity.TcpPeer.Address); + Assert.Same(identity, sessionFromIdentity.ClientIdentity); + + var direct = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.51.100.8"), 3333)); + var directSession = new NntpSession( + new IdentityOnlyConnection(direct), + NullLogger.Instance); + Assert.Equal(IPAddress.Parse("198.51.100.8"), directSession.ClientAddress); + Assert.Equal(3333, directSession.ClientPort); + Assert.Equal(direct.TcpPeer.Address, directSession.TcpPeer.Address); + Assert.Equal(direct.TcpPeer.Port, directSession.TcpPeer.Port); + } + + private static async Task WaitForRemoteCloseAsync(Socket socket) + { + var buffer = new byte[64]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + var read = await socket.ReceiveAsync(buffer.AsMemory(), SocketFlags.None, cts.Token); + if (read == 0) + { + return; + } + } + } + + private static async Task StartAcceptAsync() + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + listener.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + listener.Listen(1); + var acceptTask = AcceptOneAsync(listener, tcs); + return new AcceptFixture(listener, tcs.Task, acceptTask); + } + + private static async Task AcceptOneAsync(Socket listener, TaskCompletionSource tcs) + { + try + { + var accepted = await listener.AcceptAsync().ConfigureAwait(false); + tcs.TrySetResult(accepted); + } + // codeql[cs/catch-of-all-exceptions] Stopping the test listener can fault the accept loop with any transport exception. + catch (Exception ex) + { + tcs.TrySetException(ex); + } + } + + private sealed class AcceptFixture(Socket listener, Task accepted, Task acceptLoop) : IAsyncDisposable + { + public IPEndPoint EndPoint => (IPEndPoint)listener.LocalEndPoint!; + + public Task Accepted { get; } = accepted; + + public async ValueTask DisposeAsync() + { + listener.Dispose(); + try + { + await acceptLoop.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Listener disposed. + } + } + } + + private sealed class IdentityOnlyConnection : INntpConnection + { + public IdentityOnlyConnection(ConnectionClientIdentity identity) + { + ClientIdentity = identity; + } + + public System.IO.Pipelines.PipeReader Input => throw new NotSupportedException(); + + public System.IO.Pipelines.PipeWriter Output => throw new NotSupportedException(); + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => CancellationToken.None; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs new file mode 100644 index 00000000..43957973 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs @@ -0,0 +1,323 @@ +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Deterministic regression coverage for ConnectionByteTransport quiescence admission. +/// +public sealed class ConnectionByteTransportQuiescenceTests +{ + [Fact] + public async Task AfterPauseReads_TryCommitReadToApplication_StashesClientHello() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + await transport.PauseReadsAsync(CancellationToken.None); + + var clientHello = new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00 }; + Assert.False(transport.TryCommitReadToApplication(clientHello)); + + var prefix = transport.TakePendingUpgradePrefix(); + Assert.Equal(clientHello, prefix.ToArray()); + } + + [Fact] + public async Task Quiesce_DuringHistoricalAdmitWindow_DoesNotStartStreamIoAfterIdle() + { + // Reproduces the pre-fix window: WaitUntilActive observed Active, then Quiesce ran and + // declared idle before outstanding++. With atomic admission, stream I/O must not start. + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesceCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesceCompleted.TrySetResult(); + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[16], readCts.Token).AsTask(); + + await quiesceCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.ConcurrentOps); + + Assert.False(readTask.IsCompleted); + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => readTask); + Assert.Equal(0, stream.ReadStarts); + } + + [Fact] + public async Task Quiesce_WaitsForAdmittedRead_UntilStreamReadReturns() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var readEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancelObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowReadReturn = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + stream.OnReadAsync = async ct => + { + readEntered.TrySetResult(); + try + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Cancel was requested, but the operation still owns the stream until we return. + cancelObserved.TrySetResult(); + await allowReadReturn.Task.ConfigureAwait(false); + throw; + } + + return 0; + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[8], readCts.Token).AsTask(); + await readEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + using var quiesceCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var quiesceTask = transport.QuiesceAsync(quiesceCts.Token); + + await cancelObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(quiesceTask.IsCompleted); + Assert.Equal(1, stream.ConcurrentOps); + + allowReadReturn.TrySetResult(); + await quiesceTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ConcurrentOps); + await Assert.ThrowsAnyAsync(() => readTask); + } + + [Fact] + public async Task Quiesce_WaitsForAdmittedWrite_UntilStreamWriteReturns() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var writeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + stream.OnWriteAsync = async _ => + { + writeEntered.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var writeTask = transport.WriteAsync(new byte[] { 1, 2, 3 }, writeCts.Token).AsTask(); + await writeEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + using var quiesceCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var quiesceTask = transport.QuiesceAsync(quiesceCts.Token); + await Task.Yield(); + Assert.False(quiesceTask.IsCompleted); + Assert.Equal(1, stream.ConcurrentOps); + + releaseWrite.TrySetResult(); + await quiesceTask.WaitAsync(TimeSpan.FromSeconds(5)); + await writeTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ConcurrentOps); + } + + [Fact] + public async Task QuiescedIdle_SimulatedHandshake_CannotOverlapPlaintextStreamIo() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + await transport.QuiesceAsync(CancellationToken.None); + Assert.Same(transport.RateLimiter, transport.TakeQuiescedStreamForTlsWrap()); + Assert.Equal(0, stream.ConcurrentOps); + + stream.ArmPostQuiesceTracking(); + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var blockedRead = transport.ReadAsync(new byte[16], readCts.Token).AsTask(); + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var blockedWrite = transport.WriteAsync(new byte[] { 1 }, writeCts.Token).AsTask(); + + // Ownership window equivalent to AuthenticateAsServerAsync: stream must stay untouched. + for (var i = 0; i < 64; i++) + { + Assert.Equal(0, stream.PostQuiesceIoStarts); + Assert.Equal(0, stream.ConcurrentOps); + Assert.False(blockedRead.IsCompleted); + Assert.False(blockedWrite.IsCompleted); + await Task.Yield(); + } + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => blockedRead); + await Assert.ThrowsAnyAsync(() => blockedWrite); + Assert.Equal(0, stream.PostQuiesceIoStarts); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.WriteStarts); + } + + [Fact] + public async Task Quiesce_RepeatedAdmitRace_NeverStartsIoAfterIdle() + { + for (var iteration = 0; iteration < 64; iteration++) + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesced = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesced.TrySetResult(); + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[8], readCts.Token).AsTask(); + await quiesced.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.ConcurrentOps); + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => readTask); + Assert.Equal(0, stream.ReadStarts); + } + } + + [Fact] + public async Task QuiescedIdle_WriteAdmitRace_NeverStartsWriteAfterIdle() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.ReaderService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesced = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesced.TrySetResult(); + }; + + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var writeTask = transport.WriteAsync(new byte[] { 9 }, writeCts.Token).AsTask(); + await quiesced.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.WriteStarts); + Assert.Equal(0, stream.ConcurrentOps); + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => writeTask); + Assert.Equal(0, stream.WriteStarts); + } + + private sealed class ControllableStream : Stream + { + private int _concurrentOps; + private int _readStarts; + private int _writeStarts; + private int _postQuiesceIoStarts; + private bool _trackPostQuiesce; + + public Func>? OnReadAsync { get; set; } + public Func? OnWriteAsync { get; set; } + + public int ConcurrentOps => Volatile.Read(ref _concurrentOps); + public int ReadStarts => Volatile.Read(ref _readStarts); + public int WriteStarts => Volatile.Read(ref _writeStarts); + public int PostQuiesceIoStarts => Volatile.Read(ref _postQuiesceIoStarts); + + public void ArmPostQuiesceTracking() => _trackPostQuiesce = true; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _readStarts); + if (_trackPostQuiesce) + { + Interlocked.Increment(ref _postQuiesceIoStarts); + } + + Interlocked.Increment(ref _concurrentOps); + try + { + if (OnReadAsync is { } onRead) + { + return await onRead(cancellationToken).ConfigureAwait(false); + } + + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + return 0; + } + finally + { + Interlocked.Decrement(ref _concurrentOps); + } + } + + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _writeStarts); + if (_trackPostQuiesce) + { + Interlocked.Increment(ref _postQuiesceIoStarts); + } + + Interlocked.Increment(ref _concurrentOps); + try + { + if (OnWriteAsync is { } onWrite) + { + await onWrite(cancellationToken).ConfigureAwait(false); + return; + } + + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _concurrentOps); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs new file mode 100644 index 00000000..dbf54979 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs @@ -0,0 +1,291 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +[Collection(nameof(TransportTestHostCollection))] +public sealed class DeflateUpgradeTransportTests +{ + [Fact] + public async Task UpgradeToDeflate_Plain_RoundTripsApplicationBytes() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.False(server.IsCompressed); + + var before = "PLAIN-BEFORE"u8.ToArray(); + await clientSocket.SendAsync(before); + Assert.Equal(before, await TransportTestShared.ReadExactAsync(server.Input, before.Length)); + + var identity = server.ClientIdentity; + var local = server.LocalEndPoint; + var remote = server.RemoteEndPoint; + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + Assert.False(server.IsTls); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(local, server.LocalEndPoint); + Assert.Equal(remote, server.RemoteEndPoint); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + + var payload = "COMPRESSED-PAYLOAD-12345"u8.ToArray(); + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + + var outbound = "SERVER-TO-CLIENT"u8.ToArray(); + await server.Output.WriteAsync(outbound); + await server.Output.FlushAsync(); + var received = new byte[outbound.Length]; + Assert.Equal(outbound.Length, await ReadExactAsync(clientDeflate, received)); + Assert.Equal(outbound, received); + } + + [Fact] + public async Task UpgradeToDeflate_AfterTls_LayersDeflateAboveTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var tlsUpgrade = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await tlsUpgrade; + Assert.True(server.IsTls); + Assert.False(server.IsCompressed); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsTls); + Assert.True(server.IsCompressed); + + // NntpDeflateStream takes ownership of the SslStream. + await using var clientDeflate = new NntpDeflateStream(ssl); + var payload = new byte[] { 0x10, 0x20, 0x30, 0x40 }; + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_AfterDeflate_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("DEFLATE", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_AlreadyCompressed_Throws() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + await server.UpgradeToDeflateAsync(); + + await Assert.ThrowsAsync(() => server.UpgradeToDeflateAsync()); + } + + [Fact] + public async Task UpgradeToDeflate_UnconsumedInput_RefusesAndKeepsUncompressed() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await clientSocket.SendAsync("UNCONSUMED"u8.ToArray()); + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek)) + { + var hasData = !peek.Buffer.IsEmpty; + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + if (hasData) + { + break; + } + } + + await Task.Yield(); + } + + var ex = await Assert.ThrowsAsync(() => server.UpgradeToDeflateAsync()); + Assert.Contains("unconsumed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsCompressed); + Assert.False(server.ConnectionClosed.IsCancellationRequested); + Assert.Equal("UNCONSUMED"u8.ToArray(), await TransportTestShared.ReadExactAsync(server.Input, 10)); + } + + [Fact] + public async Task UpgradeToDeflate_PreservesTrustedProxyClientIdentity() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartPlainWithProxyAsync(trusted); + + using var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await clientSocket.ConnectAsync(host.EndPoint); + await clientSocket.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.88 127.0.0.1 4242 119\r\n")); + + await using var server = await host.AcceptAsync(); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.88"), server.ClientIdentity.ClientAddress); + Assert.Equal(4242, server.ClientIdentity.ClientPort); + var identity = server.ClientIdentity; + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(IPAddress.Parse("203.0.113.88"), server.ClientIdentity.ClientAddress); + Assert.Equal(4242, server.ClientIdentity.ClientPort); + } + + [Fact] + public async Task UpgradeToDeflate_MalformedPeerData_FailsTerminally() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.UpgradeToDeflateAsync(); + await clientSocket.SendAsync(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01 }); + clientSocket.Shutdown(SocketShutdown.Send); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await Assert.ThrowsAnyAsync(async () => + { + while (!cts.Token.IsCancellationRequested) + { + var result = await server.Input.ReadAsync(cts.Token); + if (result.IsCompleted) + { + throw new InvalidDataException("Input completed after malformed DEFLATE."); + } + + server.Input.AdvanceTo(result.Buffer.End); + } + }); + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_CancelledDuringQuiesce_FailsTerminally() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => + server.UpgradeToDeflateAsync(cts.Token)); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToDeflate_ConcurrentWithTlsUpgrade_SecondRejected() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var tls = server.UpgradeToTlsAsync(host.CertificateProvider!); + var deflate = server.UpgradeToDeflateAsync(); + + var secondEx = await Assert.ThrowsAsync(() => deflate); + Assert.Contains("already in progress", secondEx.Message, StringComparison.OrdinalIgnoreCase); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await tls; + Assert.True(server.IsTls); + Assert.False(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_FlushedUncompressedOutbound_IsDeliveredBeforeCompression() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "206 Compression active\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var receivedNotice = new byte[notice.Length]; + var noticeTotal = 0; + using (var noticeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (noticeTotal < notice.Length) + { + var n = await clientSocket.ReceiveAsync(receivedNotice.AsMemory(noticeTotal), noticeCts.Token); + Assert.True(n > 0); + noticeTotal += n; + } + } + + Assert.Equal(notice, receivedNotice); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + var payload = "after"u8.ToArray(); + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + private static async Task ReadExactAsync(Stream stream, Memory buffer) + { + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await stream.ReadAsync(buffer[total..], cts.Token); + Assert.True(n > 0); + total += n; + } + + return total; + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs new file mode 100644 index 00000000..d72ce685 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs @@ -0,0 +1,144 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Deterministic Input.Reader ownership: the application consumer examines/advances; +/// CompleteAsync (including ObservePump after receive EOF) must not complete the reader +/// while ReadAsync can still run. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpConnectionInputReaderOwnershipTests +{ + [Fact] + public async Task CompleteAsync_WhileInputReadAsyncPending_DoesNotThrowReaderCompleted() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inboundRead = server.Input.ReadAsync().AsTask(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + var complete = server.CompleteAsync(); + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "CompleteAsync completed Input.Reader while ReadAsync was outstanding: " + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + await complete.WaitAsync(timeout.Token); + + Assert.False(server.InputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ReceiveEof_WhileInputReadAsyncPending_DoesNotCompleteReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inboundRead = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Receive EOF completed Input.Reader while ReadAsync was outstanding: " + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + Assert.False(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task CompleteInputReader_AfterConsumerStops_IsOnceAndAllowsNoFurtherRead() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + server.AttachInputReaderConsumer(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + Assert.False(server.InputReaderCompletedForTests); + + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + + var completed = false; + try + { + _ = await server.Input.ReadAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task ConcurrentCompleteAsync_DoesNotCompleteInputReaderWhileConsumerAttached() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + server.AttachInputReaderConsumer(); + var inboundRead = server.Input.ReadAsync().AsTask(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + var first = server.CompleteAsync(); + var second = server.CompleteAsync(); + await second.WaitAsync(timeout.Token); + await first.WaitAsync(timeout.Token); + + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Concurrent CompleteAsync completed Input.Reader while a consumer was attached: " + + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + Assert.False(server.InputReaderCompletedForTests); + + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs new file mode 100644 index 00000000..3a68cc56 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs @@ -0,0 +1,260 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Deterministic Input.Writer ownership: ReceiveAsync GetMemory/Advance/completes; +/// CompleteAsync must not complete the writer while a receive reservation is outstanding. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpConnectionInputWriterOwnershipTests +{ + [Fact] + public async Task CompleteAsync_AfterReadReturns_DoesNotInvalidateAdvance() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.AfterStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync(new byte[] { 0x41 }); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await entered.Task.WaitAsync(timeout.Token); + + await server.CompleteInputReaderAsync(); + var complete = server.CompleteAsync(); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.InputWriterCompletedForTests); + Assert.False(complete.IsCompleted); + Assert.False(server.ReceivePumpTaskForTests!.IsCompleted); + + release.TrySetResult(); + await complete.WaitAsync(timeout.Token); + + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task IdleTimeout_WhileReadOutstanding_DoesNotLogReceivePumpError() + { + var logger = new CapturingConnectionLogger(); + var clock = new ControllableTimeProvider(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var session = new NntpSession( + server, + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(2), + timeProvider: clock); + var run = session.RunAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + transport.AfterStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync(new byte[] { (byte)'D' }); + await entered.Task.WaitAsync(timeout.Token); + + clock.Advance(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + Assert.False(server.InputWriterCompletedForTests); + + release.TrySetResult(); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(TcpDisconnectReason.IdleTimeout, server.DisconnectReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task Quit_WhileNextReadOutstanding_DoesNotLogReceivePumpError() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var session = new NntpSession(server, NullLogger.Instance); + var run = session.RunAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync("QUIT\r\n"u8.ToArray()); + await entered.Task.WaitAsync(timeout.Token); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + Assert.False(server.InputWriterCompletedForTests); + + release.TrySetResult(); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task RemoteEof_CompletesWriterOnce_AndKeepsRemoteClosed() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inbound = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await inbound.WaitAsync(timeout.Token); + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + await WaitUntilAsync(static s => s.InputWriterCompletedForTests, server, timeout.Token); + Assert.Equal(TcpDisconnectReason.RemoteClosed, server.DisconnectReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + Assert.False(server.InputReaderCompletedForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task ReceiveException_StillCompletesWriter() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = static () => + throw new IOException("forced-receive-failure"); + + await client.SendAsync(new byte[] { 0x01 }); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync( + static s => s.InputWriterCompletedForTests && s.IsCompleted, + server, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + Assert.True(server.InputWriterCompletedForTests); + Assert.Equal(TcpDisconnectReason.ReceiveError, server.DisconnectReasonForTests); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + var receive = server.ReceivePumpTaskForTests; + Assert.NotNull(receive); + Assert.True(receive.IsFaulted); + Assert.Contains( + receive.Exception!.Flatten().InnerExceptions, + static ex => ex is IOException); + } + + private static void AssertNoAdvanceRangeFault(Task? receive) + { + Assert.NotNull(receive); + if (!receive.IsFaulted) + { + return; + } + + foreach (var ex in receive.Exception!.Flatten().InnerExceptions) + { + Assert.False( + ex is ArgumentOutOfRangeException aor + && string.Equals(aor.ParamName, "bytes", StringComparison.Ordinal), + "Receive pump must not Advance after Input.Writer was completed."); + } + } + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class CapturingConnectionLogger : ILogger + { + public List PumpErrors { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var formatted = formatter(state, exception); + if (formatted.Contains("pump ended with an error", StringComparison.Ordinal)) + { + PumpErrors.Add(formatted); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs new file mode 100644 index 00000000..b50c25e6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs @@ -0,0 +1,133 @@ +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Deterministic Output.Reader ownership: SendAsync examines/advances/completes; +/// CompleteAsync must not complete the reader while the send pump holds a ReadAsync result. +/// +public sealed class NntpConnectionOutputReaderOwnershipTests +{ + [Fact] + public async Task CompleteAsync_WhileSendBlockedInWrite_DoesNotCompleteReaderBeforeAdvanceTo() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var enteredWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = async () => + { + enteredWrite.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + await server.Output.WriteAsync(new byte[] { 0x01, 0x02, 0x03 }); + await server.Output.FlushAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await enteredWrite.Task.WaitAsync(timeout.Token); + + var complete = server.CompleteAsync(); + await WaitUntilAsync(static s => s.OutputWriterCompletedForTests, server, timeout.Token); + + Assert.True(server.OutputWriterCompletedForTests); + Assert.False(complete.IsCompleted); + Assert.False(server.SendPumpTaskForTests!.IsCompleted); + Assert.False(server.OutputReaderCompletedForTests); + + releaseWrite.TrySetResult(); + await complete.WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task CompleteAsync_WhileSendIdleInReadAsync_CompletesCleanly() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(static s => s.SendPumpAwaitingOutputForTests == 1, server, timeout.Token); + + await server.CompleteAsync().WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ConcurrentCompleteAsync_IsSingleFlight_AndDoesNotRaceOutputReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var enteredWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = async () => + { + enteredWrite.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + await server.Output.WriteAsync(new byte[] { 0x11 }); + await server.Output.FlushAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await enteredWrite.Task.WaitAsync(timeout.Token); + + var first = server.CompleteAsync(); + await WaitUntilAsync(static s => s.IsCompleted, server, timeout.Token); + + var second = server.CompleteAsync(); + await second.WaitAsync(timeout.Token); + Assert.False(first.IsCompleted); + + releaseWrite.TrySetResult(); + await first.WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + } + + private static void AssertNoAdvanceToAfterReaderCompleted(Task? send) + { + Assert.NotNull(send); + if (!send.IsFaulted) + { + return; + } + + foreach (var ex in send.Exception!.Flatten().InnerExceptions) + { + Assert.False( + ex is InvalidOperationException + && ex.Message.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal), + "Send pump must not fault because Output.Reader was completed before AdvanceTo."); + } + } + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpDeflateStreamTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpDeflateStreamTests.cs new file mode 100644 index 00000000..9a32a20e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpDeflateStreamTests.cs @@ -0,0 +1,250 @@ +using System.IO.Compression; +using System.Text; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Codec-level tests for raw DEFLATE used by NNTP COMPRESS (RFC 8054 §4 / RFC 1951). +/// +public sealed class NntpDeflateStreamTests +{ + [Fact] + public void DeflateStream_ProducesRawDeflate_NotZlibOrGzip() + { + var plain = "hello hello hello hello NNTP raw deflate"u8.ToArray(); + using var ms = new MemoryStream(); + using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) + { + ds.Write(plain); + } + + var compressed = ms.ToArray(); + Assert.NotEmpty(compressed); + // zlib wrapper starts with CMF/FLG typically 0x78; gzip with 0x1F 0x8B. + Assert.NotEqual(0x78, compressed[0]); + Assert.False(compressed.Length >= 2 && compressed[0] == 0x1F && compressed[1] == 0x8B); + + using var zlibMs = new MemoryStream(); + using (var zs = new ZLibStream(zlibMs, CompressionLevel.Optimal, leaveOpen: true)) + { + zs.Write(plain); + } + + Assert.Equal(0x78, zlibMs.ToArray()[0]); + Assert.ThrowsAny(() => + { + using var reader = new ZLibStream(new MemoryStream(compressed), CompressionMode.Decompress); + reader.CopyTo(Stream.Null); + }); + } + + [Theory] + [InlineData("")] + [InlineData("a")] + [InlineData("hello world")] + [InlineData("AAAAAAABBBBBBCCCCCC")] + public async Task RoundTrip_VariousPayloads(string text) + { + var plain = Encoding.UTF8.GetBytes(text); + await AssertRoundTripAsync(plain); + } + + [Fact] + public async Task RoundTrip_BinaryAndHighEntropy() + { + var plain = new byte[4096]; + Random.Shared.NextBytes(plain); + await AssertRoundTripAsync(plain); + } + + [Fact] + public async Task RoundTrip_LargeStreaming() + { + var plain = new byte[256 * 1024]; + for (var i = 0; i < plain.Length; i++) + { + plain[i] = (byte)('A' + (i % 26)); + } + + await AssertRoundTripAsync(plain, writeChunk: 17, readChunk: 31); + } + + [Fact] + public async Task Flush_MakesCompressedBytesVisibleWithoutDispose() + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(peer.Server); + await using var client = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(peer.Client); + + var payload = "flush-boundary-test-payload"u8.ToArray(); + await server.WriteAsync(payload); + await server.FlushAsync(); + + var buffer = new byte[payload.Length]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await client.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + + Assert.Equal(payload, buffer); + } + + [Fact] + public async Task MalformedCompressedInput_ThrowsWithoutSilentCorruption() + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(peer.Server); + // Write non-DEFLATE junk on the wire (peer.Client is the raw NetworkStream). + await peer.Client.WriteAsync(new byte[] { 0x00, 0x01, 0x02, 0xFF, 0xFE }); + await peer.Client.FlushAsync(); + + var buffer = new byte[64]; + await Assert.ThrowsAnyAsync(async () => + { + _ = await server.ReadAsync(buffer); + }); + } + + [Fact] + public async Task IndependentStreams_DoNotShareDictionaryState() + { + await using var a = await ConnectedStreamPair.CreateAsync(); + await using var b = await ConnectedStreamPair.CreateAsync(); + await using var aServer = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(a.Server); + await using var aClient = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(a.Client); + await using var bServer = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(b.Server); + await using var bClient = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(b.Client); + + await aServer.WriteAsync("connection-A-payload"u8.ToArray()); + await aServer.FlushAsync(); + await bServer.WriteAsync("connection-B-different"u8.ToArray()); + await bServer.FlushAsync(); + + var bufA = new byte[20]; + var bufB = new byte[22]; + Assert.Equal(20, await ReadExactAsync(aClient, bufA)); + Assert.Equal(22, await ReadExactAsync(bClient, bufB)); + Assert.Equal("connection-A-payload"u8.ToArray(), bufA); + Assert.Equal("connection-B-different"u8.ToArray(), bufB); + } + + private static async Task AssertRoundTripAsync(byte[] plain, int writeChunk = 0, int readChunk = 0) + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(peer.Server); + await using var client = new VectorNNTPD.ReaderService.Networking.Transport.NntpDeflateStream(peer.Client); + + if (writeChunk <= 0) + { + await server.WriteAsync(plain); + } + else + { + for (var i = 0; i < plain.Length; i += writeChunk) + { + var len = Math.Min(writeChunk, plain.Length - i); + await server.WriteAsync(plain.AsMemory(i, len)); + } + } + + await server.FlushAsync(); + + var received = new byte[plain.Length]; + if (readChunk <= 0) + { + Assert.Equal(plain.Length, await ReadExactAsync(client, received)); + } + else + { + var total = 0; + while (total < received.Length) + { + var n = await client.ReadAsync(received.AsMemory(total, Math.Min(readChunk, received.Length - total))); + Assert.True(n > 0); + total += n; + } + } + + Assert.Equal(plain, received); + } + + private static async Task ReadExactAsync(Stream stream, Memory buffer) + { + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + while (total < buffer.Length) + { + var n = await stream.ReadAsync(buffer[total..], cts.Token); + if (n == 0) + { + break; + } + + total += n; + } + + return total; + } + + private sealed class ConnectedStreamPair : IAsyncDisposable + { + private readonly System.Net.Sockets.Socket _listener; + private readonly System.Net.Sockets.Socket _serverSocket; + private readonly System.Net.Sockets.Socket _clientSocket; + + private ConnectedStreamPair( + System.Net.Sockets.Socket listener, + System.Net.Sockets.Socket serverSocket, + System.Net.Sockets.Socket clientSocket, + System.Net.Sockets.NetworkStream server, + System.Net.Sockets.NetworkStream client) + { + _listener = listener; + _serverSocket = serverSocket; + _clientSocket = clientSocket; + Server = server; + Client = client; + } + + public System.Net.Sockets.NetworkStream Server { get; } + public System.Net.Sockets.NetworkStream Client { get; } + + public static async Task CreateAsync() + { + var listener = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + listener.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + listener.Listen(1); + + var clientSocket = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + var connect = clientSocket.ConnectAsync(listener.LocalEndPoint!); + var serverSocket = await listener.AcceptAsync(); + await connect; + + return new ConnectedStreamPair( + listener, + serverSocket, + clientSocket, + new System.Net.Sockets.NetworkStream(serverSocket, ownsSocket: true), + new System.Net.Sockets.NetworkStream(clientSocket, ownsSocket: true)); + } + + public async ValueTask DisposeAsync() + { + await Server.DisposeAsync(); + await Client.DisposeAsync(); + _listener.Dispose(); + _serverSocket.Dispose(); + _clientSocket.Dispose(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpTransportTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpTransportTests.cs new file mode 100644 index 00000000..3a769163 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/NntpTransportTests.cs @@ -0,0 +1,742 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Tests.Acme; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.Networking.Listeners; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +public sealed class ListenEndpointPlannerTests +{ + [Fact] + public void Star_ProducesSingleDualStackIpv6Any() + { + var bindings = ListenEndpointPlanner.Plan(["*"], 1199); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Star_IgnoresAdditionalExplicits() + { + var bindings = ListenEndpointPlanner.Plan(["*", "127.0.0.1", "::1"], 1199); + Assert.Single(bindings); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv4Any_Alone_ProducesIpv4Wildcard() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0"], 1200); + Assert.Single(bindings); + Assert.Equal(IPAddress.Any, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void Ipv6Any_Alone_ProducesDualStack() + { + var bindings = ListenEndpointPlanner.Plan(["::"], 1200); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void ExplicitIpv4_Alone() + { + var address = IPAddress.Parse("192.0.2.10"); + var bindings = ListenEndpointPlanner.Plan(["192.0.2.10"], 1200); + Assert.Single(bindings); + Assert.Equal(address, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void ExplicitIpv6_Alone() + { + var address = IPAddress.Parse("2001:db8::10"); + var bindings = ListenEndpointPlanner.Plan(["2001:db8::10"], 1200); + Assert.Single(bindings); + Assert.Equal(address, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void ExplicitAddresses_Deduplicate() + { + var bindings = ListenEndpointPlanner.Plan(["127.0.0.1", "127.0.0.1", "::1"], 1200); + Assert.Equal(2, bindings.Count); + } + + [Fact] + public void MultipleIndependentExplicits_AreRetained() + { + var bindings = ListenEndpointPlanner.Plan(["192.0.2.10", "2001:db8::10"], 1200); + Assert.Equal(2, bindings.Count); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Parse("192.0.2.10"))); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Parse("2001:db8::10"))); + } + + [Fact] + public void Ipv4AnyAndIpv6Any_AreSeparateSingleFamilySockets() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0", "::"], 1201); + Assert.Equal(2, bindings.Count); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Any) && !b.DualMode); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.IPv6Any) && !b.DualMode); + } + + [Fact] + public void Ipv6AnyDualMode_PlusExplicitIpv4_DoesNotOverlap() + { + var bindings = ListenEndpointPlanner.Plan(["::", "192.0.2.10"], 1202); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv6AnyDualMode_PlusIpv4MappedExplicit_DoesNotOverlap() + { + var bindings = ListenEndpointPlanner.Plan(["::", "::ffff:192.0.2.10"], 1203); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv4Any_PlusExplicitIpv4_SkipsExplicit() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0", "192.0.2.10"], 1204); + Assert.Single(bindings); + Assert.Equal(IPAddress.Any, bindings[0].Address); + } + + [Fact] + public void Ipv6Any_PlusExplicitIpv6_SkipsExplicit() + { + var bindings = ListenEndpointPlanner.Plan(["::", "2001:db8::10"], 1205); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } +} + +/// +/// Binds planned dual-stack endpoints on loopback/ephemeral ports (Windows dual-stack semantics). +/// +public sealed class ListenEndpointBindTests +{ + [Fact] + public async Task DualStackIpv6Any_AcceptsIpv4AndIpv6Loopback() + { + if (!Socket.OSSupportsIPv6) + { + return; + } + + var plan = ListenEndpointPlanner.Plan(["::", "127.0.0.1"], port: 1199); + Assert.Single(plan); + Assert.True(plan[0].DualMode); + + // Ephemeral port: one DualMode IPv6Any listener (no separate IPv4 socket from the planner). + var binding = new ListenBinding(IPAddress.IPv6Any, 0, DualMode: true); + await using var listener = new SocketAcceptListener( + binding, + static (_, _) => ValueTask.CompletedTask, + NullLogger.Instance); + listener.Start(); + var port = listener.LocalEndPoint.Port; + + using (var v4 = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await v4.ConnectAsync(new IPEndPoint(IPAddress.Loopback, port)); + } + + using (var v6 = new Socket(AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp)) + { + await v6.ConnectAsync(new IPEndPoint(IPAddress.IPv6Loopback, port)); + } + } +} + +public sealed class NntpPlainTransportTests +{ + [Fact] + public async Task Accept_ExchangeArbitraryBytes_Bidirectional() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var payload = new byte[] { 0x00, 0x01, 0x7F, 0x80, 0xFE, 0xFF, 0x0D, 0x0A, 0x2E, 0x2E }; + await client.SendAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + + var reply = new byte[] { 0xAA, 0x55, 0x00, 0xFF }; + await server.Output.WriteAsync(reply); + await server.Output.FlushAsync(); + var buffer = new byte[reply.Length]; + Assert.Equal(reply.Length, await client.ReceiveAsync(buffer)); + Assert.Equal(reply, buffer); + } + + [Fact] + public async Task FragmentedReceives_PreserveOrder() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await client.SendAsync(new byte[] { 0x01, 0x02 }); + await client.SendAsync(new byte[] { 0x03, 0x04, 0x05 }); + Assert.Equal( + new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }, + await TransportTestShared.ReadExactAsync(server.Input, 5)); + } + + [Fact] + public async Task MultipleConnections_AreIndependent() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var c1 = await host.ConnectPlainClientAsync(); + using var c2 = await host.ConnectPlainClientAsync(); + await using var s1 = await host.AcceptAsync(); + await using var s2 = await host.AcceptAsync(); + + await c1.SendAsync(new byte[] { 0x11 }); + await c2.SendAsync(new byte[] { 0x22 }); + Assert.Equal(new byte[] { 0x11 }, await TransportTestShared.ReadExactAsync(s1.Input, 1)); + Assert.Equal(new byte[] { 0x22 }, await TransportTestShared.ReadExactAsync(s2.Input, 1)); + } + + [Fact] + public async Task RemoteGracefulClose_CompletesInput() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Arm inbound read before FIN so Ordering A/B are both covered without sleeps. + var inboundRead = server.Input.ReadAsync().AsTask(); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + // Writer completion (receive EOF) must unblock ReadAsync as IsCompleted. + // CompleteAsync must not complete Input.Reader while this consumer is still reading. + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Input.Reader was completed while ReadAsync was outstanding: " + ex.Message); + return; + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + Assert.Fail("Timed out waiting for inbound completion after remote FIN."); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + // 2) Connection lifecycle must reach closed (may already be cancelled). + if (!server.ConnectionClosed.IsCancellationRequested) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + server.ConnectionClosed, + timeout.Token); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + Assert.Fail("Expected ConnectionClosed after remote FIN."); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(timeout.IsCancellationRequested); + } + + [Fact] + public async Task ListenerContinues_AfterPriorConnectionCloses() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var c1 = await host.ConnectPlainClientAsync(); + await using var s1 = await host.AcceptAsync(); + await s1.CompleteAsync(); + await s1.DisposeAsync(); + c1.Dispose(); + + using var c2 = await host.ConnectPlainClientAsync(); + await using var s2 = await host.AcceptAsync(); + await c2.SendAsync(new byte[] { 0x42 }); + Assert.Equal(new byte[] { 0x42 }, await TransportTestShared.ReadExactAsync(s2.Input, 1)); + } + + [Fact] + public async Task PlainListenerService_BindsAndAccepts() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.NotEmpty(service.LocalEndPoints); + + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(service.LocalEndPoints[0]); + await client.SendAsync(new byte[] { 0x01 }); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (service.ActiveConnectionCount < 1 && !cts.IsCancellationRequested) + { + await Task.Delay(10, cts.Token); + } + + Assert.True(service.ActiveConnectionCount >= 1); + await service.StopAsync(CancellationToken.None); + } +} + +public sealed class NntpTlsTransportTests +{ + [Fact] + public async Task TlsDisabled_DoesNotBind() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + await using var service = new NntpTlsListenerService( + Options.Create(options), + new TlsCertificateContextProvider(NullLogger.Instance), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.False(service.ListenersBound); + } + + [Fact] + public async Task TlsEnabledWithoutCertificate_FailsStart() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPortTls = TestHostFactory.GetFreeTcpPort(); + await using var service = new NntpTlsListenerService( + Options.Create(options), + new TlsCertificateContextProvider(NullLogger.Instance), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task SuccessfulHandshake_ExchangesBinary_AndPresentsIdentity() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + await using var client = await host.ConnectTlsClientAsync(); + Assert.NotNull(client.RemoteCertificate); + Assert.Contains( + "nntpd01.usenet.ninja", + client.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + + await using var server = await host.AcceptAsync(); + Assert.True(server.IsTls); + var payload = new byte[] { 0x00, 0x80, 0xFF, 0x0D, 0x0A, 0x2E }; + await client.Stream.WriteAsync(payload); + await client.Stream.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task TlsOutbound_MultiplePipeBatches_PreserveOrderWithoutSslFlush() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + + // Distinct PipeWriter flushes → distinct send-pump ReadAsync batches. + // Delivery must not depend on SslStream.FlushAsync (NetworkStream is unbuffered). + var batch1 = new byte[] { 0x01, 0x02, 0x03 }; + var batch2 = new byte[] { 0xFE, 0xFF, 0x00 }; + var batch3 = new byte[] { 0x0D, 0x0A, 0x2E, 0x2E }; + + await server.Output.WriteAsync(batch1); + await server.Output.FlushAsync(); + await server.Output.WriteAsync(batch2); + await server.Output.FlushAsync(); + await server.Output.WriteAsync(batch3); + await server.Output.FlushAsync(); + + var expected = batch1.Concat(batch2).Concat(batch3).ToArray(); + var received = new byte[expected.Length]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < expected.Length) + { + var n = await client.Stream.ReadAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0, "TLS stream closed before all outbound batches arrived."); + total += n; + } + + Assert.Equal(expected, received); + } + + [Fact] + public async Task FailedHandshake_DoesNotKillListener() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + using (var raw = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await raw.ConnectAsync(host.EndPoint); + await raw.SendAsync(new byte[] { 0x15, 0x00, 0x00, 0x00 }); + } + + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + await client.Stream.WriteAsync(new byte[] { 0x01 }); + await client.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0x01 }, await TransportTestShared.ReadExactAsync(server.Input, 1)); + } + + [Fact] + public async Task CertificateRotation_KeepsOldSession_NewHandshakeUsesNewCert() + { + var pfxA = TransportTestShared.CreatePfx("cert-a.usenet.ninja"); + var pfxB = TransportTestShared.CreatePfx("cert-b.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfxA); + + await using var client1 = await host.ConnectTlsClientAsync(); + Assert.Contains( + "cert-a.usenet.ninja", + client1.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + await using var server1 = await host.AcceptAsync(); + + host.PublishCertificate(pfxB); + + await using var client2 = await host.ConnectTlsClientAsync(); + Assert.Contains( + "cert-b.usenet.ninja", + client2.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + await using var server2 = await host.AcceptAsync(); + + await client1.Stream.WriteAsync(new byte[] { 0xA1 }); + await client1.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0xA1 }, await TransportTestShared.ReadExactAsync(server1.Input, 1)); + + await client2.Stream.WriteAsync(new byte[] { 0xB2 }); + await client2.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0xB2 }, await TransportTestShared.ReadExactAsync(server2.Input, 1)); + } +} + +public sealed class TlsCertificateContextProviderTests +{ + [Fact] + public void Publish_AtomicSwap_HeldLeaseRemainsUsable() + { + var provider = new TlsCertificateContextProvider(NullLogger.Instance); + provider.PublishFromPfx( + TransportTestShared.CreatePfx("a.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + using var leaseA = provider.Acquire(); + provider.PublishFromPfx( + TransportTestShared.CreatePfx("b.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + using var leaseB = provider.Acquire(); + Assert.Contains( + "a.usenet.ninja", + leaseA.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.Contains( + "b.usenet.ninja", + leaseB.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + } +} + +internal static class TransportTestShared +{ + public static byte[] CreatePfx(string cn) => + TestCertificateFactory.CreateMaterial( + [cn, "news.usenet.ninja"], + AcmeConfigurationTests.TestPfxPassword, + DateTimeOffset.UtcNow.AddDays(30)) + .PfxBytes; + + public static async Task ReadExactAsync(PipeReader reader, int count) + { + while (true) + { + var result = await reader.ReadAsync(); + if (result.Buffer.Length >= count) + { + var slice = result.Buffer.Slice(0, count); + var copy = new byte[count]; + slice.CopyTo(copy.AsSpan()); + reader.AdvanceTo(slice.End); + return copy; + } + + if (result.IsCompleted) + { + throw new InvalidOperationException($"Completed before {count} bytes arrived."); + } + + reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + } + } +} + +internal sealed class TransportTestHost : IAsyncDisposable +{ + private readonly SocketAcceptListener _listener; + private readonly TlsCertificateContextProvider? _certs; + private readonly ConcurrentQueue> _waiters = new(); + private readonly ConcurrentQueue _ready = new(); + private readonly object _gate = new(); + + private TransportTestHost(SocketAcceptListener listener, TlsCertificateContextProvider? certs) + { + _listener = listener; + _certs = certs; + } + + public IPEndPoint EndPoint => _listener.LocalEndPoint; + + public TlsCertificateContextProvider? CertificateProvider => _certs; + + public static Task StartPlainAsync(ILogger? connectionLogger = null) => + StartAsync(tls: false, pfx: null, connectionLogger: connectionLogger); + + /// Plain accept path with a certificate provider available for . + public static Task StartPlainWithCertificateAsync(byte[] pfx) => + StartAsync(tls: false, pfx); + + public static Task StartTlsAsync(byte[] pfx) => StartAsync(tls: true, pfx); + + /// + /// Starts a TLS test host that resolves PROXY identity via + /// before the TLS handshake (production listener ordering). + /// + public static Task StartTlsWithProxyAsync(byte[] pfx, ITrustedProxyHosts trustedProxyHosts) => + StartAsync(tls: true, pfx, trustedProxyHosts); + + /// Plain accept with PROXY trust evaluation and optional upgrade certificate provider. + public static Task StartPlainWithProxyAsync( + ITrustedProxyHosts trustedProxyHosts, + byte[]? upgradePfx = null) => + StartAsync(tls: false, upgradePfx, trustedProxyHosts); + + private static Task StartAsync( + bool tls, + byte[]? pfx, + ITrustedProxyHosts? trustedProxyHosts = null, + ILogger? connectionLogger = null) + { + TlsCertificateContextProvider? certs = null; + if (pfx is not null) + { + certs = new TlsCertificateContextProvider(NullLogger.Instance); + certs.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + + if (tls) + { + ArgumentNullException.ThrowIfNull(certs); + } + + trustedProxyHosts ??= new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + var logger = connectionLogger ?? NullLogger.Instance; + + TransportTestHost? host = null; + var binding = new ListenBinding(IPAddress.Loopback, 0, DualMode: false); + var listener = new SocketAcceptListener( + binding, + async (socket, ct) => + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + socket.Dispose(); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, trustedProxyHosts, ct) + .ConfigureAwait(false); + } + catch (ProxyProtocolException) + { + socket.Dispose(); + return; + } + + INntpConnection connection = tls + ? await NntpConnection.StartTlsAsync( + socket, + certs!, + preamble.Identity, + logger, + ct, + preamble.Leftover) + .ConfigureAwait(false) + : NntpConnection.StartPlain( + socket, + preamble.Identity, + logger, + preamble.Leftover); + + lock (host!._gate) + { + if (host._waiters.TryDequeue(out var waiter)) + { + waiter.TrySetResult(connection); + } + else + { + host._ready.Enqueue(connection); + } + } + }, + NullLogger.Instance); + + host = new TransportTestHost(listener, certs); + listener.Start(); + return Task.FromResult(host); + } + + public void PublishCertificate(byte[] pfx) + { + ArgumentNullException.ThrowIfNull(_certs); + _certs.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + + public async Task ConnectPlainClientAsync() + { + var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(EndPoint); + return client; + } + + public async Task ConnectTlsClientAsync() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(EndPoint); + var network = new NetworkStream(socket, ownsSocket: true); + var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + var remoteCertificate = ssl.RemoteCertificate is null + ? null + : new X509Certificate2(ssl.RemoteCertificate); + return new TlsClient(ssl, remoteCertificate); + } + + public async Task AcceptAsync() + { + TaskCompletionSource tcs; + lock (_gate) + { + if (_ready.TryDequeue(out var ready)) + { + return ready; + } + + tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Enqueue(tcs); + } + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await using var reg = cts.Token.Register( + static s => ((TaskCompletionSource)s!).TrySetCanceled(), + tcs); + return await tcs.Task; + } + + public async ValueTask DisposeAsync() + { + await _listener.DisposeAsync(); + if (_certs is not null) + { + await _certs.DisposeAsync(); + } + } + + internal sealed class TlsClient : IAsyncDisposable + { + public TlsClient(SslStream stream, X509Certificate2? remoteCertificate) + { + Stream = stream; + RemoteCertificate = remoteCertificate; + } + + public SslStream Stream { get; } + + public X509Certificate2? RemoteCertificate { get; } + + public ValueTask DisposeAsync() + { + Stream.Dispose(); + RemoteCertificate?.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SendPipeAdvancementTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SendPipeAdvancementTests.cs new file mode 100644 index 00000000..034fd5c9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SendPipeAdvancementTests.cs @@ -0,0 +1,169 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Documents outbound pipe advancement: unsent bytes must not be marked consumed while a send +/// remains logically in progress; on terminal abort the connection is completed (no retry). +/// +public sealed class SendPipeAdvancementTests +{ + [Fact] + public async Task PlainSendPattern_OnThrowAfterPartialSegments_DoesNotConsumeUnsentBytes() + { + var pipe = new Pipe(); + // Two segments in one ReadOnlySequence: force non-contiguous via two writes without flush coalesce... + // A single contiguous buffer is fine: treat first half as "sent", then throw before marking End. + var payload = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06 }; + await pipe.Writer.WriteAsync(payload); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + + var result = await pipe.Reader.ReadAsync(); + var buffer = result.Buffer; + Assert.Equal(6, buffer.Length); + + // Mirror SendSocketAsync: only advance consumed on full success. + var consumed = buffer.Start; + var transmitted = 0; + var failed = false; + try + { + foreach (var segment in buffer) + { + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + // Transmit first three bytes across possibly multiple calls, then fail. + if (transmitted >= 3) + { + throw new IOException("simulated send failure after partial transmit"); + } + + var take = Math.Min(2, Math.Min(memory.Length, 3 - transmitted)); + transmitted += take; + return ValueTask.FromResult(take); + }, + segment, + state: null, + CancellationToken.None); + } + + consumed = buffer.End; + } + catch (IOException) + { + failed = true; + } + finally + { + pipe.Reader.AdvanceTo(consumed); + } + + Assert.True(failed); + Assert.Equal(3, transmitted); + Assert.True(consumed.Equals(buffer.Start)); + + // Unsent octets remain readable (would be discarded only when the connection/pipe is completed). + var again = await pipe.Reader.ReadAsync(); + Assert.Equal(6, again.Buffer.Length); + pipe.Reader.AdvanceTo(again.Buffer.Start, again.Buffer.End); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task PlainSendPattern_OnFullSuccess_ConsumesEntireBuffer() + { + var pipe = new Pipe(); + var payload = new byte[] { 0xAA, 0xBB, 0xCC }; + await pipe.Writer.WriteAsync(payload); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + + var result = await pipe.Reader.ReadAsync(); + var buffer = result.Buffer; + var consumed = buffer.Start; + try + { + foreach (var segment in buffer) + { + await SocketPayloadSender.SendAllAsync( + static (memory, _, _) => ValueTask.FromResult(memory.Length), + segment, + state: null, + CancellationToken.None); + } + + consumed = buffer.End; + } + finally + { + pipe.Reader.AdvanceTo(consumed); + } + + Assert.True(consumed.Equals(buffer.End)); + var drained = await pipe.Reader.ReadAsync(); + Assert.True(drained.Buffer.IsEmpty); + Assert.True(drained.IsCompleted); + pipe.Reader.AdvanceTo(drained.Buffer.End); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task LiveConnection_SendAfterPeerAbort_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Abortive close so subsequent server sends fail rather than blocking forever. + client.LingerState = new LingerOption(enable: true, seconds: 0); + client.Dispose(); + + var payload = new byte[64 * 1024]; + Random.Shared.NextBytes(payload); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + server.ConnectionClosed, + timeout.Token); + + try + { + while (!linked.IsCancellationRequested) + { + await server.Output.WriteAsync(payload, linked.Token); + await server.Output.FlushAsync(linked.Token); + } + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Send pump faulted → ObservePump → CompleteAsync cancelled ConnectionClosed. + } + catch (Exception ex) when ( + server.ConnectionClosed.IsCancellationRequested + && ex is InvalidOperationException or IOException or SocketException) + { + // Peer abort completes the connection; Write/Flush may surface several transport exceptions. + } + + if (!server.ConnectionClosed.IsCancellationRequested) + { + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True( + server.ConnectionClosed.IsCancellationRequested, + "Send failure must permanently abort the connection (ConnectionClosed)."); + } +} \ No newline at end of file diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SocketPayloadSenderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SocketPayloadSenderTests.cs new file mode 100644 index 00000000..f50da116 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/SocketPayloadSenderTests.cs @@ -0,0 +1,124 @@ +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +public sealed class SocketPayloadSenderTests +{ + [Fact] + public async Task SendAllAsync_LoopsUntilEntireBufferSent() + { + var payload = new byte[100]; + for (var i = 0; i < payload.Length; i++) + { + payload[i] = (byte)i; + } + + var scripted = new Queue([37, 41, 22]); + var transmitted = new List(payload.Length); + + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + Assert.False(memory.IsEmpty); + var take = scripted.Dequeue(); + Assert.True(take <= memory.Length); + transmitted.AddRange(memory.Span[..take].ToArray()); + return ValueTask.FromResult(take); + }, + payload, + state: null, + CancellationToken.None); + + Assert.Empty(scripted); + Assert.Equal(payload, transmitted); + } + + [Fact] + public async Task SendAllAsync_ZeroByteSend_ThrowsWithoutSpinning() + { + var calls = 0; + var ex = await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (_, _, _) => + { + calls++; + return ValueTask.FromResult(0); + }, + new byte[16], + state: null, + CancellationToken.None)); + + Assert.Equal(1, calls); + Assert.Contains("zero bytes", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task SendAllAsync_HonorsCancellation() + { + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(async () => + await SocketPayloadSender.SendAllAsync( + static (_, _, _) => ValueTask.FromResult(1), + new byte[8], + state: null, + cts.Token)); + } + + [Fact] + public async Task SendAllAsync_DoesNotAdvancePastFailure_WhenUsedWithPipeSemantics() + { + // Simulate the connection send contract: only mark consumed after full success. + var payload = new byte[10]; + Random.Shared.NextBytes(payload); + var sent = new List(); + var calls = 0; + + await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + calls++; + if (calls == 1) + { + sent.AddRange(memory.Span[..4].ToArray()); + return ValueTask.FromResult(4); + } + + return ValueTask.FromResult(0); + }, + payload, + state: null, + CancellationToken.None)); + + Assert.Equal(4, sent.Count); + Assert.Equal(payload.AsSpan(0, 4).ToArray(), sent.ToArray()); + } + + [Fact] + public async Task SendAllAsync_ThrowAfterPartial_PropagatesWithoutClaimingSuccess() + { + var payload = new byte[] { 1, 2, 3, 4, 5, 6 }; + var sent = 0; + + var ex = await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + if (sent > 0) + { + throw new IOException("peer reset after partial send"); + } + + sent += memory.Length >= 3 ? 3 : memory.Length; + return ValueTask.FromResult(3); + }, + payload, + state: null, + CancellationToken.None)); + + Assert.Equal(3, sent); + Assert.Contains("peer reset", ex.Message, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs new file mode 100644 index 00000000..62c3bc34 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs @@ -0,0 +1,320 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.ReaderService.Networking.Listeners; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Exactly one Information TCP disconnect summary per established connection, with a +/// known-cause reason and no exception text on that line. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class TcpConnectionDisconnectLoggingTests +{ + [Fact] + public async Task RemoteClose_EmitsExactlyOneDisconnect_WithRemoteClosed() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inbound = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await inbound.WaitAsync(timeout.Token); + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + // IsCompleted flips before the summary is logged and a second CompleteAsync returns immediately, + // so wait on the observable (the single disconnect line) rather than on the completion flag. + await WaitUntilAsync( + () => server.IsCompleted && logger.Disconnects.Count > 0, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "RemoteClosed"); + Assert.Equal(TcpDisconnectReason.RemoteClosed, server.DisconnectReasonForTests); + Assert.Empty(logger.DisconnectExceptions); + } + + [Fact] + public async Task LocalCompleteAsync_EmitsExactlyOneDisconnect_WithLocalClose() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "LocalClose"); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + Assert.Empty(logger.DisconnectExceptions); + } + + [Fact] + public async Task ReceivePumpError_EmitsExactlyOneDisconnect_WithReceiveError_AndKeepsDebugException() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = static () => + throw new IOException("forced-receive-failure"); + + // The in-flight receive already passed the probe; one octet starts the next read. + await client.SendAsync(new byte[] { 0x01 }); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync( + () => server.IsCompleted && logger.Disconnects.Count == 1 && logger.PumpErrors.Count > 0, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "ReceiveError"); + Assert.Equal(TcpDisconnectReason.ReceiveError, server.DisconnectReasonForTests); + Assert.DoesNotContain("forced-receive-failure", line, StringComparison.Ordinal); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + Assert.Contains( + logger.PumpExceptions, + static e => e.Contains("forced-receive-failure", StringComparison.Ordinal)); + } + + [Fact] + public async Task SendPumpError_EmitsExactlyOneDisconnect_WithSendError_AndKeepsDebugException() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = static () => + throw new IOException("forced-send-failure"); + + // The send pump can fail and complete the output pipe before WriteAsync/FlushAsync observe it + // (timing depends on JIT warm-up); the pipe then surfaces the pump's exception to the producer. + try + { + await server.Output.WriteAsync(new byte[] { 0x01 }); + await server.Output.FlushAsync(); + } + catch (IOException ex) when (ex.Message == "forced-send-failure") + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => server.IsCompleted, timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "SendError"); + Assert.Equal(TcpDisconnectReason.SendError, server.DisconnectReasonForTests); + Assert.DoesNotContain("forced-send-failure", line, StringComparison.Ordinal); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + Assert.Contains( + logger.PumpExceptions, + static e => e.Contains("forced-send-failure", StringComparison.Ordinal)); + } + + [Fact] + public async Task RequestCloseThenComplete_EmitsExactlyOneDisconnect_WithProtocolClose() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession(server, NullLogger.Instance); + + session.RequestClose(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "ProtocolClose"); + Assert.Equal(TcpDisconnectReason.ProtocolClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task ConcurrentCompleteAndDispose_EmitsExactlyOneDisconnect() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.DisposeAsync().AsTask().WaitAsync(timeout.Token); + + Assert.Single(logger.Disconnects); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task IdleTimeout_EmitsDisconnect_WithIdleTimeoutReason() + { + var logger = new CapturingConnectionLogger(); + var clock = new ControllableTimeProvider(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession( + server, + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(2), + timeProvider: clock); + var run = session.RunAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + clock.Advance(TimeSpan.FromSeconds(2)); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(TcpDisconnectReason.IdleTimeout, server.DisconnectReasonForTests); + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "IdleTimeout"); + Assert.Empty(logger.DisconnectExceptions); + Assert.Null(logger.LastException); + Assert.Equal(LogLevel.Information, logger.LastLevel); + } + + [Fact] + public void GeneratedDisconnectMessage_IsSingleLine_WithStructuredReason() + { + var logger = new CapturingConnectionLogger(); + NetworkingLogMessages.TcpConnectionDisconnected( + logger, + "69.80.99.16:49136", + "0.0.0.0:119", + "RemoteClosed"); + + var line = Assert.Single(logger.Disconnects); + Assert.Equal(LogLevel.Information, logger.LastLevel); + Assert.Equal(1425, logger.LastEventId); + Assert.Equal( + "TCP connection disconnected: remote=69.80.99.16:49136 local=0.0.0.0:119 reason=RemoteClosed", + line); + Assert.DoesNotContain('\n', line); + Assert.Null(logger.LastException); + Assert.Equal("69.80.99.16:49136", logger.LastRemote); + Assert.Equal("RemoteClosed", logger.LastReason); + } + + private static void AssertSingleLineDisconnect(string line, NntpConnection server, string reason) + { + Assert.DoesNotContain('\n', line); + Assert.StartsWith("TCP connection disconnected:", line, StringComparison.Ordinal); + Assert.Contains("remote=", line, StringComparison.Ordinal); + Assert.Contains("reason=" + reason, line, StringComparison.Ordinal); + var remote = ConnectionAcceptanceLogging.FormatEndpoint(server.RemoteEndPoint); + Assert.Contains(remote, line, StringComparison.Ordinal); + Assert.DoesNotContain("at VectorNNTP", line, StringComparison.Ordinal); + Assert.DoesNotContain("StackTrace", line, StringComparison.Ordinal); + } + + private static async Task WaitUntilAsync(Func predicate, CancellationToken cancellationToken) + { + while (!predicate()) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class CapturingConnectionLogger : ILogger + { + public List Disconnects { get; } = []; + + public List DisconnectExceptions { get; } = []; + + public List PumpErrors { get; } = []; + + public List PumpExceptions { get; } = []; + + public LogLevel LastLevel { get; private set; } + + public int LastEventId { get; private set; } + + public Exception? LastException { get; private set; } + + public string? LastRemote { get; private set; } + + public string? LastReason { get; private set; } + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var formatted = formatter(state, exception); + LastLevel = logLevel; + LastEventId = eventId.Id; + LastException = exception; + if (state is IReadOnlyList> properties) + { + LastRemote = properties.FirstOrDefault(static p => p.Key == "Remote").Value as string; + LastReason = properties.FirstOrDefault(static p => p.Key == "Reason").Value as string; + } + + if (formatted.Contains("TCP connection disconnected:", StringComparison.Ordinal)) + { + Disconnects.Add(formatted); + if (exception is not null) + { + DisconnectExceptions.Add(exception); + } + } + + if (formatted.Contains("pump ended with an error", StringComparison.Ordinal)) + { + PumpErrors.Add(formatted); + if (exception is not null) + { + PumpExceptions.Add(exception.ToString()); + } + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs new file mode 100644 index 00000000..7a69e056 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs @@ -0,0 +1,195 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Tests.Acme; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +public sealed class TlsCertificateLifetimeTests +{ + [Fact] + public async Task ExistingLease_SurvivesPublication_UntilReleased() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + TlsCertificateHolder holderA; + using (var leaseA = provider.Acquire()) + { + holderA = leaseA.Holder; + Assert.Equal(0, holderA.DisposeCount); + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + Assert.Contains( + "a.usenet.ninja", + leaseA.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.False(holderA.IsDisposed); + Assert.Equal(0, holderA.DisposeCount); + } + + Assert.Equal(1, holderA.DisposeCount); + Assert.True(holderA.IsDisposed); + } + + [Fact] + public async Task NewAcquire_AfterPublish_ReceivesNewGeneration() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + int generationA; + using (var leaseA = provider.Acquire()) + { + generationA = leaseA.Generation; + } + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + using var leaseB = provider.Acquire(); + Assert.NotEqual(generationA, leaseB.Generation); + Assert.Contains( + "b.usenet.ninja", + leaseB.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task FailedPublication_LeavesPriorContextActive() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + using var leaseBefore = provider.Acquire(); + var generationA = leaseBefore.Generation; + + Assert.Throws(() => + provider.PublishFromPfx([0x01, 0x02, 0x03, 0x04], AcmeConfigurationTests.TestPfxPassword)); + + using var leaseAfter = provider.Acquire(); + Assert.Equal(generationA, leaseAfter.Generation); + Assert.Contains( + "a.usenet.ninja", + leaseAfter.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.False(leaseBefore.HolderIsDisposed); + } + + [Fact] + public async Task LeaseDoubleDispose_DoesNotDoubleDisposeHolder() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + // codeql[cs/missed-using-statement] Lease must be disposed twice explicitly before publication retires the holder. + var lease = provider.Acquire(); + var holder = lease.Holder; + + lease.Dispose(); + // codeql[cs/dispose-not-called-on-throw] Intentional double-dispose contract exercise for TlsCertificateLease. + lease.Dispose(); + Assert.Equal(0, holder.DisposeCount); + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public async Task ProviderDispose_BlocksAcquire_AndDisposesAfterLeasesReleased() + { + var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + TlsCertificateHolder holder; + using (var lease = provider.Acquire()) + { + holder = lease.Holder; + + await provider.DisposeAsync(); + Assert.Throws(() => provider.Acquire()); + Assert.False(holder.IsDisposed); + } + + Assert.True(holder.IsDisposed); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public void RetiredHolder_RejectsAddRef() + { + var holder = TlsCertificateHolder.CreateFromPfx( + CreatePfx("a.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + holder.Retire(); + Assert.Throws(holder.AddRef); + holder.Release(); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public async Task ConcurrentAcquirePublishRelease_NeverUsesDisposedHolder() + { + await using var provider = CreateProvider(); + var pfxA = CreatePfx("a.usenet.ninja"); + var pfxB = CreatePfx("b.usenet.ninja"); + provider.PublishFromPfx(pfxA, AcmeConfigurationTests.TestPfxPassword); + + var errors = new ConcurrentBag(); + var observedGenerations = new ConcurrentBag(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + var tasks = new Task[8]; + + for (var t = 0; t < tasks.Length; t++) + { + var publish = t % 2 == 0; + tasks[t] = Task.Run(() => + { + try + { + var flip = 0; + while (!cts.IsCancellationRequested) + { + if (publish) + { + var pfx = Interlocked.Increment(ref flip) % 2 == 0 ? pfxA : pfxB; + provider.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + else + { + using var lease = provider.Acquire(); + observedGenerations.Add(lease.Generation); + if (lease.HolderIsDisposed) + { + throw new InvalidOperationException("Acquired a disposed holder."); + } + + _ = lease.Context.TargetCertificate.Thumbprint; + } + } + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Provider disposal at test end can race the last iterations. + } + // codeql[cs/catch-of-all-exceptions] Concurrent stress worker must capture any provider disposal race exception. + catch (Exception ex) + { + errors.Add(ex); + } + }, CancellationToken.None); + } + + await Task.Delay(200); + await cts.CancelAsync(); + await Task.WhenAll(tasks); + + Assert.Empty(errors); + Assert.NotEmpty(observedGenerations); + } + + private static TlsCertificateContextProvider CreateProvider() => + new(NullLogger.Instance); + + private static byte[] CreatePfx(string cn) => + TestCertificateFactory.CreateMaterial( + [cn, "news.usenet.ninja"], + AcmeConfigurationTests.TestPfxPassword, + DateTimeOffset.UtcNow.AddDays(30)) + .PfxBytes; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs new file mode 100644 index 00000000..72c7c82b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs @@ -0,0 +1,475 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +public sealed class TlsUpgradeTransportTests +{ + [Fact] + public async Task UpgradeToTls_SameSocket_PlainThenEncryptedTraffic() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.False(server.IsTls); + + var localBefore = server.LocalEndPoint; + var remoteBefore = server.RemoteEndPoint; + var identityBefore = server.ClientIdentity; + + var plain = "PLAIN-BEFORE"u8.ToArray(); + await clientSocket.SendAsync(plain); + Assert.Equal(plain, await TransportTestShared.ReadExactAsync(server.Input, plain.Length)); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Equal(localBefore, server.LocalEndPoint); + Assert.Equal(remoteBefore, server.RemoteEndPoint); + Assert.Same(identityBefore, server.ClientIdentity); + Assert.False(server.ClientIdentity.UsedProxyHeader); + + var encrypted = new byte[] { 0x10, 0x20, 0x30, 0x0D, 0x0A }; + await ssl.WriteAsync(encrypted); + await ssl.FlushAsync(); + Assert.Equal(encrypted, await TransportTestShared.ReadExactAsync(server.Input, encrypted.Length)); + + var outbound = new byte[] { 0xAA, 0xBB, 0xCC }; + await server.Output.WriteAsync(outbound); + await server.Output.FlushAsync(); + var received = new byte[outbound.Length]; + var total = 0; + while (total < received.Length) + { + var n = await ssl.ReadAsync(received.AsMemory(total)); + Assert.True(n > 0); + total += n; + } + + Assert.Equal(outbound, received); + } + + [Fact] + public async Task ImplicitTls_ClientWithoutCertificate_HandshakeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfx); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + + Assert.True(server.IsTls); + Assert.Null(client.Stream.LocalCertificate); + Assert.NotNull(client.RemoteCertificate); + + var payload = new byte[] { 0x01, 0x02, 0x03 }; + await client.Stream.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_ClientWithoutCertificate_HandshakeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + // Explicitly no client certificate. + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Null(ssl.LocalCertificate); + } + + [Fact] + public async Task UpgradeToTls_PreservesTrustedProxyClientIdentity() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartPlainWithProxyAsync(trusted, pfx); + + using var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await clientSocket.ConnectAsync(host.EndPoint); + await clientSocket.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.77 127.0.0.1 51515 119\r\n")); + + await using var server = await host.AcceptAsync(); + Assert.False(server.IsTls); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.77"), server.ClientIdentity.ClientAddress); + Assert.Equal(51515, server.ClientIdentity.ClientPort); + var identity = server.ClientIdentity; + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(IPAddress.Parse("203.0.113.77"), server.ClientIdentity.ClientAddress); + Assert.Equal(51515, server.ClientIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, server.ClientIdentity.TcpPeer.Address); + + var payload = new byte[] { 0x01, 0x02 }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task After382_WithReadsStillActive_ClientHelloEntersApplicationInput() + { + // Documents the historical failure mode: writing 382 while reads remain Active lets + // ClientHello commit into application Input and fail EnsureApplicationInputDrainedForUpgrade. + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + // Minimal TLS record header (Handshake / TLS 1.2) — enough to prove content-type 0x16. + var clientHelloPrefix = new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00 }; + await clientSocket.SendAsync(clientHelloPrefix); + + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + byte first = 0; + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek) && !peek.Buffer.IsEmpty) + { + first = peek.Buffer.FirstSpan[0]; + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + break; + } + + await Task.Yield(); + } + + Assert.Equal(0x16, first); + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("unconsumed plaintext", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsTls); + Assert.False(server.IsCompleted); + } + + [Fact] + public async Task PauseReads_Before382_ClientHelloDoesNotEnterApplicationInput() + { + // Required STARTTLS order: PauseReads → write 382 → ClientHello must not hit Input. + // With reads paused, the receive pump does not commit socket octets to Input; SslStream + // reads ClientHello from the network stream (or PrefixedStream if an in-flight read raced). + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.PauseReadsAsync(); + + var idleBefore = server.OutboundIdleVersion; + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + await server.WaitForOutboundDeliveryAsync(idleBefore); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + // Start the client handshake immediately after 382 — the historical race window. + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + var clientAuth = ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + + // While handshake is in flight, application Input must not observe ClientHello. + for (var i = 0; i < 20; i++) + { + if (server.Input.TryRead(out var peek)) + { + Assert.True( + peek.Buffer.IsEmpty, + "ClientHello must not enter application Input after PauseReads."); + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + } + + if (upgradeTask.IsCompleted && clientAuth.IsCompleted) + { + break; + } + + await Task.Yield(); + } + + await clientAuth; + await upgradeTask; + + Assert.True(server.IsTls); + Assert.False(server.Input.TryRead(out var leftover) && !leftover.Buffer.IsEmpty); + } + + [Fact] + public async Task PauseReadsBefore382_ThenClientHello_UpgradeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Correct STARTTLS handoff: pause reads BEFORE 382 can reach the peer. + await server.PauseReadsAsync(); + var idleBefore = server.OutboundIdleVersion; + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + await server.WaitForOutboundDeliveryAsync(idleBefore); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.False(server.Input.TryRead(out var leftover) && !leftover.Buffer.IsEmpty); + + var payload = new byte[] { 0x0A, 0x0B }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_UnconsumedInput_RefusesUpgradeAndKeepsPlaintext() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await clientSocket.SendAsync("UNCONSUMED"u8.ToArray()); + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek)) + { + var hasData = !peek.Buffer.IsEmpty; + // Leave bytes unconsumed and unexamined so UpgradeToTlsAsync can still observe them. + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + if (hasData) + { + break; + } + } + + await Task.Yield(); + } + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("unconsumed plaintext", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsTls); + // Precondition failure refuses the upgrade without tearing down plaintext transport. + Assert.False(server.ConnectionClosed.IsCancellationRequested); + + // Session can still consume the buffered plaintext. + Assert.Equal("UNCONSUMED"u8.ToArray(), await TransportTestShared.ReadExactAsync(server.Input, 10)); + } + + [Fact] + public async Task UpgradeToTls_InvalidClientHello_FailsWithoutPlaintextFallback() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_ClientClosesDuringHandshake_FailsTerminally() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + await Task.Yield(); + clientSocket.Dispose(); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_Cancelled_FailsTerminally() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_AlreadyTls_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfx); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.True(server.IsTls); + + await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + } + + [Fact] + public async Task UpgradeToTls_AfterComplete_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + await server.CompleteAsync(); + + await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + } + + [Fact] + public async Task UpgradeToTls_ConcurrentCalls_SecondRejected() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var first = server.UpgradeToTlsAsync(host.CertificateProvider!); + var second = server.UpgradeToTlsAsync(host.CertificateProvider!); + + var secondEx = await Assert.ThrowsAsync(() => second); + Assert.Contains("already in progress", secondEx.Message, StringComparison.OrdinalIgnoreCase); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await first; + Assert.True(server.IsTls); + } + + [Fact] + public async Task UpgradeToTls_FlushedPlaintextOutbound_IsDeliveredBeforeHandshake() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "382 Go ahead\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var receivedNotice = new byte[notice.Length]; + var noticeTotal = 0; + using (var noticeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (noticeTotal < notice.Length) + { + var n = await clientSocket.ReceiveAsync( + receivedNotice.AsMemory(noticeTotal), + noticeCts.Token); + Assert.True(n > 0); + noticeTotal += n; + } + } + + Assert.Equal(notice, receivedNotice); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + Assert.True(server.IsTls); + } + + private static SslClientAuthenticationOptions CreateClientSslOptions( + System.Security.Cryptography.X509Certificates.X509Certificate2? clientCertificate) => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + ClientCertificates = clientCertificate is null + ? [] + : [clientCertificate], + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportIoSessionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportIoSessionTests.cs new file mode 100644 index 00000000..3ff61938 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportIoSessionTests.cs @@ -0,0 +1,235 @@ +using System.Buffers; +using System.IO.Pipelines; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// Serializes static enable/disable so tests do not race. +[Collection(nameof(TransportIoSessionTests))] +public sealed class TransportIoSessionTests +{ + [Fact] + public void CreateSession_WhenDisabled_ReturnsNull() + { + TransportIoProbe.Disable(); + Assert.False(TransportIoProbe.IsEnabled); + Assert.Null(TransportIoProbe.CreateSession()); + } + + [Fact] + public void RecordReceive_AggregatesBytesAndSyncCounts() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordReceive(4096, 4096, ts, ts, sync: true); + session.RecordReceive(4096, 2048, ts, ts, sync: false); + + var snap = session.Snapshot(); + Assert.Equal(2, snap.Rx.Ops); + Assert.Equal(6144, snap.Rx.Bytes); + Assert.Equal(8192, snap.Rx.Requested); + Assert.Equal(2048, snap.Rx.MinBytes); + Assert.Equal(4096, snap.Rx.MaxBytes); + Assert.Equal(1, snap.Rx.Sync); + Assert.Equal(1, snap.Rx.Async); + Assert.Equal(2, snap.Rx.OpUs.Length); + } + + [Fact] + public void RecordSend_AggregatesAndReportsPercentiles() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + for (var i = 0; i < 10; i++) + { + session.RecordSend(65536, 65536, ts, ts, sync: true); + } + + var snap = session.Snapshot(); + Assert.Equal(10, snap.Tx.Ops); + Assert.Equal(655360, snap.Tx.Bytes); + Assert.Equal(65536, snap.Tx.MinBytes); + Assert.Equal(65536, snap.Tx.MaxBytes); + Assert.Equal(10, snap.Tx.Sync); + Assert.Equal(0, snap.Tx.Async); + var stats = TransportIoReport.Stats(snap.Tx.OpUs); + Assert.True(stats.P50 >= 0); + Assert.True(stats.P99 >= stats.P50); + Assert.True(stats.Max >= stats.P99); + } + + [Fact] + public void Percentile_KnownSeries() + { + var samples = new[] { 10, 20, 30, 40, 50 }; + Assert.Equal(10, TransportIoReport.Percentile(samples, 0)); + Assert.Equal(30, TransportIoReport.Percentile(samples, 0.5)); + Assert.Equal(50, TransportIoReport.Percentile(samples, 1)); + Assert.Equal(0, TransportIoReport.Percentile([], 0.5)); + } + + [Fact] + public void ExcessOperations_UpdateCounters_ButCapSamples() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + for (var i = 0; i < TransportIoSession.SampleCapacity + 25; i++) + { + session.RecordReceive(100, 100, ts, ts, sync: true); + } + + var snap = session.Snapshot(); + Assert.Equal(TransportIoSession.SampleCapacity + 25, snap.Rx.Ops); + Assert.Equal(TransportIoSession.SampleCapacity, snap.Rx.OpUs.Length); + Assert.Equal((TransportIoSession.SampleCapacity + 25) * 100L, snap.Rx.Bytes); + } + + [Fact] + public void RecordTxReadShape_SingleSegment() + { + var session = new TransportIoSession(); + session.RecordTxReadShape(new ReadOnlySequence(new byte[65536])); + session.RecordTxReadShape(ReadOnlySequence.Empty); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.TxRead.Reads); + Assert.Equal(1, snap.TxRead.EmptyReads); + Assert.Equal(65536, snap.TxRead.Bytes); + Assert.Equal(1, snap.TxRead.Segments); + Assert.Equal(65536, snap.TxRead.MinBytes); + Assert.Equal(65536, snap.TxRead.MaxBytes); + Assert.Equal(1, snap.TxRead.MinSegments); + Assert.Equal(1, snap.TxRead.MaxSegments); + Assert.Equal(65536, snap.TxRead.MinSegmentBytes); + Assert.Equal(65536, snap.TxRead.MaxSegmentBytes); + } + + [Fact] + public async Task RecordTxReadShape_MultiSegmentFromPipe() + { + var options = new PipeOptions( + pauseWriterThreshold: 4 * 1024 * 1024, + resumeWriterThreshold: 2 * 1024 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false); + var pipe = new Pipe(options); + var chunk = new byte[65536]; + for (var i = 0; i < 4; i++) + { + var memory = pipe.Writer.GetMemory(chunk.Length); + chunk.CopyTo(memory); + pipe.Writer.Advance(chunk.Length); + _ = await pipe.Writer.FlushAsync(); + } + + var result = await pipe.Reader.ReadAsync(); + var session = new TransportIoSession(); + session.RecordTxReadShape(result.Buffer); + pipe.Reader.AdvanceTo(result.Buffer.End); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.TxRead.Reads); + Assert.Equal(4 * 65536, snap.TxRead.Bytes); + Assert.True(snap.TxRead.Segments >= 4, "expected at least one segment per 64 KiB flush"); + Assert.Equal(snap.TxRead.Segments, snap.TxRead.MaxSegments); + Assert.True(snap.TxRead.MaxSegmentBytes >= 65536 || snap.TxRead.Segments > 4); + } + + [Fact] + public void Write_ProducesReportFile() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordSend(1024, 1024, ts, ts, sync: true); + var dir = Path.Join(Path.GetTempPath(), "vectornntp-transport-io-" + Guid.NewGuid().ToString("N")); + try + { + var path = session.Write(dir, "198.18.0.66:1234"); + Assert.True(File.Exists(path)); + var text = File.ReadAllText(path); + Assert.Contains("TX (stream WriteAsync)", text, StringComparison.Ordinal); + Assert.Contains("operations:", text, StringComparison.Ordinal); + Assert.Contains("TX SendAsync loop stages", text, StringComparison.Ordinal); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + } + + [Fact] + public void EnableDisable_TogglesSessionCreation() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-transport-io-" + Guid.NewGuid().ToString("N")); + try + { + TransportIoProbe.Enable(dir); + Assert.True(TransportIoProbe.IsEnabled); + Assert.NotNull(TransportIoProbe.CreateSession()); + } + finally + { + TransportIoProbe.Disable(); + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + + Assert.False(TransportIoProbe.IsEnabled); + Assert.Null(TransportIoProbe.CreateSession()); + } + + [Fact] + public void RecordSendLoopStages_AccumulateWithoutPerOpObjects() + { + var session = new TransportIoSession(); + var t0 = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordSendLoopStart(); + session.RecordReadToFirstWrite(t0, t0 + 1000); + session.RecordBetweenWrites(t0, t0 + 500); + session.RecordTransportWrite(t0, t0 + 2000); + session.RecordLastWriteToAdvance(t0, t0 + 300); + session.RecordAdvanceTo(t0, t0 + 100); + session.RecordAdvanceToNextRead(t0, t0 + 400); + session.RecordSendLoopEnd(); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.Loop.AdvanceCount); + Assert.Equal(1, snap.Loop.BetweenWriteCount); + Assert.Equal(1, snap.Loop.TransportWriteCount); + Assert.True(snap.Loop.SendAsyncElapsed >= TimeSpan.Zero); + Assert.True(snap.Loop.AdvanceTo >= TimeSpan.Zero); + Assert.True(snap.Loop.TransportWrite >= TimeSpan.Zero); + _ = Assert.Single(snap.Loop.AdvanceUs); + var text = TransportIoReport.Format(snap, "loop-test"); + Assert.Contains("AdvanceTo calls:", text, StringComparison.Ordinal); + Assert.Contains("ReadAsync waiting:", text, StringComparison.Ordinal); + Assert.Contains("TX write coalesce:", text, StringComparison.Ordinal); + Assert.Contains("TX write granularity", text, StringComparison.Ordinal); + } + + [Fact] + public void RecordCopy_AggregatesBytesAndDuration() + { + var session = new TransportIoSession { WriteGranularityTargetBytes = 256 * 1024 }; + var t0 = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordCopy(65536, t0, t0 + 1000); + session.RecordCopy(65536, t0, t0 + 2000); + + var snap = session.Snapshot(); + Assert.Equal(256 * 1024, snap.Coalesce.TargetBytes); + Assert.Equal(2, snap.Coalesce.CopyCount); + Assert.Equal(131072, snap.Coalesce.CopyBytes); + Assert.True(snap.Coalesce.CopyTime >= TimeSpan.Zero); + Assert.Equal(2, snap.Coalesce.CopyUs.Length); + var text = TransportIoReport.Format(snap, "copy-test"); + Assert.Contains("copy bytes:", text, StringComparison.Ordinal); + Assert.Contains("target:", text, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportTestHostCollection.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportTestHostCollection.cs new file mode 100644 index 00000000..fb1c9c64 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TransportTestHostCollection.cs @@ -0,0 +1,7 @@ +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Serializes tests that bind ephemeral loopback sockets via . +/// +[CollectionDefinition(nameof(TransportTestHostCollection), DisableParallelization = true)] +public sealed class TransportTestHostCollection; diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs new file mode 100644 index 00000000..612c075c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs @@ -0,0 +1,169 @@ +using System.IO.Pipelines; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Serializes the static TX-pipe pause experiment so tests do not leak into production defaults. +/// +[Collection(nameof(TxPipePauseExperimentTests))] +public sealed class TxPipePauseExperimentTests +{ + public TxPipePauseExperimentTests() => TxPipePauseExperiment.Clear(); + + [Fact] + public void Create_RemainsProduction64KiB_WhenExperimentIsSet() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + var input = NntpPipeOptions.Create(); + Assert.Equal(NntpPipeOptions.PauseWriterThreshold, input.PauseWriterThreshold); + Assert.Equal(NntpPipeOptions.ResumeWriterThreshold, input.ResumeWriterThreshold); + Assert.Equal(64 * 1024, input.PauseWriterThreshold); + Assert.Equal(32 * 1024, input.ResumeWriterThreshold); + Assert.Same(PipeScheduler.ThreadPool, input.ReaderScheduler); + Assert.Same(PipeScheduler.ThreadPool, input.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void CreateOutput_Default_IsProduction64KiB() + { + TxPipePauseExperiment.Clear(); + Assert.False(TxPipePauseExperiment.IsConfigured); + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(64 * 1024, output.PauseWriterThreshold); + Assert.Equal(32 * 1024, output.ResumeWriterThreshold); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + + [Fact] + public void CreateOutput_AppliesAllowlistedPause_AndHalfResume() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + Assert.True(TxPipePauseExperiment.TryGetEffective(out var pause, out var resume)); + Assert.Equal(4 * 1024 * 1024, pause); + Assert.Equal(2 * 1024 * 1024, resume); + + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(4 * 1024 * 1024, output.PauseWriterThreshold); + Assert.Equal(2 * 1024 * 1024, output.ResumeWriterThreshold); + Assert.Equal(NntpPipeOptions.MinimumSegmentSize, output.MinimumSegmentSize); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void Set_64KiB_MatchesProductionThresholds() + { + try + { + TxPipePauseExperiment.Set(64 * 1024); + Assert.True(TxPipePauseExperiment.IsConfigured); + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(NntpPipeOptions.PauseWriterThreshold, output.PauseWriterThreshold); + Assert.Equal(NntpPipeOptions.ResumeWriterThreshold, output.ResumeWriterThreshold); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void Set_RejectsValuesOutsideAllowlist() + { + Assert.Throws(() => TxPipePauseExperiment.Set(1)); + Assert.Throws(() => TxPipePauseExperiment.Set(16 * 1024 * 1024)); + Assert.Throws(() => TxPipePauseExperiment.Set(96 * 1024)); + Assert.False(TxPipePauseExperiment.IsConfigured); + Assert.Equal(64 * 1024, NntpPipeOptions.CreateOutput().PauseWriterThreshold); + } + + [Theory] + [InlineData("64KiB", 64 * 1024)] + [InlineData("128k", 128 * 1024)] + [InlineData("256KiB", 256 * 1024)] + [InlineData("512KiB", 512 * 1024)] + [InlineData("1MiB", 1024 * 1024)] + [InlineData("2m", 2 * 1024 * 1024)] + [InlineData("4MiB", 4 * 1024 * 1024)] + [InlineData("8MiB", 8 * 1024 * 1024)] + [InlineData("4194304", 4 * 1024 * 1024)] + public void TryParseAllowed_AcceptsSweepTokens(string token, long expected) + { + Assert.True(TxPipePauseExperiment.TryParseAllowed(token, out var parsed)); + Assert.Equal(expected, parsed); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("16MiB")] + [InlineData("96KiB")] + [InlineData("")] + public void TryParseAllowed_RejectsAccidentalValues(string token) + { + Assert.False(TxPipePauseExperiment.TryParseAllowed(token, out _)); + } + + [Fact] + public async Task CreateOutput_4MiB_DoesNotBackpressureAfterOne64KiBChunk() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + var pipe = new Pipe(NntpPipeOptions.CreateOutput()); + WriteZeros(pipe.Writer, 64 * 1024); + var flush = pipe.Writer.FlushAsync(); + Assert.True(flush.IsCompletedSuccessfully); + pipe.Reader.AdvanceTo((await pipe.Reader.ReadAsync()).Buffer.End); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public async Task Create_Production64KiB_BackpressuresAfterOne64KiBChunk() + { + TxPipePauseExperiment.Clear(); + var pipe = new Pipe(NntpPipeOptions.Create()); + WriteZeros(pipe.Writer, 64 * 1024); + var flush = pipe.Writer.FlushAsync(); + Assert.False(flush.IsCompleted); + + var read = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(read.Buffer.End); + var result = await flush; + Assert.False(result.IsCompleted); + + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + private static void WriteZeros(PipeWriter writer, int bytes) + { + var span = writer.GetSpan(bytes); + span[..bytes].Clear(); + writer.Advance(bytes); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteCoalescerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteCoalescerTests.cs new file mode 100644 index 00000000..2197cc2e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteCoalescerTests.cs @@ -0,0 +1,318 @@ +using System.Buffers; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Exact-byte and flush-shape contracts for the diagnostic TX write coalescer. +/// +public sealed class TxWriteCoalescerTests +{ + [Theory] + [InlineData(64 * 1024)] + [InlineData(128 * 1024)] + [InlineData(256 * 1024)] + [InlineData(512 * 1024)] + [InlineData(1024 * 1024)] + [InlineData(4 * 1024 * 1024)] + public void Copy_ExactTarget_FillsOnce_PreservesBytes(int target) + { + var coalescer = new TxWriteCoalescer(target); + var source = Pattern(target, seed: 7); + Assert.Equal(target, coalescer.Copy(source, io: null)); + Assert.True(coalescer.IsFull); + Assert.Equal(0, coalescer.RemainingCapacity); + Assert.True(source.AsSpan().SequenceEqual(coalescer.PendingMemory.Span)); + coalescer.Clear(); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public void Copy_MultipleSegments_PreservesOrder_NoDuplicationOrLoss() + { + const int target = 256 * 1024; + var coalescer = new TxWriteCoalescer(target); + var expected = Pattern(target, seed: 11); + var copied = 0; + foreach (var chunk in expected.Chunk(64 * 1024)) + { + copied += coalescer.Copy(chunk, io: null); + } + + Assert.Equal(target, copied); + Assert.True(coalescer.IsFull); + Assert.True(expected.AsSpan().SequenceEqual(coalescer.PendingMemory.Span)); + } + + [Fact] + public void Copy_PartialFinalAggregate_LeavesRemainder() + { + var coalescer = new TxWriteCoalescer(128 * 1024); + var first = Pattern(64 * 1024, seed: 3); + var tail = Pattern(100, seed: 9); + Assert.Equal(first.Length, coalescer.Copy(first, io: null)); + Assert.False(coalescer.IsFull); + Assert.Equal(tail.Length, coalescer.Copy(tail, io: null)); + Assert.Equal(first.Length + tail.Length, coalescer.PendingCount); + Assert.True(first.AsSpan().SequenceEqual(coalescer.PendingMemory.Span[..first.Length])); + Assert.True(tail.AsSpan().SequenceEqual(coalescer.PendingMemory.Span[first.Length..])); + } + + [Fact] + public void Copy_DoesNotOverflow_WhenSegmentExceedsRemaining() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var oversized = Pattern((64 * 1024) + 16, seed: 5); + Assert.Equal(64 * 1024, coalescer.Copy(oversized, io: null)); + Assert.True(coalescer.IsFull); + Assert.Equal(0, coalescer.Copy(oversized.AsSpan(64 * 1024), io: null)); + Assert.True(oversized.AsSpan(0, 64 * 1024).SequenceEqual(coalescer.PendingMemory.Span)); + } + + [Fact] + public async Task Drain_FlushesFullAggregates_ThenPartialRemainder() + { + const int target = 128 * 1024; + var coalescer = new TxWriteCoalescer(target); + var payload = Pattern((128 * 1024) + 64, seed: 13); + var sequence = ToSequence(payload.AsMemory(0, 64 * 1024), payload.AsMemory(64 * 1024, 64 * 1024), payload.AsMemory(128 * 1024)); + var writes = new List(); + + await DrainAsync(sequence, coalescer, writes, flushRemaining: true, CancellationToken.None); + + Assert.Equal(2, writes.Count); + Assert.Equal(target, writes[0].Length); + Assert.Equal(64, writes[1].Length); + Assert.Equal(payload, writes.SelectMany(static w => w).ToArray()); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_WithoutFlushRemaining_HoldsPartialAcrossCalls() + { + var coalescer = new TxWriteCoalescer(256 * 1024); + var first = Pattern(64 * 1024, seed: 17); + var second = Pattern(64 * 1024, seed: 19); + var writes = new List(); + + await DrainAsync(new ReadOnlySequence(first), coalescer, writes, flushRemaining: false, CancellationToken.None); + Assert.Empty(writes); + Assert.Equal(first.Length, coalescer.PendingCount); + + await DrainAsync(new ReadOnlySequence(second), coalescer, writes, flushRemaining: false, CancellationToken.None); + Assert.Empty(writes); + Assert.Equal(first.Length + second.Length, coalescer.PendingCount); + + await DrainAsync(ReadOnlySequence.Empty, coalescer, writes, flushRemaining: true, CancellationToken.None); + Assert.Single(writes); + Assert.Equal(first.Concat(second).ToArray(), writes[0]); + } + + [Fact] + public async Task Drain_TransportFailure_DoesNotClear_AndDoesNotWriteTwice() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var payload = Pattern(64 * 1024, seed: 23); + var writes = 0; + await Assert.ThrowsAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(payload), + coalescer, + writeAsync: (_, _) => + { + writes++; + throw new IOException("simulated transport failure"); + }, + flushRemaining: true, + io: null, + CancellationToken.None); + }); + + Assert.Equal(1, writes); + Assert.Equal(payload.Length, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_Cancellation_DoesNotWrite() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var writes = 0; + await Assert.ThrowsAnyAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(Pattern(64 * 1024, seed: 29)), + coalescer, + writeAsync: (_, token) => + { + writes++; + token.ThrowIfCancellationRequested(); + return ValueTask.CompletedTask; + }, + flushRemaining: true, + io: null, + cts.Token); + }); + + Assert.Equal(0, writes); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_WriteCanceled_LeavesPendingUnwritten() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var writes = 0; + await Assert.ThrowsAnyAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(Pattern(64 * 1024, seed: 29)), + coalescer, + writeAsync: (_, _) => + { + writes++; + throw new OperationCanceledException(); + }, + flushRemaining: true, + io: null, + CancellationToken.None); + }); + + Assert.Equal(1, writes); + Assert.Equal(64 * 1024, coalescer.PendingCount); + } + + [Fact] + public void RecordCopy_IsMeasuredSeparately() + { + var io = new TransportIoSession(); + var coalescer = new TxWriteCoalescer(64 * 1024); + var source = Pattern(64 * 1024, seed: 31); + Assert.Equal(source.Length, coalescer.Copy(source, io)); + var snap = io.Snapshot(); + Assert.Equal(1, snap.Coalesce.CopyCount); + Assert.Equal(source.Length, snap.Coalesce.CopyBytes); + Assert.True(snap.Coalesce.CopyTime >= TimeSpan.Zero); + Assert.Single(snap.Coalesce.CopyUs); + } + + [Fact] + public void Constructor_RejectsNonAllowlistedTarget() + { + Assert.Throws(() => new TxWriteCoalescer(96 * 1024)); + } + + private static async Task DrainAsync( + ReadOnlySequence buffer, + TxWriteCoalescer coalescer, + List writes, + bool flushRemaining, + CancellationToken cancellationToken) + { + await TxWriteCoalesceTestDrain.RunAsync( + buffer, + coalescer, + writeAsync: (memory, _) => + { + writes.Add(memory.ToArray()); + return ValueTask.CompletedTask; + }, + flushRemaining, + io: null, + cancellationToken); + } + + private static byte[] Pattern(int length, byte seed) + { + var bytes = new byte[length]; + for (var i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)(seed + (i * 31)); + } + + return bytes; + } + + private static ReadOnlySequence ToSequence(params ReadOnlyMemory[] parts) + { + if (parts.Length == 0) + { + return ReadOnlySequence.Empty; + } + + if (parts.Length == 1) + { + return new ReadOnlySequence(parts[0]); + } + + var first = new BufferSegment(parts[0]); + var last = first; + for (var i = 1; i < parts.Length; i++) + { + last = last.Append(parts[i]); + } + + return new ReadOnlySequence(first, 0, last, last.Memory.Length); + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(ReadOnlyMemory memory) => Memory = memory; + + public BufferSegment Append(ReadOnlyMemory memory) + { + var next = new BufferSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length, + }; + Next = next; + return next; + } + } +} + +/// +/// Test-only drain that mirrors NntpConnection aggregate copy/write without a live pipe. +/// +internal static class TxWriteCoalesceTestDrain +{ + public static async ValueTask RunAsync( + ReadOnlySequence buffer, + TxWriteCoalescer coalescer, + Func, CancellationToken, ValueTask> writeAsync, + bool flushRemaining, + TransportIoSession? io, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var offset = 0; + while (offset < segment.Length) + { + cancellationToken.ThrowIfCancellationRequested(); + if (coalescer.RemainingCapacity == 0) + { + await writeAsync(coalescer.PendingMemory, cancellationToken).ConfigureAwait(false); + coalescer.Clear(); + } + + offset += coalescer.Copy(segment.Span[offset..], io); + } + } + + if (coalescer.IsFull || (flushRemaining && coalescer.PendingCount > 0)) + { + cancellationToken.ThrowIfCancellationRequested(); + await writeAsync(coalescer.PendingMemory, cancellationToken).ConfigureAwait(false); + coalescer.Clear(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs new file mode 100644 index 00000000..8f995743 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs @@ -0,0 +1,323 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Live contracts for diagnostic write coalescing. +/// Uses per-transport so other tests keep +/// the production one-segment-one-write path. +/// +public sealed class TxWriteGranularityConnectionTests +{ + [Fact] + public async Task Disabled_WritesEachSegmentImmediately() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + Assert.False(TxWriteGranularityExperiment.TryResolveTarget(transport, out _)); + + var writes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + + var first = Pattern(64 * 1024, 1); + var second = Pattern(64 * 1024, 2); + await WriteSegmentAsync(server.Output, first, timeout.Token); + await WriteSegmentAsync(server.Output, second, timeout.Token); + + var received = await ReadExactAsync(client, first.Length + second.Length, timeout.Token); + Assert.Equal(first.Concat(second).ToArray(), received); + Assert.Equal(2, Volatile.Read(ref writes)); + } + + [Fact] + public async Task Disabled_SmallFlush_HitsTransportImmediately() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.BeforeStreamWriteProbe = () => + { + entered.TrySetResult(); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + await server.Output.WriteAsync(new byte[] { 0x01, 0x02, 0x03 }, timeout.Token); + await server.Output.FlushAsync(timeout.Token); + await entered.Task.WaitAsync(timeout.Token); + Assert.Equal(new byte[] { 0x01, 0x02, 0x03 }, await ReadExactAsync(client, 3, timeout.Token)); + } + + [Theory] + [InlineData(64 * 1024, 1)] + [InlineData(128 * 1024, 2)] + [InlineData(256 * 1024, 4)] + [InlineData(512 * 1024, 8)] + [InlineData(1024 * 1024, 16)] + [InlineData(4 * 1024 * 1024, 64)] + public async Task Coalesce_ExactTarget_OneWrite_PreservesBytes(int target, int chunks) + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var writes = 0; + var writeBytes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, target); + try + { + var expected = new byte[target]; + for (var i = 0; i < chunks; i++) + { + var chunk = Pattern(64 * 1024, (byte)(i + 1)); + chunk.CopyTo(expected.AsSpan(i * 64 * 1024)); + var span = server.Output.GetSpan(chunk.Length); + chunk.CopyTo(span); + server.Output.Advance(chunk.Length); + writeBytes += chunk.Length; + } + + var read = ReadExactAsync(client, target, timeout.Token); + await server.Output.FlushAsync(timeout.Token); + + var received = await read; + Assert.Equal(expected, received); + Assert.Equal(target, writeBytes); + Assert.Equal(1, Volatile.Read(ref writes)); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_PartialFinal_FlushesWithoutWaitingForMore() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var writes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 256 * 1024); + try + { + var chunk = Pattern(64 * 1024, 41); + await WriteSegmentAsync(server.Output, chunk, timeout.Token); + var received = await ReadExactAsync(client, chunk.Length, timeout.Token); + Assert.Equal(chunk, received); + Assert.Equal(1, Volatile.Read(ref writes)); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_TransportFailure_CompletesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + transport.BeforeStreamWriteProbe = () => throw new IOException("simulated coalesce write failure"); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 64 * 1024); + try + { + try + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 8), timeout.Token); + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or OperationCanceledException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + await WaitUntilAsync( + static s => s.ConnectionClosed.IsCancellationRequested, + server, + timeout.Token); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_Cancellation_DoesNotHang() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 256 * 1024); + try + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 12), timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.OutputReaderCompletedForTests); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_PeerDisconnect_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 64 * 1024); + try + { + client.LingerState = new LingerOption(enable: true, seconds: 0); + client.Dispose(); + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(server.ConnectionClosed, timeout.Token); + try + { + while (!linked.IsCancellationRequested) + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 15), linked.Token); + } + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (Exception ex) when ( + server.ConnectionClosed.IsCancellationRequested + && ex is InvalidOperationException or IOException or SocketException) + { + _ = ex; // observed; expected during test teardown/cleanup + } + + if (!server.ConnectionClosed.IsCancellationRequested) + { + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + private static async Task WriteSegmentAsync(PipeWriter writer, ReadOnlyMemory chunk, CancellationToken cancellationToken) + { + var span = writer.GetSpan(chunk.Length); + chunk.Span.CopyTo(span); + writer.Advance(chunk.Length); + await writer.FlushAsync(cancellationToken); + } + + private static byte[] Pattern(int length, byte seed) + { + var bytes = new byte[length]; + for (var i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)(seed + (i * 31)); + } + + return bytes; + } + + private static async Task ReadExactAsync(Socket client, int count, CancellationToken cancellationToken) + { + var buffer = new byte[count]; + var read = 0; + while (read < count) + { + var n = await client.ReceiveAsync(buffer.AsMemory(read), cancellationToken); + if (n == 0) + { + throw new InvalidOperationException($"Peer closed after {read} of {count} bytes."); + } + + read += n; + } + + return buffer; + } + + private static async Task WaitIdleAsync(NntpConnection server, CancellationToken cancellationToken) => + await WaitUntilAsync(static s => s.SendPumpAwaitingOutputForTests == 1, server, cancellationToken); + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs new file mode 100644 index 00000000..286cde33 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs @@ -0,0 +1,83 @@ +using VectorNNTPD.ReaderService.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Networking.Transport; + +/// +/// Serializes process-wide write-granularity experiment state. These tests do not create +/// instances while +/// is active. +/// +[Collection(nameof(TxWriteGranularityExperimentTests))] +public sealed class TxWriteGranularityExperimentTests +{ + public TxWriteGranularityExperimentTests() => TxWriteGranularityExperiment.Clear(); + + [Fact] + public void Default_IsInactive() + { + TxWriteGranularityExperiment.Clear(); + Assert.False(TxWriteGranularityExperiment.IsConfigured); + Assert.Null(TxWriteGranularityExperiment.ConfiguredTargetBytes); + Assert.False(TxWriteGranularityExperiment.TryGetTarget(out var target)); + Assert.Equal(0, target); + Assert.False(TxWriteGranularityExperiment.TryResolveTarget(transport: null, out _)); + } + + [Theory] + [InlineData(64 * 1024)] + [InlineData(128 * 1024)] + [InlineData(256 * 1024)] + [InlineData(512 * 1024)] + [InlineData(1024 * 1024)] + [InlineData(4 * 1024 * 1024)] + public void Set_AllowlistedTargets_AreResolved(int targetBytes) + { + try + { + TxWriteGranularityExperiment.Set(targetBytes); + Assert.True(TxWriteGranularityExperiment.IsConfigured); + Assert.True(TxWriteGranularityExperiment.TryGetTarget(out var resolved)); + Assert.Equal(targetBytes, resolved); + Assert.Equal(targetBytes, TxWriteGranularityExperiment.ConfiguredTargetBytes); + } + finally + { + TxWriteGranularityExperiment.Clear(); + } + } + + [Fact] + public void Set_RejectsValuesOutsideAllowlist() + { + Assert.Throws(() => TxWriteGranularityExperiment.Set(1)); + Assert.Throws(() => TxWriteGranularityExperiment.Set(2 * 1024 * 1024)); + Assert.Throws(() => TxWriteGranularityExperiment.Set(8 * 1024 * 1024)); + Assert.False(TxWriteGranularityExperiment.IsConfigured); + } + + [Theory] + [InlineData("64KiB", 64 * 1024)] + [InlineData("128k", 128 * 1024)] + [InlineData("256KiB", 256 * 1024)] + [InlineData("512KiB", 512 * 1024)] + [InlineData("1MiB", 1024 * 1024)] + [InlineData("4MiB", 4 * 1024 * 1024)] + [InlineData("4194304", 4 * 1024 * 1024)] + public void TryParseAllowed_AcceptsSweepTokens(string token, int expected) + { + Assert.True(TxWriteGranularityExperiment.TryParseAllowed(token, out var parsed)); + Assert.Equal(expected, parsed); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("2MiB")] + [InlineData("8MiB")] + [InlineData("96KiB")] + [InlineData("")] + public void TryParseAllowed_RejectsAccidentalValues(string token) + { + Assert.False(TxWriteGranularityExperiment.TryParseAllowed(token, out _)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs new file mode 100644 index 00000000..4d55d49c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs @@ -0,0 +1,289 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Newsgroups; + +public sealed class NewsgroupCatalogueServiceTests +{ + [Fact] + public async Task StartAsync_LoadsCompleteSnapshot_BeforeReturning() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = + [ + new NntpGroupRow("comp.risks", "Risks", 10, 2, (byte)'m'), + new NntpGroupRow("misc.test", "Testing", 5, 1, (byte)'y'), + ], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await catalogue.StartAsync(CancellationToken.None); + + Assert.True(catalogue.HasSnapshot); + var snapshot = catalogue.Current; + Assert.Equal(2, snapshot.Groups.Count); + Assert.Equal("comp.risks", snapshot.Groups[0].GroupName); + Assert.Equal("misc.test", snapshot.Groups[1].GroupName); + Assert.Equal(1, factory.QueryNewsgroupsCount); + Assert.Equal(1, Assert.Single(factory.Connections.Skip(1)).DisposeCount); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenQueryFails() + { + var factory = new FakeNntpDbConnectionFactory + { + QueryNewsgroupsException = new NntpDbUnavailableException("down"), + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenRowIsMalformed() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("", "desc", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task StartAsync_PreservesExtendedPostingStatuses() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = + [ + new NntpGroupRow("group.closed", "", 4, 1, (byte)'x'), + new NntpGroupRow("group.junk", "", 5, 2, (byte)'j'), + ], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await catalogue.StartAsync(CancellationToken.None); + + var snapshot = catalogue.Current; + Assert.Equal(NewsgroupPostingStatus.NoPostingOrPeerArticles, snapshot.Groups[0].PostingStatus); + Assert.Equal(NewsgroupPostingStatus.PeerOnly, snapshot.Groups[1].PostingStatus); + Assert.True(snapshot.Groups[0].ListActiveLine.Span.SequenceEqual("group.closed 4 1 x\r\n"u8)); + Assert.True(snapshot.Groups[1].ListActiveLine.Span.SequenceEqual("group.junk 5 2 j\r\n"u8)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenEqualsAliasStatus() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("alias.group", "", 1, 1, (byte)'=')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task Refresh_Success_AtomicallyReplacesCurrent() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("old.group", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var first = catalogue.Current; + + factory.Newsgroups = [new NntpGroupRow("new.group", "", 8, 3, (byte)'n')]; + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => + { + var current = catalogue.Current; + return !ReferenceEquals(current, first) + && current.Groups.Count == 1 + && current.Groups[0].GroupName == "new.group"; + }); + + var second = catalogue.Current; + Assert.NotSame(first, second); + Assert.Equal("old.group", Assert.Single(first.Groups).GroupName); + Assert.Equal("new.group", Assert.Single(second.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_Failure_RetainsPreviousSnapshot() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("keep.group", "", 2, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var first = catalogue.Current; + + factory.QueryNewsgroupsException = new NntpDbUnavailableException("down"); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => factory.QueryNewsgroupsCount >= 2); + + Assert.Same(first, catalogue.Current); + Assert.Equal("keep.group", Assert.Single(catalogue.Current.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_DoesNotOverlap() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("g", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + Assert.Equal(1, factory.QueryNewsgroupsCount); + + factory.BlockQueryNewsgroups = new TaskCompletionSource(); + factory.QueryNewsgroupsStarted = new TaskCompletionSource(); + clock.Advance(TimeSpan.FromMinutes(5)); + await factory.QueryNewsgroupsStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(catalogue.IsRefreshing); + clock.Advance(TimeSpan.FromMinutes(5)); + Assert.Equal(1, factory.QueryNewsgroupsCount); + + factory.BlockQueryNewsgroups.SetResult(); + await WaitUntilAsync(() => !catalogue.IsRefreshing); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Current_CanBeReadConcurrently_WhileRefreshPublishes() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("first.group", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readers = Enumerable.Range(0, 4).Select(_ => Task.Run(() => + { + while (!cts.IsCancellationRequested) + { + var snapshot = catalogue.Current; + _ = snapshot.ListActiveComplete.Length; + _ = snapshot.ListNewsgroupsComplete.Length; + } + }, cts.Token)).ToArray(); + + factory.Newsgroups = [new NntpGroupRow("second.group", "", 3, 1, (byte)'n')]; + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => factory.QueryNewsgroupsCount >= 2); + cts.Cancel(); + try + { + await Task.WhenAll(readers); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Readers observe the test timeout token. + } + + Assert.Equal("second.group", Assert.Single(catalogue.Current.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopAsync_CancelsInFlightRefresh() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("g", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + + factory.BlockQueryNewsgroups = new TaskCompletionSource(); + factory.QueryNewsgroupsStarted = new TaskCompletionSource(); + clock.Advance(TimeSpan.FromMinutes(5)); + await factory.QueryNewsgroupsStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var stop = catalogue.StopAsync(CancellationToken.None); + factory.BlockQueryNewsgroups.SetCanceled(); + await stop.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static NntpDbService CreateDb(FakeNntpDbConnectionFactory factory) + { + return new NntpDbService( + factory, + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }), + NullLogger.Instance); + } + + private static NewsgroupCatalogueService CreateCatalogue( + NntpDbService db, + TimeProvider? clock = null, + TimeSpan? interval = null) + { + return new NewsgroupCatalogueService( + db, + NullLogger.Instance, + clock ?? TimeProvider.System, + interval ?? NewsgroupCatalogueService.RefreshInterval); + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupSnapshotTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupSnapshotTests.cs new file mode 100644 index 00000000..2e69d1a1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NewsgroupSnapshotTests.cs @@ -0,0 +1,260 @@ +using System.Runtime.InteropServices; +using System.Text; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Newsgroups; + +public sealed class NewsgroupSnapshotTests +{ + [Fact] + public void Create_MapsFields_PreservesUnsignedCountsAndPostingStatus() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition( + "misc.test", + "General testing", + CountHigh: 4_294_967_295UL, + CountLow: 1, + NewsgroupPostingStatus.Moderated), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal("misc.test", group.GroupName); + Assert.Equal("General testing", group.GroupDescription); + Assert.Equal(4_294_967_295UL, group.CountHigh); + Assert.Equal(1UL, group.CountLow); + Assert.Equal(NewsgroupPostingStatus.Moderated, group.PostingStatus); + Assert.Equal(4_294_967_295UL, group.EstimatedCount); + } + + [Fact] + public void Create_SortsDeterministically_ByOrdinalIgnoreCaseThenOrdinal() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("comp.risks", "", 2, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("alt.test", "", 2, 1, NewsgroupPostingStatus.Prohibited), + new NewsgroupDefinition("Misc.Test", "", 2, 1, NewsgroupPostingStatus.Moderated), + ]); + + Assert.Equal(["alt.test", "comp.risks", "Misc.Test"], snapshot.Groups.Select(g => g.GroupName)); + } + + [Fact] + public void TryGet_IsOrdinalCaseInsensitive_AndPreservesOriginalName() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.TEST", "desc", 10, 2, NewsgroupPostingStatus.Allowed), + ]); + + Assert.True(snapshot.TryGet("MISC.test"u8, out var group)); + Assert.Equal("misc.TEST", group.GroupName); + Assert.True(group.GroupNameBytes.Span.SequenceEqual("misc.TEST"u8)); + } + + [Theory] + [InlineData(NewsgroupPostingStatus.Allowed, (byte)'y', "group.example 100 1 y\r\n")] + [InlineData(NewsgroupPostingStatus.Prohibited, (byte)'n', "group.example 100 1 n\r\n")] + [InlineData(NewsgroupPostingStatus.Moderated, (byte)'m', "group.example 100 1 m\r\n")] + [InlineData(NewsgroupPostingStatus.NoPostingOrPeerArticles, (byte)'x', "group.example 100 1 x\r\n")] + [InlineData(NewsgroupPostingStatus.PeerOnly, (byte)'j', "group.example 100 1 j\r\n")] + public void ListActiveLine_EmitsEachSupportedStatusOctet( + NewsgroupPostingStatus status, + byte expectedOctet, + string expectedLine) + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("group.example", "", 100, 1, status), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(status, group.PostingStatus); + Assert.Equal(expectedOctet, (byte)group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual(Encoding.ASCII.GetBytes(expectedLine))); + } + + [Fact] + public void ListActiveLine_IsExactUtf8WithCrlf() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "ignored", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + ]); + + var line = Assert.Single(snapshot.Groups).ListActiveLine; + Assert.True(line.Span.SequenceEqual("misc.test 3002322 3000234 y\r\n"u8)); + Assert.True(snapshot.ListActiveComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test 3002322 3000234 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + Assert.True(Assert.Single(snapshot.Groups).ListCountsLine.Span.SequenceEqual( + "misc.test 3002322 3000234 2089 y\r\n"u8)); + Assert.True(snapshot.ListCountsComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test 3002322 3000234 2089 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + } + + [Fact] + public void ListNewsgroupsLine_EncodesUtf8Description() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "Café résumé £", 1, 1, NewsgroupPostingStatus.Allowed), + ]); + + var expected = Encode("misc.test Café résumé £\r\n"); + Assert.True(Assert.Single(snapshot.Groups).ListNewsgroupsLine.Span.SequenceEqual(expected)); + Assert.True(snapshot.ListNewsgroupsComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, expected, NntpResponses.MultilineTerminator))); + } + + [Fact] + public void PrecomputedLines_AreStableReferences() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("a.test", "d", 2, 1, NewsgroupPostingStatus.Allowed), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.True(group.ListActiveLine.Equals(snapshot.Groups[0].ListActiveLine)); + Assert.True(snapshot.ListActiveComplete.Equals(snapshot.ListActiveComplete)); + Assert.True(MemoryMarshal.TryGetArray(group.ListActiveLine, out var first)); + Assert.True(MemoryMarshal.TryGetArray(snapshot.Groups[0].ListActiveLine, out var second)); + Assert.Same(first.Array, second.Array); + } + + [Fact] + public void Create_FromDatabaseRows_UsesExactQueryFields() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NntpGroupRow("tx.natives.recovery", "Texas", 89, 56, (byte)'n'), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(NewsgroupPostingStatus.Prohibited, group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual("tx.natives.recovery 89 56 n\r\n"u8)); + } + + [Theory] + [InlineData((byte)'x', NewsgroupPostingStatus.NoPostingOrPeerArticles, "group.example 100 1 x\r\n")] + [InlineData((byte)'j', NewsgroupPostingStatus.PeerOnly, "group.example 100 1 j\r\n")] + public void Create_PreservesExtendedStatus_AndListActiveBytes( + byte statusOctet, + NewsgroupPostingStatus expected, + string expectedLine) + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NntpGroupRow("group.example", "", 100, 1, statusOctet), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(expected, group.PostingStatus); + Assert.Equal(statusOctet, (byte)group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual(Encoding.ASCII.GetBytes(expectedLine))); + } + + [Fact] + public void Create_RejectsInvalidPostingStatus_AndDoesNotPublishPartial() + { + var ex = Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NntpGroupRow("good.group", "", 1, 1, (byte)'y'), + new NntpGroupRow("bad.group", "", 1, 1, (byte)'z'), + ])); + Assert.Contains("posting_status", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Create_RejectsEqualsAliasForm() + { + var ex = Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NntpGroupRow("alias.group", "", 1, 1, (byte)'='), + ])); + Assert.Contains("posting_status", ex.Message, StringComparison.Ordinal); + Assert.False(NewsgroupPostingStatusOctets.IsSupported((byte)'=')); + } + + [Fact] + public void Create_RejectsDuplicateCaseInsensitiveNames() + { + Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "", 1, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("MISC.TEST", "", 2, 1, NewsgroupPostingStatus.Allowed), + ])); + } + + [Fact] + public void PublishedSnapshot_RemainsUnchanged_WhenAnotherIsConstructed() + { + var first = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("old.group", "old", 5, 1, NewsgroupPostingStatus.Allowed), + ]); + var captured = first; + var second = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("new.group", "new", 9, 2, NewsgroupPostingStatus.Moderated), + ]); + + Assert.Equal("old.group", Assert.Single(captured.Groups).GroupName); + Assert.Equal("new.group", Assert.Single(second.Groups).GroupName); + Assert.True(captured.ListActiveComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "old.group 5 1 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + } + + [Theory] + [InlineData(0UL, 0UL, 0UL)] + [InlineData(5UL, 5UL, 1UL)] + [InlineData(4000UL, 3999UL, 0UL)] + [InlineData(234UL, 567UL, 334UL)] + [InlineData(0UL, ulong.MaxValue, ulong.MaxValue)] + public void EstimateCount_FollowsRfcEmptyAndSpanRules(ulong low, ulong high, ulong expected) + { + Assert.Equal(expected, NewsgroupSnapshot.EstimateCount(low, high)); + } + + [Fact] + public void Empty_HasTerminatorOnlyListBodies() + { + Assert.Empty(NewsgroupSnapshot.Empty.Groups); + Assert.True(NewsgroupSnapshot.Empty.ListActiveComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator))); + Assert.True(NewsgroupSnapshot.Empty.ListCountsComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator))); + } + + private static byte[] Encode(string text) => Encoding.UTF8.GetBytes(text); + + private static byte[] Concat(ReadOnlyMemory first, ReadOnlySpan second, ReadOnlyMemory third) + { + var result = new byte[first.Length + second.Length + third.Length]; + first.Span.CopyTo(result); + second.CopyTo(result.AsSpan(first.Length)); + third.Span.CopyTo(result.AsSpan(first.Length + second.Length)); + return result; + } + + private static byte[] Concat(ReadOnlyMemory first, ReadOnlyMemory second) + { + var result = new byte[first.Length + second.Length]; + first.Span.CopyTo(result); + second.Span.CopyTo(result.AsSpan(first.Length)); + return result; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NntpWildmatTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NntpWildmatTests.cs new file mode 100644 index 00000000..fb406995 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Newsgroups/NntpWildmatTests.cs @@ -0,0 +1,52 @@ +using System.Text; +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.Tests.Newsgroups; + +public sealed class NntpWildmatTests +{ + [Theory] + [InlineData("abc")] + [InlineData("a*")] + [InlineData("a*,!*b")] + [InlineData("*.recovery")] + [InlineData("*")] + public void TryValidate_AcceptsRfcPatterns(string wildmat) + { + Assert.True(NntpWildmat.TryValidate(Encoding.ASCII.GetBytes(wildmat))); + } + + [Theory] + [InlineData("u[ks].*")] + [InlineData(@"a\b")] + [InlineData("a]b")] + [InlineData("")] + [InlineData("a*,")] + [InlineData("!*b")] + public void TryValidate_RejectsForbiddenOrEmptyPatterns(string wildmat) + { + Assert.False(NntpWildmat.TryValidate(Encoding.ASCII.GetBytes(wildmat))); + } + + [Theory] + [InlineData("misc.test", "misc.test", true)] + [InlineData("MISC.TEST", "misc.test", true)] + [InlineData("misc.test", "MISC.TEST", true)] + [InlineData("alt.rfc-writers.recovery", "*.recovery", true)] + [InlineData("tx.natives.recovery", "*.recovery", true)] + [InlineData("misc.test", "*.recovery", false)] + [InlineData("aaa", "a*,!*b", true)] + [InlineData("abb", "a*,!*b", false)] + [InlineData("ccb", "a*,!*b,*c*", true)] + [InlineData("xxx", "a*,!*b,*c*", false)] + [InlineData("misc.test", "misc.?est", true)] + [InlineData("misc.test", "misc.??st", true)] + [InlineData("misc.test", "misc.?", false)] + public void IsMatch_FollowsRfcExamples_CaseInsensitive(string text, string wildmat, bool expected) + { + var textBytes = Encoding.ASCII.GetBytes(text); + var pattern = Encoding.ASCII.GetBytes(wildmat); + Assert.Equal(expected, NntpWildmat.IsMatch(textBytes, pattern)); + Assert.Equal(expected, NntpWildmat.IsMatchValidated(textBytes, pattern)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsLogReaderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsLogReaderTests.cs new file mode 100644 index 00000000..4f0c3482 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsLogReaderTests.cs @@ -0,0 +1,187 @@ +using System.Text; +using VectorNNTPD.ReaderService.Ninpaths; + +namespace VectorNNTPD.ReaderService.Tests.Ninpaths; + +public sealed class NinpathsLogReaderTests +{ + [Fact] + public void EmptyFile_YieldsNoArticles() + { + var stats = Read(""); + Assert.Equal(0, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void SingleHopPath_ChopsTheOnlyElement() + { + var stats = Read("Path: only\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void MultiHopPath_RegistersAllButTheLastHop() + { + var stats = Read("Path: a!b!c\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("a", out var a)); + Assert.True(stats.TryGetSentTo("b", out var b)); + Assert.False(stats.TryGetSentTo("c", out _)); + Assert.Equal(1, a); + Assert.Equal(1, b); + Assert.True(stats.TryGetRelation("a", "b", out var tally)); + Assert.Equal(1, tally); + } + + [Fact] + public void NotForMail_IsDroppedAsTheFinalElement() + { + var stats = Read("Path: news.example!peer.example!not-for-mail\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("news.example", out _)); + Assert.True(stats.TryGetSentTo("peer.example", out _)); + Assert.False(stats.TryGetSentTo("not-for-mail", out _)); + } + + [Fact] + public void RepeatedSite_IncrementsSentTo_WithoutSelfRelation() + { + var stats = Read("Path: a!a!b\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(1, stats.SiteCount); + Assert.True(stats.TryGetSentTo("a", out var sent)); + Assert.Equal(2, sent); + Assert.False(stats.TryGetRelation("a", "a", out _)); + Assert.False(stats.TryGetSentTo("b", out _)); + } + + [Fact] + public void MultipleUnrelatedPaths_AccumulateSitesAndRelations() + { + var stats = Read("Path: a!b!z\nPath: c!d!z\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.Equal(4, stats.SiteCount); + Assert.True(stats.TryGetRelation("a", "b", out var ab)); + Assert.True(stats.TryGetRelation("c", "d", out var cd)); + Assert.Equal(1, ab); + Assert.Equal(1, cd); + Assert.False(stats.TryGetRelation("a", "c", out _)); + } + + [Fact] + public void WhitespaceTerminatesThePath_AndDoesNotSkipLeadingSpaceAfterPrefix() + { + var stats = Read("Path: a!b!c extra\nPath: skipped-as-empty!x\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("b", out _)); + Assert.False(stats.TryGetSentTo("c", out _)); + Assert.False(stats.TryGetSentTo("skipped-as-empty", out _)); + } + + [Fact] + public void PathPrefixIsCaseInsensitive() + { + var stats = Read("path: a!b!c\nPATH: d!e!f\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("d", out _)); + } + + [Fact] + public void EmptyPathLine_CountsAnArticleWithoutSites() + { + var stats = Read("Path: \n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void LineWithoutNewlineAtEof_IsBogus() + { + var stats = Read("Path: a!b!c"); + Assert.Equal(0, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void ConsecutiveBangs_AreSkipped() + { + var stats = Read("Path: a!!b!z\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetRelation("a", "b", out var tally)); + Assert.Equal(1, tally); + Assert.False(stats.TryGetSentTo(string.Empty, out _)); + } + + [Fact] + public void OverlongHostToken_AbortsRestOfPath_ButStillCountsTheArticle() + { + var longHop = new string('x', NinpathsConstants.MaxHostChars + 1); + var stats = Read($"Path: keep!{longHop}!later!end\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("keep", out var sent)); + Assert.Equal(1, sent); + Assert.False(stats.TryGetSentTo("later", out _)); + Assert.False(stats.TryGetSentTo(longHop, out _)); + } + + [Fact] + public void MaxHostCharsToken_IsAccepted() + { + var hop = new string('y', NinpathsConstants.MaxHostChars); + var stats = Read($"Path: {hop}!end\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo(hop, out var sent)); + Assert.Equal(1, sent); + } + + [Fact] + public void SiteNamesAreCaseSensitive() + { + var stats = Read("Path: News!news!end\n"); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("News", out _)); + Assert.True(stats.TryGetSentTo("news", out _)); + } + + [Fact] + public void LargeSyntheticStream_DoesNotRetainPerArticleState() + { + var builder = new StringBuilder(capacity: 80 * 20_000); + for (var i = 0; i < 20_000; i++) + { + builder.Append("Path: site-one!site-two!not-for-mail\n"); + } + + var stats = Read(builder.ToString()); + Assert.Equal(20_000, stats.TotalArticles); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("site-one", out var one)); + Assert.True(stats.TryGetSentTo("site-two", out var two)); + Assert.Equal(20_000, one); + Assert.Equal(20_000, two); + Assert.True(stats.TryGetRelation("site-one", "site-two", out var tally)); + Assert.Equal(20_000, tally); + } + + [Fact] + public void FgetsWindowWithoutWhitespace_SkipsTheNextWindowThatHasWhitespace() + { + var bogus = new string('z', NinpathsConstants.MaxFgetsChars); + var stats = Read(bogus + "\nPath: a!b!c\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("b", out _)); + } + + private static NinpathsStatistics Read(string text) + { + var bytes = Encoding.Latin1.GetBytes(text); + using var stream = new MemoryStream(bytes, writable: false); + return NinpathsLogReader.Read(stream); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs new file mode 100644 index 00000000..18201e3e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs @@ -0,0 +1,313 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.Ninpaths; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace VectorNNTPD.ReaderService.Tests.Ninpaths; + +public sealed class NinpathsProcessingServiceTests +{ + [Fact] + public void Top1000_WhitespaceOnly_IsDisabled() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = [" ", "\t"]; + Assert.False(NinpathsTop1000.IsEnabled(options)); + Assert.Empty(NinpathsTop1000.GetRecipients(options)); + } + + [Fact] + public void Top1000_DedupesAndKeepsValidMailboxes() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["a@example.com", " ", "A@example.com", "b@example.com"]; + Assert.Equal(["a@example.com", "b@example.com"], NinpathsTop1000.GetRecipients(options)); + } + + [Fact] + public async Task Handler_ReturnsWithoutParsing_UntilTheWorkerRuns() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + + handler.OnCompletedFile(path); + + Assert.Equal(1, service.PendingCount); + Assert.Empty(email.Sent); + Assert.True(File.Exists(path)); + + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + + var message = Assert.Single(email.Sent); + Assert.Equal("inpaths nntpd01.usenet.ninja", message.Subject); + var body = Encoding.Latin1.GetString(message.Body.Span); + Assert.StartsWith("!!NINP 3.1.1 ", body, StringComparison.Ordinal); + Assert.Contains("!!NLREC", body, StringComparison.Ordinal); + Assert.Contains("!!NLEND", body, StringComparison.Ordinal); + Assert.DoesNotContain("survey@example.com", body, StringComparison.Ordinal); + } + + [Fact] + public async Task MultipleRecipients_ShareOneGeneratedReport() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["one@example.com", "two@example.com"]); + var handler = CreateHandler(service, ["one@example.com", "two@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, email.SendCount); + var message = Assert.Single(email.Sent); + Assert.Equal(["one@example.com", "two@example.com"], message.To.Select(static a => a.Address).ToArray()); + Assert.Equal("us-ascii", message.Charset); + Assert.Contains(message.Headers, static h => h.Name == "Auto-Submitted" && h.Value == "auto-generated"); + } + + [Fact] + public async Task EmailFailure_IsNonFatal_AndDoesNotFaultExecution() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService + { + Throw = new InvalidOperationException("smtp exploded"), + }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (service.PendingCount > 0 && !wait.IsCancellationRequested) + { + await Task.Delay(10, wait.Token); + } + + Assert.NotNull(service.Execution); + Assert.False(service.Execution.IsFaulted); + Assert.False(service.Execution.IsCompleted); + await service.StopAsync(CancellationToken.None); + Assert.False(service.Execution.IsFaulted); + } + + [Fact] + public async Task EmailDisabledStatus_IsNonFatal() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService { Status = EmailEnqueueStatus.Disabled }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Disabled, email.Status); + Assert.False(service.Execution!.IsFaulted); + } + + [Fact] + public async Task DisabledTop1000_DoesNotOpenOrEnqueue() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, []); + var handler = CreateHandler(service, []); + handler.OnCompletedFile(path); + Assert.Equal(0, service.PendingCount); + Assert.Empty(email.Sent); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DuplicateCompletedFile_IsNotProcessedTwice() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + handler.OnCompletedFile(path); + handler.OnCompletedFile(path); + Assert.Equal(1, service.PendingCount); + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Equal(1, email.SendCount); + } + + [Fact] + public async Task CompletedSourceRemainsReadableAfterDelete_ViaShareDeleteHandle() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + handler.OnCompletedFile(path); + File.Delete(path); + Assert.False(File.Exists(path)); + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Single(email.Sent); + } + + [Fact] + public async Task GzipStillRuns_WhileNinpathsHoldsTheFile() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService { Hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + + handler.OnCompletedFile(path); + Assert.True(new GzipLogFileCompressor().TryCompressAndReplace(path)); + + Assert.False(File.Exists(path)); + var archive = Path.Join(dir.Path, NntpdFileLogging.GzipArchiveFileName(path)); + Assert.True(File.Exists(archive)); + + email.Hold.SetResult(); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Single(email.Sent); + } + + [Fact] + public async Task EmptySurvey_DoesNotSendEmail() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", ""); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (service.PendingCount > 0 && !wait.IsCancellationRequested) + { + await Task.Delay(10, wait.Token); + } + + await service.StopAsync(CancellationToken.None); + Assert.Empty(email.Sent); + } + + [Fact] + public void Host_RegistersNinpathsHandlerAndService() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + Assert.IsType(host.Services.GetRequiredService()); + var services = host.Services.GetServices().Select(static s => s.GetType()).ToArray(); + var email = Array.IndexOf(services, typeof(EmailDeliveryService)); + var ninpaths = Array.IndexOf(services, typeof(NinpathsProcessingService)); + Assert.True(email >= 0); + Assert.Equal(email + 1, ninpaths); + } + + [Fact] + public void ReportMail_DoesNotEmbedRecipientAddressesInTheBody() + { + var report = "!!NINP 3.1.1 1 2 0 1 2\n\n!!NLREC\n\n!!NLEND 0\n"u8.ToArray(); + var message = NinpathsReportMail.TryCreate( + report, + "nntpd01.usenet.ninja", + "noreply@usenet.ninja", + ["one@example.com", "two@example.com"]); + Assert.NotNull(message); + var body = Encoding.Latin1.GetString(message.Body.Span); + Assert.DoesNotContain("one@example.com", body, StringComparison.Ordinal); + Assert.DoesNotContain("two@example.com", body, StringComparison.Ordinal); + Assert.Equal("inpaths nntpd01.usenet.ninja", message.Subject); + } + + private static NinpathsProcessingService CreateService(RecordingEmailService email, string[] top1000) + { + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.Top1000 = top1000; + var emailOptions = new EmailOptions { DefaultFrom = "noreply@usenet.ninja" }; + return new NinpathsProcessingService( + email, + Options.Create(nntpd), + Options.Create(emailOptions), + NullLogger.Instance, + new UtcTimeProvider(new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero))); + } + + private static NinpathsCompletedFileHandler CreateHandler(NinpathsProcessingService service, string[] top1000) + { + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.Top1000 = top1000; + return new NinpathsCompletedFileHandler( + service, + Options.Create(nntpd), + NullLogger.Instance); + } + + private sealed class UtcTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utc; + + public UtcTimeProvider(DateTimeOffset utc) => _utc = utc; + + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + + public override DateTimeOffset GetUtcNow() => _utc; + } + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "ninpaths-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public string Write(string name, string content) + { + var file = System.IO.Path.Join(Path, name); + File.WriteAllBytes(file, Encoding.Latin1.GetBytes(content)); + return file; + } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsReportFormatterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsReportFormatterTests.cs new file mode 100644 index 00000000..f9e169cb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Ninpaths/NinpathsReportFormatterTests.cs @@ -0,0 +1,139 @@ +using System.Text; +using VectorNNTPD.ReaderService.Ninpaths; + +namespace VectorNNTPD.ReaderService.Tests.Ninpaths; + +public sealed class NinpathsReportFormatterTests +{ + [Fact] + public void Format_EmptyTraffic_ReturnsEmptyBuffer() + { + var stats = Read(""); + Assert.Empty(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + } + + [Fact] + public void Format_WritesInn311CompactDump() + { + var stats = Read("Path: alpha!beta!omega\nPath: alpha!beta!omega\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 100, 200, 200)); + Assert.StartsWith("!!NINP 3.1.1 100 200 2 2 200\n", dump, StringComparison.Ordinal); + Assert.Contains("!!NLREC\n", dump, StringComparison.Ordinal); + Assert.EndsWith("!!NLEND 1\n", dump, StringComparison.Ordinal); + Assert.Contains("alpha 2", dump, StringComparison.Ordinal); + Assert.Contains("beta 2", dump, StringComparison.Ordinal); + Assert.DoesNotContain("omega", dump, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd", dump, StringComparison.Ordinal); + Assert.DoesNotContain("ZCZC", dump, StringComparison.Ordinal); + + Assert.True(stats.TryGetSentTo("alpha", out _)); + var alpha = stats.Find("alpha"u8)!; + var beta = stats.Find("beta"u8)!; + Assert.Contains( + $":{alpha.Number}!{beta.Number},2", + dump, + StringComparison.Ordinal); + } + + [Fact] + public void Format_OmitsCountWhenTallyIsOne() + { + var stats = Read("Path: left!right!end\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + var left = stats.Find("left"u8)!; + var right = stats.Find("right"u8)!; + Assert.Contains($":{left.Number}!{right.Number}", dump, StringComparison.Ordinal); + Assert.DoesNotContain($":{left.Number}!{right.Number},", dump, StringComparison.Ordinal); + Assert.EndsWith("!!NLEND 1\n", dump, StringComparison.Ordinal); + } + + [Fact] + public void Format_NumbersSitesInInnHashWalkOrder() + { + var stats = Read("Path: a!b!z\nPath: c!d!z\n"); + _ = NinpathsReportFormatter.Format(stats, 1, 2, 2); + long expected = 0; + for (var i = 0; i < stats.Buckets.Length; i++) + { + for (var site = stats.Buckets[i]; site is not null; site = site.BucketNext) + { + Assert.Equal(expected, site.Number); + expected++; + } + } + + Assert.Equal(stats.SiteCount, expected); + } + + [Fact] + public void Format_RelationOrderIsInnPrependOrder() + { + var stats = Read("Path: hub!first!end\nPath: hub!second!end\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + var hub = stats.Find("hub"u8)!; + var first = stats.Find("first"u8)!; + var second = stats.Find("second"u8)!; + var expected = $":{hub.Number}!{second.Number}!{first.Number}"; + Assert.Contains(expected, dump, StringComparison.Ordinal); + } + + [Fact] + public void Format_DoesNotRereadTheSource() + { + var stats = Read("Path: a!b!c\n"); + var first = NinpathsReportFormatter.Format(stats, 5, 6, 6); + var second = NinpathsReportFormatter.Format(stats, 5, 6, 6); + Assert.Equal(first, second); + } + + [Fact] + public void ReportPeriod_UsesLocalMidnightFromInpathsFileName() + { + var time = new UtcTimeProvider(new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero)); + var (start, end, average) = NinpathsReportPeriod.Resolve( + Path.Join("logs", "inpaths-20260927.log"), + time); + Assert.Equal( + new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero).ToUnixTimeSeconds(), + start); + Assert.Equal(time.GetUtcNow().ToUnixTimeSeconds(), end); + Assert.Equal(end, average); + } + + [Fact] + public void ReportPeriod_UnknownFileName_UsesDumpTimeForAllFields() + { + var time = new UtcTimeProvider(new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero)); + var unix = time.GetUtcNow().ToUnixTimeSeconds(); + var (start, end, average) = NinpathsReportPeriod.Resolve("not-a-survey.txt", time); + Assert.Equal(unix, start); + Assert.Equal(unix, end); + Assert.Equal(unix, average); + } + + [Fact] + public void HeaderFields_AreSpaceSeparatedInInnOrder() + { + var stats = Read("Path: a!b!c\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 10, 20, 15)); + var header = dump.Split('\n')[0].Split(' '); + Assert.Equal(["!!NINP", "3.1.1", "10", "20", "2", "1", "15"], header); + } + + private static NinpathsStatistics Read(string text) + { + using var stream = new MemoryStream(Encoding.Latin1.GetBytes(text), writable: false); + return NinpathsLogReader.Read(stream); + } + + private sealed class UtcTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utc; + + public UtcTimeProvider(DateTimeOffset utc) => _utc = utc; + + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + + public override DateTimeOffset GetUtcNow() => _utc; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs new file mode 100644 index 00000000..49f45a3e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.ReaderService.Tests.NntpDb; + +public sealed class MySqlNntpDbConnectionTests +{ + [Fact] + public async Task OpenAsync_MalformedConnectionString_ThrowsConfigurationException() + { + var factory = new MySqlNntpDbConnectionFactory(); + var ex = await Assert.ThrowsAsync( + () => factory.OpenAsync(TestHostFactory.MalformedNntpDbConnectionString, CancellationToken.None)); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Contains("initialization string", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.IsType(ex.InnerException); + } + + [Fact] + public async Task OpenAsync_MalformedConnectionString_DoesNotExposePassword() + { + var factory = new MySqlNntpDbConnectionFactory(); + var ex = await Assert.ThrowsAsync( + () => factory.OpenAsync( + TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + CancellationToken.None)); + + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.Reason, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("Password=", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ConvertSelectOneScalar_AcceptsInt32() + { + Assert.Equal(1, MySqlNntpDbConnection.ConvertSelectOneScalar(1)); + } + + [Fact] + public void ConvertSelectOneScalar_AcceptsMySqlInt64() + { + Assert.Equal(1, MySqlNntpDbConnection.ConvertSelectOneScalar(1L)); + } + + [Theory] + [InlineData((ulong)4_294_967_295)] + public void ConvertUnsignedCount_PreservesUInt32Max(ulong expected) + { + Assert.Equal(expected, MySqlNntpDbConnection.ConvertUnsignedCount(uint.MaxValue, "count_high", "g")); + } + + [Fact] + public void ConvertUnsignedCount_PreservesUInt64() + { + Assert.Equal(ulong.MaxValue, MySqlNntpDbConnection.ConvertUnsignedCount(ulong.MaxValue, "count_high", "g")); + } + + [Fact] + public void ConvertUnsignedCount_RejectsNegativeAndFloatingPoint() + { + Assert.Throws( + () => MySqlNntpDbConnection.ConvertUnsignedCount(-1, "count_low", "g")); + Assert.Throws( + () => MySqlNntpDbConnection.ConvertUnsignedCount(1.5d, "count_low", "g")); + } + + [Theory] + [InlineData((byte)'y')] + [InlineData((byte)'n')] + [InlineData((byte)'m')] + [InlineData((byte)'x')] + [InlineData((byte)'j')] + public void ConvertPostingStatus_AcceptsSingleOctet(byte status) + { + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus(status, "g")); + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus(((char)status).ToString(), "g")); + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus((char)status, "g")); + } + + [Theory] + [InlineData((byte)'z')] + [InlineData((byte)'Y')] + [InlineData((byte)'=')] + public void ConvertPostingStatus_RejectsUnknownAndEquals(byte status) + { + Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus(status, "g")); + Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus(((char)status).ToString(), "g")); + } + + [Fact] + public void ConvertPostingStatus_RejectsEqualsAliasString() + { + var ex = Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus("=other.group", "g")); + Assert.Contains("=", ex.Message, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbOptionsTests.cs new file mode 100644 index 00000000..e9f603f4 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbOptionsTests.cs @@ -0,0 +1,188 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Hosting; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.NntpDb; + +public sealed class NntpDbOptionsTests +{ + [Fact] + public void Defaults_MatchProductionValues() + { + var options = new NntpDbOptions(); + Assert.Equal(TimeSpan.FromSeconds(15), options.StartupTimeout); + Assert.Equal("NntpDb", NntpDbOptions.SectionName); + Assert.Equal("NntpDB", NntpDbOptions.ConnectionStringName); + Assert.Equal("ConnectionStrings__NntpDB", NntpDbOptions.ConnectionStringEnvironmentVariable); + } + + [Fact] + public void Validate_Succeeds_ForDefaultsWithConnectionString() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenConnectionStringMissing() + { + var result = new NntpDbOptionsValidator().Validate(null, new NntpDbOptions()); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("ConnectionStrings:NntpDB", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("Password", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_Fails_WhenConnectionStringIsMalformed() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.MalformedNntpDbConnectionString }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("NntpDB connection string is invalid", StringComparison.Ordinal)); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("initialization string", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains(TestHostFactory.MalformedNntpDbConnectionString, StringComparison.Ordinal)); + } + + [Fact] + public void Validate_MalformedConnectionString_DoesNotExposePassword() + { + var options = new NntpDbOptions + { + ConnectionString = TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains(TestHostFactory.FakeNntpDbPassword, StringComparison.Ordinal)); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains("Password=", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_Succeeds_WhenConnectionStringIsAcceptedByMySqlConnector() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Succeeded); + NntpDbConnectionString.Validate(options.ConnectionString); + } + + [Fact] + public void Validate_Fails_WhenStartupTimeoutNotPositive() + { + var options = new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.Zero, + }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Failed); + } + + [Fact] + public void Bind_ReadsConnectionStringsNntpDB() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:NntpDB"] = TestHostFactory.TestNntpDbConnectionString, + ["NntpDb:StartupTimeout"] = "00:00:20", + }) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddNntpDbOptions(); + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(TestHostFactory.TestNntpDbConnectionString, options.ConnectionString); + Assert.Equal(TimeSpan.FromSeconds(20), options.StartupTimeout); + } + + [Fact] + public void ProductionAppsettings_DoesNotContainConnectionStrings() + { + var path = FindProductionAppsettings(); + using var document = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.False(document.RootElement.TryGetProperty("ConnectionStrings", out _)); + Assert.False(File.ReadAllText(path).Contains("GrabberDB", StringComparison.Ordinal)); + + var nntpDb = document.RootElement.GetProperty("NntpDb"); + Assert.Equal("00:00:15", nntpDb.GetProperty("StartupTimeout").GetString()); + Assert.False(nntpDb.TryGetProperty("ConnectionString", out _)); + Assert.False(nntpDb.TryGetProperty("MinimumPoolSize", out _)); + Assert.False(nntpDb.TryGetProperty("MaximumPoolSize", out _)); + Assert.False(nntpDb.TryGetProperty("MaximumIdleTime", out _)); + Assert.False(nntpDb.TryGetProperty("MaintenanceInterval", out _)); + Assert.False(nntpDb.TryGetProperty("AcquisitionTimeout", out _)); + } + + [Fact] + public void Same_connection_strings_nntpdb_value_targets_nntpdb_catalog() + { + const string connectionString = + "Server=198.18.0.70;Port=3306;Database=nntpdb;User ID=nntpd;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=32;ConnectionIdleTimeout=300;"; + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = connectionString, + }) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddNntpDbOptions(); + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(connectionString, options.ConnectionString); + Assert.Equal("nntpdb", new MySqlConnector.MySqlConnectionStringBuilder(options.ConnectionString).Database); + Assert.Equal("ConnectionStrings__NntpDB", NntpDbOptions.ConnectionStringEnvironmentVariable); + } + + [Fact] + public void Factory_DoesNotOverrideSuppliedPoolingSetting() + { + using var disabled = MySqlNntpDbConnectionFactory.CreateConnection( + "Server=db;Database=nntpdb;User ID=nntpd;Pooling=false;"); + var disabledPooling = new MySqlConnector.MySqlConnectionStringBuilder(disabled.ConnectionString).Pooling; + Assert.False(disabledPooling); + + using var enabled = MySqlNntpDbConnectionFactory.CreateConnection( + "Server=db;Database=nntpdb;User ID=nntpd;Pooling=true;"); + var enabledPooling = new MySqlConnector.MySqlConnectionStringBuilder(enabled.ConnectionString).Pooling; + Assert.True(enabledPooling); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbServiceTests.cs new file mode 100644 index 00000000..8304ca42 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpDbServiceTests.cs @@ -0,0 +1,422 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.NntpDb; + +public sealed class NntpDbServiceTests +{ + [Fact] + public async Task StartAsync_OpensConnection_SelectsOne_AndDisposesLogicalConnection() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenCount); + var connection = Assert.Single(factory.Connections); + Assert.Equal(1, connection.SelectOneCount); + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task StartAsync_Fails_WhenMySqlIsUnavailable() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbUnavailableException("down"), + }; + var service = CreateService(factory, startupTimeout: TimeSpan.FromMilliseconds(1)); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("startup", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_FailsImmediately_WithoutRetry() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbUnavailableException("should not be reached"), + }; + var clock = new ControllableTimeProvider(); + var logger = new CollectingLogger(); + var service = CreateService( + factory, + startupTimeout: TimeSpan.FromMinutes(1), + clock: clock, + connectionString: TestHostFactory.MalformedNntpDbConnectionString, + logger: logger); + + var starting = service.StartAsync(CancellationToken.None); + var ex = await Assert.ThrowsAsync( + () => starting.WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Contains("initialization string", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.Contains("index", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.IsType(ex.InnerException); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + Assert.Contains( + logger.Messages, + static message => message.Contains("NntpDB connection string is invalid", StringComparison.Ordinal)); + Assert.DoesNotContain( + logger.Messages, + static message => message.Contains("startup connectivity retry", StringComparison.Ordinal)); + Assert.Equal(1, logger.Messages.Count(static message => + message.Contains("NntpDB connection string is invalid", StringComparison.Ordinal))); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_DoesNotExposePassword() + { + var factory = new FakeNntpDbConnectionFactory(); + var logger = new CollectingLogger(); + var service = CreateService( + factory, + connectionString: TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + logger: logger); + + var ex = await Assert.ThrowsAsync( + () => service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + AssertSecretNotExposed(ex.Message); + AssertSecretNotExposed(ex.Reason); + AssertSecretNotExposed(ex.ToString()); + foreach (var message in logger.Messages) + { + AssertSecretNotExposed(message); + } + + foreach (var logged in logger.Exceptions) + { + AssertSecretNotExposed(logged.ToString()); + } + + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + } + + [Fact] + public async Task StartAsync_ValidConnectionString_ProceedsPastParsing() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory, connectionString: TestHostFactory.TestNntpDbConnectionString); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.Equal(1, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenAttemptCount); + Assert.Equal(1, factory.OpenCount); + } + + [Fact] + public async Task StartAsync_CanceledToken_IsNotConvertedToConfigurationFailure() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory, connectionString: TestHostFactory.TestNntpDbConnectionString); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => service.StartAsync(cts.Token)); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.False(service.HasStarted); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_WithRealFactory_DoesNotOpen() + { + var clock = new ControllableTimeProvider(); + var service = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.MalformedNntpDbConnectionString, + StartupTimeout = TimeSpan.FromMinutes(1), + }), + NullLogger.Instance, + clock); + + var ex = await Assert.ThrowsAsync( + () => service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.False(service.HasStarted); + } + + [Fact] + public async Task Lifecycle_MalformedConnectionString_FailsStartupWithConfigurationException() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService( + factory, + startupTimeout: TimeSpan.FromMinutes(1), + connectionString: TestHostFactory.MalformedNntpDbConnectionString); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartAsync_FailsImmediately_OnAuthenticationFailure() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbAuthenticationException("denied"), + }; + var service = CreateService(factory, startupTimeout: TimeSpan.FromMinutes(1)); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task StartAsync_FailsImmediately_WhenSelectOneReturnsWrongValue() + { + var factory = new FakeNntpDbConnectionFactory { NextHealthCheckResult = 0 }; + var service = CreateService(factory); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("SELECT 1", ex.Message, StringComparison.Ordinal); + Assert.False(service.HasStarted); + Assert.Equal(1, Assert.Single(factory.Connections).DisposeCount); + } + + [Fact] + public async Task StartAsync_FailsImmediately_WhenSelectOneThrows() + { + var factory = new FakeNntpDbConnectionFactory + { + NextHealthCheckException = new NntpDbUnavailableException("MySQL health query SELECT 1 failed."), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasStarted); + Assert.Equal(1, Assert.Single(factory.Connections).DisposeCount); + } + + [Fact] + public async Task StartAsync_RetriesTransientOpenFailure_ThenSucceeds() + { + var factory = new FakeNntpDbConnectionFactory { RemainingOpenFailures = 1 }; + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.Equal(0, factory.RemainingOpenFailures); + Assert.Equal(2, factory.OpenAttemptCount); + Assert.Equal(2, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenCount); + Assert.Equal(1, Assert.Single(factory.Connections).SelectOneCount); + Assert.Equal(1, factory.Connections[0].DisposeCount); + } + + [Fact] + public async Task StartAsync_CancellationDuringOpen_ResetsState_AndIsRetryable() + { + var factory = new FakeNntpDbConnectionFactory + { + BlockOpen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + OpenStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var starting = service.StartAsync(cts.Token); + await factory.OpenStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => starting); + factory.BlockOpen.TrySetResult(); + factory.BlockOpen = null; + factory.OpenStarted = null; + + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + Assert.Empty(factory.Connections); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, Assert.Single(factory.Connections).SelectOneCount); + } + + [Fact] + public async Task StartAsync_CancellationDuringSelectOne_DisposesConnection_AndRetriesFullStartup() + { + var factory = new FakeNntpDbConnectionFactory + { + BlockSelectOne = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + SelectOneStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var starting = service.StartAsync(cts.Token); + await factory.SelectOneStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => starting); + factory.BlockSelectOne.TrySetResult(); + factory.BlockSelectOne = null; + factory.SelectOneStarted = null; + + var first = Assert.Single(factory.Connections); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(0, first.SelectOneCount); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, factory.Connections.Sum(static connection => connection.SelectOneCount)); + Assert.All(factory.Connections, static connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task OpenAsync_CreatesLogicalConnection_AndDoesNotRetainIt() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.Equal(1, factory.OpenCount); + + await using (var connection = await service.OpenAsync()) + { + Assert.Equal(2, factory.OpenCount); + Assert.Equal(0, ((FakeNntpDbConnection)connection).DisposeCount); + } + + Assert.Equal(2, factory.OpenCount); + Assert.All(factory.Connections, static connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_RejectsNewOpens() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + await service.StopAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => service.OpenAsync().AsTask()); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task Lifecycle_StartsServiceBeforeRunning_AndStopsIt() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.True(service.HasStarted); + Assert.Equal(1, factory.OpenCount); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + await Assert.ThrowsAsync(() => service.OpenAsync().AsTask()); + } + + [Fact] + public void Options_NeverExposeSecretPropertyNamesOnValidatorFailures() + { + var options = new NntpDbOptions + { + ConnectionString = "Server=db;Password=super-secret;User ID=nntpd;", + StartupTimeout = TimeSpan.Zero, + }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("super-secret", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("Password=", StringComparison.Ordinal)); + } + + private static NntpDbService CreateService( + FakeNntpDbConnectionFactory factory, + TimeSpan? startupTimeout = null, + TimeProvider? clock = null, + string? connectionString = null, + ILogger? logger = null) + { + var options = new NntpDbOptions + { + ConnectionString = connectionString ?? TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = startupTimeout ?? TimeSpan.FromSeconds(15), + }; + return new NntpDbService( + factory, + Options.Create(options), + logger ?? NullLogger.Instance, + clock ?? TimeProvider.System); + } + + private static void AssertSecretNotExposed(string text) + { + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, text, StringComparison.Ordinal); + Assert.DoesNotContain("Password=", text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(TestHostFactory.MalformedNntpDbConnectionStringWithPassword, text, StringComparison.Ordinal); + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public List Exceptions { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + if (exception is not null) + { + Exceptions.Add(exception); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs new file mode 100644 index 00000000..4d3e2c92 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs @@ -0,0 +1,265 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.NntpDb; + +public sealed class NntpSharedConfigurationServiceTests +{ + [Fact] + public async Task Start_publishes_one_shared_configuration_row() + { + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = + [ + new NntpSharedConfigurationCandidate( + 4096, + "news.example", + "http://prom.example", + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ], + }; + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(4096, service.Current.MaxArticleBytes); + Assert.Equal("news.example", service.Current.SiteName); + Assert.Equal("http://prom.example", service.Current.PrometheusUrl); + await service.DisposeAsync(); + } + + [Fact] + public async Task Start_fails_when_the_row_is_invalid_and_does_not_publish() + { + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [], + }; + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + await service.DisposeAsync(); + } + + [Fact] + public async Task Start_copies_shared_credentials_and_does_not_log_them() + { + var nntp = Options.Create(new NntpdOptions + { + ServerId = 1, + BindPortTls = 0, + AcmeEmail = "from-env@example.test", + AcmeCertificatePassword = "from-env-pfx", + CloudFlareApiKey = "from-env-key", + }); + var rabbit = Options.Create(new RabbitMqOptions + { + Username = "from-env-user", + Password = "from-env-pass", + Hosts = ["127.0.0.1"], + VirtualHost = "/articles", + }); + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow()], + }); + var service = new NntpSharedConfigurationService( + database, + nntp, + logger, + TimeProvider.System, + TimeSpan.FromHours(1), + rabbit); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderAcmeAccount, nntp.Value.AcmeEmail); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderAcmeCertificatePassword, nntp.Value.AcmeCertificatePassword); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, nntp.Value.CloudFlareApiKey); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqUsername, rabbit.Value.Username); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, rabbit.Value.Password); + Assert.Equal("/articles", rabbit.Value.VirtualHost); + Assert.Equal("127.0.0.1", Assert.Single(rabbit.Value.Hosts!)); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderAcmeAccount, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderAcmeCertificatePassword, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderRabbitMqUsername, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain("from-env-pfx", logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain("from-env-pass", logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Invalid_shared_credentials_prevent_the_first_snapshot() + { + const string secret = "credential-must-not-be-logged-xxxxxxxxxxxxxxxxxxxx"; + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow() with { CloudFlareApiKey = secret }], + }); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + logger, + TimeProvider.System, + TimeSpan.FromHours(1)); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + Assert.Contains("cloudflareapikey", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(secret, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(secret, logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Later_invalid_refresh_retains_the_previous_credentials() + { + const string secret = "refresh-credential-must-not-be-logged-xxxxxxxxxxxx"; + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow()], + }; + var clock = new ControllableTimeProvider(); + var nntp = Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }); + var rabbit = Options.Create(new RabbitMqOptions()); + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + nntp, + logger, + clock, + TimeSpan.FromSeconds(60), + rabbit); + await service.StartAsync(CancellationToken.None); + var published = service.Current; + + factory.SharedConfigurationRows = [ValidRow() with { RabbitMqPassword = secret }]; + await WaitUntilAsync(() => clock.HasScheduledTimers); + clock.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => logger.Text.Contains("retaining the last known-good snapshot", StringComparison.Ordinal)); + + Assert.Equal(published, service.Current); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, rabbit.Value.Password); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, nntp.Value.CloudFlareApiKey); + Assert.DoesNotContain(secret, logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Open_before_startup_is_refused() + { + var database = new NntpDbService( + new FakeNntpDbConnectionFactory(), + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await Assert.ThrowsAsync(() => database.OpenAsync(CancellationToken.None).AsTask()); + await database.DisposeAsync(); + } + + private static NntpSharedConfigurationCandidate ValidRow() => + new( + 4096, + "news.example", + null, + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"); + + private static async Task WaitUntilAsync(Func condition) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(10, timeout.Token); + } + } + + private static async Task StartDatabaseAsync(FakeNntpDbConnectionFactory factory) + { + var database = new NntpDbService( + factory, + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await database.StartAsync(CancellationToken.None); + return database; + } + + private sealed class CaptureLogger : ILogger + { + private readonly List _entries = []; + + public string Text + { + get + { + lock (_entries) + { + return string.Join('\n', _entries); + } + } + } + + public IDisposable BeginScope(TState state) + where TState : notnull => + NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var text = formatter(state, exception); + if (exception is not null) + { + text = text + " " + exception; + } + + lock (_entries) + { + _entries.Add(text); + } + } + + private sealed class NullScope : IDisposable + { + internal static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/InMemoryPostFilterPolicyRepository.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/InMemoryPostFilterPolicyRepository.cs new file mode 100644 index 00000000..bd11c2b6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/InMemoryPostFilterPolicyRepository.cs @@ -0,0 +1,39 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// In-memory NntpDB stand-in for PostFilter policy load/refresh tests. +internal sealed class InMemoryPostFilterPolicyRepository : IPostFilterPolicyRepository +{ + public PostFilterPolicyRecord? Record { get; set; } + + public Exception? Exception { get; set; } + + public int LoadCount { get; private set; } + + public ValueTask LoadAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + LoadCount++; + if (Exception is not null) + { + throw Exception; + } + + if (Record is null) + { + throw new InvalidOperationException( + "nntppostfiltercurrent is missing policy_id = 1. NNTPD will not invent a local PostFilter policy."); + } + + return ValueTask.FromResult(Record); + } + + public static PostFilterPolicyRecord Create( + PostFilterOptions options, + long revision = 1, + DateTimeOffset? updatedUtc = null) => + new(revision, updatedUtc ?? DateTimeOffset.UtcNow, options); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LiveNntpPostFilterRuntimeVerificationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LiveNntpPostFilterRuntimeVerificationTests.cs new file mode 100644 index 00000000..b6a70610 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LiveNntpPostFilterRuntimeVerificationTests.cs @@ -0,0 +1,317 @@ +using System.Globalization; +using System.Net; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MySqlConnector; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// Live-runtime PostFilter verification against the real nntp database. +/// +/// +/// Requires VECTORNNTP_NNTPDB_INTEGRATION with Database=nntp. +/// Any other database name fails rather than running against a disposable +/// nntpgroups_pfval_* target. Loads revision 2 through +/// and +/// ; does not publish, update, delete, +/// or roll back PostFilter policy. The only permitted write is append-only +/// rejection evidence from the real POST/evidence path. +/// +public sealed class LiveNntpPostFilterRuntimeVerificationTests +{ + private const string DeploymentMessageId = ""; + private const string Gtube = + "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"; + + [NntpDbIntegrationFact] + public async Task LiveNntp_Revision2_IsLoadedAndEnforcedOnPost() + { + var connectionString = RequireLiveNntp(); + await AssertLiveRevisionUnchangedAsync(connectionString); + var deploymentBefore = await ReadEvidenceAsync(connectionString, DeploymentMessageId); + Assert.NotNull(deploymentBefore); + Assert.Equal(2L, deploymentBefore.Revision); + var rejectionCountBefore = await CountRejectionsAsync(connectionString); + + var leftover = ValidateProductionAppsettingsHasNoPostFilterSection(); + Assert.True(leftover.Succeeded, leftover.FailureMessage); + + var policyLog = new CollectingLogger(); + await using var nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + Options.Create(NntpDbIntegration.CreateOptions(connectionString)), + NullLogger.Instance); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + var loaded = await repository.LoadAsync(); + Assert.Equal(2, loaded.Revision); + Assert.Equal(PostFilterGateState.Active, loaded.Options.Gate); + Assert.True(loaded.Options.SpamAssassin.Enabled); + Assert.Equal(PostFilterSpamOnFailure.Reject, loaded.Options.SpamAssassin.OnFailure); + Assert.Equal(131072, loaded.Options.SpamAssassin.MaxArticleSize); + Assert.Equal(["198.18.0.70"], loaded.Options.SpamAssassin.Hosts); + Assert.Equal(["YEncoded"], loaded.Options.SpamAssassin.ExcludeArtTypes); + + await using var policy = new PostFilterPolicyService(repository, policyLog); + await policy.StartAsync(CancellationToken.None); + Assert.Equal(2, policy.Current.Revision); + Assert.Equal(PostFilterGateState.Active, policy.Current.Gate); + Assert.True(policy.Current.SpamAssassinEnabled); + Assert.Equal(PostFilterSpamOnFailure.Reject, policy.Current.SpamAssassinOnFailure); + Assert.Equal(131072, policy.Current.SpamAssassinMaxArticleSize); + Assert.Equal(["198.18.0.70"], policy.Current.SpamAssassinHosts); + Assert.Equal(ArticleType.YEncoded, policy.Current.SpamAssassinExcludeArtTypes); + Assert.Contains( + policyLog.Messages, + static message => message.Contains("revision=2", StringComparison.Ordinal) + && message.Contains("gate=Active", StringComparison.Ordinal) + && message.Contains("spamAssassin=True", StringComparison.Ordinal)); + Assert.DoesNotContain( + policyLog.Messages, + static message => message.Contains("Nntpd:PostFilter", StringComparison.Ordinal) + || message.Contains("appsettings", StringComparison.OrdinalIgnoreCase) + || message.Contains("fallback", StringComparison.OrdinalIgnoreCase)); + + var queue = new PostFilterRejectionEvidenceQueue(); + var writer = new PostFilterRejectionEvidenceService( + nntpDb, + queue, + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + await using var spamd = new SpamdCheckClient(); + var filter = new PostFilterEvaluator( + policy, + new InMemoryPostFilterQuotaStore(), + new PostFilterReservationIdentity("runtime-verify", "inc"), + spamd, + NullLogger.Instance); + + var rejectToken = "vectortest-postfilter-runtime-reject-" + Guid.NewGuid().ToString("N"); + var rejectId = "<" + rejectToken + "@example.com>"; + await using (var duplex = new PostFilterPostDuplex()) + { + var session = duplex.CreateSession( + PostFilterPostHarness.NewQueue(), + filter, + clientAddress: IPAddress.Parse("198.18.0.70"), + postFilterEvidence: queue); + await PostFilterPostHarness.PostAsync( + duplex, + session, + "441 Posting failed", + article: PostFilterPostHarness.TextArticle(Gtube + "\r\n", rejectToken)); + } + + await WaitForAsync(() => queue.Written >= 1 && queue.WriteFailures == 0); + var rejectRow = await ReadEvidenceAsync(connectionString, rejectId); + Assert.NotNull(rejectRow); + var liveCurrent = Convert.ToInt64( + await ScalarAsync(connectionString, "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"), + CultureInfo.InvariantCulture); + Assert.Equal(2L, liveCurrent); + Assert.Equal(liveCurrent, rejectRow.Revision); + Assert.Equal("198.18.0.70", rejectRow.SourceIpText); + Assert.Equal(4, rejectRow.SourceIpLength); + Assert.True(rejectRow.HasPayload); + Assert.Equal(rejectRow.ArticleSize, rejectRow.PayloadLength); + Assert.True(rejectRow.ArticleSize > 0); + Assert.Equal("SpamAssassin", rejectRow.Stage); + Assert.True( + rejectRow.Reason is "spam" or "scanner-failure", + "Unexpected PostFilter reason: " + rejectRow.Reason); + + var acceptToken = "vectortest-postfilter-runtime-accept-" + Guid.NewGuid().ToString("N"); + var acceptId = "<" + acceptToken + "@example.com>"; + await using (var duplex = new PostFilterPostDuplex()) + { + var session = duplex.CreateSession( + PostFilterPostHarness.NewQueue(), + filter, + clientAddress: IPAddress.Parse("198.18.0.70"), + postFilterEvidence: queue); + await PostFilterPostHarness.PostAsync( + duplex, + session, + "240 Article received OK", + article: PostFilterPostHarness.TextArticle("This is a short ham body.\r\n", acceptToken)); + } + + await writer.StopAsync(CancellationToken.None); + await policy.StopAsync(CancellationToken.None); + + await AssertLiveRevisionUnchangedAsync(connectionString); + var deploymentAfter = await ReadEvidenceAsync(connectionString, DeploymentMessageId); + Assert.NotNull(deploymentAfter); + Assert.Equal(deploymentBefore.RejectionId, deploymentAfter.RejectionId); + Assert.Equal(deploymentBefore.ArticleSize, deploymentAfter.ArticleSize); + Assert.Equal(deploymentBefore.Reason, deploymentAfter.Reason); + Assert.Equal(rejectionCountBefore + 1, await CountRejectionsAsync(connectionString)); + Assert.Null(await ReadEvidenceAsync(connectionString, acceptId)); + } + + private static string RequireLiveNntp() + { + var connectionString = NntpDbIntegration.TryGetConnectionString() + ?? throw new InvalidOperationException(NntpDbIntegration.SkipReason); + var database = new MySqlConnectionStringBuilder(connectionString).Database; + Assert.Equal("nntp", database); + return connectionString; + } + + private static ValidateOptionsResult ValidateProductionAppsettingsHasNoPostFilterSection() + { + var path = FindProductionAppsettings(); + Assert.True(File.Exists(path), "Production VectorNNTPD.ReaderService.json was not found: " + path); + var configuration = new ConfigurationBuilder().AddJsonFile(path, optional: false).Build(); + Assert.False(configuration.GetSection("Nntpd:PostFilter").Exists()); + return new PostFilterLeftoverConfigurationValidator(configuration).Validate(null, new NntpdOptions()); + } + + private static string FindProductionAppsettings() + { + var relative = Path.Join("src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, relative); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + return Path.GetFullPath(relative); + } + + private static async Task AssertLiveRevisionUnchangedAsync(string connectionString) + { + var current = Convert.ToInt64( + await ScalarAsync(connectionString, "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"), + CultureInfo.InvariantCulture); + Assert.Equal(2L, current); + var gate = Convert.ToString( + await ScalarAsync( + connectionString, + "SELECT gate FROM nntppostfilterpolicy WHERE revision = 2"), + CultureInfo.InvariantCulture); + Assert.Equal("Active", gate); + var sa = Convert.ToString( + await ScalarAsync( + connectionString, + "SELECT sa_enabled FROM nntppostfilterpolicy WHERE revision = 2"), + CultureInfo.InvariantCulture); + Assert.Equal("Y", sa); + var historical = Convert.ToString( + await ScalarAsync( + connectionString, + "SELECT gate FROM nntppostfilterpolicy WHERE revision = 1"), + CultureInfo.InvariantCulture); + Assert.Equal("Disabled", historical); + } + + private static async Task CountRejectionsAsync(string connectionString) => + Convert.ToInt64( + await ScalarAsync(connectionString, "SELECT COUNT(*) FROM nntppostfilterrejections"), + CultureInfo.InvariantCulture); + + private static async Task ReadEvidenceAsync(string connectionString, string messageId) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + SELECT rejection_id, revision, INET6_NTOA(source_ip), OCTET_LENGTH(source_ip), + article_size, OCTET_LENGTH(article_payload), stage, reason + FROM nntppostfilterrejections + WHERE message_id = @id + """; + command.Parameters.AddWithValue("@id", messageId); + await using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + return null; + } + + return new EvidenceRow( + reader.GetInt64(0), + reader.GetInt64(1), + reader.GetString(2), + reader.GetInt32(3), + reader.GetInt32(4), + reader.IsDBNull(5) ? 0 : reader.GetInt32(5), + !reader.IsDBNull(5), + reader.GetString(6), + reader.GetString(7)); + } + + private static async Task ScalarAsync(string connectionString, string sql) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + return await command.ExecuteScalarAsync(); + } + + private static async Task WaitForAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(20, cts.Token); + } + } + + private sealed record EvidenceRow( + long RejectionId, + long Revision, + string SourceIpText, + int SourceIpLength, + int ArticleSize, + int PayloadLength, + bool HasPayload, + string Stage, + string Reason); + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LocalSpamdServer.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LocalSpamdServer.cs new file mode 100644 index 00000000..faa61abb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/LocalSpamdServer.cs @@ -0,0 +1,320 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// Loopback SPAMC listener used by ordinary tests. It accepts real TCP connections +/// and speaks the CHECK framing the production client writes. +/// +internal sealed class LocalSpamdServer : IAsyncDisposable +{ + private const int MaxHeaderBytes = 8192; + + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _accept; + private readonly ConcurrentQueue _connections = new(); + private readonly ConcurrentQueue _requests = new(); + private readonly ConcurrentQueue _faults = new(); + private readonly TaskCompletionSource _accepted = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly bool _keepOpen; + private readonly bool _hang; + private readonly bool _dropAfterRequest; + private readonly TimeSpan _delay; + private readonly byte[]? _response; + private readonly byte[]? _subsequentResponse; + private int _connectionCount; + private int _checks; + private int _inflight; + private int _peak; + + private LocalSpamdServer( + TcpListener listener, + bool keepOpen, + bool hang, + bool dropAfterRequest, + TimeSpan delay, + byte[]? response, + byte[]? subsequentResponse) + { + _listener = listener; + _keepOpen = keepOpen; + _hang = hang; + _dropAfterRequest = dropAfterRequest; + _delay = delay; + _response = response; + _subsequentResponse = subsequentResponse; + Port = ((IPEndPoint)listener.LocalEndpoint).Port; + _accept = AcceptLoopAsync(); + } + + /// Gets the loopback port selected by the listener. + public int Port { get; } + + /// Gets how many TCP connections were accepted. + public int Connections => Volatile.Read(ref _connectionCount); + + /// Gets how many complete CHECK bodies were read. + public int Checks => Volatile.Read(ref _checks); + + /// Gets the highest number of connections inside ServeAsync at once. + public int PeakConnections => Volatile.Read(ref _peak); + + /// Completes when the first connection is accepted. + public Task Accepted => _accepted.Task; + + /// Gets CHECK payloads in arrival order. + public IReadOnlyList Requests => _requests.ToArray(); + + /// Gets protocol violations observed while reading requests. + public IReadOnlyList ProtocolFaults => _faults.ToArray(); + + /// Binds on an ephemeral port and starts accepting. + public static async Task StartAsync( + bool keepOpen, + bool hang = false, + bool dropAfterRequest = false, + TimeSpan delay = default, + byte[]? response = null, + byte[]? subsequentResponse = null) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var server = new LocalSpamdServer( + listener, + keepOpen, + hang, + dropAfterRequest, + delay, + response, + subsequentResponse); + await Task.Yield(); + return server; + } + + /// Clears recorded protocol faults after a test has asserted them. + public void AcknowledgeProtocolFaults() + { + while (_faults.TryDequeue(out _)) + { + // Drain remaining recorded faults after the test asserted them. + } + } + + /// + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync().ConfigureAwait(false); + _listener.Stop(); + try + { + await _accept.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + foreach (var connection in _connections) + { + try + { + await connection.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + _cts.Dispose(); + if (!_faults.IsEmpty) + { + throw new InvalidOperationException( + "Local SPAMD server observed protocol faults: " + string.Join("; ", ProtocolFaults)); + } + } + + private async Task AcceptLoopAsync() + { + while (!_cts.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(_cts.Token).ConfigureAwait(false); + } + catch (Exception ex) when (_cts.IsCancellationRequested) + { + _ = ex.GetType(); // observed: accept loop canceled during server stop + return; + } + + Interlocked.Increment(ref _connectionCount); + _accepted.TrySetResult(); + var serve = Task.Run(() => ServeAsync(client), CancellationToken.None); + _connections.Enqueue(serve); + } + } + + private async Task ServeAsync(TcpClient client) + { + using (client) + { + var inflight = Interlocked.Increment(ref _inflight); + ObservePeak(inflight); + try + { + await using var stream = client.GetStream(); + do + { + if (_hang) + { + await Task.Delay(Timeout.Infinite, _cts.Token).ConfigureAwait(false); + } + + if (!await TryReadCheckAsync(stream).ConfigureAwait(false)) + { + return; + } + + Interlocked.Increment(ref _checks); + if (_dropAfterRequest) + { + return; + } + + if (_delay > TimeSpan.Zero) + { + await Task.Delay(_delay, _cts.Token).ConfigureAwait(false); + } + + var check = Volatile.Read(ref _checks); + var bytes = check == 1 || _subsequentResponse is null + ? _response ?? "SPAMD/1.1 0 EX_OK\r\nSpam: False ; 0.0 / 5.0\r\n\r\n"u8.ToArray() + : _subsequentResponse; + await stream.WriteAsync(bytes, _cts.Token).ConfigureAwait(false); + await stream.FlushAsync(_cts.Token).ConfigureAwait(false); + } + while (_keepOpen); + } + catch (OperationCanceledException ex) when (_cts.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + Interlocked.Decrement(ref _inflight); + } + } + } + + private async Task TryReadCheckAsync(NetworkStream stream) + { + var buffer = new byte[MaxHeaderBytes]; + var total = 0; + var headerEnd = -1; + while (total < buffer.Length && headerEnd < 0) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), _cts.Token).ConfigureAwait(false); + if (read == 0) + { + if (total == 0) + { + return false; + } + + _faults.Enqueue("unexpected-eof-in-headers"); + return false; + } + + total += read; + headerEnd = buffer.AsSpan(0, total).IndexOf("\r\n\r\n"u8); + } + + if (headerEnd < 0) + { + _faults.Enqueue("header-too-long"); + return false; + } + + var headerText = Encoding.ASCII.GetString(buffer, 0, headerEnd); + if (!headerText.StartsWith("CHECK SPAMC/", StringComparison.Ordinal) + && !headerText.StartsWith("SYMBOLS SPAMC/", StringComparison.Ordinal)) + { + _faults.Enqueue("unexpected-command"); + return false; + } + + int? length = null; + foreach (var line in headerText.Split("\r\n")) + { + if (!line.StartsWith("Content-length:", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!int.TryParse(line["Content-length:".Length..].Trim(), out var parsed) || parsed < 0) + { + _faults.Enqueue("invalid-content-length"); + return false; + } + + length = parsed; + } + + if (length is not int contentLength) + { + _faults.Enqueue("missing-content-length"); + return false; + } + + var body = new byte[contentLength]; + var already = total - (headerEnd + 4); + if (already > contentLength) + { + _faults.Enqueue("body-longer-than-content-length"); + return false; + } + + if (already > 0) + { + buffer.AsSpan(headerEnd + 4, already).CopyTo(body); + } + + var filled = already; + while (filled < contentLength) + { + var read = await stream.ReadAsync(body.AsMemory(filled), _cts.Token).ConfigureAwait(false); + if (read == 0) + { + _faults.Enqueue("short-body"); + return false; + } + + filled += read; + } + + _requests.Enqueue(body); + return true; + } + + private void ObservePeak(int inflight) + { + while (true) + { + var peak = Volatile.Read(ref _peak); + if (inflight <= peak || Interlocked.CompareExchange(ref _peak, inflight, peak) == peak) + { + return; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterEvidenceAndLivePolicyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterEvidenceAndLivePolicyTests.cs new file mode 100644 index 00000000..27ff9046 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterEvidenceAndLivePolicyTests.cs @@ -0,0 +1,282 @@ +using System.Globalization; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +[Collection("NntpDbPostFilter")] +public sealed class MySqlPostFilterEvidenceAndLivePolicyTests +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + + public MySqlPostFilterEvidenceAndLivePolicyTests(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [NntpDbIntegrationFact] + public async Task InsertRejection_PersistsMetadataAndPayload() + { + RequireReady(); + var messageId = ""; + var payload = Encoding.UTF8.GetBytes("Message-ID: " + messageId + "\r\n\r\nbody\r\n"); + var evidence = new PostFilterRejectionEvidence( + DateTimeOffset.UtcNow, + policyRevision: 1, + accountName: "poster", + sourceAddress: IPAddress.Parse("198.51.100.10"), + artType: ArticleType.Default, + messageId: messageId, + articleSize: payload.Length, + stage: PostFilterStage.Deny, + reason: "denied", + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + articlePayload: payload); + await _fixture.InsertRejectionAsync(evidence); + + var count = await _fixture.ExecuteRawScalarAsync( + "SELECT COUNT(*) FROM nntppostfilterrejections WHERE message_id = @id AND reason = 'denied'", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(1L, Convert.ToInt64(count, CultureInfo.InvariantCulture)); + + var storedName = await _fixture.ExecuteRawScalarAsync( + "SELECT account_name FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(PostFilterAccountIdentity.FromUsername("poster"), Convert.ToString(storedName)); + Assert.NotEqual("poster", Convert.ToString(storedName)); + + var storedType = await _fixture.ExecuteRawScalarAsync( + "SELECT art_type FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal((uint)ArticleType.Default, Convert.ToUInt32(storedType, CultureInfo.InvariantCulture)); + + var storedStage = await _fixture.ExecuteRawScalarAsync( + "SELECT stage FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(nameof(PostFilterStage.Deny), Convert.ToString(storedStage)); + + var storedPayload = await _fixture.ExecuteRawScalarAsync( + "SELECT article_payload FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(payload, Assert.IsType(storedPayload)); + } + + [NntpDbIntegrationFact] + public async Task InsertRejection_AllowsNullPayload() + { + RequireReady(); + var evidence = new PostFilterRejectionEvidence( + DateTimeOffset.UtcNow, + policyRevision: 1, + accountName: null, + sourceAddress: IPAddress.Loopback, + artType: ArticleType.None, + messageId: null, + articleSize: 0, + stage: PostFilterStage.Gate, + reason: "closed", + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + articlePayload: null); + await _fixture.InsertRejectionAsync(evidence); + var storedPayload = await _fixture.ExecuteRawScalarAsync( + "SELECT article_payload FROM nntppostfilterrejections WHERE reason = 'closed' AND account_name IS NULL"); + Assert.True(storedPayload is null or DBNull); + } + + [NntpDbIntegrationFact] + public async Task PublishLiveActiveSpamAssassinRevision() + { + RequireReady(); + var revision = _fixture.NextPersistentRevision(); + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + MaxArticleSize = 131072, + Hosts = ["198.18.0.70"], + ExcludeArtTypes = ["YEncoded"], + }, + }; + await _fixture.PublishOptionsAsync(revision, options); + _fixture.RememberPublishedRevision(revision); + + var record = await _fixture.LoadAsync(); + Assert.Equal(revision, record.Revision); + Assert.Equal(PostFilterGateState.Active, record.Options.Gate); + Assert.True(record.Options.SpamAssassin.Enabled); + Assert.Equal(131072, record.Options.SpamAssassin.MaxArticleSize); + Assert.Equal(["198.18.0.70"], record.Options.SpamAssassin.Hosts); + Assert.Equal(["YEncoded"], record.Options.SpamAssassin.ExcludeArtTypes); + + await using var service = new PostFilterPolicyService( + _fixture.Repository!, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.Equal(revision, service.Current.Revision); + Assert.Equal(PostFilterGateState.Active, service.Current.Gate); + Assert.True(service.Current.SpamAssassinEnabled); + Assert.Equal(131072, service.Current.SpamAssassinMaxArticleSize); + Assert.Equal(["198.18.0.70"], service.Current.SpamAssassinHosts); + Assert.Equal(ArticleType.YEncoded, service.Current.SpamAssassinExcludeArtTypes); + await service.StopAsync(CancellationToken.None); + } + + [NntpDbIntegrationFact] + public async Task QueueService_WritesBinaryIpsAndPayload_ForPublishedRevision() + { + RequireReady(); + var revision = _fixture.NextPersistentRevision(); + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + MaxArticleSize = 131072, + Hosts = ["198.18.0.70"], + ExcludeArtTypes = ["YEncoded"], + }, + }; + await _fixture.PublishOptionsAsync(revision, options); + _fixture.RememberPublishedRevision(revision); + + var suffix = Guid.NewGuid().ToString("N"); + var payload = Encoding.UTF8.GetBytes( + "Message-ID: \r\nFrom: a@b\r\n\r\nbody\r\n"); + var v4Id = ""; + var v6Id = ""; + var mappedId = ""; + + var connectionString = NntpDbIntegration.TryGetConnectionString() + ?? throw new InvalidOperationException(NntpDbIntegration.SkipReason); + await using var nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + Options.Create(NntpDbIntegration.CreateOptions(connectionString)), + NullLogger.Instance); + await nntpDb.StartAsync(CancellationToken.None); + var queue = new PostFilterRejectionEvidenceQueue(); + var writer = new PostFilterRejectionEvidenceService( + nntpDb, + queue, + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + Assert.True(queue.TryEnqueue(Evidence(revision, IPAddress.Parse("198.18.0.70"), v4Id, payload))); + Assert.True(queue.TryEnqueue(Evidence(revision, IPAddress.Parse("2001:db8::1234"), v6Id, payload))); + Assert.True(queue.TryEnqueue( + Evidence(revision, IPAddress.Parse("::ffff:198.18.0.70"), mappedId, payload))); + await WaitForAsync(() => queue.Written == 3); + await writer.StopAsync(CancellationToken.None); + await nntpDb.DisposeAsync(); + + var current = Convert.ToInt64( + await _fixture.ExecuteRawScalarAsync("SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"), + CultureInfo.InvariantCulture); + Assert.Equal(revision, current); + + await AssertBinaryIpAsync(v4Id, "198.18.0.70", expectedLength: 4, revision, payload); + await AssertBinaryIpAsync(v6Id, "2001:db8::1234", expectedLength: 16, revision, payload); + await AssertBinaryIpAsync(mappedId, "198.18.0.70", expectedLength: 4, revision, payload); + Assert.Equal( + SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("198.18.0.70")), + SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("::ffff:198.18.0.70"))); + } + + private async Task AssertBinaryIpAsync( + string messageId, + string expectedText, + int expectedLength, + long revision, + byte[] payload) + { + var text = await _fixture.ExecuteRawScalarAsync( + "SELECT INET6_NTOA(source_ip) FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(expectedText, Convert.ToString(text, CultureInfo.InvariantCulture)); + + var length = await _fixture.ExecuteRawScalarAsync( + "SELECT OCTET_LENGTH(source_ip) FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(expectedLength, Convert.ToInt32(length, CultureInfo.InvariantCulture)); + + var storedRevision = await _fixture.ExecuteRawScalarAsync( + "SELECT revision FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(revision, Convert.ToInt64(storedRevision, CultureInfo.InvariantCulture)); + + var articleSize = await _fixture.ExecuteRawScalarAsync( + "SELECT article_size FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + var payloadLength = await _fixture.ExecuteRawScalarAsync( + "SELECT OCTET_LENGTH(article_payload) FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(payload.Length, Convert.ToInt32(articleSize, CultureInfo.InvariantCulture)); + Assert.Equal(payload.Length, Convert.ToInt32(payloadLength, CultureInfo.InvariantCulture)); + + var storedPayload = await _fixture.ExecuteRawScalarAsync( + "SELECT article_payload FROM nntppostfilterrejections WHERE message_id = @id", + command => command.Parameters.AddWithValue("@id", messageId)); + Assert.Equal(payload, Assert.IsType(storedPayload)); + } + + private static PostFilterRejectionEvidence Evidence( + long revision, + IPAddress source, + string messageId, + byte[] payload) => + new( + DateTimeOffset.UtcNow, + revision, + "poster", + source, + ArticleType.Default, + messageId, + payload.Length, + PostFilterStage.Deny, + "denied", + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + payload); + + private static async Task WaitForAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(20, cts.Token); + } + } + + private void RequireReady() + { + if (_fixture.IsConfigured) + { + return; + } + + if (NntpDbIntegration.TryGetConnectionString() is null) + { + return; + } + + Assert.Fail(_fixture.SkipReason ?? "PostFilter MySQL fixture is not configured."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyRepositoryIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyRepositoryIntegrationTests.cs new file mode 100644 index 00000000..f248a57a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyRepositoryIntegrationTests.cs @@ -0,0 +1,401 @@ +using System.Globalization; +using MySqlConnector; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +[Collection("NntpDbPostFilter")] +public sealed class MySqlPostFilterPolicyRepositoryIntegrationTests +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + + public MySqlPostFilterPolicyRepositoryIntegrationTests(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [Fact] + public void OfficialSchemaScript_ContainsRevisionKeyedTables() + { + var path = PostFilterSchemaScript.FindPath(); + Assert.EndsWith(Path.Join("docs", "schema", "postfilter.sql"), path, StringComparison.OrdinalIgnoreCase); + var sql = File.ReadAllText(path); + Assert.Contains("CREATE TABLE nntppostfilterpolicy", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfiltercurrent", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfilteraccounts", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfiltercidrs", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfilterarttypes", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfiltersahosts", sql, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE nntppostfilterrejections", sql, StringComparison.Ordinal); + Assert.Contains("source_ip VARBINARY(16) NOT NULL", sql, StringComparison.Ordinal); + Assert.Contains("account_name CHAR(32) NOT NULL", sql, StringComparison.Ordinal); + Assert.Contains("art_type ENUM(", sql, StringComparison.Ordinal); + Assert.Contains("'YEncoded'", sql, StringComparison.Ordinal); + Assert.Contains("PRIMARY KEY (revision, list_kind, account_name)", sql, StringComparison.Ordinal); + Assert.Contains("trg_nntppostfiltercurrent_revision_forward", sql, StringComparison.Ordinal); + Assert.Contains("'Disabled'", sql, StringComparison.Ordinal); + Assert.Contains("86400000, 600000", sql, StringComparison.Ordinal); + Assert.Contains("'N', NULL, 131072, 783", sql, StringComparison.Ordinal); + Assert.Contains("(1, 'sa_exclude', 'YEncoded')", sql, StringComparison.Ordinal); + Assert.Contains("INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, 1)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("INSERT INTO nntppostfiltersahosts", sql, StringComparison.Ordinal); + Assert.Contains("DELIMITER $$", sql, StringComparison.Ordinal); + Assert.Contains("END$$", sql, StringComparison.Ordinal); + Assert.Contains("DELIMITER ;", sql, StringComparison.Ordinal); + Assert.DoesNotContain("VECTORNNTP_STMT", sql, StringComparison.Ordinal); + Assert.DoesNotContain(";;", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ALTER TABLE nntpusers", sql, StringComparison.Ordinal); + Assert.Contains("nntpusers-account-art-type.sql", sql, StringComparison.Ordinal); + var statements = PostFilterSchemaScript.ReadStatements(); + Assert.Equal(11, statements.Count); + var trigger = Assert.Single( + statements, + statement => statement.StartsWith("CREATE TRIGGER trg_nntppostfiltercurrent_revision_forward", StringComparison.Ordinal)); + Assert.DoesNotContain("DELIMITER", trigger, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("$$", trigger, StringComparison.Ordinal); + Assert.Contains("BEFORE UPDATE ON nntppostfiltercurrent", trigger, StringComparison.Ordinal); + Assert.Contains("NEW.revision <= OLD.revision", trigger, StringComparison.Ordinal); + Assert.Contains("SIGNAL SQLSTATE '45000'", trigger, StringComparison.Ordinal); + } + + [NntpDbIntegrationFact] + public async Task OfficialSeedRevision1_LoadsExpectedScalars() + { + RequireReady(); + await _fixture.ExecuteRawAsync("DELETE FROM nntppostfiltercurrent WHERE policy_id = 1"); + await _fixture.ExecuteRawAsync("INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, 1)"); + var record = await _fixture.LoadAsync(); + Assert.Equal(1, record.Revision); + Assert.Equal(PostFilterGateState.Disabled, record.Options.Gate); + Assert.Equal(TimeSpan.FromDays(1), record.Options.Quota.LongWindow); + Assert.Equal(TimeSpan.FromMinutes(10), record.Options.Quota.ShortWindow); + Assert.Equal(0, record.Options.Quota.MaxMessagesLong); + Assert.Equal(0, record.Options.Quota.MaxBytesLong); + Assert.Equal(0, record.Options.Quota.MaxIdenticalLong); + Assert.Equal(0, record.Options.Quota.MaxMessagesShort); + Assert.Equal(0, record.Options.Quota.MaxBytesShort); + Assert.Equal(0, record.Options.Quota.MaxIdenticalShort); + Assert.False(record.Options.SpamAssassin.Enabled); + Assert.Null(record.Options.SpamAssassin.OnFailure); + Assert.Equal(131072, record.Options.SpamAssassin.MaxArticleSize); + Assert.Equal(783, record.Options.SpamAssassin.Port); + Assert.Equal("1.5", record.Options.SpamAssassin.ProtocolVersion); + Assert.Equal(4, record.Options.SpamAssassin.MaxConnections); + Assert.Equal(PostFilterSpamAssassinHostSelection.RoundRobin, record.Options.SpamAssassin.HostSelection); + Assert.Equal(TimeSpan.FromMilliseconds(5000), record.Options.SpamAssassin.ConnectTimeout); + Assert.Equal(TimeSpan.FromMilliseconds(30000), record.Options.SpamAssassin.OperationTimeout); + Assert.Equal(["YEncoded"], record.Options.SpamAssassin.ExcludeArtTypes); + Assert.Empty(record.Options.SpamAssassin.Hosts); + Assert.Empty(record.Options.DeniedAccounts); + Assert.Empty(record.Options.AllowlistedAccounts); + Assert.Empty(record.Options.DeniedCidrs); + Assert.Empty(record.Options.AllowlistedCidrs); + Assert.Empty(record.Options.RejectArtTypes); + var snapshot = PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + Assert.Equal(1, snapshot.Revision); + Assert.Equal(PostFilterGateState.Disabled, snapshot.Gate); + } + + [NntpDbIntegrationFact] + public async Task UnpublishedCommittedRevision_IsNeverSelected() + { + RequireReady(); + var published = _fixture.NextRevision(); + var draft = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(published); + await _fixture.InsertArtTypeAsync(published, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.PublishAsync(published); + await _fixture.InsertDisabledRevisionAsync(draft); + await _fixture.InsertAccountAsync(draft, NntpPostFilterQueries.ListKindDeny, "unpublished-poster"); + await _fixture.InsertArtTypeAsync(draft, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + + var record = await _fixture.LoadAsync(); + Assert.Equal(published, record.Revision); + Assert.DoesNotContain("unpublished-poster", record.Options.DeniedAccounts); + } + + [NntpDbIntegrationFact] + public async Task FreshDisabledSeed_LoadsCompletePolicy() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + await _fixture.InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.PublishAsync(revision); + + var record = await _fixture.LoadAsync(); + Assert.Equal(revision, record.Revision); + Assert.Equal(PostFilterGateState.Disabled, record.Options.Gate); + Assert.Equal(TimeSpan.FromDays(1), record.Options.Quota.LongWindow); + Assert.Equal(TimeSpan.FromMinutes(10), record.Options.Quota.ShortWindow); + Assert.Equal(0, record.Options.Quota.MaxMessagesLong); + Assert.Equal(0, record.Options.Quota.MaxMessagesShort); + Assert.False(record.Options.SpamAssassin.Enabled); + Assert.Null(record.Options.SpamAssassin.OnFailure); + Assert.Equal(131072, record.Options.SpamAssassin.MaxArticleSize); + Assert.Equal(783, record.Options.SpamAssassin.Port); + Assert.Equal("1.5", record.Options.SpamAssassin.ProtocolVersion); + Assert.Equal(4, record.Options.SpamAssassin.MaxConnections); + Assert.Equal(PostFilterSpamAssassinHostSelection.RoundRobin, record.Options.SpamAssassin.HostSelection); + Assert.Equal(["YEncoded"], record.Options.SpamAssassin.ExcludeArtTypes); + Assert.Empty(record.Options.SpamAssassin.Hosts); + var snapshot = PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + Assert.Equal(PostFilterGateState.Disabled, snapshot.Gate); + Assert.Equal(revision, snapshot.Revision); + } + + [NntpDbIntegrationFact] + public async Task MissingCurrent_FailsWithoutInventingPolicy() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + await _fixture.PublishAsync(revision); + await _fixture.ExecuteRawAsync("DELETE FROM nntppostfiltercurrent WHERE policy_id = 1"); + try + { + var ex = await Assert.ThrowsAsync(_fixture.LoadAsync); + Assert.Contains("nntppostfiltercurrent", ex.Message, StringComparison.Ordinal); + } + finally + { + await _fixture.ExecuteRawAsync( + "INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, @revision)", + command => command.Parameters.AddWithValue("@revision", revision)); + } + } + + [NntpDbIntegrationFact] + public async Task InvalidRevision_IsRejectedByDatabase() + { + RequireReady(); + var ex = await Assert.ThrowsAsync(() => + _fixture.ExecuteRawAsync( + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + 0, UTC_TIMESTAMP(3), 'Disabled', + 86400000, 600000, + 0, 0, 0, 0, 0, 0, + 'N', NULL, 131072, 783, + '1.5', 4, 'RoundRobin', + 5000, 30000 + ) + """)); + Assert.Contains("chk_nntppostfilterpolicy_revision", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [NntpDbIntegrationFact] + public async Task DuplicateCollectionRows_AreRejectedByDatabase() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + await _fixture.InsertAccountAsync(revision, NntpPostFilterQueries.ListKindDeny, "poster"); + await Assert.ThrowsAsync(() => + _fixture.InsertAccountAsync(revision, NntpPostFilterQueries.ListKindDeny, "poster")); + await _fixture.InsertCidrAsync(revision, NntpPostFilterQueries.ListKindAllow, "10.0.0.0/8"); + await Assert.ThrowsAsync(() => + _fixture.InsertCidrAsync(revision, NntpPostFilterQueries.ListKindAllow, "10.0.0.0/8")); + await _fixture.InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindReject, "Binary"); + await Assert.ThrowsAsync(() => + _fixture.InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindReject, "Binary")); + await _fixture.InsertHostAsync(revision, 1, "127.0.0.1"); + await Assert.ThrowsAsync(() => + _fixture.InsertHostAsync(revision, 1, "10.0.0.1")); + await Assert.ThrowsAsync(() => + _fixture.InsertHostAsync(revision, 2, "127.0.0.1")); + } + + [NntpDbIntegrationFact] + public async Task CompleteRevision_LoadsOnlyThatRevisionCollections() + { + RequireReady(); + var first = _fixture.NextRevision(); + var second = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(first); + await _fixture.InsertAccountAsync(first, NntpPostFilterQueries.ListKindDeny, "old-poster"); + await _fixture.InsertCidrAsync(first, NntpPostFilterQueries.ListKindDeny, "192.0.2.0/24"); + await _fixture.InsertArtTypeAsync(first, NntpPostFilterQueries.ListKindReject, "Html"); + await _fixture.InsertHostAsync(first, 1, "192.0.2.10"); + await _fixture.PublishAsync(first); + + await _fixture.InsertDisabledRevisionAsync(second); + await _fixture.InsertAccountAsync(second, NntpPostFilterQueries.ListKindDeny, "new-poster"); + await _fixture.InsertCidrAsync(second, NntpPostFilterQueries.ListKindAllow, "198.51.100.0/24"); + await _fixture.InsertArtTypeAsync(second, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.InsertHostAsync(second, 1, "198.51.100.10"); + await _fixture.PublishAsync(second); + + var record = await _fixture.LoadAsync(); + Assert.Equal(second, record.Revision); + Assert.Equal([PostFilterAccountIdentity.FromUsername("new-poster")], record.Options.DeniedAccounts); + Assert.DoesNotContain("new-poster", record.Options.DeniedAccounts); + Assert.DoesNotContain("old-poster", record.Options.DeniedAccounts); + Assert.DoesNotContain(PostFilterAccountIdentity.FromUsername("old-poster"), record.Options.DeniedAccounts); + Assert.Equal(["198.51.100.0/24"], record.Options.AllowlistedCidrs); + Assert.DoesNotContain("192.0.2.0/24", record.Options.DeniedCidrs); + Assert.Equal(["YEncoded"], record.Options.SpamAssassin.ExcludeArtTypes); + Assert.DoesNotContain("Html", record.Options.RejectArtTypes); + Assert.Equal(["198.51.100.10"], record.Options.SpamAssassin.Hosts); + } + + [NntpDbIntegrationFact] + public async Task UncommittedUpdate_IsInvisible_AndRollbackLeavesPrevious() + { + RequireReady(); + var published = _fixture.NextRevision(); + var draft = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(published); + await _fixture.PublishAsync(published); + var before = await _fixture.LoadAsync(); + Assert.Equal(published, before.Revision); + + await using var held = await _fixture.BeginHeldWriteAsync(); + await held.ExecuteAsync( + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + @revision, UTC_TIMESTAMP(3), 'Closed', + 86400000, 600000, + 0, 0, 0, 0, 0, 0, + 'N', NULL, 131072, 783, + '1.5', 4, 'RoundRobin', + 5000, 30000 + ) + """, + command => command.Parameters.AddWithValue("@revision", draft)); + await held.ExecuteAsync( + "UPDATE nntppostfiltercurrent SET revision = @revision WHERE policy_id = 1", + command => command.Parameters.AddWithValue("@revision", draft)); + + var unseen = await _fixture.LoadAsync(); + Assert.Equal(published, unseen.Revision); + Assert.Equal(PostFilterGateState.Disabled, unseen.Options.Gate); + + await held.RollbackAsync(); + var afterRollback = await _fixture.LoadAsync(); + Assert.Equal(published, afterRollback.Revision); + Assert.Equal(PostFilterGateState.Disabled, afterRollback.Options.Gate); + } + + [NntpDbIntegrationFact] + public async Task CommittedUpdate_IsVisibleToTwoConsumers() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + await _fixture.PublishAsync(revision); + var first = await _fixture.LoadAsync(); + var second = await _fixture.LoadAsync(); + Assert.Equal(revision, first.Revision); + Assert.Equal(revision, second.Revision); + Assert.Equal(first.Options.Gate, second.Options.Gate); + } + + [NntpDbIntegrationFact] + public async Task RevisionRegression_IsRejectedByTrigger() + { + RequireReady(); + var first = _fixture.NextRevision(); + var second = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(first); + await _fixture.PublishAsync(first); + var same = await Assert.ThrowsAsync(() => _fixture.PublishAsync(first)); + Assert.Contains("must increase", same.Message, StringComparison.OrdinalIgnoreCase); + var seedPointer = await Assert.ThrowsAsync(() => + _fixture.ExecuteRawAsync( + "UPDATE nntppostfiltercurrent SET revision = 1 WHERE policy_id = 1")); + Assert.Contains("must increase", seedPointer.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(first, (await _fixture.LoadAsync()).Revision); + + await _fixture.InsertDisabledRevisionAsync(second); + await _fixture.PublishAsync(second); + Assert.Equal(second, (await _fixture.LoadAsync()).Revision); + var lower = await Assert.ThrowsAsync(() => _fixture.PublishAsync(first)); + Assert.Contains("must increase", lower.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(second, (await _fixture.LoadAsync()).Revision); + } + + [NntpDbIntegrationFact] + public async Task InvalidCidrAndArtType_FailCompilation() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + await _fixture.InsertCidrAsync(revision, NntpPostFilterQueries.ListKindDeny, "not-a-cidr"); + await _fixture.PublishAsync(revision); + var record = await _fixture.LoadAsync(); + var cidr = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(record.Options, record.Revision)); + Assert.Contains("CIDR", cidr.Message, StringComparison.Ordinal); + + var next = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(next); + var invalidArtType = await Assert.ThrowsAsync(() => + _fixture.InsertArtTypeAsync(next, NntpPostFilterQueries.ListKindReject, "NotAnArtType")); + Assert.Contains("art_type", invalidArtType.Message, StringComparison.OrdinalIgnoreCase); + } + + [NntpDbIntegrationFact] + public async Task ShowCreate_UsesMd5AccountAndArtTypeEnumAndEvidenceTable() + { + RequireReady(); + var accounts = await _fixture.ShowCreateTableAsync("nntppostfilteraccounts"); + Assert.Contains("char(32)", accounts, StringComparison.OrdinalIgnoreCase); + var artTypes = await _fixture.ShowCreateTableAsync("nntppostfilterarttypes"); + Assert.Contains("enum(", artTypes, StringComparison.OrdinalIgnoreCase); + Assert.Contains("YEncoded", artTypes, StringComparison.Ordinal); + var evidence = await _fixture.ShowCreateTableAsync("nntppostfilterrejections"); + Assert.Contains("longblob", evidence, StringComparison.OrdinalIgnoreCase); + Assert.Contains("article_payload", evidence, StringComparison.Ordinal); + Assert.Contains("varbinary(16)", evidence, StringComparison.OrdinalIgnoreCase); + } + + [NntpDbIntegrationFact] + public async Task AccountInsert_RejectsPlaintextUsername() + { + RequireReady(); + var revision = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(revision); + var error = await Assert.ThrowsAsync(() => + _fixture.ExecuteRawAsync( + "INSERT INTO nntppostfilteraccounts (revision, list_kind, account_name) " + + "VALUES (" + revision.ToString(CultureInfo.InvariantCulture) + + ", 'deny', 'plaintext-user')")); + Assert.False(string.IsNullOrWhiteSpace(error.Message)); + } + + private void RequireReady() + { + if (_fixture.IsConfigured) + { + return; + } + + if (NntpDbIntegration.TryGetConnectionString() is null) + { + return; + } + + Assert.Fail(_fixture.SkipReason ?? "PostFilter MySQL fixture is not configured."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyServiceIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyServiceIntegrationTests.cs new file mode 100644 index 00000000..a29e3825 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterPolicyServiceIntegrationTests.cs @@ -0,0 +1,103 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +[Collection("NntpDbPostFilter")] +public sealed class MySqlPostFilterPolicyServiceIntegrationTests +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + + public MySqlPostFilterPolicyServiceIntegrationTests(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [NntpDbIntegrationFact] + public async Task LiveRepository_RefreshLastKnownGoodAndRecovery() + { + RequireReady(); + var first = _fixture.NextRevision(); + var second = _fixture.NextRevision(); + var third = _fixture.NextRevision(); + await _fixture.InsertDisabledRevisionAsync(first); + await _fixture.InsertArtTypeAsync(first, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.PublishAsync(first); + + var time = new FakeTimeProvider(); + await using var service = new PostFilterPolicyService( + _fixture.Repository!, + NullLogger.Instance, + time, + TimeSpan.FromSeconds(60)); + await service.StartAsync(CancellationToken.None); + Assert.Equal(first, service.Current.Revision); + Assert.Equal(PostFilterGateState.Disabled, service.Current.Gate); + + await _fixture.InsertDisabledRevisionAsync( + second, + command => command.CommandText = command.CommandText.Replace( + "'Disabled'", + "'Active'", + StringComparison.Ordinal)); + await _fixture.InsertArtTypeAsync(second, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.PublishAsync(second); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => service.Current.Revision == second); + Assert.Equal(PostFilterGateState.Active, service.Current.Gate); + var snapshotBeforeFailure = service.Current; + + await _fixture.ExecuteRawAsync("RENAME TABLE nntppostfiltercurrent TO nntppostfiltercurrent_hidden"); + try + { + time.Advance(TimeSpan.FromSeconds(60)); + await Task.Delay(50); + Assert.Same(snapshotBeforeFailure, service.Current); + Assert.Equal(second, service.Current.Revision); + Assert.Equal(PostFilterGateState.Active, service.Current.Gate); + } + finally + { + await _fixture.ExecuteRawAsync("RENAME TABLE nntppostfiltercurrent_hidden TO nntppostfiltercurrent"); + } + + await _fixture.InsertDisabledRevisionAsync(third); + await _fixture.InsertArtTypeAsync(third, NntpPostFilterQueries.ListKindSaExclude, "YEncoded"); + await _fixture.PublishAsync(third); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => service.Current.Revision == third); + Assert.Equal(PostFilterGateState.Disabled, service.Current.Gate); + Assert.NotSame(snapshotBeforeFailure, service.Current); + await service.StopAsync(CancellationToken.None); + } + + private void RequireReady() + { + if (_fixture.IsConfigured) + { + return; + } + + if (NntpDbIntegration.TryGetConnectionString() is null) + { + return; + } + + Assert.Fail(_fixture.SkipReason ?? "PostFilter MySQL fixture is not configured."); + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterProductionPolicyMatrixTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterProductionPolicyMatrixTests.cs new file mode 100644 index 00000000..85199276 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/MySqlPostFilterProductionPolicyMatrixTests.cs @@ -0,0 +1,67 @@ +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// Live MySQL variant of the production-policy matrix. Each case persists a +/// revision, loads it through , +/// publishes , then runs the +/// existing POST harness. +/// +[Collection("NntpDbPostFilter")] +public sealed class MySqlPostFilterProductionPolicyMatrixTests +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + + public MySqlPostFilterProductionPolicyMatrixTests(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + public static TheoryData Scenarios + { + get + { + var data = new TheoryData(); + foreach (var scenario in PostFilterProductionPolicyMatrix.All) + { + data.Add(scenario.Name); + } + + return data; + } + } + + [NntpDbIntegrationTheory] + [MemberData(nameof(Scenarios))] + public async Task ProductionMatrix_UsesMySqlRepositoryAndPolicyService(string scenario) + { + RequireReady(); + await using var policy = new MySqlPostFilterMatrixPolicy(_fixture); + var loadsBefore = policy.RepositoryLoadCount; + await PostFilterProductionPolicyMatrix.RunAsync(scenario, policy); + Assert.True( + policy.UsesPolicyService, + scenario + " did not publish through PostFilterPolicyService."); + Assert.True( + policy.RepositoryLoadCount > loadsBefore, + scenario + " did not load through MySqlPostFilterPolicyRepository."); + } + + private void RequireReady() + { + if (_fixture.IsConfigured) + { + return; + } + + if (NntpDbIntegration.TryGetConnectionString() is null) + { + return; + } + + Assert.Fail(_fixture.SkipReason ?? "PostFilter MySQL fixture is not configured."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterAccountIdentityTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterAccountIdentityTests.cs new file mode 100644 index 00000000..1e915b23 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterAccountIdentityTests.cs @@ -0,0 +1,48 @@ +using System.Security.Cryptography; +using System.Text; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterAccountIdentityTests +{ + [Fact] + public void FromUsername_IsLowercaseMd5HexOfUtf8Bytes() + { + var expected = Convert.ToHexString(MD5.HashData(Encoding.UTF8.GetBytes("poster"))).ToLowerInvariant(); + Assert.Equal(32, expected.Length); + Assert.Equal(expected, PostFilterAccountIdentity.FromUsername("poster")); + Assert.True(PostFilterAccountIdentity.IsStoredIdentifier(expected)); + } + + [Fact] + public void FromUsername_DoesNotTrim() + { + Assert.NotEqual( + PostFilterAccountIdentity.FromUsername("poster"), + PostFilterAccountIdentity.FromUsername(" poster")); + } + + [Fact] + public void FromUsername_IsOrdinalCaseSensitive() + { + Assert.NotEqual( + PostFilterAccountIdentity.FromUsername("Poster"), + PostFilterAccountIdentity.FromUsername("poster")); + } + + [Fact] + public void FromPolicyEntry_TrimsThenHashes() + { + Assert.Equal( + PostFilterAccountIdentity.FromUsername("poster"), + PostFilterAccountIdentity.FromPolicyEntry(" poster ")); + } + + [Fact] + public void FromPolicyEntry_KeepsStoredIdentifier() + { + var stored = PostFilterAccountIdentity.FromUsername("poster"); + Assert.Equal(stored, PostFilterAccountIdentity.FromPolicyEntry(stored)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterEvaluatorTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterEvaluatorTests.cs new file mode 100644 index 00000000..8461c4f2 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterEvaluatorTests.cs @@ -0,0 +1,865 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterEvaluatorTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + + [Fact] + public async Task DisabledGate_AcceptsWithoutReservation() + { + var quota = new RecordingQuotaStore(); + var filter = Create(PostFilterPolicySnapshot.Disabled, quota); + var result = await filter.EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Null(result.Lease); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task ClosedGate_RejectsWithoutReservation() + { + var quota = new RecordingQuotaStore(); + var filter = Create(Compile(new PostFilterOptions { Gate = PostFilterGateState.Closed }), quota); + var result = await filter.EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Reject, result.Decision); + Assert.Equal(PostFilterStage.Gate, result.Stage); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task DeniedAccount_RejectsBeforeQuota() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Reject, result.Decision); + Assert.Equal(PostFilterStage.Deny, result.Stage); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task DeniedAccount_DoesNotMatchDifferentCase() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["Poster"], + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request(account: "poster")); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(PostFilterStage.Complete, result.Stage); + } + + [Fact] + public async Task DeniedAccount_DoesNotTrimRequestUsername() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request(account: " poster")); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(PostFilterStage.Complete, result.Stage); + } + + [Fact] + public async Task AllowlistedAccount_MatchesHashedIdentity() + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + AllowlistedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, quota, sa).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(0, sa.Calls); + } + + [Fact] + public async Task DeniedAccount_DoesNotAffectOtherAccount() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request(account: "other")); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(PostFilterStage.Complete, result.Stage); + } + + [Fact] + public async Task ArtTypePolicy_RejectsYenc() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + RejectArtTypes = ["YEncoded"], + }); + var article = Article(type: ArticleType.YEncoded); + var result = await Create(snapshot, quota).EvaluateAsync(Request(article)); + Assert.Equal(PostFilterDecision.Reject, result.Decision); + Assert.Equal(PostFilterStage.ArtType, result.Stage); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task Allowlisted_StillReserves_AndSkipsSpamAssassin() + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + AllowlistedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, quota, sa).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.NotNull(result.Lease); + Assert.Contains("reserve", quota.Operations); + Assert.Equal(0, sa.Calls); + } + + [Fact] + public async Task NonAllowlisted_InvokesSpamAssassin() + { + var sa = new RecordingSpamAssassin(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, new RecordingQuotaStore(), sa).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task SpamAssassinSpam_ReleasesAndRejects() + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Spam("spam") }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, quota, sa).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Reject, result.Decision); + Assert.Equal(PostFilterStage.SpamAssassin, result.Stage); + Assert.Null(result.Lease); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + } + + [Theory] + [InlineData(PostFilterSpamOnFailure.Reject, PostFilterDecision.Reject)] + [InlineData(PostFilterSpamOnFailure.Accept, PostFilterDecision.Accept)] + public async Task SpamAssassinFailure_FollowsConfiguredPolicy( + PostFilterSpamOnFailure onFailure, + PostFilterDecision expected) + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Failed("down") }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = onFailure, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, quota, sa).EvaluateAsync(Request()); + Assert.Equal(expected, result.Decision); + if (expected == PostFilterDecision.Reject) + { + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Null(result.Lease); + } + else + { + Assert.Equal(new[] { "reserve" }, quota.Operations); + Assert.NotNull(result.Lease); + } + } + + [Fact] + public async Task QuotaMessageDenial_Is441PathReject() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var filter = Create(snapshot, quota); + Assert.Equal(PostFilterDecision.Accept, (await filter.EvaluateAsync(Request(account: "poster"))).Decision); + var denied = await filter.EvaluateAsync(Request(account: "poster")); + Assert.Equal(PostFilterDecision.Reject, denied.Decision); + Assert.Equal(PostFilterQuotaReserveStatus.DeniedMessagesLong, denied.QuotaStatus); + } + + [Fact] + public async Task QuotaIdenticalDenial_Is441PathReject() + { + var quota = new RecordingQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxIdenticalLong = 1 }, + }); + var filter = Create(snapshot, quota); + Assert.Equal(PostFilterDecision.Accept, (await filter.EvaluateAsync(Request())).Decision); + var denied = await filter.EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Reject, denied.Decision); + Assert.Equal(PostFilterQuotaReserveStatus.DeniedIdenticalLong, denied.QuotaStatus); + } + + [Fact] + public async Task RedisUnavailable_Rejects() + { + var quota = new RecordingQuotaStore { Unavailable = true }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Reject, result.Decision); + Assert.Equal(PostFilterQuotaReserveStatus.Unavailable, result.QuotaStatus); + } + + [Fact] + public async Task CancellationAfterReserve_Releases() + { + var quota = new RecordingQuotaStore(); + using var cts = new CancellationTokenSource(); + var sa = new RecordingSpamAssassin { Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var evaluate = Create(snapshot, quota, sa).EvaluateAsync(Request(), cts.Token).AsTask(); + await sa.BlockEntered.Task; + await cts.CancelAsync(); + sa.Block.SetCanceled(); + await Assert.ThrowsAnyAsync(() => evaluate); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + } + + [Fact] + public async Task ReserveCanceled_ReleasesWithNone() + { + var quota = new RecordingQuotaStore { CancelReserve = true }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + await Assert.ThrowsAnyAsync( + () => Create(snapshot, quota).EvaluateAsync(Request()).AsTask()); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + } + + [Fact] + public async Task SnapshotCapturedOnce_RefreshDuringSaDoesNotChangeWindows() + { + var first = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10, LongWindow = TimeSpan.FromHours(1) }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var second = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Closed, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1, LongWindow = TimeSpan.FromMinutes(1) }, + }); + var source = new MutablePolicySource(first); + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin { Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var filter = new PostFilterEvaluator( + source, + quota, + new PostFilterReservationIdentity("n1", "inc"), + sa, + NullLogger.Instance); + var evaluate = filter.EvaluateAsync(Request(), CancellationToken.None).AsTask(); + await sa.BlockEntered.Task; + source.Current = second; + sa.Block.SetResult(); + var result = await evaluate; + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.NotNull(result.Lease); + Assert.Equal(first.Windows, quota.LastWindows); + Assert.Equal(first.Ceilings, quota.LastCeilings); + Assert.Equal(first.ReservationTtlMs, quota.LastReservationTtlMs); + Assert.Equal(first.SpamAssassinHosts, sa.LastTarget.Hosts); + Assert.Equal(first.SpamAssassinOperationTimeout, sa.LastTarget.OperationTimeout); + Assert.Equal(1, sa.Calls); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await result.Lease.CommitAsync(Now, CancellationToken.None)); + Assert.Equal(first.Windows, quota.LastCommitWindows); + Assert.Equal(first.Ceilings, quota.LastCommitCeilings); + } + + [Fact] + public async Task ReserveCanceledAfterRedisWrite_Releases() + { + var redis = new FakeRedisService + { + Database = + { + ScriptStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + BlockScript = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }, + }; + var store = new RedisPostFilterQuotaStore(redis); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + using var cts = new CancellationTokenSource(); + var evaluate = Create(snapshot, store).EvaluateAsync(Request(), cts.Token).AsTask(); + await redis.Database.ScriptStarted!.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + redis.Database.BlockScript!.SetResult(); + await Assert.ThrowsAnyAsync(() => evaluate); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await store.ReserveAsync( + "poster", + new PostFilterReservationId("n2", "b", 1), + Now, + snapshot.Windows, + snapshot.Ceilings, + 1, + 1, + 0, + null)); + } + + [Theory] + [InlineData("disabled")] + [InlineData("quota")] + [InlineData("sa-skip")] + [InlineData("sa-ham")] + [InlineData("sa-reject")] + public async Task EvaluateAsync_DoesNotMutateArticleRecordBytes(string path) + { + var article = Article(); + var before = Capture(article); + var sa = new RecordingSpamAssassin(); + PostFilterPolicySnapshot snapshot; + if (path == "disabled") + { + snapshot = PostFilterPolicySnapshot.Disabled; + } + else if (path == "sa-skip") + { + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + MaxArticleSize = 1, + ExcludeArtTypes = [], + }, + }; + snapshot = Compile(options); + } + else if (path is "sa-ham" or "sa-reject") + { + sa.Result = path == "sa-reject" + ? PostFilterSpamAssassinResult.Spam("spam") + : PostFilterSpamAssassinResult.Ham(); + snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + MaxArticleSize = 0, + ExcludeArtTypes = [], + }, + }); + } + else + { + snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + } + + var result = await Create(snapshot, new RecordingQuotaStore(), sa).EvaluateAsync(Request(article)); + if (path == "sa-reject") + { + Assert.Equal(PostFilterDecision.Reject, result.Decision); + } + else + { + Assert.Equal(PostFilterDecision.Accept, result.Decision); + } + + AssertUnchanged(before, article); + } + + [Fact] + public async Task TwoNodes_ShareAccountQuota() + { + var engine = new PostFilterQuotaEngine(); + var shared = new InMemoryPostFilterQuotaStore(engine); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var a = Create(snapshot, shared, identity: new PostFilterReservationIdentity("n1", "incA")); + var b = Create(snapshot, shared, identity: new PostFilterReservationIdentity("n2", "incB")); + Assert.Equal(PostFilterDecision.Accept, (await a.EvaluateAsync(Request())).Decision); + Assert.Equal(PostFilterDecision.Reject, (await b.EvaluateAsync(Request())).Decision); + } + + [Fact] + public void ArticleRecord_IsNotMutated() + { + var article = Article(); + var hash = article.ArtHash; + var size = article.ArtSize; + var type = article.ArtType; + _ = PostFilterBodyHash.TryCompute(article); + Assert.Equal(hash, article.ArtHash); + Assert.Equal(size, article.ArtSize); + Assert.Equal(type, article.ArtType); + } + + [Fact] + public void DisabledSpamAssassin_PreservesTenSecondReservationFloor() + { + var hold = PostFilterQuotaDefaults.HoldMilliseconds(false, TimeSpan.FromSeconds(30)); + Assert.Equal((long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds, hold); + var snapshot = Compile(new PostFilterOptions { Gate = PostFilterGateState.Active }); + Assert.False(snapshot.SpamAssassinEnabled); + Assert.Equal(hold, snapshot.ReservationTtlMs); + } + + [Fact] + public void EnabledSpamAssassin_ReservationHoldCoversDefaultOperationTimeout() + { + var sa = new PostFilterSpamAssassinOptions(); + var hold = PostFilterQuotaDefaults.HoldMilliseconds(true, sa.OperationTimeout); + Assert.True(hold >= (long)sa.OperationTimeout.TotalMilliseconds); + Assert.Equal( + (long)sa.OperationTimeout.TotalMilliseconds + (long)PostFilterQuotaDefaults.ReservationHoldSkew.TotalMilliseconds, + hold); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + Assert.Equal(hold, snapshot.ReservationTtlMs); + Assert.Equal(["127.0.0.1"], snapshot.SpamAssassinHosts); + Assert.Equal(783, snapshot.SpamAssassinPort); + Assert.Equal(sa.OperationTimeout, snapshot.SpamAssassinOperationTimeout); + Assert.Equal(PostFilterSpamAssassinOptions.DefaultMaxArticleSize, snapshot.SpamAssassinMaxArticleSize); + Assert.Equal(ArticleType.YEncoded, snapshot.SpamAssassinExcludeArtTypes); + Assert.Equal(PostFilterSpamAssassinOptions.DefaultProtocolVersion, snapshot.SpamAssassinProtocolVersion); + Assert.Equal(PostFilterSpamAssassinOptions.DefaultMaxConnections, snapshot.SpamAssassinMaxConnections); + Assert.Equal(PostFilterSpamAssassinHostSelection.RoundRobin, snapshot.SpamAssassinHostSelection); + } + + [Fact] + public async Task EvaluateAsync_ReservationStaysLiveThroughConfiguredSaWait() + { + var quota = new InMemoryPostFilterQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }); + var result = await Create(snapshot, quota).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.NotNull(result.Lease); + var duringSa = Now.Add(snapshot.SpamAssassinOperationTimeout); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await result.Lease!.CommitAsync(duringSa, CancellationToken.None)); + } + + [Fact] + public async Task ExpiredReservation_DoesNotConsumeQuotaForLaterPost() + { + var quota = new InMemoryPostFilterQuotaStore(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var first = await Create(snapshot, quota).EvaluateAsync(Request()); + Assert.Equal(PostFilterDecision.Accept, first.Decision); + var expired = Now.AddMilliseconds(snapshot.ReservationTtlMs); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await first.Lease!.CommitAsync(expired, CancellationToken.None)); + var second = await Create(snapshot, quota).EvaluateAsync(Request(now: expired)); + Assert.Equal(PostFilterDecision.Accept, second.Decision); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await second.Lease!.CommitAsync(expired, CancellationToken.None)); + var third = await Create(snapshot, quota).EvaluateAsync(Request(now: expired)); + Assert.Equal(PostFilterDecision.Reject, third.Decision); + } + + [Fact] + public void Compiler_RejectsSpamAssassinOperationTimeoutAboveCap() + { + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + OperationTimeout = TimeSpan.FromMinutes(3), + }, + }; + Assert.Throws(() => PostFilterPolicyCompiler.Compile(options)); + } + + [Fact] + public void Compiler_RequiresOnFailureWhenSpamAssassinEnabled() + { + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + SpamAssassin = new PostFilterSpamAssassinOptions { Enabled = true, Hosts = ["127.0.0.1"] }, + }; + Assert.Throws(() => PostFilterPolicyCompiler.Compile(options)); + } + + private static PostFilterEvaluator Create( + PostFilterPolicySnapshot snapshot, + IPostFilterQuotaStore quota, + IPostFilterSpamAssassin? sa = null, + PostFilterReservationIdentity? identity = null) => + new( + new StaticPostFilterPolicySource(snapshot), + quota, + identity ?? new PostFilterReservationIdentity("n1", "inc"), + sa ?? new RecordingSpamAssassin(), + NullLogger.Instance); + + private static PostFilterPolicySnapshot Compile(PostFilterOptions options) => + PostFilterPolicyCompiler.Compile(options); + + private static PostFilterRequest Request( + ArticleRecord? article = null, + string account = "poster", + DateTimeOffset? now = null) => + new( + article ?? Article(), + account, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + accountPolicy: null, + now ?? Now, + ["misc.test"], + now ?? Now); + + private static ArticleSnapshot Capture(in ArticleRecord article) => + new( + article.ArtId, + article.ArtHash, + article.ArtType, + article.ArtLines, + article.ArtSize, + article.CanonicalUtc, + article.ParseStatus, + article.Fields, + article.ArtData.ToArray(), + GetBuffer(article)); + + private static void AssertUnchanged(ArticleSnapshot before, in ArticleRecord after) + { + Assert.Equal(before.ArtId, after.ArtId); + Assert.Equal(before.ArtHash, after.ArtHash); + Assert.Equal(before.ArtType, after.ArtType); + Assert.Equal(before.ArtLines, after.ArtLines); + Assert.Equal(before.ArtSize, after.ArtSize); + Assert.Equal(before.CanonicalUtc, after.CanonicalUtc); + Assert.Equal(before.ParseStatus, after.ParseStatus); + Assert.Equal(before.Fields.MessageId, after.Fields.MessageId); + Assert.Equal(before.Fields.Newsgroups, after.Fields.Newsgroups); + Assert.Equal(before.Fields.Subject, after.Fields.Subject); + Assert.Equal(before.Fields.From, after.Fields.From); + Assert.Equal(before.Fields.Date, after.Fields.Date); + Assert.Equal(before.Fields.References, after.Fields.References); + Assert.Equal(before.Fields.Path, after.Fields.Path); + Assert.True(before.Bytes.AsSpan().SequenceEqual(after.ArtData.Span)); + Assert.Same(before.Buffer, GetBuffer(after)); + } + + private static byte[] GetBuffer(in ArticleRecord article) => + System.Runtime.InteropServices.MemoryMarshal.TryGetArray(article.ArtData, out var segment) + ? segment.Array! + : throw new InvalidOperationException("ArtData is not an array segment."); + + private readonly record struct ArticleSnapshot( + ArticleId ArtId, + ulong ArtHash, + ArticleType ArtType, + int ArtLines, + int ArtSize, + DateTime CanonicalUtc, + ArticleParseStatus ParseStatus, + ArticleFieldTable Fields, + byte[] Bytes, + byte[] Buffer); + + private sealed class MutablePolicySource : IPostFilterPolicySource + { + public MutablePolicySource(PostFilterPolicySnapshot current) => Current = current; + + public PostFilterPolicySnapshot Current { get; set; } + } + + private static ArticleRecord Article(ArticleType type = ArticleType.Default) + { + var bytes = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(); + return new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 1, + artType: type, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: bytes, + fields: default); + } + + private sealed class RecordingQuotaStore : IPostFilterQuotaStore + { + private readonly InMemoryPostFilterQuotaStore _inner = new(); + + public bool Unavailable { get; set; } + + public bool CancelReserve { get; set; } + + public List Operations { get; } = []; + + public CancellationToken LastReleaseToken { get; private set; } + + public PostFilterQuotaWindows LastWindows { get; private set; } + + public PostFilterQuotaCeilings LastCeilings { get; private set; } + + public PostFilterQuotaWindows LastCommitWindows { get; private set; } + + public PostFilterQuotaCeilings LastCommitCeilings { get; private set; } + + public long LastReservationTtlMs { get; private set; } + + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + Operations.Add("reserve"); + LastWindows = windows; + LastCeilings = ceilings; + LastReservationTtlMs = reservationTtlMs; + if (CancelReserve) + { + var reserved = _inner.ReserveAsync( + accountName, + reservation, + now, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex, + CancellationToken.None, + reservationTtlMs); + if (!reserved.IsCompletedSuccessfully) + { + reserved.GetAwaiter().GetResult(); + } + + throw new OperationCanceledException(); + } + + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReserveStatus.Unavailable); + } + + return _inner.ReserveAsync( + accountName, + reservation, + now, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex, + cancellationToken, + reservationTtlMs); + } + + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + Operations.Add("commit"); + LastCommitWindows = windows; + LastCommitCeilings = ceilings; + return _inner.CommitAsync(accountName, reservation, now, windows, ceilings, cancellationToken); + } + + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) + { + Operations.Add("release"); + LastReleaseToken = cancellationToken; + return _inner.ReleaseAsync(accountName, reservation, now, cancellationToken); + } + } + + private sealed class RecordingSpamAssassin : IPostFilterSpamAssassin + { + public int Calls { get; private set; } + + public PostFilterSpamAssassinResult Result { get; set; } = PostFilterSpamAssassinResult.Ham(); + + public TaskCompletionSource? Block { get; set; } + + public TaskCompletionSource BlockEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public PostFilterSpamAssassinTarget LastTarget { get; private set; } + + public async ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = article; + _ = accountName; + _ = scanContext; + LastTarget = target; + Calls++; + if (Block is not null) + { + BlockEntered.TrySetResult(); + await Block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + return Result; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLeftoverConfigurationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLeftoverConfigurationTests.cs new file mode 100644 index 00000000..cf928001 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLeftoverConfigurationTests.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterLeftoverConfigurationTests +{ + [Fact] + public void LeftoverNntpdPostFilterSection_FailsStartupValidation() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:PostFilter:Gate"] = "Active", + ["Nntpd:PostFilter:Quota:MaxMessagesShort"] = "100", + }) + .Build(); + var validator = new PostFilterLeftoverConfigurationValidator(configuration); + var result = validator.Validate(null, new NntpdOptions()); + Assert.False(result.Succeeded); + Assert.Contains("nntppostfiltercurrent", result.FailureMessage, StringComparison.Ordinal); + } + + [Fact] + public void AbsentNntpdPostFilterSection_Succeeds() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection().Build(); + var validator = new PostFilterLeftoverConfigurationValidator(configuration); + var result = validator.Validate(null, new NntpdOptions()); + Assert.True(result.Succeeded); + } + + [Fact] + public void NntpdOptions_HasNoPostFilterProperty() + { + Assert.Null(typeof(NntpdOptions).GetProperty("PostFilter")); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLiveClusterQuotaTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLiveClusterQuotaTests.cs new file mode 100644 index 00000000..487eb957 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterLiveClusterQuotaTests.cs @@ -0,0 +1,125 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// Two reservation identities against one live Redis store, using the same +/// authenticated account name that POST passes (Authentication.Username). +/// +public sealed class PostFilterLiveClusterQuotaTests : IClassFixture +{ + private readonly PostFilterQuotaRedisIntegrationFixture _redis; + + public PostFilterLiveClusterQuotaTests(PostFilterQuotaRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public async Task TwoNodes_SameAccountUsername_ShareLiveRedisCeiling() + { + var account = await _redis.CreateAccountAsync(); + var snapshot = PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var nodeA = new PostFilterEvaluator( + new StaticPostFilterPolicySource(snapshot), + _redis.Store, + new PostFilterReservationIdentity("n1", "incA"), + new NullPostFilterSpamAssassin(), + NullLogger.Instance); + var nodeB = new PostFilterEvaluator( + new StaticPostFilterPolicySource(snapshot), + _redis.Store, + new PostFilterReservationIdentity("n2", "incB"), + new NullPostFilterSpamAssassin(), + NullLogger.Instance); + + var first = await nodeA.EvaluateAsync(Request(account)); + Assert.Equal(PostFilterDecision.Accept, first.Decision); + Assert.NotNull(first.Lease); + Assert.Equal(account, first.Lease!.AccountName); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await first.Lease.CommitAsync( + DateTimeOffset.FromUnixTimeMilliseconds(5_000), + CancellationToken.None)); + + var second = await nodeB.EvaluateAsync(Request(account)); + Assert.Equal(PostFilterDecision.Reject, second.Decision); + Assert.Equal(PostFilterQuotaReserveStatus.DeniedMessagesLong, second.QuotaStatus); + } + + [SessionStateRedisIntegrationFact] + public async Task Scenario10_CrossNode_SameAccountSharesLiveRedisCeiling_AtPost() + { + var account = await _redis.CreateAccountAsync(); + var snapshot = PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var nodeA = PostFilterPostHarness.CreateFilter( + snapshot, + new PolicyRecordingQuotaStore(_redis.Store), + identity: new PostFilterReservationIdentity("n1", "incA")); + var nodeB = PostFilterPostHarness.CreateFilter( + snapshot, + new PolicyRecordingQuotaStore(_redis.Store), + identity: new PostFilterReservationIdentity("n2", "incB")); + + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), nodeA), + "240 Article received OK", + username: account); + await using var second = new PostFilterPostDuplex(); + var queue = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(queue, nodeB), + "441 Posting failed", + username: account); + Assert.Equal(0, queue.Count); + await _redis.DeleteKeysAsync(account); + } + + private static PostFilterRequest Request(string account) => + PostFilterEvaluatorTestsRequest.Create(account); +} + +/// Shares the evaluator test article constructor without coupling session duplex. +internal static class PostFilterEvaluatorTestsRequest +{ + private static readonly DateTimeOffset Now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + + public static PostFilterRequest Create(string account) + { + var bytes = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(); + var article = new VectorNNTPD.Common.Articles.ArticleRecord( + VectorNNTPD.Common.Articles.ArticleId.FromMessageId(""u8), + artHash: 1, + artType: VectorNNTPD.Common.Articles.ArticleType.Default, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: VectorNNTPD.Common.Articles.ArticleParseStatus.CanonicalV1, + artData: bytes, + fields: default); + return new PostFilterRequest( + article, + account, + VectorNNTPD.ReaderService.Networking.Proxy.ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + accountPolicy: null, + Now, + ["misc.test"], + Now); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterMatrixPolicy.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterMatrixPolicy.cs new file mode 100644 index 00000000..04e36f89 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterMatrixPolicy.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// Publishes a matrix policy snapshot for POST-boundary scenarios. +internal interface IPostFilterMatrixPolicy +{ + /// Gets how many times the live MySQL repository was loaded. + int RepositoryLoadCount { get; } + + /// Gets whether published sources are . + bool UsesPolicyService { get; } + + Task PublishDisabledAsync(); + + Task PublishAsync(PostFilterOptions options); +} + +/// Existing in-memory compile path used by the offline matrix. +internal sealed class InMemoryPostFilterMatrixPolicy : IPostFilterMatrixPolicy +{ + public int RepositoryLoadCount { get; private set; } + + public bool UsesPolicyService => false; + + public Task PublishDisabledAsync() => + Task.FromResult( + new StaticPostFilterPolicySource(PostFilterPolicySnapshot.Disabled)); + + public Task PublishAsync(PostFilterOptions options) + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create(options), + }; + var record = repository.LoadAsync().AsTask().GetAwaiter().GetResult(); + RepositoryLoadCount += repository.LoadCount; + var snapshot = PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + return Task.FromResult(new StaticPostFilterPolicySource(snapshot)); + } +} + +/// +/// Persists each matrix policy as a new NntpDB revision, then publishes it +/// through and +/// . +/// +internal sealed class MySqlPostFilterMatrixPolicy : IPostFilterMatrixPolicy, IAsyncDisposable +{ + private readonly MySqlPostFilterPolicyIntegrationFixture _fixture; + private readonly CountingPostFilterPolicyRepository _repository; + private readonly List _services = []; + + public MySqlPostFilterMatrixPolicy(MySqlPostFilterPolicyIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + if (fixture.Repository is null) + { + throw new InvalidOperationException("PostFilter MySQL fixture is not configured."); + } + + _fixture = fixture; + _repository = new CountingPostFilterPolicyRepository(fixture.Repository); + } + + public int RepositoryLoadCount => _repository.LoadCount; + + public bool UsesPolicyService => true; + + public Task PublishDisabledAsync() => + PublishAsync(new PostFilterOptions { Gate = PostFilterGateState.Disabled }); + + public async Task PublishAsync(PostFilterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var revision = _fixture.NextRevision(); + await _fixture.PublishOptionsAsync(revision, options); + var service = new PostFilterPolicyService( + _repository, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + await service.StartAsync(CancellationToken.None); + _services.Add(service); + Assert.Equal(revision, service.Current.Revision); + Assert.Equal(options.Gate, service.Current.Gate); + return service; + } + + public async ValueTask DisposeAsync() + { + foreach (var service in _services) + { + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + } + + _services.Clear(); + } + + private sealed class CountingPostFilterPolicyRepository : IPostFilterPolicyRepository + { + private readonly IPostFilterPolicyRepository _inner; + + public CountingPostFilterPolicyRepository(IPostFilterPolicyRepository inner) + { + _inner = inner; + } + + public int LoadCount { get; private set; } + + public ValueTask LoadAsync(CancellationToken cancellationToken = default) + { + LoadCount++; + Assert.IsType(_inner); + return _inner.LoadAsync(cancellationToken); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyCompilerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyCompilerTests.cs new file mode 100644 index 00000000..800d3b00 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyCompilerTests.cs @@ -0,0 +1,106 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterPolicyCompilerTests +{ + [Fact] + public void Compile_AssignsRevision() + { + var snapshot = PostFilterPolicyCompiler.Compile(new PostFilterOptions(), revision: 12); + Assert.Equal(12, snapshot.Revision); + Assert.Equal(PostFilterGateState.Disabled, snapshot.Gate); + } + + [Fact] + public void Compile_DuplicateAccount_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + DeniedAccounts = ["poster", "poster"], + })); + Assert.Contains("duplicate", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Compile_EmptyAccount_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + AllowlistedAccounts = [" "], + })); + Assert.Contains("malformed", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Compile_InvalidCidr_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + DeniedCidrs = ["not-a-cidr"], + })); + Assert.Contains("CIDR", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_InvalidWindow_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + Quota = new PostFilterQuotaOptions { LongWindow = TimeSpan.Zero }, + })); + Assert.Contains("window", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Compile_NegativeMaxArticleSize_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + SpamAssassin = new PostFilterSpamAssassinOptions { MaxArticleSize = -1 }, + })); + Assert.Contains("MaxArticleSize", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_HashesAccountNamesToLowercaseMd5() + { + var snapshot = PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + DeniedAccounts = ["poster"], + AllowlistedAccounts = [" allowed "], + }); + Assert.Equal(PostFilterAccountIdentity.FromUsername("poster"), Assert.Single(snapshot.DeniedAccounts)); + Assert.Equal(PostFilterAccountIdentity.FromUsername("allowed"), Assert.Single(snapshot.AllowlistedAccounts)); + Assert.DoesNotContain("poster", snapshot.DeniedAccounts); + Assert.DoesNotContain("allowed", snapshot.AllowlistedAccounts); + } + + [Fact] + public void Compile_AccountMatchIsOrdinal_AfterHash() + { + var snapshot = PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + DeniedAccounts = ["Poster"], + }); + Assert.Contains(PostFilterAccountIdentity.FromUsername("Poster"), snapshot.DeniedAccounts); + Assert.DoesNotContain(PostFilterAccountIdentity.FromUsername("poster"), snapshot.DeniedAccounts); + } + + [Fact] + public void Compile_UnknownArtType_FailsCompletely() + { + var ex = Assert.Throws(() => + PostFilterPolicyCompiler.Compile(new PostFilterOptions + { + RejectArtTypes = ["NotAnArtType"], + })); + Assert.Contains("ArtType", ex.Message, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyRepositoryTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyRepositoryTests.cs new file mode 100644 index 00000000..78051b7f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyRepositoryTests.cs @@ -0,0 +1,198 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterPolicyRepositoryTests +{ + [Fact] + public async Task LoadAsync_MapsFactoryRecord() + { + var options = new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesShort = 100 }, + }; + var factory = new FakeNntpDbConnectionFactory + { + PostFilterPolicy = InMemoryPostFilterPolicyRepository.Create(options, revision: 3), + }; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + var record = await repository.LoadAsync(); + Assert.Equal(3, record.Revision); + Assert.Equal(PostFilterGateState.Active, record.Options.Gate); + Assert.Equal(["poster"], record.Options.DeniedAccounts); + Assert.Equal(100, record.Options.Quota.MaxMessagesShort); + Assert.Equal(1, factory.Connections.Sum(static connection => connection.QueryPostFilterPolicyCount)); + await nntpDb.DisposeAsync(); + } + + [Fact] + public async Task LoadAsync_MissingCurrent_ThrowsWithoutInventingPolicy() + { + var factory = new FakeNntpDbConnectionFactory { PostFilterPolicy = null }; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => repository.LoadAsync().AsTask()); + Assert.Contains("nntppostfiltercurrent", ex.Message, StringComparison.Ordinal); + await nntpDb.DisposeAsync(); + } + + [Fact] + public async Task LoadAsync_Unavailable_Propagates() + { + var factory = new FakeNntpDbConnectionFactory + { + QueryPostFilterPolicyException = new NntpDbUnavailableException("down"), + }; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + await Assert.ThrowsAsync(() => repository.LoadAsync().AsTask()); + await nntpDb.DisposeAsync(); + } + + [Fact] + public async Task LoadAsync_PublishedRevision_IgnoresOtherRevisionCollections() + { + var factory = new FakeNntpDbConnectionFactory(); + factory.PostFilterRevisions[11] = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions + { + Gate = PostFilterGateState.Closed, + DeniedAccounts = ["old-poster"], + }, + revision: 11); + factory.PostFilterRevisions[12] = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["new-poster"], + AllowlistedCidrs = ["10.0.0.0/8"], + RejectArtTypes = ["Binary"], + SpamAssassin = new PostFilterSpamAssassinOptions + { + ExcludeArtTypes = ["YEncoded"], + Hosts = ["127.0.0.1"], + }, + }, + revision: 12); + factory.PostFilterCurrentRevision = 12; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + var record = await repository.LoadAsync(); + Assert.Equal(12, record.Revision); + Assert.Equal(PostFilterGateState.Active, record.Options.Gate); + Assert.Equal(["new-poster"], record.Options.DeniedAccounts); + Assert.DoesNotContain("old-poster", record.Options.DeniedAccounts); + Assert.Equal(["10.0.0.0/8"], record.Options.AllowlistedCidrs); + Assert.Equal(["Binary"], record.Options.RejectArtTypes); + Assert.Equal(["127.0.0.1"], record.Options.SpamAssassin.Hosts); + await nntpDb.DisposeAsync(); + } + + [Fact] + public async Task LoadAsync_RevisionAdvance_ReturnsNewRevision() + { + var factory = new FakeNntpDbConnectionFactory(); + factory.PostFilterRevisions[1] = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Disabled }, + revision: 1); + factory.PostFilterRevisions[2] = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 2); + factory.PostFilterCurrentRevision = 1; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var repository = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + var first = await repository.LoadAsync(); + Assert.Equal(1, first.Revision); + factory.PostFilterCurrentRevision = 2; + var second = await repository.LoadAsync(); + Assert.Equal(2, second.Revision); + Assert.Equal(PostFilterGateState.Closed, second.Options.Gate); + await nntpDb.DisposeAsync(); + } + + [Fact] + public async Task TwoConsumers_SeeTheSamePublishedRevision() + { + var factory = new FakeNntpDbConnectionFactory(); + factory.PostFilterRevisions[4] = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 4); + factory.PostFilterCurrentRevision = 4; + var nntpDb = CreateStartedDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var first = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + var second = new MySqlPostFilterPolicyRepository( + nntpDb, + NullLogger.Instance); + + var left = await first.LoadAsync(); + var right = await second.LoadAsync(); + Assert.Equal(4, left.Revision); + Assert.Equal(4, right.Revision); + Assert.Equal(left.Options.Gate, right.Options.Gate); + await nntpDb.DisposeAsync(); + } + + [Fact] + public void Queries_BindOnePublishedRevision() + { + Assert.Contains("FROM nntppostfiltercurrent", NntpPostFilterQueries.SelectCurrentRevision, StringComparison.Ordinal); + Assert.Contains("policy_id = 1", NntpPostFilterQueries.SelectCurrentRevision, StringComparison.Ordinal); + Assert.Contains("FROM nntppostfilterpolicy", NntpPostFilterQueries.SelectPolicy, StringComparison.Ordinal); + Assert.Contains("WHERE revision = @revision", NntpPostFilterQueries.SelectPolicy, StringComparison.Ordinal); + Assert.DoesNotContain("policy_id = 1", NntpPostFilterQueries.SelectPolicy, StringComparison.Ordinal); + Assert.Contains("WHERE revision = @revision", NntpPostFilterQueries.SelectAccounts, StringComparison.Ordinal); + Assert.Contains("WHERE revision = @revision", NntpPostFilterQueries.SelectCidrs, StringComparison.Ordinal); + Assert.Contains("WHERE revision = @revision", NntpPostFilterQueries.SelectArtTypes, StringComparison.Ordinal); + Assert.Contains("WHERE revision = @revision", NntpPostFilterQueries.SelectHosts, StringComparison.Ordinal); + } + + [Fact] + public void Record_RejectsNonPositiveRevision() + { + Assert.Throws(() => + new PostFilterPolicyRecord(0, DateTimeOffset.UtcNow, new PostFilterOptions())); + } + + private static NntpDbService CreateStartedDb(FakeNntpDbConnectionFactory factory) => + new( + factory, + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }), + NullLogger.Instance); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyServiceTests.cs new file mode 100644 index 00000000..6de06420 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPolicyServiceTests.cs @@ -0,0 +1,260 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterPolicyServiceTests +{ + [Fact] + public async Task StartAsync_LoadsNntpDbPolicy_AndPublishesSnapshot() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 7), + }; + await using var service = CreateService(repository); + await service.StartAsync(CancellationToken.None); + Assert.True(service.HasSnapshot); + Assert.Equal(PostFilterGateState.Closed, service.Current.Gate); + Assert.Equal(7, service.Current.Revision); + Assert.Equal(1, repository.LoadCount); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_MissingPolicy_ThrowsAndDoesNotPublish() + { + var repository = new InMemoryPostFilterPolicyRepository(); + await using var service = CreateService(repository); + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("nntppostfiltercurrent", ex.Message, StringComparison.Ordinal); + Assert.False(service.HasSnapshot); + } + + [Fact] + public async Task StartAsync_NntpDbUnavailable_ThrowsAndDoesNotPublish() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Exception = new NntpDbUnavailableException("down"), + }; + await using var service = CreateService(repository); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + } + + [Fact] + public async Task StartAsync_InvalidPolicy_ThrowsAndDoesNotPublish() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions + { + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + Hosts = ["127.0.0.1"], + }, + }), + }; + await using var service = CreateService(repository); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + } + + [Fact] + public async Task Refresh_RevisionChange_PublishesAtomically() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 1), + }; + var time = new FakeTimeProvider(); + await using var service = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await service.StartAsync(CancellationToken.None); + Assert.Equal(1, service.Current.Revision); + + repository.Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Active }, + revision: 2); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => service.Current.Revision == 2); + Assert.Equal(PostFilterGateState.Active, service.Current.Gate); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task RefreshFailure_RetainsLastKnownGood() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 1), + }; + var time = new FakeTimeProvider(); + await using var service = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await service.StartAsync(CancellationToken.None); + + repository.Exception = new NntpDbUnavailableException("down"); + time.Advance(TimeSpan.FromSeconds(60)); + await Task.Yield(); + await Task.Yield(); + Assert.Equal(1, service.Current.Revision); + Assert.Equal(PostFilterGateState.Closed, service.Current.Gate); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_RecoversWhenNntpDbReturns() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 1), + }; + var time = new ControllableTimeProvider(); + await using var service = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => time.HasPendingTimer); + + repository.Exception = new NntpDbUnavailableException("down"); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => time.HasPendingTimer); + Assert.Equal(1, service.Current.Revision); + + repository.Exception = null; + repository.Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Active }, + revision: 2); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => service.Current.Revision == 2); + Assert.Equal(PostFilterGateState.Active, service.Current.Gate); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task ClusterConsumers_ConvergeOnSameRevision() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Disabled }, + revision: 1), + }; + var time = new FakeTimeProvider(); + await using var first = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await using var second = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await first.StartAsync(CancellationToken.None); + await second.StartAsync(CancellationToken.None); + + repository.Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 4); + time.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => first.Current.Revision == 4 && second.Current.Revision == 4); + Assert.Equal(PostFilterGateState.Closed, first.Current.Gate); + Assert.Equal(PostFilterGateState.Closed, second.Current.Gate); + await first.StopAsync(CancellationToken.None); + await second.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_SameRevision_DoesNotRepublish() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Disabled }, + revision: 1), + }; + var time = new FakeTimeProvider(); + await using var service = CreateService(repository, time, TimeSpan.FromSeconds(60)); + await service.StartAsync(CancellationToken.None); + var published = service.Current; + + repository.Record = InMemoryPostFilterPolicyRepository.Create( + new PostFilterOptions { Gate = PostFilterGateState.Closed }, + revision: 1); + time.Advance(TimeSpan.FromSeconds(60)); + await Task.Yield(); + await Task.Yield(); + Assert.Same(published, service.Current); + Assert.Equal(PostFilterGateState.Disabled, service.Current.Gate); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Evaluate_DoesNotQueryRepository_AfterSnapshotPublished() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create(new PostFilterOptions()), + }; + await using var service = CreateService(repository); + await service.StartAsync(CancellationToken.None); + var loads = repository.LoadCount; + + repository.Exception = new NntpDbUnavailableException("must-not-run"); + var snapshot = service.Current; + Assert.Equal(PostFilterGateState.Disabled, snapshot.Gate); + Assert.Equal(loads, repository.LoadCount); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Post_UsesPublishedSnapshot_WhenRepositoryIsUnavailable() + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create(new PostFilterOptions()), + }; + await using var service = CreateService(repository); + await service.StartAsync(CancellationToken.None); + repository.Exception = new NntpDbUnavailableException("must-not-run"); + var loads = repository.LoadCount; + + var quota = new PolicyRecordingQuotaStore(); + var queue = PostFilterPostHarness.NewQueue(); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(service.Current, quota)), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Equal(loads, repository.LoadCount); + await service.StopAsync(CancellationToken.None); + } + + private static PostFilterPolicyService CreateService( + IPostFilterPolicyRepository repository, + TimeProvider? time = null, + TimeSpan? interval = null) => + new( + repository, + NullLogger.Instance, + time ?? TimeProvider.System, + interval ?? TimeSpan.FromHours(1)); + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostCommandTests.cs new file mode 100644 index 00000000..d35bf6fc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostCommandTests.cs @@ -0,0 +1,1128 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterPostCommandTests +{ + private static readonly NntpAuthorization Poster = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(20); + + [Fact] + public async Task ClosedGate_Returns441_AndDoesNotEnqueue() + { + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(Closed(), quota)), "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task ClosedGate_WritesRejectedNewsWithPostFilterReason() + { + var news = new RecordingNewsLogWriter(); + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(Closed(), quota), newsLog: news), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(441, evt.ResponseCode); + Assert.Equal("closed", Encoding.ASCII.GetString(evt.Reason.Span)); + var timestamp = evt.Timestamp == default + ? new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero) + : evt.Timestamp; + var buffer = new byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + var line = Encoding.ASCII.GetString(buffer.AsSpan(0, written)); + Assert.EndsWith(" closed\n", line, StringComparison.Ordinal); + Assert.DoesNotContain("441", line, StringComparison.Ordinal); + } + + [Fact] + public async Task DisabledGate_AcceptsAndEnqueues() + { + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(PostFilterPolicySnapshot.Disabled, quota)), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task DeniedAccount_Returns441() + { + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + }); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota)), "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task DeniedAccount_EnqueuesRejectionEvidence() + { + var quota = new RecordingQuotaStore(); + var evidence = new PostFilterRejectionEvidenceQueue(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + }); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(NewQueue(), CreateFilter(snapshot, quota), postFilterEvidence: evidence), + "441 Posting failed"); + var row = Assert.Single(await DrainAsync(evidence)); + Assert.Equal(PostFilterStage.Deny, row.Stage); + Assert.Equal("denied", row.Reason); + Assert.Equal(PostFilterAccountIdentity.FromUsername("poster"), row.AccountName); + Assert.Equal(IPAddress.Loopback, row.SourceAddress); + Assert.Equal(ArticleType.Default, row.ArtType); + Assert.NotNull(row.MessageId); + Assert.True(row.ArticleSize > 0); + Assert.NotNull(row.ArticlePayload); + Assert.Null(row.SpamAssassinStatus); + } + + [Fact] + public async Task AcceptedPost_DoesNotEnqueueRejectionEvidence() + { + var evidence = new PostFilterRejectionEvidenceQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession( + NewQueue(), + CreateFilter(PostFilterPolicySnapshot.Disabled, new RecordingQuotaStore()), + postFilterEvidence: evidence), + "240 Article received OK"); + evidence.Complete(); + Assert.Null(await evidence.DequeueAsync(CancellationToken.None)); + } + + [Fact] + public async Task QueueFull_StillReturns441() + { + var evidence = new PostFilterRejectionEvidenceQueue(capacity: 1); + Assert.True(evidence.TryEnqueue(new PostFilterRejectionEvidence( + DateTimeOffset.UtcNow, + 1, + "other", + IPAddress.Loopback, + ArticleType.Default, + "", + 1, + PostFilterStage.Gate, + "closed", + null, + null, + null, + null))); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Closed, + }); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession( + NewQueue(), + CreateFilter(snapshot, new RecordingQuotaStore()), + postFilterEvidence: evidence), + "441 Posting failed"); + Assert.Equal(1, evidence.Dropped); + } + + [Fact] + public async Task TextOnlyCapability_RejectsYEncoded_WithoutPostFilter() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, CreateFilter(PostFilterPolicySnapshot.Disabled, new RecordingQuotaStore())); + session.ApplySuccessfulAuthentication( + "poster", + Poster, + new NntpAccountPolicy("poster", 0, 0, 0, 0, "c", ArticleTypeCapabilities.TextOnly)); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(YEncodedArticle() + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TextOnlyCapability_AllowsDefaultText() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, CreateFilter(PostFilterPolicySnapshot.Disabled, new RecordingQuotaStore())); + session.ApplySuccessfulAuthentication( + "poster", + Poster, + new NntpAccountPolicy("poster", 0, 0, 0, 0, "c", ArticleTypeCapabilities.TextOnly)); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(1, queue.Count); + } + + [Fact] + public async Task ArtTypePolicy_RejectsDefaultText() + { + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + RejectArtTypes = ["Default"], + }); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota)), "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + } + + [Fact] + public async Task Allowlisted_StillConsumesQuota_AndSkipsSpamAssassin() + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + AllowlistedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + SpamAssassin = EnabledSpamAssassin(), + }); + var filter = CreateFilter(snapshot, quota, sa); + await using var first = new PostDuplex(); + await PostAsync(first, first.CreateSession(NewQueue(), filter), "240 Article received OK"); + await using var second = new PostDuplex(); + var queue = NewQueue(); + await PostAsync(second, second.CreateSession(queue, filter), "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(0, sa.Calls); + Assert.Contains("reserve", quota.Operations); + } + + [Fact] + public async Task NonAllowlisted_InvokesSpamAssassin() + { + var sa = new RecordingSpamAssassin(); + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = EnabledSpamAssassin(), + }); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(NewQueue(), CreateFilter(snapshot, new RecordingQuotaStore(), sa)), + "240 Article received OK"); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task SpamAssassinSpam_Returns441_AndReleases() + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Spam("spam") }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = EnabledSpamAssassin(), + }); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota, sa)), "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + } + + [Theory] + [InlineData(PostFilterSpamOnFailure.Reject, "441 Posting failed", false)] + [InlineData(PostFilterSpamOnFailure.Accept, "240 Article received OK", true)] + public async Task SpamAssassinFailure_FollowsConfiguredPolicy( + PostFilterSpamOnFailure onFailure, + string expected, + bool enqueued) + { + var quota = new RecordingQuotaStore(); + var sa = new RecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Failed("down") }; + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = EnabledSpamAssassin(onFailure), + }); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota, sa)), expected); + Assert.Equal(enqueued ? 1 : 0, queue.Count); + if (enqueued) + { + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + } + else + { + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + } + } + + [Fact] + public async Task QuotaMessageCeiling_Returns441() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var quota = new RecordingQuotaStore(); + var filter = CreateFilter(snapshot, quota); + await using var first = new PostDuplex(); + await PostAsync(first, first.CreateSession(NewQueue(), filter), "240 Article received OK"); + await using var second = new PostDuplex(); + var queue = NewQueue(); + await PostAsync(second, second.CreateSession(queue, filter), "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task QuotaByteCeiling_Returns441() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxBytesLong = 1 }, + }); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, new RecordingQuotaStore())), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task QuotaIdenticalCeiling_Returns441() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxIdenticalLong = 1 }, + }); + var filter = CreateFilter(snapshot, new RecordingQuotaStore()); + await using var first = new PostDuplex(); + await PostAsync(first, first.CreateSession(NewQueue(), filter), "240 Article received OK"); + await using var second = new PostDuplex(); + var queue = NewQueue(); + await PostAsync(second, second.CreateSession(queue, filter), "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task RedisUnavailable_Returns441() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore { Unavailable = true }; + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota)), "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ReserveHappensBeforeTryAdmit_AndCommitAfterAccept() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var clock = new SequenceClock(); + var quota = new RecordingQuotaStore(clock); + var queue = new RecordingIngestionQueue(NewQueue(), clock); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota)), "240 Article received OK"); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.True(quota.ReserveSequence < queue.AdmitSequence); + Assert.True(quota.CommitSequence > queue.AdmitSequence); + } + + [Fact] + public async Task ActiveGate_ReservesAndCommits() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(queue, CreateFilter(snapshot, quota)), "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + } + + [Fact] + public async Task CommitNoop_StillQueuesAndReturns240() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore { CommitNoop = true }; + var queue = NewQueue(); + var metrics = new PostFilterMetrics(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, quota), metrics), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Equal(1, metrics.CommitNoop); + Assert.DoesNotContain("release", quota.Operations); + } + + [Fact] + public async Task CommitUnavailable_AfterAdmit_StillQueuesAndReturns240_WithoutRelease() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore { CommitUnavailable = true }; + var queue = NewQueue(); + var metrics = new PostFilterMetrics(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, quota), metrics), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Equal(1, metrics.CommitUnavailable); + Assert.Equal(0, metrics.CommitNoop); + Assert.DoesNotContain("release", quota.Operations); + Assert.Contains("commit", quota.Operations); + } + + [Fact] + public async Task TryAdmitFailure_Releases_AndDoesNotCommit() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(DisabledArticleIngestionQueue.Instance, CreateFilter(snapshot, quota)), + "441 Posting failed"); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.DoesNotContain("commit", quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + } + + [Fact] + public async Task SuccessfulAdmit_CommitsReservationUnits() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10, MaxBytesLong = 100_000 }, + }); + var quota = new RecordingQuotaStore(); + await using var duplex = new PostDuplex(); + await PostAsync(duplex, duplex.CreateSession(NewQueue(), CreateFilter(snapshot, quota)), "240 Article received OK"); + Assert.Equal(1, quota.LastReservedMessages); + Assert.True(quota.LastReservedBytes > 0); + Assert.Equal(quota.LastReservedMessages, quota.LastCommittedMessages); + Assert.Equal(quota.LastReservedBytes, quota.LastCommittedBytes); + Assert.False(quota.CommitReceivedUnits); + } + + [Fact] + public async Task HistoryPeekUnavailable_Returns441_WithoutReserve() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore(); + var history = new RecordingHistoryDb { PeekResult = HistoryLookupResult.Unavailable }; + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, quota), historyDb: history), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + Assert.Equal(0, history.RememberCalls); + } + + [Fact] + public async Task CancellationAfterReservation_Releases_AndDoesNotEnqueue() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore(); + var queue = NewQueue(); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession( + queue, + CreateFilter(snapshot, quota), + historyDb: history, + cancelConnectionAfterAccept: true); + session.ApplySuccessfulAuthentication("poster", Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + await run.WaitAsync(Safety); + Assert.Equal(0, queue.Count); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + Assert.Equal(0, history.RememberCalls); + } + + [Fact] + public async Task CommitThenRememberThen240() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var clock = new SequenceClock(); + var quota = new RecordingQuotaStore(clock); + var history = new RecordingHistoryDb(clock); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, quota), historyDb: history), + "240 Article received OK"); + Assert.Equal(1, history.RememberCalls); + Assert.True(quota.CommitSequence < history.RememberSequence); + Assert.DoesNotContain("release", quota.Operations); + } + + [Fact] + public async Task RememberFailureAfterCommit_DoesNotRelease() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var quota = new RecordingQuotaStore(); + var history = new RecordingHistoryDb { ThrowOnRemember = true }; + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, CreateFilter(snapshot, quota), historyDb: history); + session.ApplySuccessfulAuthentication("poster", Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + await Assert.ThrowsAnyAsync(() => run.WaitAsync(Safety)); + Assert.Equal(1, queue.Count); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + Assert.DoesNotContain("release", quota.Operations); + } + + [Fact] + public async Task QueuedArticle_KeepsOriginalArtHashAndBytes() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + await PostAsync( + duplex, + duplex.CreateSession(queue, CreateFilter(snapshot, new RecordingQuotaStore())), + "240 Article received OK"); + using var cts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(cts.Token); + Assert.NotNull(inbound); + var record = inbound!.Record; + Assert.Equal(ArticleParseStatus.CanonicalV1, record.ParseStatus); + var hash = record.ArtHash; + var id = record.ArtId; + var type = record.ArtType; + var lines = record.ArtLines; + var size = record.ArtSize; + var fields = record.Fields; + var data = record.ArtData.ToArray(); + Assert.True(record.ArtData.Equals(inbound.Payload)); + Assert.Equal(hash, record.ArtHash); + Assert.Equal(id, record.ArtId); + Assert.Equal(type, record.ArtType); + Assert.Equal(lines, record.ArtLines); + Assert.Equal(size, record.ArtSize); + Assert.Equal(fields.MessageId, record.Fields.MessageId); + Assert.True(data.AsSpan().SequenceEqual(record.ArtData.Span)); + Assert.True(record.ArtData.Span.SequenceEqual(inbound.Payload.Span)); + } + + [Fact] + public async Task TwoNodes_ShareAccountQuota() + { + var snapshot = Compile(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var shared = new InMemoryPostFilterQuotaStore(); + var nodeA = CreateFilter(snapshot, shared, identity: new PostFilterReservationIdentity("n1", "incA")); + var nodeB = CreateFilter(snapshot, shared, identity: new PostFilterReservationIdentity("n2", "incB")); + await using var first = new PostDuplex(); + await PostAsync(first, first.CreateSession(NewQueue(), nodeA), "240 Article received OK"); + await using var second = new PostDuplex(); + var queue = NewQueue(); + await PostAsync(second, second.CreateSession(queue, nodeB), "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + private static async Task PostAsync(PostDuplex duplex, NntpSession session, string expect) + { + session.ApplySuccessfulAuthentication("poster", Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal(expect, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static IPostFilter CreateFilter( + PostFilterPolicySnapshot snapshot, + IPostFilterQuotaStore quota, + IPostFilterSpamAssassin? sa = null, + PostFilterReservationIdentity? identity = null) => + new PostFilterEvaluator( + new StaticPostFilterPolicySource(snapshot), + quota, + identity ?? new PostFilterReservationIdentity("n1", "inc"), + sa ?? new RecordingSpamAssassin(), + NullLogger.Instance); + + private static PostFilterPolicySnapshot Compile(PostFilterOptions options) => + PostFilterPolicyCompiler.Compile(options); + + private static PostFilterPolicySnapshot Closed() => + Compile(new PostFilterOptions { Gate = PostFilterGateState.Closed }); + + private static PostFilterSpamAssassinOptions EnabledSpamAssassin( + PostFilterSpamOnFailure onFailure = PostFilterSpamOnFailure.Reject) => + new() + { + Enabled = true, + OnFailure = onFailure, + Hosts = ["127.0.0.1"], + }; + + private static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = NntpdOptions.DefaultMaxArticleSize }); + + private static string ValidArticle() => + "Date: " + PostRfcDate.Format(DateTimeOffset.UtcNow) + "\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: test\r\n" + + "Message-ID: <" + Guid.NewGuid().ToString("N") + "@example.com>\r\n" + + "\r\n" + + "body\r\n"; + + private static string YEncodedArticle() => + "Date: " + PostRfcDate.Format(DateTimeOffset.UtcNow) + "\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: test\r\n" + + "Message-ID: <" + Guid.NewGuid().ToString("N") + "@example.com>\r\n" + + "\r\n" + + "=ybegin line=128 size=3 name=a.bin\r\n" + + "abc\r\n" + + "=yend size=3\r\n"; + + private static async Task> DrainAsync( + PostFilterRejectionEvidenceQueue queue) + { + queue.Complete(); + var rows = new List(); + while (await queue.DequeueAsync(CancellationToken.None) is { } row) + { + rows.Add(row); + } + + return rows; + } + + private sealed class PostDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IPostFilter filter, + PostFilterMetrics? metrics = null, + IHistoryDb? historyDb = null, + bool cancelConnectionAfterAccept = false, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null, + INewsLogWriter? newsLog = null) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + IPostFilter wired = filter; + if (cancelConnectionAfterAccept) + { + wired = new CancelAfterAcceptFilter(filter, connection); + } + + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: historyDb, + postingTraceProtector: AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }), + postFilter: wired, + postFilterMetrics: metrics, + postFilterEvidence: postFilterEvidence, + newsLog: newsLog); + } + + public async Task WriteClientLineAsync(string line) => + await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class CancelAfterAcceptFilter : IPostFilter + { + private readonly IPostFilter _inner; + private readonly INntpConnection _connection; + + public CancelAfterAcceptFilter(IPostFilter inner, INntpConnection connection) + { + _inner = inner; + _connection = connection; + } + + public async ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + var result = await _inner.EvaluateAsync(request, cancellationToken).ConfigureAwait(false); + if (result.Decision == PostFilterDecision.Accept && result.Lease is not null) + { + await _connection.CompleteAsync().ConfigureAwait(false); + } + + return result; + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class SequenceClock + { + private int _value; + + public int Next() => Interlocked.Increment(ref _value); + } + + private sealed class RecordingIngestionQueue : IArticleIngestionQueue + { + private readonly IArticleIngestionQueue _inner; + private readonly SequenceClock _clock; + + public RecordingIngestionQueue(IArticleIngestionQueue inner, SequenceClock clock) + { + _inner = inner; + _clock = clock; + } + + public int TryAdmitCalls { get; private set; } + + public int AdmitSequence { get; private set; } + + public long MemoryLimitBytes => _inner.MemoryLimitBytes; + + public long QueuedBytes => _inner.QueuedBytes; + + public long PeakQueuedBytes => _inner.PeakQueuedBytes; + + public int MaxArticleBytes => _inner.MaxArticleBytes; + + public int Count => _inner.Count; + + public int PeakCount => _inner.PeakCount; + + public bool IsAccepting => _inner.IsAccepting; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) => + _inner.EnqueueAsync(article, cancellationToken); + + public bool TryProbeCapacity() => _inner.TryProbeCapacity(); + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + AdmitSequence = _clock.Next(); + TryAdmitCalls++; + return _inner.TryAdmit(article); + } + + public bool TryEnqueue(InboundArticle article) => _inner.TryEnqueue(article); + + public void Complete() => _inner.Complete(); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + _inner.DequeueAsync(cancellationToken); + } + + private sealed class RecordingQuotaStore : IPostFilterQuotaStore + { + private readonly InMemoryPostFilterQuotaStore _inner = new(); + private readonly SequenceClock? _clock; + + public RecordingQuotaStore(SequenceClock? clock = null) + { + _clock = clock; + } + + public bool Unavailable { get; set; } + + public bool CommitNoop { get; set; } + + public bool CommitUnavailable { get; set; } + + public CancellationToken LastReleaseToken { get; private set; } + + public List Operations { get; } = []; + + public int ReserveSequence { get; private set; } + + public int CommitSequence { get; private set; } + + public long LastReservedMessages { get; private set; } + + public long LastReservedBytes { get; private set; } + + public long LastCommittedMessages { get; private set; } + + public long LastCommittedBytes { get; private set; } + + public bool CommitReceivedUnits { get; private set; } + + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + Operations.Add("reserve"); + ReserveSequence = _clock?.Next() ?? Operations.Count; + LastReservedMessages = messages; + LastReservedBytes = bytes; + _ = reservationTtlMs; + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReserveStatus.Unavailable); + } + + return _inner.ReserveAsync( + accountName, + reservation, + now, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex, + cancellationToken, + reservationTtlMs); + } + + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + Operations.Add("commit"); + CommitSequence = _clock?.Next() ?? Operations.Count; + LastCommittedMessages = LastReservedMessages; + LastCommittedBytes = LastReservedBytes; + if (CommitNoop) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Noop); + } + + if (CommitUnavailable) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Unavailable); + } + + return _inner.CommitAsync(accountName, reservation, now, windows, ceilings, cancellationToken); + } + + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) + { + Operations.Add("release"); + LastReleaseToken = cancellationToken; + return _inner.ReleaseAsync(accountName, reservation, now, cancellationToken); + } + } + + private sealed class RecordingHistoryDb : IHistoryDb + { + private readonly SequenceClock? _clock; + + public RecordingHistoryDb(SequenceClock? clock = null) + { + _clock = clock; + } + + public int RememberCalls { get; private set; } + + public int RememberSequence { get; private set; } + + public bool ThrowOnRemember { get; set; } + + public HistoryLookupResult PeekResult { get; set; } = HistoryLookupResult.Unseen; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(PeekResult); + + public void Remember(ReadOnlyMemory messageId) + { + RememberCalls++; + RememberSequence = _clock?.Next() ?? RememberCalls; + if (ThrowOnRemember) + { + throw new InvalidOperationException("history remember failed"); + } + } + + public bool ContainsLocal(in HistoryDigest digest) => false; + } + + private sealed class RecordingSpamAssassin : IPostFilterSpamAssassin + { + public int Calls { get; private set; } + + public PostFilterSpamAssassinResult Result { get; set; } = PostFilterSpamAssassinResult.Ham(); + + public ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = article; + _ = accountName; + _ = target; + _ = scanContext; + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + return ValueTask.FromResult(Result); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostHarness.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostHarness.cs new file mode 100644 index 00000000..ba4924e6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterPostHarness.cs @@ -0,0 +1,440 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// Shared POST-boundary harness for PostFilter policy-matrix tests. +internal static class PostFilterPostHarness +{ + public static readonly NntpAuthorization Poster = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + public static readonly TimeSpan Safety = TimeSpan.FromSeconds(20); + + public static IPostFilter CreateFilter( + PostFilterPolicySnapshot snapshot, + IPostFilterQuotaStore quota, + IPostFilterSpamAssassin? sa = null, + PostFilterReservationIdentity? identity = null) => + CreateFilter( + new StaticPostFilterPolicySource(snapshot), + quota, + sa, + identity); + + public static IPostFilter CreateFilter( + IPostFilterPolicySource source, + IPostFilterQuotaStore quota, + IPostFilterSpamAssassin? sa = null, + PostFilterReservationIdentity? identity = null) => + new PostFilterEvaluator( + source, + quota, + identity ?? new PostFilterReservationIdentity("n1", "inc"), + sa ?? new PolicyRecordingSpamAssassin(), + NullLogger.Instance); + + public static PostFilterPolicySnapshot Compile(PostFilterOptions options, long revision = 1) + { + var repository = new InMemoryPostFilterPolicyRepository + { + Record = InMemoryPostFilterPolicyRepository.Create(options, revision), + }; + var record = repository.LoadAsync().AsTask().GetAwaiter().GetResult(); + return PostFilterPolicyCompiler.Compile(record.Options, record.Revision); + } + + public static PostFilterSpamAssassinOptions EnabledSpamAssassin( + PostFilterSpamOnFailure onFailure = PostFilterSpamOnFailure.Reject, + int? maxArticleSize = null, + string[]? excludeArtTypes = null) => + new() + { + Enabled = true, + OnFailure = onFailure, + Hosts = ["127.0.0.1"], + MaxArticleSize = maxArticleSize ?? 0, + ExcludeArtTypes = excludeArtTypes ?? [], + }; + + public static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = NntpdOptions.DefaultMaxArticleSize }); + + public static string TextArticle(string body = "body\r\n", string? messageId = null) => + "Date: " + PostRfcDate.Format(DateTimeOffset.UtcNow) + "\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: test\r\n" + + "Message-ID: <" + (messageId ?? Guid.NewGuid().ToString("N")) + "@example.com>\r\n" + + "\r\n" + + body; + + public static async Task PostAsync( + PostFilterPostDuplex duplex, + NntpSession session, + string expect, + string username = "poster", + string? article = null) + { + session.ApplySuccessfulAuthentication(username, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync((article ?? TextArticle()) + ".\r\n"); + Assert.Equal(expect, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } +} + +/// In-memory NNTP duplex used by PostFilter POST-boundary tests. +internal sealed class PostFilterPostDuplex : IAsyncDisposable +{ + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IPostFilter filter, + PostFilterMetrics? metrics = null, + IHistoryDb? historyDb = null, + bool cancelConnectionAfterAccept = false, + IPAddress? clientAddress = null, + IPostFilterRejectionEvidenceQueue? postFilterEvidence = null) + { + var address = clientAddress ?? IPAddress.Loopback; + var connection = new PolicyPipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(address, 119))); + IPostFilter wired = filter; + if (cancelConnectionAfterAccept) + { + wired = new PolicyCancelAfterAcceptFilter(filter, connection); + } + + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: historyDb, + postingTraceProtector: AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }), + postFilter: wired, + postFilterMetrics: metrics, + postFilterEvidence: postFilterEvidence); + } + + public async Task WriteClientLineAsync(string line) => + await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(PostFilterPostHarness.Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } +} + +/// Records quota operations and optionally delegates to an inner store. +internal sealed class PolicyRecordingQuotaStore : IPostFilterQuotaStore +{ + private readonly IPostFilterQuotaStore _inner; + + public PolicyRecordingQuotaStore(IPostFilterQuotaStore? inner = null) + { + _inner = inner ?? new InMemoryPostFilterQuotaStore(); + } + + public bool Unavailable { get; set; } + + public bool CommitNoop { get; set; } + + public bool CommitUnavailable { get; set; } + + public CancellationToken LastReleaseToken { get; private set; } + + public List Operations { get; } = []; + + public string? LastAccountName { get; private set; } + + public long LastReservedMessages { get; private set; } + + public long LastReservedBytes { get; private set; } + + public string? LastBodyHex { get; private set; } + + public ValueTask ReserveAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + long messages, + long bytes, + int mpUnits, + string? bodyHex, + CancellationToken cancellationToken = default, + long reservationTtlMs = 0) + { + Operations.Add("reserve"); + LastAccountName = accountName; + LastReservedMessages = messages; + LastReservedBytes = bytes; + LastBodyHex = bodyHex; + _ = reservationTtlMs; + if (Unavailable) + { + return ValueTask.FromResult(PostFilterQuotaReserveStatus.Unavailable); + } + + return _inner.ReserveAsync( + accountName, + reservation, + now, + windows, + ceilings, + messages, + bytes, + mpUnits, + bodyHex, + cancellationToken, + reservationTtlMs); + } + + public ValueTask CommitAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + PostFilterQuotaWindows windows, + PostFilterQuotaCeilings ceilings, + CancellationToken cancellationToken = default) + { + Operations.Add("commit"); + if (CommitNoop) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Noop); + } + + if (CommitUnavailable) + { + return ValueTask.FromResult(PostFilterQuotaCommitStatus.Unavailable); + } + + return _inner.CommitAsync(accountName, reservation, now, windows, ceilings, cancellationToken); + } + + public ValueTask ReleaseAsync( + string accountName, + PostFilterReservationId reservation, + DateTimeOffset now, + CancellationToken cancellationToken = default) + { + Operations.Add("release"); + LastReleaseToken = cancellationToken; + return _inner.ReleaseAsync(accountName, reservation, now, cancellationToken); + } +} + +/// Records SPAMD CHECK calls and builds the disposable scan representation. +internal sealed class PolicyRecordingSpamAssassin : IPostFilterSpamAssassin +{ + public int Calls { get; private set; } + + public PostFilterSpamAssassinResult Result { get; set; } = PostFilterSpamAssassinResult.Ham(); + + public byte[]? LastArtData { get; private set; } + + public byte[]? LastScan { get; private set; } + + public ArticleId LastArtId { get; private set; } + + public ulong LastArtHash { get; private set; } + + public ArticleType LastArtType { get; private set; } + + public int LastArtSize { get; private set; } + + public ArticleFieldTable LastFields { get; private set; } + + public ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = accountName; + _ = target; + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + LastArtData = article.ArtData.ToArray(); + LastArtId = article.ArtId; + LastArtHash = article.ArtHash; + LastArtType = article.ArtType; + LastArtSize = article.ArtSize; + LastFields = article.Fields; + LastScan = SpamdScanArticleBuilder.Build(article, scanContext); + Assert.True(LastArtData.AsSpan().SequenceEqual(article.ArtData.Span)); + return ValueTask.FromResult(Result); + } +} + +/// Records History Peek / Remember for POST-boundary assertions. +internal sealed class PolicyRecordingHistoryDb : IHistoryDb +{ + public int RememberCalls { get; private set; } + + public HistoryLookupResult PeekResult { get; set; } = HistoryLookupResult.Unseen; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(PeekResult); + + public void Remember(ReadOnlyMemory messageId) + { + _ = messageId; + RememberCalls++; + } + + public bool ContainsLocal(in HistoryDigest digest) => false; +} + +internal sealed class PolicyCancelAfterAcceptFilter : IPostFilter +{ + private readonly IPostFilter _inner; + private readonly INntpConnection _connection; + + public PolicyCancelAfterAcceptFilter(IPostFilter inner, INntpConnection connection) + { + _inner = inner; + _connection = connection; + } + + public async ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + var result = await _inner.EvaluateAsync(request, cancellationToken).ConfigureAwait(false); + if (result.Decision == PostFilterDecision.Accept && result.Lease is not null) + { + await _connection.CompleteAsync().ConfigureAwait(false); + } + + return result; + } +} + +internal sealed class PolicyPipeConnection : INntpConnection +{ + private readonly CancellationTokenSource _cts = new(); + + public PolicyPipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrix.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrix.cs new file mode 100644 index 00000000..43b38b34 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrix.cs @@ -0,0 +1,725 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// Shared POST-boundary production-policy matrix bodies. +/// +internal static class PostFilterProductionPolicyMatrix +{ + public static readonly (string Name, Func Run)[] All = + [ + ("Scenario01_DisabledGate_KeepsExistingPostPath", Scenario01_DisabledGate_KeepsExistingPostPath), + ("Scenario02_ClosedGate_Returns441_WithoutReserveOrSpamAssassin", Scenario02_ClosedGate_Returns441_WithoutReserveOrSpamAssassin), + ("Scenario03_AccountAndCidrDeny_Returns441_WithoutReserve", Scenario03_AccountAndCidrDeny_Returns441_WithoutReserve), + ("Scenario04_ArtTypeDeny_Returns441_WithoutReserve", Scenario04_ArtTypeDeny_Returns441_WithoutReserve), + ("Scenario05_Allowlist_SkipsSpamAssassin_KeepsDenyAndQuota", Scenario05_Allowlist_SkipsSpamAssassin_KeepsDenyAndQuota), + ("Scenario06_MessageCeiling_SecondPostReturns441_WithoutSpamAssassin", Scenario06_MessageCeiling_SecondPostReturns441_WithoutSpamAssassin), + ("Scenario07_ByteCeiling_UsesArticleRecordArtSize", Scenario07_ByteCeiling_UsesArticleRecordArtSize), + ("Scenario08_IdenticalCeiling_IsBodyHashNotArtHash", Scenario08_IdenticalCeiling_IsBodyHashNotArtHash), + ("Scenario09_AccountIsolation_SameBodyDoesNotShareQuota", Scenario09_AccountIsolation_SameBodyDoesNotShareQuota), + ("Scenario10_CrossNode_SameAccountSharesInMemoryQuota", Scenario10_CrossNode_SameAccountSharesInMemoryQuota), + ("Scenario11_EligibleHam_CallsSpamAssassin_CommitsAndReturns240", Scenario11_EligibleHam_CallsSpamAssassin_CommitsAndReturns240), + ("Scenario12_SpamResult_Returns441_ReleasesAndDoesNotRemember", Scenario12_SpamResult_Returns441_ReleasesAndDoesNotRemember), + ("Scenario13_ArticleTooLargeForSpamAssassin_IsEligibilitySkip", Scenario13_ArticleTooLargeForSpamAssassin_IsEligibilitySkip), + ("Scenario14_ExcludedArtType_SkipsSpamAssassin", Scenario14_ExcludedArtType_SkipsSpamAssassin), + ("Scenario15_AllowlistedAccount_BypassesSpamAssassinOnly", Scenario15_AllowlistedAccount_BypassesSpamAssassinOnly), + ("Scenario16_SpamAssassinOnFailureReject", static policy => Scenario16And17_SpamAssassinOnFailure(policy, PostFilterSpamOnFailure.Reject, "441 Posting failed", false)), + ("Scenario17_SpamAssassinOnFailureAccept", static policy => Scenario16And17_SpamAssassinOnFailure(policy, PostFilterSpamOnFailure.Accept, "240 Article received OK", true)), + ("Scenario18_RedisUnavailableAtReserve_FailsClosedWithoutSpamAssassin", Scenario18_RedisUnavailableAtReserve_FailsClosedWithoutSpamAssassin), + ("Scenario19_CommitUnavailableAfterAdmit_KeepsQueueAnd240_WithoutRelease", Scenario19_CommitUnavailableAfterAdmit_KeepsQueueAnd240_WithoutRelease), + ("Scenario20_TryAdmitFailure_ReleasesReservation", Scenario20_TryAdmitFailure_ReleasesReservation), + ("Scenario21_CancellationAfterReserve_ReleasesWithNone", Scenario21_CancellationAfterReserve_ReleasesWithNone), + ("Scenario22_CompletePath_ReservesScansAdmitsCommitsRemembers", Scenario22_CompletePath_ReservesScansAdmitsCommitsRemembers), + ("Scenario23And24_ArticleRecordUnchanged_AndScanIsDisposable", Scenario23And24_ArticleRecordUnchanged_AndScanIsDisposable), + ]; + + public static Task RunAsync(string name, IPostFilterMatrixPolicy policy) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(policy); + var match = All.Single(candidate => candidate.Name == name); + return match.Run(policy); + } + + public static async Task Scenario01_DisabledGate_KeepsExistingPostPath(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(await policy.PublishDisabledAsync(), quota, sa), + historyDb: history), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Empty(quota.Operations); + Assert.Equal(0, sa.Calls); + Assert.Equal(1, history.RememberCalls); + await AssertQueuedCanonicalAsync(queue); + } + + public static async Task Scenario02_ClosedGate_Returns441_WithoutReserveOrSpamAssassin(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Closed, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession(queue, PostFilterPostHarness.CreateFilter(snapshot, quota, sa)), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + Assert.Equal(0, sa.Calls); + } + + public static async Task Scenario03_AccountAndCidrDeny_Returns441_WithoutReserve(IPostFilterMatrixPolicy policy) + { + var accountQuota = new PolicyRecordingQuotaStore(); + var accountSa = new PolicyRecordingSpamAssassin(); + var accountQueue = PostFilterPostHarness.NewQueue(); + var accountSnapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var accountDuplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + accountDuplex, + accountDuplex.CreateSession( + accountQueue, + PostFilterPostHarness.CreateFilter(accountSnapshot, accountQuota, accountSa)), + "441 Posting failed"); + Assert.Equal(0, accountQueue.Count); + Assert.Empty(accountQuota.Operations); + Assert.Equal(0, accountSa.Calls); + + var cidrQuota = new PolicyRecordingQuotaStore(); + var cidrSa = new PolicyRecordingSpamAssassin(); + var cidrQueue = PostFilterPostHarness.NewQueue(); + var cidrSnapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedCidrs = ["127.0.0.0/8"], + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var cidrDuplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + cidrDuplex, + cidrDuplex.CreateSession( + cidrQueue, + PostFilterPostHarness.CreateFilter(cidrSnapshot, cidrQuota, cidrSa)), + "441 Posting failed"); + Assert.Equal(0, cidrQueue.Count); + Assert.Empty(cidrQuota.Operations); + Assert.Equal(0, cidrSa.Calls); + } + + public static async Task Scenario04_ArtTypeDeny_Returns441_WithoutReserve(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + RejectArtTypes = ["Default"], + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession(queue, PostFilterPostHarness.CreateFilter(snapshot, quota, sa)), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Empty(quota.Operations); + Assert.Equal(0, sa.Calls); + } + + public static async Task Scenario05_Allowlist_SkipsSpamAssassin_KeepsDenyAndQuota(IPostFilterMatrixPolicy policy) + { + var denyQuota = new PolicyRecordingQuotaStore(); + var denySa = new PolicyRecordingSpamAssassin(); + var denyQueue = PostFilterPostHarness.NewQueue(); + var denySnapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + DeniedAccounts = ["poster"], + AllowlistedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var denyDuplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + denyDuplex, + denyDuplex.CreateSession( + denyQueue, + PostFilterPostHarness.CreateFilter(denySnapshot, denyQuota, denySa)), + "441 Posting failed"); + Assert.Equal(0, denyQueue.Count); + Assert.Empty(denyQuota.Operations); + Assert.Equal(0, denySa.Calls); + + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + AllowlistedAccounts = ["poster"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota, sa); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), filter), + "240 Article received OK"); + await using var second = new PostFilterPostDuplex(); + var rejected = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(rejected, filter), + "441 Posting failed"); + Assert.Equal(0, rejected.Count); + Assert.Equal(0, sa.Calls); + Assert.Equal(new[] { "reserve", "commit", "reserve" }, quota.Operations); + } + + public static async Task Scenario06_MessageCeiling_SecondPostReturns441_WithoutSpamAssassin(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota, sa); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), filter), + "240 Article received OK"); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + var saCallsAfterAccept = sa.Calls; + await using var second = new PostFilterPostDuplex(); + var queue = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(queue, filter), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(saCallsAfterAccept, sa.Calls); + Assert.Equal(new[] { "reserve", "commit", "reserve" }, quota.Operations); + Assert.DoesNotContain("release", quota.Operations); + } + + public static async Task Scenario07_ByteCeiling_UsesArticleRecordArtSize(IPostFilterMatrixPolicy policy) + { + var probeQuota = new PolicyRecordingQuotaStore(); + var probeQueue = PostFilterPostHarness.NewQueue(); + var probeSnapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxBytesLong = 1_000_000 }, + }); + await using var probe = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + probe, + probe.CreateSession(probeQueue, PostFilterPostHarness.CreateFilter(probeSnapshot, probeQuota)), + "240 Article received OK"); + var inbound = await DequeueAsync(probeQueue); + Assert.Equal(inbound.Record.ArtSize, probeQuota.LastReservedBytes); + Assert.Equal(1, probeQuota.LastReservedMessages); + + var quota = new PolicyRecordingQuotaStore(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxBytesLong = inbound.Record.ArtSize }, + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), filter), + "240 Article received OK"); + await using var second = new PostFilterPostDuplex(); + var rejected = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(rejected, filter), + "441 Posting failed"); + Assert.Equal(0, rejected.Count); + Assert.Equal(inbound.Record.ArtSize, quota.LastReservedBytes); + } + + public static async Task Scenario08_IdenticalCeiling_IsBodyHashNotArtHash(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxIdenticalLong = 1 }, + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota); + const string body = "shared-canonical-body\r\n"; + var firstQueue = PostFilterPostHarness.NewQueue(); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(firstQueue, filter), + "240 Article received OK", + article: PostFilterPostHarness.TextArticle(body)); + var firstInbound = await DequeueAsync(firstQueue); + var bodyHex = PostFilterBodyHash.TryCompute(firstInbound.Record); + Assert.False(string.IsNullOrWhiteSpace(bodyHex)); + Assert.Equal(bodyHex, quota.LastBodyHex); + Assert.NotEqual(firstInbound.Record.ArtHash.ToString("x16"), quota.LastBodyHex); + + await using var sameBody = new PostFilterPostDuplex(); + var rejected = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + sameBody, + sameBody.CreateSession(rejected, filter), + "441 Posting failed", + article: PostFilterPostHarness.TextArticle(body)); + Assert.Equal(0, rejected.Count); + + await using var otherBody = new PostFilterPostDuplex(); + var accepted = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + otherBody, + otherBody.CreateSession(accepted, filter), + "240 Article received OK", + article: PostFilterPostHarness.TextArticle("different-body\r\n")); + Assert.Equal(1, accepted.Count); + var second = await DequeueAsync(accepted); + Assert.NotEqual(firstInbound.Record.ArtHash, second.Record.ArtHash); + Assert.NotEqual(bodyHex, PostFilterBodyHash.TryCompute(second.Record)); + } + + public static async Task Scenario09_AccountIsolation_SameBodyDoesNotShareQuota(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota); + const string body = "isolated-body\r\n"; + await using var alice = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + alice, + alice.CreateSession(PostFilterPostHarness.NewQueue(), filter), + "240 Article received OK", + username: "alice", + article: PostFilterPostHarness.TextArticle(body)); + Assert.Equal("alice", quota.LastAccountName); + await using var bob = new PostFilterPostDuplex(); + var bobQueue = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + bob, + bob.CreateSession(bobQueue, filter), + "240 Article received OK", + username: "bob", + article: PostFilterPostHarness.TextArticle(body)); + Assert.Equal(1, bobQueue.Count); + Assert.Equal("bob", quota.LastAccountName); + Assert.Equal(new[] { "reserve", "commit", "reserve", "commit" }, quota.Operations); + } + + public static async Task Scenario10_CrossNode_SameAccountSharesInMemoryQuota(IPostFilterMatrixPolicy policy) + { + var shared = new InMemoryPostFilterQuotaStore(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + }); + var nodeA = PostFilterPostHarness.CreateFilter( + snapshot, + shared, + identity: new PostFilterReservationIdentity("n1", "incA")); + var nodeB = PostFilterPostHarness.CreateFilter( + snapshot, + shared, + identity: new PostFilterReservationIdentity("n2", "incB")); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), nodeA), + "240 Article received OK"); + await using var second = new PostFilterPostDuplex(); + var queue = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(queue, nodeB), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + } + + public static async Task Scenario11_EligibleHam_CallsSpamAssassin_CommitsAndReturns240(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota, sa), + historyDb: history), + "240 Article received OK"); + Assert.Equal(1, sa.Calls); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + Assert.Equal(1, history.RememberCalls); + var inbound = await AssertQueuedCanonicalAsync(queue); + Assert.Equal(sa.LastArtHash, inbound.Record.ArtHash); + Assert.Equal(sa.LastArtId, inbound.Record.ArtId); + Assert.True(sa.LastArtData!.AsSpan().SequenceEqual(inbound.Record.ArtData.Span)); + Assert.False(sa.LastScan!.AsSpan().SequenceEqual(inbound.Record.ArtData.Span)); + } + + public static async Task Scenario12_SpamResult_Returns441_ReleasesAndDoesNotRemember(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Spam("spam") }; + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota, sa), + historyDb: history), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(1, sa.Calls); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(0, history.RememberCalls); + Assert.NotNull(sa.LastArtData); + Assert.NotNull(sa.LastScan); + Assert.False(sa.LastScan.AsSpan().SequenceEqual(sa.LastArtData)); + Assert.Equal(sa.LastArtSize, sa.LastArtData.Length); + } + + public static async Task Scenario13_ArticleTooLargeForSpamAssassin_IsEligibilitySkip(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Failed("must-not-run") }; + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(maxArticleSize: 64), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession(queue, PostFilterPostHarness.CreateFilter(snapshot, quota, sa)), + "240 Article received OK"); + Assert.Equal(0, sa.Calls); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + var inbound = await AssertQueuedCanonicalAsync(queue); + Assert.True(inbound.Record.ArtSize >= 64); + } + + public static async Task Scenario14_ExcludedArtType_SkipsSpamAssassin(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Failed("must-not-run") }; + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(excludeArtTypes: ["Default"]), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession(queue, PostFilterPostHarness.CreateFilter(snapshot, quota, sa)), + "240 Article received OK"); + Assert.Equal(0, sa.Calls); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + var inbound = await AssertQueuedCanonicalAsync(queue); + Assert.Equal(ArticleType.Default, inbound.Record.ArtType); + } + + public static async Task Scenario15_AllowlistedAccount_BypassesSpamAssassinOnly(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + AllowlistedCidrs = ["127.0.0.0/8"], + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 1 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + var filter = PostFilterPostHarness.CreateFilter(snapshot, quota, sa); + await using var first = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + first, + first.CreateSession(PostFilterPostHarness.NewQueue(), filter, clientAddress: IPAddress.Loopback), + "240 Article received OK"); + Assert.Equal(0, sa.Calls); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + await using var second = new PostFilterPostDuplex(); + var queue = PostFilterPostHarness.NewQueue(); + await PostFilterPostHarness.PostAsync( + second, + second.CreateSession(queue, filter), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(0, sa.Calls); + } + + public static async Task Scenario16And17_SpamAssassinOnFailure( + IPostFilterMatrixPolicy policy, + PostFilterSpamOnFailure onFailure, + string expected, + bool enqueued) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Failed("down") }; + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(onFailure), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota, sa), + historyDb: history), + expected); + Assert.Equal(1, sa.Calls); + Assert.Equal(enqueued ? 1 : 0, queue.Count); + Assert.Equal(enqueued ? 1 : 0, history.RememberCalls); + Assert.Equal(enqueued ? new[] { "reserve", "commit" } : new[] { "reserve", "release" }, quota.Operations); + } + + public static async Task Scenario18_RedisUnavailableAtReserve_FailsClosedWithoutSpamAssassin(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore { Unavailable = true }; + var sa = new PolicyRecordingSpamAssassin(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession(queue, PostFilterPostHarness.CreateFilter(snapshot, quota, sa)), + "441 Posting failed"); + Assert.Equal(0, queue.Count); + Assert.Equal(new[] { "reserve" }, quota.Operations); + Assert.DoesNotContain("commit", quota.Operations); + Assert.Equal(0, sa.Calls); + } + + public static async Task Scenario19_CommitUnavailableAfterAdmit_KeepsQueueAnd240_WithoutRelease(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore { CommitUnavailable = true }; + var metrics = new PostFilterMetrics(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota), + metrics), + "240 Article received OK"); + Assert.Equal(1, queue.Count); + Assert.Equal(1, metrics.CommitUnavailable); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + Assert.DoesNotContain("release", quota.Operations); + } + + public static async Task Scenario20_TryAdmitFailure_ReleasesReservation(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + DisabledArticleIngestionQueue.Instance, + PostFilterPostHarness.CreateFilter(snapshot, quota)), + "441 Posting failed"); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + Assert.DoesNotContain("commit", quota.Operations); + } + + public static async Task Scenario21_CancellationAfterReserve_ReleasesWithNone(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + }); + await using var duplex = new PostFilterPostDuplex(); + var session = duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota), + historyDb: history, + cancelConnectionAfterAccept: true); + session.ApplySuccessfulAuthentication("poster", PostFilterPostHarness.Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(PostFilterPostHarness.TextArticle() + ".\r\n"); + await run.WaitAsync(PostFilterPostHarness.Safety); + Assert.Equal(0, queue.Count); + Assert.Equal(new[] { "reserve", "release" }, quota.Operations); + Assert.Equal(CancellationToken.None, quota.LastReleaseToken); + Assert.Equal(0, history.RememberCalls); + } + + public static async Task Scenario22_CompletePath_ReservesScansAdmitsCommitsRemembers(IPostFilterMatrixPolicy policy) + { + var quota = new PolicyRecordingQuotaStore(); + var sa = new PolicyRecordingSpamAssassin(); + var history = new PolicyRecordingHistoryDb(); + var queue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var duplex = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + duplex, + duplex.CreateSession( + queue, + PostFilterPostHarness.CreateFilter(snapshot, quota, sa), + historyDb: history), + "240 Article received OK"); + Assert.Equal(1, sa.Calls); + Assert.Equal(new[] { "reserve", "commit" }, quota.Operations); + Assert.Equal(1, history.RememberCalls); + var inbound = await AssertQueuedCanonicalAsync(queue); + Assert.True(inbound.Record.ArtData.Equals(inbound.Payload)); + Assert.True(sa.LastArtData!.AsSpan().SequenceEqual(inbound.Record.ArtData.Span)); + Assert.False(ReferenceEquals(sa.LastScan, sa.LastArtData)); + } + + public static async Task Scenario23And24_ArticleRecordUnchanged_AndScanIsDisposable(IPostFilterMatrixPolicy policy) + { + var hamSa = new PolicyRecordingSpamAssassin(); + var hamQuota = new PolicyRecordingQuotaStore(); + var hamQueue = PostFilterPostHarness.NewQueue(); + var snapshot = await policy.PublishAsync(new PostFilterOptions + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10 }, + SpamAssassin = PostFilterPostHarness.EnabledSpamAssassin(), + }); + await using var ham = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + ham, + ham.CreateSession(hamQueue, PostFilterPostHarness.CreateFilter(snapshot, hamQuota, hamSa)), + "240 Article received OK"); + var queued = await AssertQueuedCanonicalAsync(hamQueue); + Assert.Equal(hamSa.LastArtHash, queued.Record.ArtHash); + Assert.Equal(hamSa.LastArtId, queued.Record.ArtId); + Assert.Equal(hamSa.LastArtType, queued.Record.ArtType); + Assert.Equal(hamSa.LastArtSize, queued.Record.ArtSize); + Assert.Equal(hamSa.LastFields.MessageId, queued.Record.Fields.MessageId); + Assert.True(hamSa.LastArtData!.AsSpan().SequenceEqual(queued.Record.ArtData.Span)); + Assert.False(hamSa.LastScan!.AsSpan().SequenceEqual(queued.Record.ArtData.Span)); + Assert.Contains("Received:", System.Text.Encoding.ASCII.GetString(hamSa.LastScan!), StringComparison.Ordinal); + + var spamSa = new PolicyRecordingSpamAssassin { Result = PostFilterSpamAssassinResult.Spam("spam") }; + var spamQuota = new PolicyRecordingQuotaStore(); + await using var spam = new PostFilterPostDuplex(); + await PostFilterPostHarness.PostAsync( + spam, + spam.CreateSession( + PostFilterPostHarness.NewQueue(), + PostFilterPostHarness.CreateFilter(snapshot, spamQuota, spamSa)), + "441 Posting failed"); + Assert.Equal(1, spamSa.Calls); + Assert.Equal(spamSa.LastArtSize, spamSa.LastArtData!.Length); + Assert.False(spamSa.LastScan!.AsSpan().SequenceEqual(spamSa.LastArtData)); + Assert.NotEqual(0UL, spamSa.LastArtHash); + } + + private static async Task AssertQueuedCanonicalAsync(ArticleIngestionQueue queue) + { + var inbound = await DequeueAsync(queue); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Record.ArtData.Equals(inbound.Payload)); + Assert.True(inbound.Record.ArtSize > 0); + return inbound; + } + + private static async Task DequeueAsync(ArticleIngestionQueue queue) + { + using var cts = new CancellationTokenSource(PostFilterPostHarness.Safety); + var inbound = await queue.DequeueAsync(cts.Token); + Assert.NotNull(inbound); + return inbound!; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrixTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrixTests.cs new file mode 100644 index 00000000..a2d72fc2 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterProductionPolicyMatrixTests.cs @@ -0,0 +1,109 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// POST-boundary production-policy matrix for PostFilter v1. +/// Test-local snapshots only; does not change production Gate=Disabled. +/// +public sealed class PostFilterProductionPolicyMatrixTests +{ + private static readonly IPostFilterMatrixPolicy Policy = new InMemoryPostFilterMatrixPolicy(); + + [Fact] + public Task Scenario01_DisabledGate_KeepsExistingPostPath() => + PostFilterProductionPolicyMatrix.Scenario01_DisabledGate_KeepsExistingPostPath(Policy); + + [Fact] + public Task Scenario02_ClosedGate_Returns441_WithoutReserveOrSpamAssassin() => + PostFilterProductionPolicyMatrix.Scenario02_ClosedGate_Returns441_WithoutReserveOrSpamAssassin(Policy); + + [Fact] + public Task Scenario03_AccountAndCidrDeny_Returns441_WithoutReserve() => + PostFilterProductionPolicyMatrix.Scenario03_AccountAndCidrDeny_Returns441_WithoutReserve(Policy); + + [Fact] + public Task Scenario04_ArtTypeDeny_Returns441_WithoutReserve() => + PostFilterProductionPolicyMatrix.Scenario04_ArtTypeDeny_Returns441_WithoutReserve(Policy); + + [Fact] + public Task Scenario05_Allowlist_SkipsSpamAssassin_KeepsDenyAndQuota() => + PostFilterProductionPolicyMatrix.Scenario05_Allowlist_SkipsSpamAssassin_KeepsDenyAndQuota(Policy); + + [Fact] + public Task Scenario06_MessageCeiling_SecondPostReturns441_WithoutSpamAssassin() => + PostFilterProductionPolicyMatrix.Scenario06_MessageCeiling_SecondPostReturns441_WithoutSpamAssassin(Policy); + + [Fact] + public Task Scenario07_ByteCeiling_UsesArticleRecordArtSize() => + PostFilterProductionPolicyMatrix.Scenario07_ByteCeiling_UsesArticleRecordArtSize(Policy); + + [Fact] + public Task Scenario08_IdenticalCeiling_IsBodyHashNotArtHash() => + PostFilterProductionPolicyMatrix.Scenario08_IdenticalCeiling_IsBodyHashNotArtHash(Policy); + + [Fact] + public Task Scenario09_AccountIsolation_SameBodyDoesNotShareQuota() => + PostFilterProductionPolicyMatrix.Scenario09_AccountIsolation_SameBodyDoesNotShareQuota(Policy); + + [Fact] + public Task Scenario10_CrossNode_SameAccountSharesInMemoryQuota() => + PostFilterProductionPolicyMatrix.Scenario10_CrossNode_SameAccountSharesInMemoryQuota(Policy); + + [Fact] + public Task Scenario11_EligibleHam_CallsSpamAssassin_CommitsAndReturns240() => + PostFilterProductionPolicyMatrix.Scenario11_EligibleHam_CallsSpamAssassin_CommitsAndReturns240(Policy); + + [Fact] + public Task Scenario12_SpamResult_Returns441_ReleasesAndDoesNotRemember() => + PostFilterProductionPolicyMatrix.Scenario12_SpamResult_Returns441_ReleasesAndDoesNotRemember(Policy); + + [Fact] + public Task Scenario13_ArticleTooLargeForSpamAssassin_IsEligibilitySkip() => + PostFilterProductionPolicyMatrix.Scenario13_ArticleTooLargeForSpamAssassin_IsEligibilitySkip(Policy); + + [Fact] + public Task Scenario14_ExcludedArtType_SkipsSpamAssassin() => + PostFilterProductionPolicyMatrix.Scenario14_ExcludedArtType_SkipsSpamAssassin(Policy); + + [Fact] + public Task Scenario15_AllowlistedAccount_BypassesSpamAssassinOnly() => + PostFilterProductionPolicyMatrix.Scenario15_AllowlistedAccount_BypassesSpamAssassinOnly(Policy); + + [Theory] + [InlineData(PostFilterSpamOnFailure.Reject, "441 Posting failed", false)] + [InlineData(PostFilterSpamOnFailure.Accept, "240 Article received OK", true)] + public Task Scenario16And17_SpamAssassinOnFailure( + PostFilterSpamOnFailure onFailure, + string expected, + bool enqueued) => + PostFilterProductionPolicyMatrix.Scenario16And17_SpamAssassinOnFailure( + Policy, + onFailure, + expected, + enqueued); + + [Fact] + public Task Scenario18_RedisUnavailableAtReserve_FailsClosedWithoutSpamAssassin() => + PostFilterProductionPolicyMatrix.Scenario18_RedisUnavailableAtReserve_FailsClosedWithoutSpamAssassin(Policy); + + [Fact] + public Task Scenario19_CommitUnavailableAfterAdmit_KeepsQueueAnd240_WithoutRelease() => + PostFilterProductionPolicyMatrix.Scenario19_CommitUnavailableAfterAdmit_KeepsQueueAnd240_WithoutRelease(Policy); + + [Fact] + public Task Scenario20_TryAdmitFailure_ReleasesReservation() => + PostFilterProductionPolicyMatrix.Scenario20_TryAdmitFailure_ReleasesReservation(Policy); + + [Fact] + public Task Scenario21_CancellationAfterReserve_ReleasesWithNone() => + PostFilterProductionPolicyMatrix.Scenario21_CancellationAfterReserve_ReleasesWithNone(Policy); + + [Fact] + public Task Scenario22_CompletePath_ReservesScansAdmitsCommitsRemembers() => + PostFilterProductionPolicyMatrix.Scenario22_CompletePath_ReservesScansAdmitsCommitsRemembers(Policy); + + [Fact] + public Task Scenario23And24_ArticleRecordUnchanged_AndScanIsDisposable() => + PostFilterProductionPolicyMatrix.Scenario23And24_ArticleRecordUnchanged_AndScanIsDisposable(Policy); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterRejectionEvidenceQueueTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterRejectionEvidenceQueueTests.cs new file mode 100644 index 00000000..77085d88 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterRejectionEvidenceQueueTests.cs @@ -0,0 +1,126 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterRejectionEvidenceQueueTests +{ + [Fact] + public void TryEnqueue_DropsWhenFull_AndCounts() + { + var queue = new PostFilterRejectionEvidenceQueue(capacity: 1); + Assert.True(queue.TryEnqueue(Evidence("closed"))); + Assert.False(queue.TryEnqueue(Evidence("denied"))); + Assert.Equal(1, queue.Count); + Assert.Equal(1, queue.Dropped); + } + + [Fact] + public async Task Writer_PersistsAndCountsSuccess() + { + var factory = new FakeNntpDbConnectionFactory(); + await using var nntpDb = CreateDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var queue = new PostFilterRejectionEvidenceQueue(); + var service = new PostFilterRejectionEvidenceService( + nntpDb, + queue, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.True(queue.TryEnqueue(Evidence("spam"))); + await WaitForAsync(() => queue.Written == 1); + Assert.Equal(1, queue.Written); + Assert.Equal("spam", Assert.Single(factory.Connections.SelectMany(static connection => connection.Rejections)).Reason); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Writer_DatabaseFailure_DoesNotThrowToProducer() + { + var factory = new FakeNntpDbConnectionFactory + { + InsertPostFilterRejectionException = new NntpDbUnavailableException("down"), + }; + await using var nntpDb = CreateDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var queue = new PostFilterRejectionEvidenceQueue(); + var service = new PostFilterRejectionEvidenceService( + nntpDb, + queue, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.True(queue.TryEnqueue(Evidence("closed"))); + await WaitForAsync(() => queue.WriteFailures == 1); + Assert.Equal(0, queue.Written); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Stop_DrainsQueuedEvidence() + { + var factory = new FakeNntpDbConnectionFactory(); + await using var nntpDb = CreateDb(factory); + await nntpDb.StartAsync(CancellationToken.None); + var queue = new PostFilterRejectionEvidenceQueue(); + var service = new PostFilterRejectionEvidenceService( + nntpDb, + queue, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.True(queue.TryEnqueue(Evidence("closed"))); + Assert.True(queue.TryEnqueue(Evidence("denied"))); + await service.StopAsync(CancellationToken.None); + Assert.Equal(2, factory.Connections.Sum(static connection => connection.InsertPostFilterRejectionCount)); + Assert.Equal(0, queue.Count); + } + + [Fact] + public void DisabledQueue_DoesNotChangeDecision() + { + Assert.True(DisabledPostFilterRejectionEvidenceQueue.Instance.TryEnqueue(Evidence("closed"))); + } + + private static NntpDbService CreateDb(FakeNntpDbConnectionFactory factory) => + new( + factory, + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }), + NullLogger.Instance); + + private static PostFilterRejectionEvidence Evidence(string reason) => + new( + DateTimeOffset.UtcNow, + policyRevision: 2, + accountName: "poster", + sourceAddress: IPAddress.Loopback, + artType: ArticleType.Default, + messageId: "", + articleSize: 4, + stage: PostFilterStage.Gate, + reason: reason, + spamAssassinStatus: null, + spamAssassinScore: null, + spamAssassinThreshold: null, + articlePayload: "test"u8.ToArray()); + + private static async Task WaitForAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamAssassinEligibilityTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamAssassinEligibilityTests.cs new file mode 100644 index 00000000..27306dbb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamAssassinEligibilityTests.cs @@ -0,0 +1,266 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.PostFilter.Quota; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class PostFilterSpamAssassinEligibilityTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + private const int Kilobyte768 = 768 * 1024; + + [Fact] + public async Task SmallEligibleArticle_CallsSpamAssassin() + { + var sa = new RecordingSpamAssassin(); + var result = await EvaluateAsync(EnabledOptions(), ArticleOfSize(200), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(1, sa.Calls); + Assert.NotNull(result.Lease); + } + + [Fact] + public async Task ArticleExactlyAtMaximum_IsSkipped() + { + var sa = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.MaxArticleSize = 256; + var result = await EvaluateAsync(options, ArticleOfSize(256), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(0, sa.Calls); + Assert.NotNull(result.Lease); + } + + [Fact] + public async Task ArticleOneByteBelowMaximum_IsScanned() + { + var sa = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.MaxArticleSize = 256; + var result = await EvaluateAsync(options, ArticleOfSize(255), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task ArticleAboveMaximum_DoesNotCallSpamAssassin() + { + var sa = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.MaxArticleSize = 256; + var result = await EvaluateAsync(options, ArticleOfSize(257), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(0, sa.Calls); + Assert.NotNull(result.Lease); + } + + [Fact] + public async Task YEncoded_IsExcludedByDefault() + { + var sa = new RecordingSpamAssassin(); + var result = await EvaluateAsync(EnabledOptions(), ArticleOfSize(200, ArticleType.YEncoded), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(0, sa.Calls); + } + + [Fact] + public async Task Binary_IsScannedUnlessExcluded() + { + var scanned = new RecordingSpamAssassin(); + await EvaluateAsync(EnabledOptions(), ArticleOfSize(200, ArticleType.Binary), scanned); + Assert.Equal(1, scanned.Calls); + + var skipped = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.ExcludeArtTypes = ["Binary"]; + await EvaluateAsync(options, ArticleOfSize(200, ArticleType.Binary), skipped); + Assert.Equal(0, skipped.Calls); + } + + [Fact] + public async Task MultipleExcludedFlags_SkipWhenAnyMatch() + { + var sa = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.ExcludeArtTypes = ["YEncoded", "Binary", "Mime"]; + await EvaluateAsync(options, ArticleOfSize(200, ArticleType.Mime | ArticleType.Default), sa); + Assert.Equal(0, sa.Calls); + await EvaluateAsync(options, ArticleOfSize(200, ArticleType.Default), sa); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task ZeroMaximum_DisablesSizeGate() + { + var sa = new RecordingSpamAssassin(); + var options = EnabledOptions(); + options.SpamAssassin.MaxArticleSize = 0; + var result = await EvaluateAsync(options, ArticleOfSize(Kilobyte768), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task EligibilityUsesCapturedSnapshot() + { + var first = PostFilterPolicyCompiler.Compile(EnabledOptions()); + var secondOptions = EnabledOptions(); + secondOptions.SpamAssassin.MaxArticleSize = 1; + var second = PostFilterPolicyCompiler.Compile(secondOptions); + var source = new MutablePolicySource(first); + var sa = new RecordingSpamAssassin { Block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var filter = new PostFilterEvaluator( + source, + new InMemoryPostFilterQuotaStore(), + new PostFilterReservationIdentity("n1", "inc"), + sa, + NullLogger.Instance); + var evaluate = filter.EvaluateAsync(Request(ArticleOfSize(200)), CancellationToken.None).AsTask(); + await sa.BlockEntered.Task; + source.Current = second; + sa.Block.SetResult(); + Assert.Equal(PostFilterDecision.Accept, (await evaluate).Decision); + Assert.Equal(1, sa.Calls); + Assert.Equal(first.SpamAssassinHosts, sa.LastTarget.Hosts); + Assert.Equal(first.SpamAssassinMaxConnections, sa.LastTarget.MaxConnections); + var skipped = await filter.EvaluateAsync(Request(ArticleOfSize(200))); + Assert.Equal(PostFilterDecision.Accept, skipped.Decision); + Assert.NotNull(skipped.Lease); + Assert.Equal(1, sa.Calls); + } + + [Fact] + public async Task DefaultPolicy_DoesNotSend768KbToSpamAssassin() + { + var sa = new RecordingSpamAssassin(); + var snapshot = PostFilterPolicyCompiler.Compile(EnabledOptions()); + Assert.Equal(PostFilterSpamAssassinOptions.DefaultMaxArticleSize, snapshot.SpamAssassinMaxArticleSize); + Assert.Equal(ArticleType.YEncoded, snapshot.SpamAssassinExcludeArtTypes); + var result = await EvaluateAsync(EnabledOptions(), ArticleOfSize(Kilobyte768), sa); + Assert.Equal(PostFilterDecision.Accept, result.Decision); + Assert.Equal(0, sa.Calls); + Assert.True(Kilobyte768 >= snapshot.SpamAssassinMaxArticleSize); + } + + [Fact] + public void Compiler_RejectsNegativeMaxArticleSize() + { + var options = EnabledOptions(); + options.SpamAssassin.MaxArticleSize = -1; + Assert.Throws(() => PostFilterPolicyCompiler.Compile(options)); + } + + [Fact] + public void Compiler_RejectsMaxConnectionsOutsideRange() + { + var options = EnabledOptions(); + options.SpamAssassin.MaxConnections = 0; + Assert.Throws(() => PostFilterPolicyCompiler.Compile(options)); + options.SpamAssassin.MaxConnections = 33; + Assert.Throws(() => PostFilterPolicyCompiler.Compile(options)); + } + + private static async Task EvaluateAsync( + PostFilterOptions options, + ArticleRecord article, + RecordingSpamAssassin sa) + { + var filter = new PostFilterEvaluator( + new StaticPostFilterPolicySource(PostFilterPolicyCompiler.Compile(options)), + new InMemoryPostFilterQuotaStore(), + new PostFilterReservationIdentity("n1", "inc"), + sa, + NullLogger.Instance); + return await filter.EvaluateAsync(Request(article)); + } + + private static PostFilterOptions EnabledOptions() => + new() + { + Gate = PostFilterGateState.Active, + Quota = new PostFilterQuotaOptions { MaxMessagesLong = 10_000 }, + SpamAssassin = new PostFilterSpamAssassinOptions + { + Enabled = true, + OnFailure = PostFilterSpamOnFailure.Reject, + Hosts = ["127.0.0.1"], + }, + }; + + private static PostFilterRequest Request(ArticleRecord article) => + new( + article, + "poster", + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + accountPolicy: null, + Now, + ["misc.test"], + Now, + "nntpd01.usenet.ninja"); + + private static ArticleRecord ArticleOfSize(int size, ArticleType type = ArticleType.Default) + { + var header = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\n"u8; + if (size < header.Length) + { + throw new ArgumentOutOfRangeException(nameof(size), size, "Size must cover the header block."); + } + + var bytes = new byte[size]; + header.CopyTo(bytes); + bytes.AsSpan(header.Length).Fill((byte)'x'); + return new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 1, + artType: type, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: bytes, + fields: default); + } + + private sealed class MutablePolicySource : IPostFilterPolicySource + { + public MutablePolicySource(PostFilterPolicySnapshot current) => Current = current; + + public PostFilterPolicySnapshot Current { get; set; } + } + + private sealed class RecordingSpamAssassin : IPostFilterSpamAssassin + { + public int Calls { get; private set; } + + public PostFilterSpamAssassinTarget LastTarget { get; private set; } + + public TaskCompletionSource? Block { get; set; } + + public TaskCompletionSource BlockEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask CheckAsync( + ArticleRecord article, + string? accountName, + PostFilterSpamAssassinTarget target, + SpamdScanContext scanContext, + CancellationToken cancellationToken = default) + { + _ = article; + _ = accountName; + _ = scanContext; + LastTarget = target; + Calls++; + if (Block is not null) + { + BlockEntered.TrySetResult(); + await Block.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + return PostFilterSpamAssassinResult.Ham(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamdLiveTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamdLiveTests.cs new file mode 100644 index 00000000..ab500327 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/PostFilterSpamdLiveTests.cs @@ -0,0 +1,295 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +/// +/// SPAMD CHECK coverage against a loopback server. Real SpamAssassin is opt-in only. +/// +public sealed class PostFilterSpamdLiveTests +{ + private static readonly DateTimeOffset Now = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task LocalSpamd_Check_AcceptsEmailLikeScanAsHam() + { + var original = Encoding.ASCII.GetBytes( + "Path: news.example!not-for-mail\r\n" + + "From: ham@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: lab ham\r\n" + + "Message-ID: \r\n" + + "Date: 1 Jan 2026 00:00:00 +0000\r\n" + + "Xref: nntpd01.usenet.ninja misc.test:1\r\n" + + "Injection-Info: nntpd01.usenet.ninja; posting-account=poster\r\n" + + "X-Trace: secret-trace\r\n" + + "X-Complaints-To: abuse@example.com\r\n" + + "NNTP-Posting-Host: 203.0.113.9\r\n" + + "\r\n" + + "This is a short ham body.\r\n"); + var copy = original.ToArray(); + var article = Article(original, "", artHash: 1); + var context = new SpamdScanContext(IPAddress.Parse("203.0.113.8"), "nntpd01.usenet.ninja", Now); + var scan = SpamdScanArticleBuilder.Build(article, context); + var scanText = Encoding.ASCII.GetString(scan); + Assert.Equal(1, CountOccurrences(scanText, "Received:")); + Assert.Contains("Received: from [203.0.113.8]", scanText, StringComparison.Ordinal); + Assert.Contains("To: usenet@nntpd01.usenet.ninja", scanText, StringComparison.Ordinal); + Assert.Contains("X-Usenet-Newsgroups: misc.test", scanText, StringComparison.Ordinal); + Assert.Contains("Message-ID: ", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("Path:", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("X-Complaints-To:", scanText, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host:", scanText, StringComparison.Ordinal); + Assert.True(copy.AsSpan().SequenceEqual(article.ArtData.Span)); + + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var result = await client.CheckAsync(article, "poster", Target(server.Port, 1), context, CancellationToken.None); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status); + Assert.Equal(0m, result.Score); + Assert.Equal(5.0m, result.Threshold); + Assert.True(copy.AsSpan().SequenceEqual(article.ArtData.Span)); + Assert.Equal(1, metrics.CheckRequests); + Assert.Equal(1, metrics.ConnectionsEstablished); + Assert.Equal(1, server.Checks); + Assert.Equal(1, server.Connections); + var captured = Assert.Single(server.Requests); + Assert.True(scan.AsSpan().SequenceEqual(captured)); + } + + [Fact] + public async Task LocalSpamd_SequentialChecks_ReuseOneConnection() + { + var bytes = Encoding.ASCII.GetBytes( + "From: ham@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: reuse\r\n" + + "Message-ID: \r\n" + + "Date: 1 Jan 2026 00:00:00 +0000\r\n" + + "\r\n" + + "short\r\n"); + var article = Article(bytes, "", artHash: 2); + var context = new SpamdScanContext(IPAddress.Loopback, "nntpd01.usenet.ninja", Now); + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, 1); + for (var i = 0; i < 20; i++) + { + var result = await client.CheckAsync(article, "poster", target, context); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status); + } + + Assert.Equal(20, metrics.CheckRequests); + Assert.Equal(1, metrics.ConnectionsEstablished); + Assert.Equal(19, metrics.ConnectionReuses); + Assert.Equal(20, server.Checks); + Assert.Equal(1, server.Connections); + Assert.True(bytes.AsSpan().SequenceEqual(article.ArtData.Span)); + } + + [Fact] + public async Task LocalSpamd_SpamResponse_ParsesScoreAndThreshold() + { + var spam = "SPAMD/1.1 0 EX_OK\r\nSpam: True ; 6.5 / 5.0\r\n\r\n"u8.ToArray(); + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, response: spam); + await using var client = new SpamdCheckClient(); + var result = await client.CheckAsync(SampleArticle(), "poster", Target(server.Port, 1), SampleContext()); + Assert.Equal(PostFilterSpamAssassinStatus.Spam, result.Status); + Assert.Equal(6.5m, result.Score); + Assert.Equal(5.0m, result.Threshold); + Assert.Equal(1, server.Checks); + } + + [Fact] + public async Task LocalSpamd_ServerError_IsFailedAndNotReused() + { + var error = "SPAMD/1.1 69 EX_TEMPFAIL\r\n\r\n"u8.ToArray(); + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, response: error); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var result = await client.CheckAsync(SampleArticle(), "poster", Target(server.Port, 1), SampleContext()); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Contains("EX_TEMPFAIL", result.Detail, StringComparison.Ordinal); + Assert.Equal(1, metrics.Evictions); + } + + [Fact] + public async Task LocalSpamd_PrematureDisconnect_IsClosed() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: false, dropAfterRequest: true); + await using var client = new SpamdCheckClient(); + var result = await client.CheckAsync(SampleArticle(), "poster", Target(server.Port, 1), SampleContext()); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Equal("closed", result.Detail); + Assert.Equal(1, server.Checks); + Assert.Single(server.Requests); + } + + [Fact] + public async Task LocalSpamd_MalformedRequest_IsRecorded() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: false); + using var tcp = new TcpClient(); + await tcp.ConnectAsync(IPAddress.Loopback, server.Port); + await using var stream = tcp.GetStream(); + var garbage = "NOT-SPAMC\r\n\r\n"u8.ToArray(); + await stream.WriteAsync(garbage); + await stream.FlushAsync(); + tcp.Client.Shutdown(SocketShutdown.Send); + await server.Accepted.WaitAsync(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(() => server.ProtocolFaults.Count > 0); + Assert.Contains("unexpected-command", server.ProtocolFaults); + server.AcknowledgeProtocolFaults(); + } + + [SpamdIntegrationFact] + public async Task RealSpamd_Check_AcceptsEmailLikeScanAsHam() + { + var (host, port) = SpamdIntegrationFactAttribute.RequireEndpoint(); + var original = Encoding.ASCII.GetBytes( + "From: ham@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: lab ham\r\n" + + "Message-ID: \r\n" + + "Date: 1 Jan 2026 00:00:00 +0000\r\n" + + "\r\n" + + "This is a short ham body.\r\n"); + var article = Article(original, "", artHash: 1); + var context = new SpamdScanContext(IPAddress.Parse("203.0.113.8"), "nntpd01.usenet.ninja", Now); + var scan = SpamdScanArticleBuilder.Build(article, context); + var target = Target(port, 1, host); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var result = await client.CheckAsync(article, "poster", target, context, CancellationToken.None); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status); + Assert.True(original.AsSpan().SequenceEqual(article.ArtData.Span)); + + var symbols = await RequestSymbolsAsync(host, port, scan); + Assert.Contains("SPAMD/", symbols, StringComparison.Ordinal); + Assert.Contains("EX_OK", symbols, StringComparison.Ordinal); + Assert.DoesNotContain("NO_RECEIVED", symbols, StringComparison.Ordinal); + Assert.DoesNotContain("MISSING_HEADERS", symbols, StringComparison.Ordinal); + Assert.Equal(1, metrics.CheckRequests); + Assert.Equal(1, metrics.ConnectionsEstablished); + } + + [SpamdIntegrationFact] + public async Task RealSpamd_SequentialChecks_ReportConnectionReuse() + { + var (host, port) = SpamdIntegrationFactAttribute.RequireEndpoint(); + var bytes = Encoding.ASCII.GetBytes( + "From: ham@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: reuse\r\n" + + "Message-ID: \r\n" + + "Date: 1 Jan 2026 00:00:00 +0000\r\n" + + "\r\n" + + "short\r\n"); + var article = Article(bytes, "", artHash: 2); + var context = new SpamdScanContext(IPAddress.Loopback, "nntpd01.usenet.ninja", Now); + var target = Target(port, 1, host); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + for (var i = 0; i < 20; i++) + { + var result = await client.CheckAsync(article, "poster", target, context); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status); + } + + Assert.Equal(20, metrics.CheckRequests); + Assert.True(metrics.ConnectionsEstablished >= 1); + Assert.True(metrics.ConnectionsEstablished <= 20); + } + + private static PostFilterSpamAssassinTarget Target(int port, int maxConnections, string host = "127.0.0.1") => + new( + [host], + port, + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(30), + PostFilterSpamAssassinOptions.DefaultProtocolVersion, + maxConnections, + PostFilterSpamAssassinHostSelection.RoundRobin); + + private static ArticleRecord SampleArticle() => + Article( + "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(), + "", + 1); + + private static SpamdScanContext SampleContext() => + new(IPAddress.Loopback, "nntpd01.usenet.ninja", Now); + + private static ArticleRecord Article(byte[] bytes, string messageId, ulong artHash) => + new( + ArticleId.FromMessageId(Encoding.ASCII.GetBytes(messageId)), + artHash, + ArticleType.Default, + artLines: 1, + Now.UtcDateTime, + ArticleParseStatus.CanonicalV1, + bytes, + fields: default); + + private static int CountOccurrences(string text, string value) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(value, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += value.Length; + } + + return count; + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private static async Task RequestSymbolsAsync(string host, int port, byte[] scan) + { + using var tcp = new TcpClient(); + using var connectCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await tcp.ConnectAsync(host, port, connectCts.Token); + await using var stream = tcp.GetStream(); + var header = Encoding.ASCII.GetBytes( + $"SYMBOLS SPAMC/1.5\r\nContent-length: {scan.Length}\r\n\r\n"); + await stream.WriteAsync(header); + await stream.WriteAsync(scan); + await stream.FlushAsync(); + tcp.Client.Shutdown(SocketShutdown.Send); + var buffer = new byte[4096]; + var total = 0; + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + while (total < buffer.Length) + { + var read = await stream.ReadAsync(buffer.AsMemory(total), readCts.Token); + if (read == 0) + { + break; + } + + total += read; + } + + return Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaEngineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaEngineTests.cs new file mode 100644 index 00000000..6081e5b0 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaEngineTests.cs @@ -0,0 +1,825 @@ +using System.Reflection; +using System.Text; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter.Quota; + +/// +/// In-memory algorithm contracts for accept-quota. Does not require live Redis. +/// +public sealed class PostFilterQuotaEngineTests +{ + private const long Now = 5_000; + private const long Ttl = 10_000; + private const string BodyA = "0123456789abcdef"; + private const string BodyB = "fedcba9876543210"; + + private static readonly PostFilterQuotaWindows Windows = new(10_000, 1_000); + + [Fact] + public void AccountHash_MatchesSessionAdmissionConvention() + { + const string account = "alice"; + var session = Encoding.ASCII.GetString(SessionStateKeys.CreateSession(account)); + var quota = Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateQuota(account)); + var multipost = Encoding.ASCII.GetString(PostFilterQuotaKeys.CreateMultipost(account)); + Assert.StartsWith("nntpd:sess:", session, StringComparison.Ordinal); + Assert.StartsWith("nntpd:pf:q:", quota, StringComparison.Ordinal); + Assert.StartsWith("nntpd:pf:m:", multipost, StringComparison.Ordinal); + Assert.Equal(session["nntpd:sess:".Length..], quota["nntpd:pf:q:".Length..]); + Assert.Equal(quota["nntpd:pf:q:".Length..], multipost["nntpd:pf:m:".Length..]); + } + + [Fact] + public void Reserve_OneAccepted() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Assert.True(engine.HasLiveReservation(Q("acct"), id, Now)); + } + + [Fact] + public void Reserve_DeniedAtExactMessageCeiling() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public async Task Reserve_TwoConcurrentRaceForLastSlot() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + var accepted = 0; + await Task.WhenAll( + Task.Run(() => Try(engine, "acct", Id("n1", "a", 1), ceilings, ref accepted)), + Task.Run(() => Try(engine, "acct", Id("n1", "a", 2), ceilings, ref accepted))); + Assert.Equal(1, accepted); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public async Task Reserve_RaceAcrossNodeIds() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + var accepted = 0; + await Task.WhenAll( + Task.Run(() => Try(engine, "acct", Id("n1", "a", 1), ceilings, ref accepted)), + Task.Run(() => Try(engine, "acct", Id("n2", "a", 1), ceilings, ref accepted))); + Assert.Equal(1, accepted); + } + + [Fact] + public async Task Reserve_RaceAcrossIncarnations() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + var accepted = 0; + await Task.WhenAll( + Task.Run(() => Try(engine, "acct", Id("n1", "inc1", 1), ceilings, ref accepted)), + Task.Run(() => Try(engine, "acct", Id("n1", "inc2", 1), ceilings, ref accepted))); + Assert.Equal(1, accepted); + } + + [Fact] + public void Reserve_MultipleConcurrentFromOneProcess() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 3); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 3), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 4), ceilings: ceilings)); + Assert.Equal(3, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public void Reserve_MessageCeiling() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 2); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 3), ceilings: ceilings)); + } + + [Fact] + public void Reserve_ByteCeiling() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Bytes(maxL: 100); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings, bytes: 60)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedBytesLong, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings, bytes: 50)); + } + + [Fact] + public void Reserve_MultipostCeiling() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Identical(maxL: 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedIdenticalLong, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings, mpUnits: 1, bodyHex: BodyA)); + } + + [Fact] + public void Reserve_LongAndShortSimultaneously() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 1, 0, 0); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesShort, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + } + + [Fact] + public void Reserve_DisabledDimensionsAreSkipped() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = new PostFilterQuotaCeilings(0, 0, 0, 0, 0, 0); + for (var i = 1; i <= 20; i++) + { + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", i), ceilings: ceilings)); + } + } + + [Fact] + public void Reserve_DenialOrder_IsDeterministic() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = new PostFilterQuotaCeilings(1, 10, 1, 1, 10, 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings, bytes: 10, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings, bytes: 10, mpUnits: 1, bodyHex: BodyA)); + } + + [Fact] + public void Reserve_IdempotentSameToken() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public void Reserve_ConflictDifferentUnits() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, bytes: 10)); + Assert.Equal(PostFilterQuotaReserveCodes.Conflict, Reserve(engine, "acct", id, bytes: 20)); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public void Reserve_ConflictDifferentBodyHash() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Identical(maxL: 5); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, ceilings: ceilings, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal( + PostFilterQuotaReserveCodes.Conflict, + Reserve(engine, "acct", id, ceilings: ceilings, mpUnits: 1, bodyHex: BodyB)); + } + + [Fact] + public void Release_SuccessAndDuplicate() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Assert.Equal(PostFilterQuotaReleaseCodes.Released, engine.Release(Q("acct"), M("acct"), id, Now)); + Assert.Equal(PostFilterQuotaReleaseCodes.Noop, engine.Release(Q("acct"), M("acct"), id, Now)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 2))); + } + + [Fact] + public void Release_AfterCommit_IsNoop() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.Equal(PostFilterQuotaReleaseCodes.Noop, engine.Release(Q("acct"), M("acct"), id, Now)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Release_AfterExpiry_IsNoop() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Assert.Equal(PostFilterQuotaReleaseCodes.Noop, engine.Release(Q("acct"), M("acct"), id, Now + Ttl)); + Assert.False(engine.HasReservationField(Q("acct"), id)); + } + + [Fact] + public void Release_StaleCannotAffectNewerReservation() + { + var engine = new PostFilterQuotaEngine(); + var oldId = Id("n1", "a", 1); + var newId = Id("n1", "a", 2); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", oldId)); + Assert.Equal(PostFilterQuotaReleaseCodes.Released, engine.Release(Q("acct"), M("acct"), oldId, Now)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", newId)); + Assert.Equal(PostFilterQuotaReleaseCodes.Noop, engine.Release(Q("acct"), M("acct"), oldId, Now)); + Assert.True(engine.HasLiveReservation(Q("acct"), newId, Now)); + } + + [Fact] + public void Commit_Success() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = new PostFilterQuotaCeilings(10, 10_000, 0, 10, 10_000, 0); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings, bytes: 40)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.False(engine.HasReservationField(Q("acct"), id)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(40, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Commit_Duplicate_DoesNotDoubleCount() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", id, ceilings)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Commit_AfterRelease_IsNoop() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaReleaseCodes.Released, engine.Release(Q("acct"), M("acct"), id, Now)); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", id, ceilings)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Commit_AfterExpiry_IsNoop() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", id, ceilings, Now + Ttl)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now + Ttl))); + } + + [Fact] + public void DerivedHold_StaysLiveThroughSaOperationTimeout_ThenExpires() + { + var hold = PostFilterQuotaDefaults.HoldMilliseconds(true, TimeSpan.FromSeconds(30)); + Assert.Equal(31_000, hold); + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + engine.Reserve(Q("acct"), M("acct"), id, Now, hold, Windows, ceilings, 1, 1, 0, string.Empty)); + Assert.Equal( + PostFilterQuotaCommitCodes.Committed, + engine.Commit(Q("acct"), M("acct"), id, Now + 30_000, Windows, ceilings)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now + 30_000))); + } + + [Fact] + public void ExpiredReservation_DoesNotConsumeQuotaForLaterPost() + { + var engine = new PostFilterQuotaEngine(); + var first = Id("n1", "a", 1); + var second = Id("n1", "a", 2); + var ceilings = Messages(maxL: 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", first, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", first, ceilings, Now + Ttl)); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", second, nowMs: Now + Ttl, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", second, ceilings, Now + Ttl)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now + Ttl))); + } + + [Fact] + public void Commit_StaleGeneration_IsNoop() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + engine.SeedReservation(Q("acct"), id, Now + Ttl, storedGeneration: 99, 1, 10, 0, string.Empty); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", id, ceilings)); + Assert.True(engine.HasReservationField(Q("acct"), id)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Commit_DoesNotAcceptCallerSuppliedUnits() + { + var method = typeof(IPostFilterQuotaStore).GetMethod(nameof(IPostFilterQuotaStore.CommitAsync)); + Assert.NotNull(method); + foreach (var parameter in method.GetParameters()) + { + Assert.False( + parameter.Name is "messages" or "bytes" or "mpUnits" or "bodyHex", + parameter.Name); + } + + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = new PostFilterQuotaCeilings(10, 10_000, 0, 10, 10_000, 0); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, ceilings: ceilings, bytes: 7)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.Equal(7, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void BucketBoundary_ReservationRemainsEffective_AndCommitChargesCurrent() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 10); + var id = Id("n1", "a", 1); + engine.SeedCommitted(Q("acct"), PostFilterQuotaKeys.CommittedMessagesField('L', 0), 9); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, nowMs: 9_999, ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 2), nowMs: 9_999, ceilings: ceilings)); + + var after = 10_002; + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n2", "b", 1), nowMs: after, ceilings: ceilings)); + var accepted = 1; + for (var g = 2; g <= 12; g++) + { + if (Reserve(engine, "acct", Id("n2", "b", g), nowMs: after, ceilings: ceilings) + == PostFilterQuotaReserveCodes.Accept) + { + accepted++; + } + } + + Assert.Equal(9, accepted); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings, after)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', 0)); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', 1)); + } + + [Fact] + public void Multipost_DifferentHashesIndependent() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Identical(maxL: 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings, mpUnits: 1, bodyHex: BodyB)); + } + + [Fact] + public async Task Multipost_SameHashRacesAcrossNodes() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Identical(maxL: 1); + var accepted = 0; + await Task.WhenAll( + Task.Run(() => TryMp(engine, "acct", Id("n1", "a", 1), ceilings, BodyA, ref accepted)), + Task.Run(() => TryMp(engine, "acct", Id("n2", "b", 1), ceilings, BodyA, ref accepted))); + Assert.Equal(1, accepted); + } + + [Fact] + public void ExpiredReservations_ArePruned() + { + var engine = new PostFilterQuotaEngine(); + var first = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", first)); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 2), nowMs: Now + Ttl)); + Assert.False(engine.HasReservationField(Q("acct"), first)); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public void MultipleExpiredReservations_ArePruned() + { + var engine = new PostFilterQuotaEngine(); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1))); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 2))); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 3), nowMs: Now + Ttl)); + Assert.Equal(1, engine.ReservationFieldCount(Q("acct"))); + } + + [Fact] + public void CrashBeforeCommit_ExpiresThenUnderCounts() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 10); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Noop, Commit(engine, "acct", id, ceilings, Now + Ttl)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now + Ttl))); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 2), nowMs: Now + Ttl, ceilings: ceilings)); + } + + [Fact] + public void CrashBeforeRelease_ExpiresAndRecovers() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 2), nowMs: Now + Ttl, ceilings: ceilings)); + } + + [Fact] + public void RepeatedWindows_RemainBounded() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 5); + for (var window = 0; window < 8; window++) + { + var now = (window * Windows.LongWindowMs) + 100; + var id = Id("n1", "a", window + 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, nowMs: now, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings, now)); + } + + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', 7)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', 0)); + Assert.True(engine.QuotaFieldCount(Q("acct")) <= 4); + } + + [Fact] + public void WindowExpiry_DropsEndedCommittedUsage() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 3), nowMs: 10_000, ceilings: ceilings)); + } + + [Fact] + public void ConcurrentReserveAndCommit() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 10); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id, ceilings: ceilings)); + Parallel.Invoke( + () => Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)), + () => Reserve(engine, "acct", Id("n1", "a", 2), ceilings: ceilings)); + var committed = engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now)); + var reserved = engine.ReservationFieldCount(Q("acct")); + Assert.Equal(1, committed); + Assert.True(reserved is 0 or 1); + Assert.True(committed + reserved <= 2); + } + + [Fact] + public void ConcurrentReserveAndRelease() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", id)); + Parallel.Invoke( + () => engine.Release(Q("acct"), M("acct"), id, Now), + () => Reserve(engine, "acct", Id("n1", "a", 2))); + Assert.True(engine.ReservationFieldCount(Q("acct")) <= 1); + } + + [Fact] + public void ConcurrentStaleReleaseAndNewReserve() + { + var engine = new PostFilterQuotaEngine(); + var stale = Id("n1", "old", 1); + var fresh = Id("n1", "new", 1); + Assert.Equal(PostFilterQuotaReserveCodes.Accept, Reserve(engine, "acct", stale)); + Assert.Equal(PostFilterQuotaReleaseCodes.Released, engine.Release(Q("acct"), M("acct"), stale, Now)); + Parallel.Invoke( + () => engine.Release(Q("acct"), M("acct"), stale, Now), + () => Reserve(engine, "acct", fresh)); + Assert.True(engine.HasLiveReservation(Q("acct"), fresh, Now)); + } + + [Fact] + public void AccountsAreIsolated() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "alice", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "bob", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "alice", Id("n1", "a", 2), ceilings: ceilings)); + } + + [Fact] + public void SameAccount_VisibleAcrossNodeIdentities() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Messages(maxL: 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", Id("n1", "a", 1), ceilings: ceilings)); + Assert.Equal( + PostFilterQuotaReserveCodes.DeniedMessagesLong, + Reserve(engine, "acct", Id("n2", "b", 1), ceilings: ceilings)); + } + + [Fact] + public async Task Reserve_TwoConcurrentRaceForLastByteSlot() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = Bytes(maxL: 40); + var accepted = 0; + await Task.WhenAll( + Task.Run(() => Try(engine, "acct", Id("n1", "a", 1), ceilings, ref accepted, bytes: 40)), + Task.Run(() => Try(engine, "acct", Id("n2", "b", 1), ceilings, ref accepted, bytes: 40))); + Assert.Equal(1, accepted); + } + + [Fact] + public void Commit_DisabledCeilingsWriteNoCommittedFields() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + var reserveCeilings = new PostFilterQuotaCeilings(10, 10_000, 2, 10, 10_000, 2); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, ceilings: reserveCeilings, bytes: 40, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal( + PostFilterQuotaCommitCodes.Committed, + Commit(engine, "acct", id, new PostFilterQuotaCeilings(0, 0, 0, 0, 0, 0))); + Assert.False(engine.HasReservationField(Q("acct"), id)); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.CommittedIdentical(M("acct"), BodyA, 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.QuotaFieldCount(Q("acct"))); + Assert.Equal(0, engine.MultipostFieldCount(M("acct"))); + } + + [Fact] + public void Commit_CeilingMagnitudeDoesNotChangeReservedUnits() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve( + engine, + "acct", + id, + ceilings: new PostFilterQuotaCeilings(10, 10_000, 0, 10, 10_000, 0), + messages: 2, + bytes: 40)); + Assert.Equal( + PostFilterQuotaCommitCodes.Committed, + Commit(engine, "acct", id, new PostFilterQuotaCeilings(1, 1, 0, 1, 1, 0))); + Assert.Equal(2, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(40, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(2, engine.CommittedMessages(Q("acct"), 'S', Windows.BucketId('S', Now))); + Assert.Equal(40, engine.CommittedBytes(Q("acct"), 'S', Windows.BucketId('S', Now))); + } + + [Fact] + public void Commit_MissingReservation_CannotInventUsageViaCeilings() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + Assert.Equal( + PostFilterQuotaCommitCodes.Noop, + Commit(engine, "acct", id, new PostFilterQuotaCeilings(999, 999_999, 999, 999, 999_999, 999))); + Assert.Equal(0, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.QuotaFieldCount(Q("acct"))); + } + + [Fact] + public void Commit_ReservationBodyWithSeparator_StillReadsUnits() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + const string body = "ab|cd|ef"; + engine.SeedReservation(Q("acct"), id, Now + Ttl, 1, 1, 9, 1, body); + Assert.Equal( + PostFilterQuotaCommitCodes.Committed, + Commit(engine, "acct", id, new PostFilterQuotaCeilings(10, 10_000, 2, 0, 0, 0))); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(9, engine.CommittedBytes(Q("acct"), 'L', Windows.BucketId('L', Now))); + Assert.Equal(1, engine.CommittedIdentical(M("acct"), body, 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void Commit_EmptyBodyMultipost_DoesNotWriteIdenticalField() + { + var engine = new PostFilterQuotaEngine(); + var id = Id("n1", "a", 1); + engine.SeedReservation(Q("acct"), id, Now + Ttl, 1, 1, 1, 1, string.Empty); + Assert.Equal( + PostFilterQuotaCommitCodes.Committed, + Commit(engine, "acct", id, new PostFilterQuotaCeilings(10, 0, 2, 0, 0, 0))); + Assert.Equal(0, engine.MultipostFieldCount(M("acct"))); + Assert.Equal(1, engine.CommittedMessages(Q("acct"), 'L', Windows.BucketId('L', Now))); + } + + [Fact] + public void IdleKeyTtl_OutlivesCurrentBucketsAndReservation() + { + var reservationTtl = (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds; + (long LongMs, long ShortMs, long NowMs)[] cases = + [ + (10_000, 1_000, 0), + (10_000, 1_000, 5_000), + (10_000, 1_000, 9_999), + (86_400_000, 600_000, 0), + (86_400_000, 600_000, 86_399_999), + (1, 1, 0), + (1_000, 10_000, 7_500), + ]; + + foreach (var (longMs, shortMs, nowMs) in cases) + { + var windows = new PostFilterQuotaWindows(longMs, shortMs); + var idle = Math.Max(longMs, shortMs) + reservationTtl + PostFilterQuotaDefaults.IdleTtlSkewMs; + var keyExpiry = nowMs + idle; + foreach (var window in new[] { PostFilterQuotaKeys.LongWindow, PostFilterQuotaKeys.ShortWindow }) + { + var bucketEnd = (windows.BucketId(window, nowMs) + 1) * windows.WindowMs(window); + Assert.True( + keyExpiry >= bucketEnd + reservationTtl, + $"window {window} L={longMs} S={shortMs} now={nowMs}"); + } + + Assert.True(keyExpiry > nowMs + reservationTtl); + } + } + + [Fact] + public void Commit_WritesIdenticalOnlyWhenEnabled() + { + var engine = new PostFilterQuotaEngine(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 2, 10, 0, 0); + var id = Id("n1", "a", 1); + Assert.Equal( + PostFilterQuotaReserveCodes.Accept, + Reserve(engine, "acct", id, ceilings: ceilings, mpUnits: 1, bodyHex: BodyA)); + Assert.Equal(PostFilterQuotaCommitCodes.Committed, Commit(engine, "acct", id, ceilings)); + Assert.Equal(1, engine.CommittedIdentical(M("acct"), BodyA, 'L', Windows.BucketId('L', Now))); + Assert.Equal(0, engine.CommittedIdentical(M("acct"), BodyA, 'S', Windows.BucketId('S', Now))); + } + + [Fact] + public async Task InMemoryStore_MapsCodesAndUnavailable() + { + var store = new InMemoryPostFilterQuotaStore(); + var id = Id("n1", "a", 1); + var ceilings = Messages(maxL: 1); + var now = DateTimeOffset.FromUnixTimeMilliseconds(Now); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await store.ReserveAsync("acct", id, now, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await store.CommitAsync("acct", id, now, Windows, ceilings)); + store.Unavailable = true; + Assert.Equal( + PostFilterQuotaReserveStatus.Unavailable, + await store.ReserveAsync("acct", Id("n1", "a", 2), now, Windows, ceilings, 1, 1, 0, null)); + } + + private static PostFilterReservationId Id(string node, string incarnation, long generation) => + new(node, incarnation, generation); + + private static string Q(string account) => InMemoryPostFilterQuotaStore.QuotaKey(account); + + private static string M(string account) => InMemoryPostFilterQuotaStore.MultipostKey(account); + + private static PostFilterQuotaCeilings Messages(long maxL, long maxS = 0) => + new(maxL, 0, 0, maxS, 0, 0); + + private static PostFilterQuotaCeilings Bytes(long maxL) => new(0, maxL, 0, 0, 0, 0); + + private static PostFilterQuotaCeilings Identical(long maxL) => new(0, 0, maxL, 0, 0, 0); + + private static long Reserve( + PostFilterQuotaEngine engine, + string account, + PostFilterReservationId id, + long nowMs = Now, + PostFilterQuotaCeilings? ceilings = null, + long messages = 1, + long bytes = 1, + int mpUnits = 0, + string bodyHex = "") => + engine.Reserve( + Q(account), + M(account), + id, + nowMs, + Ttl, + Windows, + ceilings ?? Messages(10), + messages, + bytes, + mpUnits, + bodyHex); + + private static long Commit( + PostFilterQuotaEngine engine, + string account, + PostFilterReservationId id, + PostFilterQuotaCeilings ceilings, + long nowMs = Now) => + engine.Commit(Q(account), M(account), id, nowMs, Windows, ceilings); + + private static void Try( + PostFilterQuotaEngine engine, + string account, + PostFilterReservationId id, + PostFilterQuotaCeilings ceilings, + ref int accepted, + long bytes = 1) + { + if (Reserve(engine, account, id, ceilings: ceilings, bytes: bytes) == PostFilterQuotaReserveCodes.Accept) + { + Interlocked.Increment(ref accepted); + } + } + + private static void TryMp( + PostFilterQuotaEngine engine, + string account, + PostFilterReservationId id, + PostFilterQuotaCeilings ceilings, + string body, + ref int accepted) + { + if (Reserve(engine, account, id, ceilings: ceilings, mpUnits: 1, bodyHex: body) + == PostFilterQuotaReserveCodes.Accept) + { + Interlocked.Increment(ref accepted); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaLuaStaticAuditTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaLuaStaticAuditTests.cs new file mode 100644 index 00000000..b4a71020 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaLuaStaticAuditTests.cs @@ -0,0 +1,268 @@ +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter.Quota; + +/// Static Lua/C# quota parity. These tests do not execute live Redis. +public sealed class PostFilterQuotaLuaStaticAuditTests +{ + [Fact] + public async Task ProductionStore_DispatchesExactQuotaScripts() + { + var redis = new RecordingRedis(); + var store = new RedisPostFilterQuotaStore(redis); + var now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + var windows = new PostFilterQuotaWindows(10_000, 1_000); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 10, 0, 0); + var id = new PostFilterReservationId("n1", "a", 1); + + _ = await store.ReserveAsync("alice", id, now, windows, ceilings, 1, 1, 0, null); + Assert.Same(PostFilterQuotaScripts.Reserve, redis.LastScript); + + _ = await store.CommitAsync("alice", id, now, windows, ceilings); + Assert.Same(PostFilterQuotaScripts.Commit, redis.LastScript); + + _ = await store.ReleaseAsync("alice", id, now); + Assert.Same(PostFilterQuotaScripts.Release, redis.LastScript); + } + + [Fact] + public void Scripts_UseSingleEval_NoClientGetIncr() + { + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("redis.call('GET'", script, StringComparison.Ordinal); + Assert.DoesNotContain("redis.call('INCR'", script, StringComparison.Ordinal); + Assert.DoesNotContain("redis.call('KEYS'", script, StringComparison.Ordinal); + Assert.DoesNotContain("EVAL", script, StringComparison.Ordinal); + } + } + + [Fact] + public void Scripts_UseOnlyDocumentedRedisCommands() + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "HGETALL", "HGET", "HSET", "HDEL", "HLEN", "HINCRBY", "DEL", "EXISTS", "PEXPIRE", + }; + foreach (var script in AllScripts()) + { + foreach (var command in EnumerateRedisCalls(script)) + { + Assert.Contains(command, allowed); + } + } + } + + [Fact] + public void Reserve_ChecksBeforeWrite_AndDenyWritesNothing() + { + var script = PostFilterQuotaScripts.Reserve; + var hset = script.IndexOf("redis.call('HSET'", StringComparison.Ordinal); + var deny = script.IndexOf("if maxML > 0", StringComparison.Ordinal); + Assert.True(deny > 0 && deny < hset); + Assert.Contains("return 1", script, StringComparison.Ordinal); + Assert.Contains("return 7", script, StringComparison.Ordinal); + } + + [Fact] + public void Commit_ReadsUnitsFromReservation_NotArgv() + { + Assert.Contains("rec.messages", PostFilterQuotaScripts.Commit, StringComparison.Ordinal); + Assert.Contains("rec.bytes", PostFilterQuotaScripts.Commit, StringComparison.Ordinal); + Assert.DoesNotContain("tonumber(ARGV[13])", PostFilterQuotaScripts.Commit, StringComparison.Ordinal); + } + + [Fact] + public void Release_NeverIncrCommitted() + { + Assert.DoesNotContain("HINCRBY", PostFilterQuotaScripts.Release, StringComparison.Ordinal); + Assert.DoesNotContain("c:m:", PostFilterQuotaScripts.Release, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreFailureMapping_IsUnavailable() + { + var now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + var windows = new PostFilterQuotaWindows(10_000, 1_000); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 10, 0, 0); + var id = new PostFilterReservationId("n1", "a", 1); + + var unavailable = new RecordingRedis { BeginSucceeds = false }; + var store = new RedisPostFilterQuotaStore(unavailable); + Assert.Equal( + PostFilterQuotaReserveStatus.Unavailable, + await store.ReserveAsync("alice", id, now, windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Unavailable, + await store.CommitAsync("alice", id, now, windows, ceilings)); + + var redisDown = new RecordingRedis { EvaluateException = new RedisUnavailableException("timeout") }; + store = new RedisPostFilterQuotaStore(redisDown); + Assert.Equal( + PostFilterQuotaReserveStatus.Unavailable, + await store.ReserveAsync("alice", id, now, windows, ceilings, 1, 1, 0, null)); + + var canceled = new RecordingRedis { EvaluateException = new OperationCanceledException() }; + store = new RedisPostFilterQuotaStore(canceled); + await Assert.ThrowsAsync( + () => store.ReserveAsync("alice", id, now, windows, ceilings, 1, 1, 0, null).AsTask()); + } + + [Fact] + public async Task ReserveCanceledAfterRedisWrite_IsStillReleasable() + { + var redis = new FakeRedisService + { + Database = + { + ScriptStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + BlockScript = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }, + }; + var store = new RedisPostFilterQuotaStore(redis); + var now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + var windows = new PostFilterQuotaWindows(10_000, 1_000); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + var id = new PostFilterReservationId("n1", "a", 1); + using var cts = new CancellationTokenSource(); + var reserve = store.ReserveAsync( + "alice", + id, + now, + windows, + ceilings, + 1, + 1, + 0, + null, + cts.Token).AsTask(); + await redis.Database.ScriptStarted!.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + redis.Database.BlockScript!.SetResult(); + await Assert.ThrowsAnyAsync(() => reserve); + Assert.Equal(PostFilterQuotaReleaseStatus.Released, await store.ReleaseAsync("alice", id, now)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await store.ReserveAsync( + "alice", + new PostFilterReservationId("n2", "b", 1), + now, + windows, + ceilings, + 1, + 1, + 0, + null)); + } + + [Fact] + public async Task FakeRedis_DispatchesQuotaScriptsToEngine() + { + var redis = new FakeRedisService(); + var store = new RedisPostFilterQuotaStore(redis); + var now = DateTimeOffset.FromUnixTimeMilliseconds(5_000); + var windows = new PostFilterQuotaWindows(10_000, 1_000); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + var first = new PostFilterReservationId("n1", "a", 1); + var second = new PostFilterReservationId("n2", "b", 1); + + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await store.ReserveAsync("alice", first, now, windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.DeniedMessagesLong, + await store.ReserveAsync("alice", second, now, windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await store.CommitAsync("alice", first, now, windows, ceilings)); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await store.CommitAsync("alice", first, now, windows, ceilings)); + } + + private static IEnumerable AllScripts() + { + yield return PostFilterQuotaScripts.Reserve; + yield return PostFilterQuotaScripts.Commit; + yield return PostFilterQuotaScripts.Release; + } + + private static IEnumerable EnumerateRedisCalls(string script) + { + const string prefix = "redis.call('"; + var start = 0; + while (true) + { + var index = script.IndexOf(prefix, start, StringComparison.Ordinal); + if (index < 0) + { + yield break; + } + + var from = index + prefix.Length; + var end = script.IndexOf('\'', from); + yield return script[from..end]; + start = end + 1; + } + } + + private sealed class RecordingRedis : IRedisService, IRedisDatabase + { + public string? LastScript { get; private set; } + + public bool BeginSucceeds { get; set; } = true; + + public Exception? EvaluateException { get; set; } + + public IRedisDatabase Database => this; + + public bool IsUnavailable => !BeginSucceeds; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return BeginSucceeds; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public Task PingAsync(CancellationToken cancellationToken = default) => + Task.FromResult(TimeSpan.FromMilliseconds(1)); + + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + LastScript = script; + if (EvaluateException is not null) + { + throw EvaluateException; + } + + return ValueTask.FromResult(0L); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaRedisLuaIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaRedisLuaIntegrationTests.cs new file mode 100644 index 00000000..4ae5c1fd --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/Quota/PostFilterQuotaRedisLuaIntegrationTests.cs @@ -0,0 +1,394 @@ +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter.Quota; + +/// +/// Live EVAL of through +/// against the dedicated test Redis. +/// +public sealed class PostFilterQuotaRedisLuaIntegrationTests : IClassFixture +{ + private const long Now = 5_000; + private const string BodyA = "0123456789abcdef"; + private static readonly DateTimeOffset At = DateTimeOffset.FromUnixTimeMilliseconds(Now); + private static readonly PostFilterQuotaWindows Windows = new(10_000, 1_000); + + private readonly PostFilterQuotaRedisIntegrationFixture _redis; + + public PostFilterQuotaRedisLuaIntegrationTests(PostFilterQuotaRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public async Task Reserve_AcceptThenCommit_WritesCurrentBucket() + { + var account = await _redis.CreateAccountAsync(); + var id = new PostFilterReservationId("n1", "a", 1); + var ceilings = new PostFilterQuotaCeilings(10, 10_000, 0, 10, 10_000, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 40, 0, null)); + Assert.NotNull(await _redis.ReadQuotaFieldAsync(account, id.Field)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, id, At, Windows, ceilings)); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, id.Field)); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + Assert.Equal("40", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedBytesField('L', 0))); + var ttl = await _redis.QuotaKeyTtlAsync(account); + Assert.NotNull(ttl); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Reserve_ExactCeiling_SecondDeniedWritesNothing() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, Id(1), At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.DeniedMessagesLong, + await _redis.Store.ReserveAsync(account, Id(2), At, Windows, ceilings, 1, 1, 0, null)); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, Id(2).Field)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Reserve_ConcurrentRaceForLastSlot() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + var results = await Task.WhenAll( + _redis.Store.ReserveAsync(account, new PostFilterReservationId("n1", "a", 1), At, Windows, ceilings, 1, 1, 0, null).AsTask(), + _redis.Store.ReserveAsync(account, new PostFilterReservationId("n2", "b", 1), At, Windows, ceilings, 1, 1, 0, null).AsTask()); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.Accepted)); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.DeniedMessagesLong)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Reserve_IdempotentAndConflict() + { + var account = await _redis.CreateAccountAsync(); + var id = Id(1); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 10, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 10, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.Conflict, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 20, 0, null)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Commit_DuplicateAndAfterRelease_AreNoop() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + var id = Id(1); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, id, At, Windows, ceilings)); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, id, At, Windows, ceilings)); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + + var released = Id(2); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, released, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaReleaseStatus.Released, + await _redis.Store.ReleaseAsync(account, released, At)); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, released, At, Windows, ceilings)); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Commit_AfterExpiry_CreatesNoUsage() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + var id = Id(1); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 1, 0, null)); + var expired = DateTimeOffset.FromUnixTimeMilliseconds(Now + 10_000); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, id, expired, Windows, ceilings)); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 1))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Release_StaleDoesNotTouchNewer() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + var oldId = Id(1); + var newId = Id(2); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, oldId, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal(PostFilterQuotaReleaseStatus.Released, await _redis.Store.ReleaseAsync(account, oldId, At)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, newId, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal(PostFilterQuotaReleaseStatus.Noop, await _redis.Store.ReleaseAsync(account, oldId, At)); + Assert.NotNull(await _redis.ReadQuotaFieldAsync(account, newId.Field)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task BucketBoundary_LiveReservationCountsInNewBucket() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + await _redis.SeedQuotaAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0), "9"); + var straddler = new PostFilterReservationId("n1", "a", 1); + var late = DateTimeOffset.FromUnixTimeMilliseconds(9_999); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, straddler, late, Windows, ceilings, 1, 1, 0, null)); + var after = DateTimeOffset.FromUnixTimeMilliseconds(10_002); + var accepted = 0; + for (var g = 1; g <= 12; g++) + { + var status = await _redis.Store.ReserveAsync( + account, + new PostFilterReservationId("n2", "b", g), + after, + Windows, + ceilings, + 1, + 1, + 0, + null); + if (status == PostFilterQuotaReserveStatus.Accepted) + { + accepted++; + } + } + + Assert.Equal(9, accepted); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, straddler, after, Windows, ceilings)); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 1))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Multipost_SameHashRacesAcrossNodes() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(0, 0, 1, 0, 0, 0); + var results = await Task.WhenAll( + _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n1", "a", 1), At, Windows, ceilings, 1, 1, 1, BodyA).AsTask(), + _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n2", "b", 1), At, Windows, ceilings, 1, 1, 1, BodyA).AsTask()); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.Accepted)); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.DeniedIdenticalLong)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task AccountsAreIsolated() + { + var alice = await _redis.CreateAccountAsync(); + var bob = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(alice, Id(1), At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(bob, Id(1), At, Windows, ceilings, 1, 1, 0, null)); + await _redis.DeleteKeysAsync(alice); + await _redis.DeleteKeysAsync(bob); + } + + [SessionStateRedisIntegrationFact] + public async Task Reserve_ConcurrentRaceForLastByteSlot() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(0, 40, 0, 0, 0, 0); + var results = await Task.WhenAll( + _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n1", "a", 1), At, Windows, ceilings, 1, 40, 0, null).AsTask(), + _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n2", "b", 1), At, Windows, ceilings, 1, 40, 0, null).AsTask()); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.Accepted)); + Assert.Equal(1, results.Count(static status => status == PostFilterQuotaReserveStatus.DeniedBytesLong)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ReserveOnNodeA_CommitOnNodeB() + { + var account = await _redis.CreateAccountAsync(); + var token = new PostFilterReservationId("nodeA", "incA", 1); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, token, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, token, At, Windows, ceilings)); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ReserveOnNodeA_ReleaseOnNodeB() + { + var account = await _redis.CreateAccountAsync(); + var token = new PostFilterReservationId("nodeA", "incA", 1); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, token, At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal(PostFilterQuotaReleaseStatus.Released, await _redis.Store.ReleaseAsync(account, token, At)); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, token.Field)); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, token, At, Windows, ceilings)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task SameAccount_VisibleAcrossNodeIdentities() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n1", "a", 1), At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaReserveStatus.DeniedMessagesLong, + await _redis.Store.ReserveAsync( + account, new PostFilterReservationId("n2", "b", 1), At, Windows, ceilings, 1, 1, 0, null)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Commit_DisabledDimensionsAreNotWritten() + { + var account = await _redis.CreateAccountAsync(); + var id = Id(1); + var reserveCeilings = new PostFilterQuotaCeilings(10, 10_000, 2, 10, 10_000, 2); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, reserveCeilings, 1, 40, 1, BodyA)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, id, At, Windows, new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0))); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 0))); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedBytesField('L', 0))); + Assert.Null(await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('S', 0))); + Assert.Null(await _redis.ReadMultipostFieldAsync(account, PostFilterQuotaKeys.MultipostField(BodyA, 'L', 0))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task QuotaKeyTtl_IsIdleGarbageCollection_NotWindowExpiry() + { + var account = await _redis.CreateAccountAsync(); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, Id(1), At, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, Id(1), At, Windows, ceilings)); + + var ttl = await _redis.QuotaKeyTtlAsync(account); + Assert.NotNull(ttl); + var idleMs = Math.Max(Windows.LongWindowMs, Windows.ShortWindowMs) + + (long)PostFilterQuotaDefaults.ReservationTtl.TotalMilliseconds + + PostFilterQuotaDefaults.IdleTtlSkewMs; + var remainingLongBucketMs = Windows.LongWindowMs - Now; + Assert.True(ttl.Value.TotalMilliseconds > remainingLongBucketMs); + Assert.InRange(ttl.Value.TotalMilliseconds, idleMs - 2_000, idleMs); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task DerivedHold_StaysLiveThroughSaOperationTimeout_ThenExpires() + { + var account = await _redis.CreateAccountAsync(); + var id = Id(1); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + var hold = PostFilterQuotaDefaults.HoldMilliseconds(true, TimeSpan.FromSeconds(30)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, id, At, Windows, ceilings, 1, 1, 0, null, reservationTtlMs: hold)); + var duringSa = DateTimeOffset.FromUnixTimeMilliseconds(Now + 30_000); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, id, duringSa, Windows, ceilings)); + Assert.Equal("1", await _redis.ReadQuotaFieldAsync(account, PostFilterQuotaKeys.CommittedMessagesField('L', 3))); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ExpiredReservation_DoesNotConsumeQuotaForLaterPost() + { + var account = await _redis.CreateAccountAsync(); + var first = Id(1); + var second = Id(2); + var ceilings = new PostFilterQuotaCeilings(1, 0, 0, 0, 0, 0); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, first, At, Windows, ceilings, 1, 1, 0, null)); + var expired = DateTimeOffset.FromUnixTimeMilliseconds(Now + 10_000); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, first, expired, Windows, ceilings)); + Assert.Equal( + PostFilterQuotaReserveStatus.Accepted, + await _redis.Store.ReserveAsync(account, second, expired, Windows, ceilings, 1, 1, 0, null)); + Assert.Equal( + PostFilterQuotaCommitStatus.Committed, + await _redis.Store.CommitAsync(account, second, expired, Windows, ceilings)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Commit_StaleGeneration_IsNoop() + { + var account = await _redis.CreateAccountAsync(); + var id = Id(1); + var ceilings = new PostFilterQuotaCeilings(10, 0, 0, 0, 0, 0); + await _redis.SeedQuotaAsync( + account, + id.Field, + PostFilterQuotaKeys.PackReservation(Now + 10_000, 99, 1, 1, 0, string.Empty)); + Assert.Equal( + PostFilterQuotaCommitStatus.Noop, + await _redis.Store.CommitAsync(account, id, At, Windows, ceilings)); + Assert.NotNull(await _redis.ReadQuotaFieldAsync(account, id.Field)); + await _redis.DeleteKeysAsync(account); + } + + private static PostFilterReservationId Id(long generation) => new("n1", "a", generation); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdScanArticleBuilderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdScanArticleBuilderTests.cs new file mode 100644 index 00000000..2477e2a7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdScanArticleBuilderTests.cs @@ -0,0 +1,114 @@ +using System.Net; +using System.Runtime.InteropServices; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class SpamdScanArticleBuilderTests +{ + private static readonly DateTimeOffset Now = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public void Build_AddsSyntheticHeaders_StripsNntpHeaders_PreservesBodyAndArticle() + { + var original = Encoding.ASCII.GetBytes( + "Path: news.example!not-for-mail\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test,alt.test\r\n" + + "Subject: hello\r\n" + + "Message-ID: \r\n" + + "Date: 1 Jan 2026 00:00:00 +0000\r\n" + + "Xref: nntpd01.usenet.ninja misc.test:1\r\n" + + "Injection-Info: nntpd01.usenet.ninja; posting-account=poster\r\n" + + "X-Trace: secret-trace\r\n" + + "X-Complaints-To: abuse@example.com\r\n" + + "NNTP-Posting-Host: 203.0.113.9\r\n" + + "\r\n" + + "body-bytes\r\n"); + var copy = original.ToArray(); + var article = new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 42, + artType: ArticleType.Default, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: original, + fields: default); + var context = new SpamdScanContext(IPAddress.Parse("203.0.113.8"), Fqdn, Now); + + var scan = SpamdScanArticleBuilder.Build(article, context); + var text = Encoding.ASCII.GetString(scan); + + Assert.StartsWith("Received: from [203.0.113.8] by nntpd01.usenet.ninja with NNTP id ; Thu, 01 Jan 2026 00:00:00 +0000\r\n", text, StringComparison.Ordinal); + Assert.Contains("To: usenet@nntpd01.usenet.ninja\r\n", text, StringComparison.Ordinal); + Assert.Contains("X-Usenet-Newsgroups: misc.test,alt.test\r\n", text, StringComparison.Ordinal); + Assert.Contains("From: poster@example.com\r\n", text, StringComparison.Ordinal); + Assert.Contains("Newsgroups: misc.test,alt.test\r\n", text, StringComparison.Ordinal); + Assert.Contains("Subject: hello\r\n", text, StringComparison.Ordinal); + Assert.Contains("Message-ID: \r\n", text, StringComparison.Ordinal); + Assert.Contains("Date: 1 Jan 2026 00:00:00 +0000\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("Path:", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", text, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", text, StringComparison.Ordinal); + Assert.DoesNotContain("X-Complaints-To:", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host:", text, StringComparison.Ordinal); + Assert.DoesNotContain("Received-By:", text, StringComparison.Ordinal); + Assert.EndsWith("\r\n\r\nbody-bytes\r\n", text, StringComparison.Ordinal); + Assert.True(original.AsSpan().SequenceEqual(article.ArtData.Span)); + Assert.True(copy.AsSpan().SequenceEqual(article.ArtData.Span)); + Assert.Equal(42UL, article.ArtHash); + Assert.Equal(ArticleType.Default, article.ArtType); + Assert.Equal(1, article.ArtLines); + Assert.Equal(copy.Length, article.ArtSize); + Assert.False(ReferenceEquals(scan, original)); + Assert.True(MemoryMarshal.TryGetArray(article.ArtData, out var segment)); + Assert.Same(original, segment.Array); + } + + [Fact] + public void Build_AddsDateWhenOriginalDateAbsent() + { + var original = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\n\r\nplain\r\n"u8.ToArray(); + var article = new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 2, + artType: ArticleType.Default, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: original, + fields: default); + var scan = Encoding.ASCII.GetString( + SpamdScanArticleBuilder.Build( + article, + new SpamdScanContext(IPAddress.Loopback, Fqdn, Now))); + Assert.Contains("Date: Thu, 01 Jan 2026 00:00:00 +0000\r\n", scan, StringComparison.Ordinal); + Assert.True(original.AsSpan().SequenceEqual(article.ArtData.Span)); + } + + [Fact] + public void Build_MessageIdRemainsValid_AndScanIsIndependent() + { + var original = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\nxyz\r\n"u8.ToArray(); + var article = new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 3, + artType: ArticleType.Default, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: original, + fields: default); + var first = SpamdScanArticleBuilder.Build(article, new SpamdScanContext(IPAddress.Loopback, Fqdn, Now)); + first[0] = (byte)'X'; + var second = SpamdScanArticleBuilder.Build(article, new SpamdScanContext(IPAddress.Loopback, Fqdn, Now)); + Assert.Equal((byte)'R', second[0]); + Assert.Contains("Message-ID: \r\n", Encoding.ASCII.GetString(second), StringComparison.Ordinal); + Assert.Equal((byte)'F', article.ArtData.Span[0]); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdTransportTests.cs b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdTransportTests.cs new file mode 100644 index 00000000..cd7b3fa1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/PostFilter/SpamdTransportTests.cs @@ -0,0 +1,206 @@ +using System.Net; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.PostFilter; + +public sealed class SpamdTransportTests +{ + private static readonly DateTimeOffset Now = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task SequentialChecks_ReuseOneConnection() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, maxConnections: 1); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, (await CheckAsync(client, target)).Status); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, (await CheckAsync(client, target)).Status); + Assert.Equal(2, server.Checks); + Assert.Equal(1, server.Connections); + Assert.Equal(1, metrics.ConnectionsEstablished); + Assert.Equal(2, metrics.CheckRequests); + Assert.Equal(1, metrics.ConnectionReuses); + } + + [Fact] + public async Task BrokenConnection_IsEvictedAndReplaced() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: false); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, maxConnections: 1); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, (await CheckAsync(client, target)).Status); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, (await CheckAsync(client, target)).Status); + Assert.Equal(2, server.Checks); + Assert.Equal(2, server.Connections); + Assert.Equal(2, metrics.ConnectionsEstablished); + Assert.True(metrics.Evictions >= 1); + } + + [Fact] + public async Task ProtocolError_DoesNotReuseConnection() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, response: "NOPE\r\n\r\n"u8.ToArray()); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var result = await CheckAsync(client, Target(server.Port, maxConnections: 1)); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Contains("malformed", result.Detail, StringComparison.OrdinalIgnoreCase); + Assert.Equal(1, metrics.Evictions); + } + + [Fact] + public async Task Timeout_IsClassifiedAndEvicts() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, hang: true); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, maxConnections: 1, operation: TimeSpan.FromMilliseconds(200)); + var result = await CheckAsync(client, target); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Equal("timeout", result.Detail); + Assert.True(metrics.Evictions >= 1); + } + + [Fact] + public async Task Cancellation_IsRethrown() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, hang: true); + await using var client = new SpamdCheckClient(); + using var cts = new CancellationTokenSource(); + var check = CheckAsync(client, Target(server.Port, operation: TimeSpan.FromSeconds(5)), cts.Token); + await server.Accepted; + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await check); + } + + [Fact] + public void HostSelection_RoundRobinRotates_FailoverStaysAtZero() + { + var rr = 0; + Assert.Equal(0, SpamdHostSelector.NextStart(PostFilterSpamAssassinHostSelection.RoundRobin, 2, ref rr)); + Assert.Equal(1, SpamdHostSelector.NextStart(PostFilterSpamAssassinHostSelection.RoundRobin, 2, ref rr)); + Assert.Equal(0, SpamdHostSelector.NextStart(PostFilterSpamAssassinHostSelection.RoundRobin, 2, ref rr)); + var failover = 99; + Assert.Equal(0, SpamdHostSelector.NextStart(PostFilterSpamAssassinHostSelection.Failover, 3, ref failover)); + Assert.Equal(0, SpamdHostSelector.NextStart(PostFilterSpamAssassinHostSelection.Failover, 3, ref failover)); + Assert.Equal(99, failover); + } + + [Fact] + public async Task Failover_SkipsUnreachableHost() + { + await using var good = await LocalSpamdServer.StartAsync(keepOpen: true); + await using var client = new SpamdCheckClient(); + var result = await client.CheckAsync( + Article(), + "poster", + new PostFilterSpamAssassinTarget( + ["203.0.113.1", "127.0.0.1"], + good.Port, + TimeSpan.FromMilliseconds(250), + TimeSpan.FromSeconds(3), + "1.5", + 1, + PostFilterSpamAssassinHostSelection.Failover), + ScanContext()); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status); + Assert.Equal(1, good.Checks); + } + + [Fact] + public async Task ConcurrentChecks_HonorPoolSize() + { + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, delay: TimeSpan.FromMilliseconds(80)); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, maxConnections: 2); + var tasks = Enumerable.Range(0, 4).Select(_ => CheckAsync(client, target).AsTask()).ToArray(); + var results = await Task.WhenAll(tasks); + Assert.All(results, result => Assert.Equal(PostFilterSpamAssassinStatus.Ham, result.Status)); + Assert.Equal(4, metrics.CheckRequests); + Assert.True(server.PeakConnections <= 2); + Assert.True(metrics.ConnectionsEstablished <= 2); + } + + [Fact] + public async Task DisposeAsync_RejectsFurtherChecks() + { + await using var client = new SpamdCheckClient(); + await client.DisposeAsync(); + var result = await CheckAsync(client, Target(9)); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Equal("disposed", result.Detail); + } + + [Fact] + public async Task LeftoverBytes_AreProtocolFailure() + { + var extra = "SPAMD/1.1 0 EX_OK\r\nSpam: False ; 0.0 / 5.0\r\n\r\nEXTRA"u8.ToArray(); + await using var server = await LocalSpamdServer.StartAsync(keepOpen: true, response: extra); + await using var client = new SpamdCheckClient(); + var result = await CheckAsync(client, Target(server.Port)); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, result.Status); + Assert.Equal("leftover", result.Detail); + } + + [Fact] + public async Task LeftoverBytes_DoNotPoisonTheNextCheck() + { + var leftover = "SPAMD/1.1 0 EX_OK\r\nSpam: False ; 0.0 / 5.0\r\n\r\nSPAMD/1.1 0 EX_OK\r\nSpam: False ; 0.0 / 5.0\r\n\r\n"u8.ToArray(); + var ham = "SPAMD/1.1 0 EX_OK\r\nSpam: False ; 0.0 / 5.0\r\n\r\n"u8.ToArray(); + await using var server = await LocalSpamdServer.StartAsync( + keepOpen: true, + response: leftover, + subsequentResponse: ham); + var metrics = new SpamdTransportMetrics(); + await using var client = new SpamdCheckClient(metrics); + var target = Target(server.Port, maxConnections: 1); + var first = await CheckAsync(client, target); + Assert.Equal(PostFilterSpamAssassinStatus.Failed, first.Status); + Assert.Equal("leftover", first.Detail); + var second = await CheckAsync(client, target); + Assert.Equal(PostFilterSpamAssassinStatus.Ham, second.Status); + Assert.Equal(2, server.Connections); + Assert.True(metrics.Evictions >= 1); + } + + private static ValueTask CheckAsync( + SpamdCheckClient client, + PostFilterSpamAssassinTarget target, + CancellationToken cancellationToken = default) => + client.CheckAsync(Article(), "poster", target, ScanContext(), cancellationToken); + + private static PostFilterSpamAssassinTarget Target( + int port, + int maxConnections = 4, + TimeSpan? operation = null) => + new( + ["127.0.0.1"], + port, + TimeSpan.FromSeconds(2), + operation ?? TimeSpan.FromSeconds(5), + "1.5", + maxConnections, + PostFilterSpamAssassinHostSelection.RoundRobin); + + private static SpamdScanContext ScanContext() => + new(IPAddress.Loopback, "nntpd01.usenet.ninja", Now); + + private static ArticleRecord Article() + { + var bytes = "From: a@b\r\nNewsgroups: misc.test\r\nSubject: t\r\nMessage-ID: \r\nDate: 1 Jan 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(); + return new ArticleRecord( + ArticleId.FromMessageId(""u8), + artHash: 1, + artType: ArticleType.Default, + artLines: 1, + canonicalUtc: Now.UtcDateTime, + parseStatus: ArticleParseStatus.CanonicalV1, + artData: bytes, + fields: default); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs new file mode 100644 index 00000000..26c2a52d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs @@ -0,0 +1,36 @@ +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +/// +/// Regression: definitive provider 430 / +/// must not schedule another backbone attempt. +/// +public sealed class ArticleWorkAggregatePolicyTests +{ + [Fact] + public void ArticleNotFound_is_attempt_terminal_and_does_not_try_next_backbone() + { + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.ArticleNotFound)); + } + + [Fact] + public void InvalidArticle_and_InvalidRequest_may_try_next_backbone() + { + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.InvalidArticle)); + Assert.True(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.InvalidArticle)); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.InvalidRequest)); + Assert.True(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.InvalidRequest)); + } + + [Fact] + public void Success_completes_lookup_without_trying_next() + { + Assert.True(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.Success)); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.Success)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.Success)); + } + +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs new file mode 100644 index 00000000..69e56d0e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs @@ -0,0 +1,612 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkRpcClientTests +{ + private static readonly byte[] MessageIdBytes = "<12345@example.invalid>"u8.ToArray(); + private const string MessageId = "<12345@example.invalid>"; + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public async Task No_active_consumers_publishes_nothing() + { + await using var harness = CreateHarness(eligible: []); + var result = await harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Contains("No BackFiller", result.Error, StringComparison.OrdinalIgnoreCase); + Assert.Empty(harness.Publisher.Publications); + } + + [Fact] + public async Task One_active_backbone_receives_exactly_one_request() + { + await using var harness = CreateHarness(Eligible("Giganews", 10)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + AssertAmqpContract(publication); + Assert.Equal("Giganews", ReadBackbone(publication)); + harness.DeliverSuccess(publication, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Two_active_backbones_first_attempt_goes_to_exactly_one() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(1); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + harness.DeliverSuccess(harness.Publisher.Publications[0], "Giganews"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task First_not_found_ends_lookup_without_publishing_to_another_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("missing", result.Error); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => p.Exchange == "backfiller.eweka"); + } + + [Fact] + public async Task First_not_found_never_retries_same_or_other_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 1)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + } + + [Fact] + public async Task First_eligible_not_found_returns_not_found_without_second_publish() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 2), Eligible("Eweka", 2)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(1); + harness.DeliverOutcome(harness.Publisher.Publications[0], ArticleWorkOutcome.ArticleNotFound, "a"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("a", result.Error); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Zero_consumer_backbone_is_never_selected() + { + await using var harness = CreateHarness(Eligible("Eweka", 5)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + Assert.DoesNotContain(harness.Publisher.Publications, static p => p.Exchange.Contains("giganews", StringComparison.Ordinal)); + harness.DeliverSuccess(publication, "Eweka"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task Success_stops_immediately_without_further_publishes() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverSuccess(first, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Cancellation_stops_without_launching_another_attempt() + { + await using var harness = CreateHarness(Eligible("Giganews", 10)); + using var cts = new CancellationTokenSource(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, cts.Token); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await lookup); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Attempt_timeout_without_response_does_not_start_another_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(ArticleWorkRpcTiming.LookupDeadline); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True( + result.Error is not null + && (result.Error.Contains("timed out", StringComparison.OrdinalIgnoreCase) + || result.Error.Contains("deadline elapsed", StringComparison.OrdinalIgnoreCase)), + result.Error); + Assert.Single(harness.Publisher.Publications); + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Lookup_completes_by_five_second_scheduler_deadline() + { + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10)], + nextInt: static _ => 0); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True(harness.Time.GetUtcNow() - started <= ArticleWorkRpcTiming.LookupDeadline); + Assert.Single(harness.Publisher.Publications); + Assert.DoesNotContain("absolute", result.Error, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("60", result.Error, StringComparison.Ordinal); + } + + [Fact] + public async Task Definitive_not_found_before_deadline_ends_lookup_immediately() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Single(harness.Publisher.Publications); + Assert.True(harness.Time.GetUtcNow() - started < ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Invalid_article_before_deadline_advances_to_next_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad yenc"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverSuccess(second, "Eweka"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(2, harness.Publisher.Publications.Count); + Assert.True(harness.Time.GetUtcNow() - started < ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Success_before_deadline_terminates_immediately() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromMilliseconds(250)); + harness.DeliverSuccess(first, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public void Lookup_deadline_is_five_seconds_with_no_separate_sixty_second_ceiling() + { + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + Assert.Null(typeof(ArticleWorkRpcTiming).GetField( + "AbsoluteLifetime", + System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic)); + } + + [Fact] + public void Weighted_selection_is_approximately_two_to_one_for_100_vs_50() + { + var roll = 0; + var selector = new WeightedBackboneSelector(exclusiveMax => + { + var value = roll % exclusiveMax; + roll++; + return value; + }); + var candidates = new[] + { + Eligible("Giganews", 100), + Eligible("Eweka", 50), + }; + var counts = new Dictionary(StringComparer.Ordinal); + const int iterations = 15_000; + for (var i = 0; i < iterations; i++) + { + var pick = selector.Select(candidates)!.Value.Backbone; + counts[pick] = counts.GetValueOrDefault(pick) + 1; + } + + var ratio = counts["Giganews"] / (double)counts["Eweka"]; + Assert.InRange(ratio, 1.7, 2.3); + } + + [Fact] + public async Task Invalid_article_is_attempt_terminal_and_tries_next() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad yenc"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverSuccess(second, "Eweka"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task Repeated_lookups_with_single_active_backbone_never_touch_others() + { + await using var harness = CreateHarness(Eligible("Giganews", 25)); + for (var i = 0; i < 8; i++) + { + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews", skip: i); + Assert.Equal("backfiller.giganews", publication.Exchange); + harness.DeliverSuccess(publication, "Giganews"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + Assert.Equal(8, harness.Publisher.Publications.Count); + Assert.All( + harness.Publisher.Publications, + static p => Assert.Equal("backfiller.giganews", p.Exchange)); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => + p.Exchange.Contains("eweka", StringComparison.Ordinal) + || p.Exchange.Contains("highwinds", StringComparison.Ordinal) + || p.Exchange.Contains("storage", StringComparison.Ordinal)); + } + + [Fact] + public async Task Zero_consumer_and_storage_never_receive_attempts_even_when_listed() + { + await using var harness = CreateHarness( + [ + Eligible("Giganews", 100), + Eligible("Eweka", 50), + Eligible("Highwinds", 0), + new BackfillEligibleBackbone( + "Storage", + StorageArticleRetrievalTopology.EntityName, + StorageArticleRetrievalTopology.EntityName, + StorageArticleRetrievalTopology.EntityName, + 0), + ], + CreateSequentialRolls(0, 0)); + + for (var i = 0; i < 6; i++) + { + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(i + 1); + var publication = harness.Publisher.Publications[i]; + Assert.Contains( + publication.Exchange, + new[] { "backfiller.giganews", "backfiller.eweka" }, + StringComparer.Ordinal); + harness.DeliverSuccess(publication, ReadBackbone(publication)); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => + p.Exchange.Equals("backfiller.highwinds", StringComparison.Ordinal) + || p.Exchange.Equals(StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal)); + } + + [Fact] + public async Task Three_backbone_invalid_article_chain_never_reuses_attempted_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10), Eligible("Highwinds", 10)], + CreateSequentialRolls(0, 0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad-g"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverOutcome(second, ArticleWorkOutcome.InvalidArticle, "bad-e"); + var third = await harness.Publisher.WaitForPublicationAsync("backfiller.highwinds"); + harness.DeliverSuccess(third, "Highwinds"); + + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal("Highwinds", result.Backbone); + Assert.Equal(3, harness.Publisher.Publications.Count); + Assert.Equal( + new[] { "backfiller.giganews", "backfiller.eweka", "backfiller.highwinds" }, + harness.Publisher.Publications.Select(static p => p.Exchange).ToArray()); + Assert.Equal( + Guid.Parse(first.RequestId), + Guid.Parse(second.RequestId)); + Assert.Equal( + Guid.Parse(first.RequestId), + Guid.Parse(third.RequestId)); + Assert.NotEqual(first.CorrelationId, second.CorrelationId); + Assert.NotEqual(second.CorrelationId, third.CorrelationId); + Assert.Equal(1, harness.Publisher.Publications.Count(static p => p.Exchange == "backfiller.giganews")); + } + + [Fact] + public async Task Global_deadline_bounds_total_lookup_not_per_attempt() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10), Eligible("Highwinds", 10)], + CreateSequentialRolls(0, 0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + // InvalidArticle (not NotFound) may advance; NotFound would end after the first publish. + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad"); + + await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.Time.Advance(TimeSpan.FromSeconds(4)); + var result = await lookup; + + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal(2, harness.Publisher.Publications.Count); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => p.Exchange == "backfiller.highwinds"); + Assert.True(harness.Time.GetUtcNow() - started <= ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Not_found_on_first_of_three_backbones_publishes_only_once() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 5), Eligible("Eweka", 5), Eligible("Highwinds", 5)], + CreateSequentialRolls(0, 0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + await harness.Publisher.WaitForPublicationCountAsync(1); + harness.DeliverOutcome(harness.Publisher.Publications[0], ArticleWorkOutcome.ArticleNotFound, "a"); + + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("a", result.Error); + Assert.Single(harness.Publisher.Publications); + } + + private static Func CreateSequentialRolls(params int[] rolls) + { + var index = 0; + return exclusiveMax => + { + _ = exclusiveMax; + var value = rolls[Math.Min(index, rolls.Length - 1)]; + index++; + return value; + }; + } + + private static BackfillEligibleBackbone Eligible(string backbone, int consumers) + { + var entity = BackfillArticleRetrievalTopology.BuildProviderEntityName(backbone); + return new BackfillEligibleBackbone(backbone, entity, entity, entity, consumers); + } + + private static Harness CreateHarness(params BackfillEligibleBackbone[] eligible) => + CreateHarness(eligible, nextInt: static _ => 0); + + private static Harness CreateHarness(BackfillEligibleBackbone[] eligible, Func nextInt) => + CreateHarness(eligible, nextInt, currentGeneration: null); + + private static Harness CreateHarness( + BackfillEligibleBackbone[] eligible, + Func? nextInt, + Func? currentGeneration) + { + var time = new FakeTimeProvider(); + var publisher = new RecordingArticleWorkRpcPublisher(); + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var availability = new StaticBackfillConsumerAvailability(eligible); + var selector = new WeightedBackboneSelector(nextInt ?? (static _ => 0)); + var client = new ArticleWorkRpcClient( + publisher, + router, + availability, + selector, + time, + NullLogger.Instance, + currentGeneration); + return new Harness(time, publisher, router, client, availability); + } + + private static void AssertAmqpContract(FakeRabbitMqRpcPublication publication) + { + Assert.False(string.IsNullOrWhiteSpace(publication.ReplyTo)); + Assert.Equal(ArticleWorkWireProtocol.JsonContentType, publication.ContentType); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + Assert.True(Guid.TryParse(publication.RequestId, out var requestId)); + Assert.NotEqual(Guid.Empty, requestId); + Assert.True(Guid.TryParse(publication.CorrelationId, out var correlationId)); + Assert.NotEqual(Guid.Empty, correlationId); + Assert.Equal(publication.Exchange, publication.RoutingKey); + using var document = JsonDocument.Parse(publication.Body); + Assert.Equal(requestId, document.RootElement.GetProperty("requestId").GetGuid()); + } + + private static string ReadBackbone(FakeRabbitMqRpcPublication publication) => + JsonDocument.Parse(publication.Body).RootElement.GetProperty("backbone").GetString()!; + + private static byte[] SuccessBody(Guid requestId, string backbone) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"Success","fqdn":"{{SuccessFqdn}}","vatpPort":{{SuccessVatpPort}},"articleId":"{{SuccessArticleIdHex}}"}"""); + } + + private static byte[] FailureBody(Guid requestId, ArticleWorkOutcome outcome, string error, string backbone) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"{{outcome}}","error":"{{error}}"}"""); + } + + private sealed class Harness : IAsyncDisposable + { + internal Harness( + FakeTimeProvider time, + RecordingArticleWorkRpcPublisher publisher, + ArticleWorkRpcResponseRouter router, + ArticleWorkRpcClient client, + StaticBackfillConsumerAvailability availability) + { + Time = time; + Publisher = publisher; + Router = router; + Client = client; + Availability = availability; + } + + internal FakeTimeProvider Time { get; } + + internal RecordingArticleWorkRpcPublisher Publisher { get; } + + internal ArticleWorkRpcResponseRouter Router { get; } + + internal ArticleWorkRpcClient Client { get; } + + internal StaticBackfillConsumerAvailability Availability { get; } + + internal void DeliverSuccess(FakeRabbitMqRpcPublication publication, string backbone, Guid? requestId = null) + { + var id = requestId ?? Guid.Parse(publication.RequestId); + Router.Dispatch(publication.CorrelationId, SuccessBody(id, backbone), id.ToString("D"), 0); + } + + internal void DeliverOutcome(FakeRabbitMqRpcPublication publication, ArticleWorkOutcome outcome, string error) + { + var requestId = Guid.Parse(publication.RequestId); + var backbone = ReadBackbone(publication); + Router.Dispatch( + publication.CorrelationId, + FailureBody(requestId, outcome, error, backbone), + requestId.ToString("D"), + 0); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class RecordingArticleWorkRpcPublisher : IArticleWorkRpcPublisher + { + private readonly SemaphoreSlim _publishedPulse = new(0, int.MaxValue); + + public string ReplyTo => "nntpd.01.test.rpc"; + + public List Publications { get; } = []; + + public Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + lock (Publications) + { + Publications.Add(new FakeRabbitMqRpcPublication( + exchange, + routingKey, + correlationId, + requestId.ToString("D"), + ReplyTo, + ArticleWorkWireProtocol.JsonContentType, + Expiration: string.Empty, + body.ToArray())); + } + + _publishedPulse.Release(); + return Task.CompletedTask; + } + + public async Task WaitForPublicationAsync(string exchange, int skip = 0) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + FakeRabbitMqRpcPublication? match; + lock (Publications) + { + match = Publications.Where(publication => publication.Exchange == exchange).Skip(skip).FirstOrDefault(); + } + + if (match is not null) + { + return match; + } + + await _publishedPulse.WaitAsync(cts.Token).ConfigureAwait(false); + } + } + + public async Task WaitForPublicationCountAsync(int count) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + lock (Publications) + { + if (Publications.Count >= count) + { + return; + } + } + + await _publishedPulse.WaitAsync(cts.Token).ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs new file mode 100644 index 00000000..ab655a1f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs @@ -0,0 +1,169 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkRpcResponseRouterTests +{ + private const string MessageId = "<12345@example.invalid>"; + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public void StaleGeneration_CannotCompleteNewerGenerationRequest() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 2); + + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 1); + Assert.False(operation.IsCompleted); + + Assert.Equal(1, router.OutstandingCount); + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 2); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal("cache.requests", result.SourceExchange); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void Success_RemovesAllLocalCorrelationsImmediately() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var first = Guid.NewGuid().ToString("D"); + var second = Guid.NewGuid().ToString("D"); + router.Register(first, operation, "cache.requests", generation: 0); + router.Register(second, operation, "backfiller.eweka", generation: 0); + Assert.Equal(2, router.OutstandingCount); + + router.Dispatch(first, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void ResponseRacingWithLocalCompletion_CannotResurrectLookup() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + operation.TryComplete(ArticleWorkRpcResult.NotFound(requestId, MessageId, "deadline")); + router.UnregisterAll(operation); + router.Register(Guid.NewGuid().ToString("D"), operation, "backfiller.eweka", generation: 0); + Assert.Equal(0, router.OutstandingCount); + + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void ConcurrentSuccess_CompletesExactlyOnce_AndClearsCorrelations() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var first = Guid.NewGuid().ToString("D"); + var second = Guid.NewGuid().ToString("D"); + var third = Guid.NewGuid().ToString("D"); + router.Register(first, operation, "backfiller.eweka", generation: 0); + router.Register(second, operation, "backfiller.giganews", generation: 0); + router.Register(third, operation, "backfiller.abavia", generation: 0); + + Parallel.Invoke( + () => router.Dispatch(first, SuccessBody(requestId, "Eweka"), requestId.ToString("D"), 0), + () => router.Dispatch(second, SuccessBody(requestId, "Giganews"), requestId.ToString("D"), 0), + () => router.Dispatch(third, SuccessBody(requestId, "Abavia"), requestId.ToString("D"), 0)); + + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.True( + result.SourceExchange is "backfiller.eweka" or "backfiller.giganews" or "backfiller.abavia", + result.SourceExchange); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void MismatchedAmqpRequestId_IsIgnored() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(requestId), Guid.NewGuid().ToString("D"), deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + Assert.Equal(1, router.OutstandingCount); + } + + [Fact] + public void MissingAmqpRequestId_IsIgnored() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(requestId), amqpRequestId: null, deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + } + + [Fact] + public void JsonRequestIdMismatch_IsIgnoredEvenWhenAmqpMatchesAnotherValue() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var otherRequestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(otherRequestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + } + + [Fact] + public void ArticleNotFound_CompletesAttempt_ButIsNotLookupSuccess() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "backfiller.giganews", generation: 0); + + router.Dispatch(correlationId, FailureBody(requestId, "ArticleNotFound"), requestId.ToString("D"), 0); + Assert.True(operation.IsCompleted); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.ArticleNotFound)); + Assert.True(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.Success)); + } + + private static byte[] SuccessBody(Guid requestId, string backbone = "Storage") + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"Success","fqdn":"{{SuccessFqdn}}","vatpPort":{{SuccessVatpPort}},"articleId":"{{SuccessArticleIdHex}}"}"""); + } + + private static byte[] FailureBody(Guid requestId, string outcome) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"Storage","outcome":"{{outcome}}","error":"missing"}"""); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs new file mode 100644 index 00000000..03d002b5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs @@ -0,0 +1,287 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.RabbitMq.Management; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +[Collection(SerilogCollection.Name)] +public sealed class ArticleWorkRpcServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresExclusiveReplyQueue_AndConsumesOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + var rpc = CreateRpcService(rabbit); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Equal(2, connection.RpcChannels.Count); + var consume = connection.RpcChannels[1]; + var queue = Assert.Single(consume.Queues); + Assert.Equal(rpc.CurrentReplyTo, queue.Name); + Assert.False(queue.Durable); + Assert.True(queue.Exclusive); + Assert.True(queue.AutoDelete); + Assert.Equal(queue.Name, consume.ConsumedQueue); + Assert.Matches("^nntpd\\.01\\.[0-9a-f]{32}\\.rpc$", queue.Name); + Assert.Null(queue.Arguments); + + await rpc.StopAsync(CancellationToken.None); + Assert.Null(rpc.CurrentReplyTo); + Assert.Equal(1, connection.RpcChannels[0].DisposeCount); + Assert.Equal(1, consume.DisposeCount); + } + + [Fact] + public async Task ConnectionReplacement_DisposesStaleSession_AndDoesNotCorruptNewGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory()); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + var first = factory.Connections[0]; + var firstReplyTo = rpc.CurrentReplyTo; + Assert.Equal(1, rpc.CurrentSessionGeneration); + var firstPublish = first.RpcChannels[0]; + + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.SimulateLost(); + await factory.Connected.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitForAsync(() => rpc.CurrentSessionGeneration == 2 && firstPublish.DisposeCount == 1); + + Assert.Equal(2, rpc.CurrentSessionGeneration); + Assert.Equal(1, firstPublish.DisposeCount); + Assert.True(first.RpcChannels.TrueForAll(static channel => channel.DisposeCount == 1)); + var second = factory.LastConnection ?? throw new InvalidOperationException("Expected replacement connection."); + Assert.Equal(2, second.RpcChannels.Count); + Assert.NotNull(firstReplyTo); + Assert.False(string.IsNullOrWhiteSpace(rpc.CurrentReplyTo)); + await Assert.ThrowsAsync( + () => firstPublish.PublishAsync( + "cache.requests", + "cache.requests", + Guid.NewGuid().ToString("D"), + Guid.NewGuid().ToString("D"), + firstReplyTo!, + ArticleWorkWireProtocol.JsonContentType, + ArticleWorkRpcAmqp.ExpirationMilliseconds, + ReadOnlyMemory.Empty, + CancellationToken.None)); + Assert.Empty(second.RpcChannels[0].Publications); + + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopAsync_CancelsPendingLookup_AndClearsCorrelations() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory()); + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var lookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => factory.LastConnection!.RpcChannels[0].Publications.Count == 1); + var publication = Assert.Single(factory.LastConnection!.RpcChannels[0].Publications); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + Assert.True(Guid.TryParse(publication.RequestId, out _)); + Assert.Equal(ArticleWorkWireProtocol.JsonContentType, publication.ContentType); + Assert.Equal(rpc.CurrentReplyTo, publication.ReplyTo); + Assert.True(rpc.OutstandingCorrelations >= 1); + + await rpc.StopAsync(CancellationToken.None); + await Assert.ThrowsAnyAsync(() => lookup); + Assert.Equal(0, rpc.OutstandingCorrelations); + } + + [Fact] + public async Task WorkerResponse_CarriesOneSecondExpiration_AndCompletesLookup() + { + var time = new FakeTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory(), time); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var lookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => factory.LastConnection!.RpcChannels[0].Publications.Count == 1); + var publication = Assert.Single(factory.LastConnection!.RpcChannels[0].Publications); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + + var consume = factory.LastConnection.RpcChannels[1]; + await consume.DeliverAsync( + publication.CorrelationId, + SuccessBody(publication.RequestId), + requestId: publication.RequestId, + expiration: ArticleWorkRpcAmqp.ResponseExpirationMilliseconds); + var result = await lookup; + + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(ArticleWorkRpcAmqp.ResponseExpirationMilliseconds, consume.LastDelivery?.Expiration); + Assert.Equal(0, rpc.OutstandingCorrelations); + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StaleGenerationDelivery_CannotCompleteNewerLookup() + { + var time = new FakeTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory(), time); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + var first = factory.Connections[0]; + var firstLookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => first.RpcChannels[0].Publications.Count == 1); + var firstPublication = Assert.Single(first.RpcChannels[0].Publications); + + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.SimulateLost(); + await factory.Connected.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitForAsync(() => rpc.CurrentSessionGeneration == 2); + + var secondLookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + var second = factory.LastConnection ?? throw new InvalidOperationException("Expected replacement connection."); + await WaitForAsync(() => second.RpcChannels[0].Publications.Count == 1); + var secondPublication = Assert.Single(second.RpcChannels[0].Publications); + + await second.RpcChannels[1].DeliverAsync( + firstPublication.CorrelationId, + SuccessBody(firstPublication.RequestId), + requestId: firstPublication.RequestId); + Assert.False(secondLookup.IsCompleted); + + await first.RpcChannels[1].DeliverAsync( + secondPublication.CorrelationId, + SuccessBody(secondPublication.RequestId), + requestId: secondPublication.RequestId); + Assert.False(secondLookup.IsCompleted); + + await second.RpcChannels[1].DeliverAsync( + secondPublication.CorrelationId, + SuccessBody(secondPublication.RequestId), + requestId: secondPublication.RequestId); + var result = await secondLookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(secondPublication.RequestId, result.RequestId.ToString("D"), StringComparer.Ordinal); + + time.Advance(ArticleWorkRpcTiming.LookupDeadline); + var firstResult = await firstLookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, firstResult.Outcome); + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public void RpcChannel_DoesNotExposeQueueCleanup() + { + var methods = typeof(IRabbitMqRpcChannel).GetMethods().Select(static method => method.Name).ToHashSet(StringComparer.Ordinal); + Assert.DoesNotContain("QueueDeleteAsync", methods); + Assert.DoesNotContain("QueuePurgeAsync", methods); + Assert.DoesNotContain("ExchangeDeleteAsync", methods); + } + + [Fact] + public async Task Host_RegistersRpcService_ImmediatelyAfterTopology() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + // NntpDB and nntpsharedconfig are registered ahead of Cloudflare, Redis, and RabbitMQ. + Assert.Equal(typeof(RabbitMqTopologyService), services[5].GetType()); + Assert.Equal(typeof(VectorNNTPD.ReaderService.Storage.StorageServerFleetConsumerService), services[6].GetType()); + Assert.Equal(typeof(VectorNNTPD.ReaderService.Storage.StorageArticleLookupService), services[7].GetType()); + Assert.Equal(typeof(ArticleWorkRpcService), services[8].GetType()); + Assert.Same( + host.Services.GetRequiredService(), + services[8]); + } + + private static ArticleWorkRpcService CreateRpcService( + IRabbitMqService rabbit, + IRabbitMqManagementQueueInventory? inventory = null, + TimeProvider? time = null) => + new( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + inventory ?? new EmptyManagementInventory(), + NullLogger.Instance, + time); + + private static IRabbitMqManagementQueueInventory CreateActiveInventory() + { + var queues = BackfillArticleRetrievalTopology.Definitions + .Select(static definition => new RabbitMqManagementQueueInfo(definition.QueueName, 1)) + .ToArray(); + return new StaticManagementInventory(queues); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static async Task WaitForAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(10, cts.Token).ConfigureAwait(false); + } + } + + private static byte[] SuccessBody(string requestId) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"<12345@example.invalid>","backbone":"Storage","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + } + + private sealed class EmptyManagementInventory : IRabbitMqManagementQueueInventory + { + public Task> ListQueuesAsync(CancellationToken cancellationToken) => + Task.FromResult>([]); + } + + private sealed class StaticManagementInventory(IReadOnlyList queues) + : IRabbitMqManagementQueueInventory + { + public Task> ListQueuesAsync(CancellationToken cancellationToken) => + Task.FromResult(queues); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs new file mode 100644 index 00000000..8c9fcde1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs @@ -0,0 +1,144 @@ +using System.Text; +using System.Text.Json; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkWireProtocolTests +{ + private static readonly Guid RequestId = Guid.Parse("7c1cb8a0-95f9-4c13-8e53-339773e3afaa"); + + [Fact] + public void SerializeRequestV1_IsCompactCanonicalJson_WithoutTransportFields() + { + var request = new ArticleWorkRequest(1, RequestId, "<12345@example.invalid>", "Giganews"); + var bytes = ArticleWorkWireProtocol.SerializeRequestV1(request); + var json = Encoding.UTF8.GetString(bytes); + + Assert.Equal( + """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews"}""", + json); + Assert.DoesNotContain('\n', json); + Assert.DoesNotContain("CorrelationId", json, StringComparison.Ordinal); + Assert.DoesNotContain("ReplyTo", json, StringComparison.Ordinal); + Assert.DoesNotContain("correlationId", json, StringComparison.Ordinal); + Assert.DoesNotContain("replyTo", json, StringComparison.Ordinal); + + using var document = JsonDocument.Parse(bytes); + Assert.Equal(1, document.RootElement.GetProperty("version").GetInt32()); + Assert.Equal(RequestId, document.RootElement.GetProperty("requestId").GetGuid()); + Assert.Equal("<12345@example.invalid>", document.RootElement.GetProperty("messageId").GetString()); + Assert.Equal("Giganews", document.RootElement.GetProperty("backbone").GetString()); + Assert.Equal(4, document.RootElement.EnumerateObject().Count()); + } + + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public void TryParseResponseV1_AcceptsCanonicalSuccess() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""u8; + Assert.True(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Equal(string.Empty, reason); + Assert.NotNull(response); + Assert.Equal(ArticleWorkOutcome.Success, response.Outcome); + Assert.Equal(RequestId, response.RequestId); + Assert.Equal("<12345@example.invalid>", response.MessageId); + Assert.Equal("Giganews", response.Backbone); + Assert.Equal(SuccessFqdn, response.Fqdn); + Assert.Equal(SuccessVatpPort, response.VatpPort); + Assert.Equal( + VectorNNTPD.Common.Articles.ArticleId.ParseLowerHex(SuccessArticleIdHex), + response.ArticleId); + Assert.Null(response.Error); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithoutArticleId() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("articleId", reason, StringComparison.Ordinal); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithInvalidArticleId() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"NOT-HEX"}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("articleId", reason, StringComparison.Ordinal); + } + + [Theory] + [InlineData("nodots")] + [InlineData("")] + [InlineData("Backfiller01.usenet.ninja")] + public void TryParseResponseV1_RejectsInvalidFqdn(string fqdn) + { + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"{{fqdn}}","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("fqdn", reason, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(65536)] + public void TryParseResponseV1_RejectsInvalidVatpPort(int port) + { + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":{{port}},"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("vatpPort", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_AcceptsCanonicalArticleNotFound() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"ArticleNotFound","error":"No article with that message-id"}"""u8; + Assert.True(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out _)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, response!.Outcome); + Assert.Null(response.Fqdn); + Assert.Null(response.VatpPort); + Assert.Null(response.ArticleId); + Assert.Equal("No article with that message-id", response.Error); + } + + [Fact] + public void TryParseResponseV1_RejectsLegacyUriField() + { + var legacy = "vatp" + "://" + "backfiller01.usenet.ninja:119/" + SuccessArticleIdHex; + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"{{SuccessArticleIdHex}}","uri":"{{legacy}}"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("uri", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_RejectsUriOnlySuccess() + { + var legacy = "vatp" + "://" + "backfiller01.usenet.ninja:119/" + SuccessArticleIdHex; + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","uri":"{{legacy}}","articleId":"{{SuccessArticleIdHex}}"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("uri", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithError() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14","error":"nope"}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("error", reason, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs new file mode 100644 index 00000000..09ce8b78 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs @@ -0,0 +1,475 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq.ArticleWork; + +/// +/// Proves Management API–backed BackFiller availability discovery (~5s cache, +/// last-known-good on failure, no AMQP passive probing). +/// +public sealed class BackfillConsumerAvailabilityTests +{ + [Fact] + public async Task Cache_is_lazy_no_management_call_without_GetEligibleBackbones() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 10)); + await using var availability = CreateAvailability(inventory, time); + + await Task.Delay(20); + Assert.Equal(0, inventory.CallCount); + } + + [Fact] + public async Task First_GetEligible_refreshes_once_and_reuses_within_refresh_interval() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 100), + Queue("backfiller.eweka", 50), + Queue("backfiller.highwinds", 0)); + await using var availability = CreateAvailability(inventory, time); + + var first = availability.GetEligibleBackbones(); + Assert.Equal(2, first.Count); + Assert.Contains(first, static b => b.Backbone == "Giganews" && b.ConsumerCount == 100); + Assert.Contains(first, static b => b.Backbone == "Eweka" && b.ConsumerCount == 50); + Assert.DoesNotContain(first, static b => b.Backbone.Equals("Highwinds", StringComparison.OrdinalIgnoreCase)); + Assert.Equal(1, inventory.CallCount); + + for (var i = 0; i < 20; i++) + { + time.Advance(TimeSpan.FromMilliseconds(100)); + _ = availability.GetEligibleBackbones(); + } + + Assert.Equal(1, inventory.CallCount); + } + + [Fact] + public async Task After_refresh_interval_next_GetEligible_requeries_and_picks_up_changes() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 10)); + await using var availability = CreateAvailability(inventory, time); + + var initial = availability.GetEligibleBackbones(); + Assert.Single(initial); + Assert.Equal("Giganews", initial[0].Backbone); + + inventory.SetQueues( + Queue("backfiller.eweka", 25), + Queue("backfiller.giganews", 0)); + + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval - TimeSpan.FromMilliseconds(1)); + var stillCached = availability.GetEligibleBackbones(); + Assert.Single(stillCached); + Assert.Equal("Giganews", stillCached[0].Backbone); + Assert.Equal(1, inventory.CallCount); + + time.Advance(TimeSpan.FromMilliseconds(1)); + var refreshed = availability.GetEligibleBackbones(); + Assert.Single(refreshed); + Assert.Equal("Eweka", refreshed[0].Backbone); + Assert.Equal(25, refreshed[0].ConsumerCount); + Assert.Equal(2, inventory.CallCount); + } + + [Fact] + public async Task Missing_provider_queue_is_not_eligible() + { + var inventory = new FakeManagementInventory(Queue("backfiller.eweka", 7)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.DoesNotContain(eligible, static b => b.Backbone == "Abavia"); + Assert.DoesNotContain(eligible, static b => b.Backbone == "Giganews"); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 7); + } + + [Fact] + public async Task Existing_provider_queue_with_zero_consumers_is_not_eligible() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 0), + Queue("backfiller.eweka", 7)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.DoesNotContain(eligible, static b => b.Backbone == "Giganews"); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 7); + } + + [Fact] + public async Task Non_backfiller_queues_are_ignored() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 5), + Queue(StorageArticleRetrievalTopology.EntityName, 99), + Queue("overviewdb.queue", 3), + Queue("unrelated.queue", 12)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.Single(eligible); + Assert.Equal("Giganews", eligible[0].Backbone); + Assert.DoesNotContain( + eligible, + static b => b.QueueName.Equals(StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal)); + } + + [Fact] + public async Task All_twelve_queues_with_mixed_consumers_only_positive_are_eligible() + { + var rows = BackfillArticleRetrievalTopology.Definitions + .Select((definition, index) => Queue(definition.QueueName, (uint)(index % 3 == 0 ? 0 : index + 1))) + .ToArray(); + var inventory = new FakeManagementInventory(rows); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.All(eligible, static b => Assert.True(b.ConsumerCount > 0)); + Assert.Equal( + BackfillArticleRetrievalTopology.Definitions.Count(static d => + { + var index = Array.IndexOf(BackfillArticleRetrievalTopology.Providers, d.Provider); + return index % 3 != 0; + }), + eligible.Count); + Assert.Equal(BackfillArticleRetrievalTopology.Definitions.Count, rows.Length); + } + + [Fact] + public async Task Provider_names_match_canonical_topology_entity_names() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.gtt", 4), + Queue("backfiller.usenetnode1", 2), + Queue("backfiller.baseip", 1)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + Assert.Contains(eligible, static b => b.Backbone == "GTT" && b.QueueName == "backfiller.gtt"); + Assert.Contains(eligible, static b => b.Backbone == "UsenetNode1" && b.QueueName == "backfiller.usenetnode1"); + Assert.Contains(eligible, static b => b.Backbone == "BaseIP" && b.QueueName == "backfiller.baseip"); + } + + [Fact] + public async Task Weighted_selection_receives_management_eligible_set() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 100), + Queue("backfiller.eweka", 50)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + var eligible = availability.GetEligibleBackbones(); + var selector = new WeightedBackboneSelector(static _ => 0); + + var selected = selector.Select(eligible); + + // Nullable value type: HasValue/GetValueOrDefault (Assert.NotNull does not narrow for CodeQL). + Assert.True(selected.HasValue); + var choice = selected.GetValueOrDefault(); + // Definitions order places Eweka before Giganews; roll 0 selects the first positive weight. + Assert.Equal("Eweka", choice.Backbone); + Assert.Equal(50, choice.ConsumerCount); + Assert.Equal(2, eligible.Count); + Assert.Contains(eligible, static b => b.Backbone == "Giganews" && b.ConsumerCount == 100); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 50); + } + + [Fact] + public async Task Concurrent_GetEligible_does_not_issue_multiple_simultaneous_refreshes() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 3)) + { + Delay = TimeSpan.FromMilliseconds(100), + }; + await using var availability = CreateAvailability(inventory, time); + + var tasks = Enumerable.Range(0, 16) + .Select(_ => Task.Run(() => availability.GetEligibleBackbones())) + .ToArray(); + var results = await Task.WhenAll(tasks); + + Assert.Equal(1, inventory.CallCount); + Assert.All(results, static r => Assert.Single(r)); + Assert.Equal(1, inventory.MaxConcurrentCalls); + } + + [Fact] + public async Task Failed_refresh_retains_last_successful_snapshot() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 9)); + await using var availability = CreateAvailability(inventory, time); + + var first = availability.GetEligibleBackbones(); + Assert.Single(first); + Assert.Equal("Giganews", first[0].Backbone); + + inventory.Fault = new HttpRequestException("Management API returned 503 (ServiceUnavailable) for queue inventory."); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var retained = availability.GetEligibleBackbones(); + + Assert.Single(retained); + Assert.Equal("Giganews", retained[0].Backbone); + Assert.Equal(9, retained[0].ConsumerCount); + Assert.Equal(2, inventory.CallCount); + } + + [Fact] + public async Task Successful_refresh_replaces_previous_snapshot() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 1)); + await using var availability = CreateAvailability(inventory, time); + _ = availability.GetEligibleBackbones(); + + inventory.SetQueues(Queue("backfiller.eweka", 8)); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var next = availability.GetEligibleBackbones(); + + Assert.Single(next); + Assert.Equal("Eweka", next[0].Backbone); + Assert.Equal(8, next[0].ConsumerCount); + } + + [Fact] + public async Task Cancellation_during_refresh_propagates() + { + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 1)) + { + Delay = TimeSpan.FromSeconds(30), + }; + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + using var cts = new CancellationTokenSource(); + var refresh = availability.RefreshNowAsync(cts.Token); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(async () => await refresh); + } + + [Fact] + public async Task Malformed_inventory_failure_does_not_fabricate_empty_snapshot_after_success() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 4)); + await using var availability = CreateAvailability(inventory, time); + _ = availability.GetEligibleBackbones(); + + inventory.Fault = new InvalidOperationException( + "RabbitMQ Management API returned a malformed queue inventory payload."); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var retained = availability.GetEligibleBackbones(); + + Assert.Single(retained); + Assert.Equal("Giganews", retained[0].Backbone); + } + + [Fact] + public async Task Availability_service_does_not_use_amqp_passive_queue_declare() + { + var source = await File.ReadAllTextAsync( + Path.Join( + RepoPaths.FindRepoRoot(), + "src", + "VectorNNTPD.ReaderService", + "RabbitMq", + "ArticleWork", + "BackfillConsumerAvailability.cs")); + + Assert.DoesNotContain("QueueDeclarePassiveAsync", source, StringComparison.Ordinal); + Assert.DoesNotContain("CreateTopologyChannelAsync", source, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_lists_queues_for_encoded_vhost_with_basic_auth() + { + var handler = new CapturingHandler( + """ + [{"name":"backfiller.giganews","consumers":100},{"name":"backfiller.eweka","consumers":50}] + """); + var options = RabbitMqOptionsTests.CreateValid(); + options.VirtualHost = "/"; + options.Username = "mgmt-user"; + options.Password = "mgmt-secret"; + using var http = new HttpClient(handler) + { + BaseAddress = new Uri("http://127.0.0.1:15672/"), + }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var queues = await client.ListQueuesAsync(CancellationToken.None); + + Assert.Equal(2, queues.Count); + Assert.Equal("/api/queues/%2F", handler.LastPathAndQuery); + Assert.NotNull(handler.LastAuthorization); + Assert.Equal("Basic", handler.LastAuthorization!.Scheme); + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(handler.LastAuthorization.Parameter!)); + Assert.Equal("mgmt-user:mgmt-secret", decoded); + // codeql[cs/call-to-object-tostring] Assert default Object.ToString does not embed credentials. + Assert.DoesNotContain("mgmt-secret", handler.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_auth_failure_does_not_embed_credentials_in_exception() + { + var handler = new CapturingHandler("{}", HttpStatusCode.Unauthorized); + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "mgmt-user"; + options.Password = "super-secret-password"; + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://127.0.0.1:15672/") }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var ex = await Assert.ThrowsAsync( + () => client.ListQueuesAsync(CancellationToken.None)); + + Assert.DoesNotContain("super-secret-password", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("mgmt-user:super", ex.Message, StringComparison.Ordinal); + Assert.Contains("401", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_rejects_malformed_json() + { + var handler = new CapturingHandler("{not-json"); + var options = RabbitMqOptionsTests.CreateValid(); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://127.0.0.1:15672/") }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var ex = await Assert.ThrowsAsync( + () => client.ListQueuesAsync(CancellationToken.None)); + Assert.Contains("malformed", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + private static BackfillConsumerAvailabilityService CreateAvailability( + FakeManagementInventory inventory, + FakeTimeProvider time) => + new(inventory, NullLogger.Instance, time); + + private static RabbitMqManagementQueueInfo Queue(string name, uint consumers) => + new(name, checked((int)consumers)); + + private sealed class FakeManagementInventory : IRabbitMqManagementQueueInventory + { + private readonly object _gate = new(); + private IReadOnlyList _queues; + private int _inflight; + + public FakeManagementInventory(params RabbitMqManagementQueueInfo[] queues) + { + _queues = queues; + } + + public int CallCount { get; private set; } + + public int MaxConcurrentCalls { get; private set; } + + public TimeSpan Delay { get; set; } + + public Exception? Fault { get; set; } + + public void SetQueues(params RabbitMqManagementQueueInfo[] queues) + { + lock (_gate) + { + _queues = queues; + Fault = null; + } + } + + public async Task> ListQueuesAsync( + CancellationToken cancellationToken) + { + var inflight = Interlocked.Increment(ref _inflight); + try + { + MaxConcurrentCalls = Math.Max(MaxConcurrentCalls, inflight); + CallCount++; + if (Delay > TimeSpan.Zero) + { + await Task.Delay(Delay, cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (Fault is not null) + { + throw Fault; + } + + return _queues; + } + } + finally + { + Interlocked.Decrement(ref _inflight); + } + } + } + + private sealed class CapturingHandler : HttpMessageHandler + { + private readonly string _body; + private readonly HttpStatusCode _status; + + public CapturingHandler(string body, HttpStatusCode status = HttpStatusCode.OK) + { + _body = body; + _status = status; + } + + public string? LastPathAndQuery { get; private set; } + + public System.Net.Http.Headers.AuthenticationHeaderValue? LastAuthorization { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + LastPathAndQuery = request.RequestUri?.PathAndQuery; + LastAuthorization = request.Headers.Authorization; + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + var response = new HttpResponseMessage(_status) + { + Content = new StringContent(_body, Encoding.UTF8, "application/json"), + }; + return Task.FromResult(response); + } + } +} + +/// Resolves the repository root from the test assembly location. +file static class RepoPaths +{ + internal static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not locate VectorNNTPD.slnx from the test base directory."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs new file mode 100644 index 00000000..b751fb2c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs @@ -0,0 +1,123 @@ +using VectorNNTPD.ReaderService.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class BackfillArticleRetrievalTopologyTests +{ + private static readonly string[] ExpectedProviders = + [ + "Abavia", + "Altopia", + "BaseIP", + "Eweka", + "Elbracht", + "Giganews", + "GTT", + "Highwinds", + "ItsHosted", + "Novia", + "UExpress", + "UsenetNode1", + ]; + + [Fact] + public void Providers_AreExactlyTheTwelveBackFillerIdentifiers() + { + Assert.Equal(ExpectedProviders, BackfillArticleRetrievalTopology.Providers); + Assert.Equal(12, BackfillArticleRetrievalTopology.Definitions.Count); + Assert.Equal(ExpectedProviders, BackfillArticleRetrievalTopology.Definitions.Select(static d => d.Provider)); + Assert.DoesNotContain("storage", BackfillArticleRetrievalTopology.Providers, StringComparer.OrdinalIgnoreCase); + Assert.DoesNotContain("storage.requests", BackfillArticleRetrievalTopology.Providers, StringComparer.Ordinal); + Assert.DoesNotContain( + BackfillArticleRetrievalTopology.Definitions, + static definition => + definition.ExchangeName == StorageArticleRetrievalTopology.EntityName + || definition.QueueName == StorageArticleRetrievalTopology.EntityName + || definition.ExchangeName.Contains("storage.requests", StringComparison.Ordinal) + || definition.ExchangeName.Contains("grabbers.", StringComparison.Ordinal)); + } + + [Fact] + public void Definitions_UseNormalizedBackFillerEntityNamesAndFanoutBinding() + { + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + var entityName = $"backfiller.{definition.Provider.ToLowerInvariant()}"; + Assert.Equal(entityName, definition.ExchangeName); + Assert.Equal(entityName, definition.QueueName); + Assert.Equal(entityName, definition.RoutingKey); + Assert.Equal(definition.ExchangeName, definition.QueueName); + Assert.Equal(definition.QueueName, definition.RoutingKey); + Assert.DoesNotContain("grabbers.", definition.ExchangeName, StringComparison.Ordinal); + Assert.Equal("fanout", definition.ExchangeType); + Assert.True(definition.ExchangeDurable); + Assert.False(definition.ExchangeAutoDelete); + Assert.True(definition.QueueDurable); + Assert.False(definition.QueueExclusive); + Assert.False(definition.QueueAutoDelete); + } + + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName("Abavia")); + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName(" Abavia ")); + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName("AbAvIa")); + Assert.Equal("backfiller.gtt", BackfillArticleRetrievalTopology.BuildProviderEntityName("GTT")); + Assert.Equal("backfiller.usenetnode1", BackfillArticleRetrievalTopology.BuildProviderEntityName("UsenetNode1")); + Assert.Equal("backfiller.baseip", BackfillArticleRetrievalTopology.BuildProviderEntityName("BaseIP")); + Assert.Equal("backfiller.itshosted", BackfillArticleRetrievalTopology.BuildProviderEntityName("ItsHosted")); + Assert.Equal("backfiller.uexpress", BackfillArticleRetrievalTopology.BuildProviderEntityName("UExpress")); + } + + [Fact] + public void Definitions_DeclareQuorumQueuesWithQueueTypeArgument() + { + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + Assert.NotNull(definition.QueueArguments); + Assert.True(definition.QueueDurable); + Assert.False(definition.QueueExclusive); + Assert.False(definition.QueueAutoDelete); + Assert.True(definition.QueueArguments.TryGetValue( + BackfillArticleRetrievalTopology.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(BackfillArticleRetrievalTopology.QuorumQueueType, queueType); + Assert.False(definition.QueueArguments.ContainsKey("x-message-ttl")); + Assert.False(definition.QueueArguments.ContainsKey("x-expires")); + Assert.Single(definition.QueueArguments); + } + } + + [Fact] + public void BuildDefinitions_PreservesProviderCasing_AndNormalizesOnlyEntityNames() + { + var definition = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions(["Giganews"])); + Assert.Equal("Giganews", definition.Provider); + Assert.Equal("backfiller.giganews", definition.ExchangeName); + Assert.Equal("backfiller.giganews", definition.QueueName); + Assert.Equal("backfiller.giganews", definition.RoutingKey); + } + + [Fact] + public void BuildDefinitions_NormalizesWhitespaceAndMixedCaseProviderNames() + { + var definition = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions([" Abavia "])); + Assert.Equal("Abavia", definition.Provider); + Assert.Equal("backfiller.abavia", definition.ExchangeName); + Assert.Equal("backfiller.abavia", definition.QueueName); + Assert.Equal("backfiller.abavia", definition.RoutingKey); + + var mixed = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions(["AbAvIa"])); + Assert.Equal("AbAvIa", mixed.Provider); + Assert.Equal("backfiller.abavia", mixed.ExchangeName); + } + + [Fact] + public void BuildDefinitions_SkipsBlanksAndDuplicateProviders() + { + var definitions = BackfillArticleRetrievalTopology.BuildDefinitions( + ["Giganews", " ", "giganews", "Eweka", "Giganews"]); + + Assert.Equal(2, definitions.Count); + Assert.Equal("Giganews", definitions[0].Provider); + Assert.Equal("Eweka", definitions[1].Provider); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqOptionsTests.cs new file mode 100644 index 00000000..002ddf8d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqOptionsTests.cs @@ -0,0 +1,359 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class RabbitMqOptionsTests +{ + [Fact] + public void SectionName_IsRabbitMQ() + { + Assert.Equal("RabbitMQ", RabbitMqOptions.SectionName); + } + + [Fact] + public void Defaults_MatchBackFillerContract() + { + var options = new RabbitMqOptions(); + + Assert.Equal(1024, options.WorkRequestMaxPayloadBytes); + Assert.Equal(60, options.ChannelLeaseTimeoutSeconds); + Assert.Equal(30, options.RpcTimeoutSeconds); + Assert.Equal(30, options.ConnectionBlockedTimeoutSeconds); + Assert.Empty(options.Hosts!); + Assert.Null(options.Username); + Assert.Null(options.Password); + Assert.Equal("/", options.VirtualHost); + Assert.True(options.EnableSsl); + Assert.Equal(5672, options.Port); + Assert.Equal(512, options.ChannelPoolSize); + Assert.Equal(4, options.MinConnections); + Assert.Equal(16, options.MaxConnections); + Assert.Equal(1024, options.MaxPendingLeaseWaiters); + Assert.Equal(300, options.ConnectionScaleDownIdleSeconds); + Assert.Equal(30, options.ScaleDownCooldownSeconds); + Assert.Equal(5, options.NetworkRecoveryIntervalSeconds); + Assert.Equal(250, options.PoolReconnectBaseDelayMs); + Assert.Equal(30000, options.PoolReconnectMaxDelayMs); + Assert.Equal(300, options.MinimumConnectionLifetimeSeconds); + Assert.Equal(10, options.PublishConfirmTimeoutSeconds); + Assert.Equal(30, options.MaximumShutdownDrainTimeoutSeconds); + Assert.Equal(0.75, options.DegradedThreshold); + Assert.Equal(5, options.UnhealthyThreshold); + Assert.Equal(60, options.RequestedHeartbeatSeconds); + Assert.Equal(30, options.SocketTimeoutSeconds); + Assert.Equal(2047, options.RequestedChannelMax); + Assert.Null(options.ConsumerPrefetchCount); + Assert.Null(options.DiagnosticPayloadCorrelationId); + Assert.NotNull(options.Management); + Assert.Null(options.Management.BaseUrl); + Assert.Equal(5, options.Management.RequestTimeoutSeconds); + } + + [Fact] + public void Bind_ReadsIdenticalShape() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["RabbitMQ:Hosts:0"] = "rabbit-01.example.net", + ["RabbitMQ:Hosts:1"] = "10.0.0.8", + ["RabbitMQ:Port"] = "5671", + ["RabbitMQ:Username"] = "nntparticles", + ["RabbitMQ:Password"] = "unit-test-password-not-real", + ["RabbitMQ:VirtualHost"] = "/articles", + ["RabbitMQ:EnableSsl"] = "true", + ["RabbitMQ:ChannelLeaseTimeoutSeconds"] = "90", + ["RabbitMQ:RpcTimeoutSeconds"] = "45", + ["RabbitMQ:ConnectionBlockedTimeoutSeconds"] = "40", + ["RabbitMQ:WorkRequestMaxPayloadBytes"] = "2048", + ["RabbitMQ:ChannelPoolSize"] = "256", + ["RabbitMQ:MinConnections"] = "2", + ["RabbitMQ:MaxConnections"] = "8", + ["RabbitMQ:MaxPendingLeaseWaiters"] = "64", + ["RabbitMQ:ConnectionScaleDownIdleSeconds"] = "120", + ["RabbitMQ:ScaleDownCooldownSeconds"] = "15", + ["RabbitMQ:NetworkRecoveryIntervalSeconds"] = "8", + ["RabbitMQ:PoolReconnectBaseDelayMs"] = "100", + ["RabbitMQ:PoolReconnectMaxDelayMs"] = "5000", + ["RabbitMQ:MinimumConnectionLifetimeSeconds"] = "90", + ["RabbitMQ:PublishConfirmTimeoutSeconds"] = "12", + ["RabbitMQ:MaximumShutdownDrainTimeoutSeconds"] = "20", + ["RabbitMQ:DegradedThreshold"] = "0.5", + ["RabbitMQ:UnhealthyThreshold"] = "4", + ["RabbitMQ:RequestedHeartbeatSeconds"] = "30", + ["RabbitMQ:SocketTimeoutSeconds"] = "15", + ["RabbitMQ:RequestedChannelMax"] = "1024", + ["RabbitMQ:ConsumerPrefetchCount"] = "50", + ["RabbitMQ:DiagnosticPayloadCorrelationId"] = "ee9f7a44-a5d6-4603-ba22-c5e0803fc834", + ["RabbitMQ:Management:BaseUrl"] = "http://rabbit-01.example.net:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + }) + .Build(); + + var options = new RabbitMqOptions(); + configuration.GetSection(RabbitMqOptions.SectionName).Bind(options); + + Assert.Equal(["rabbit-01.example.net", "10.0.0.8"], options.Hosts!); + Assert.Equal(5671, options.Port); + Assert.Equal("nntparticles", options.Username); + Assert.Equal("unit-test-password-not-real", options.Password); + Assert.Equal("/articles", options.VirtualHost); + Assert.True(options.EnableSsl); + Assert.Equal(90, options.ChannelLeaseTimeoutSeconds); + Assert.Equal(45, options.RpcTimeoutSeconds); + Assert.Equal(40, options.ConnectionBlockedTimeoutSeconds); + Assert.Equal(2048, options.WorkRequestMaxPayloadBytes); + Assert.Equal(256, options.ChannelPoolSize); + Assert.Equal(2, options.MinConnections); + Assert.Equal(8, options.MaxConnections); + Assert.Equal(64, options.MaxPendingLeaseWaiters); + Assert.Equal(120, options.ConnectionScaleDownIdleSeconds); + Assert.Equal(15, options.ScaleDownCooldownSeconds); + Assert.Equal(8, options.NetworkRecoveryIntervalSeconds); + Assert.Equal(100, options.PoolReconnectBaseDelayMs); + Assert.Equal(5000, options.PoolReconnectMaxDelayMs); + Assert.Equal(90, options.MinimumConnectionLifetimeSeconds); + Assert.Equal(12, options.PublishConfirmTimeoutSeconds); + Assert.Equal(20, options.MaximumShutdownDrainTimeoutSeconds); + Assert.Equal(0.5, options.DegradedThreshold); + Assert.Equal(4, options.UnhealthyThreshold); + Assert.Equal(30, options.RequestedHeartbeatSeconds); + Assert.Equal(15, options.SocketTimeoutSeconds); + Assert.Equal(1024, options.RequestedChannelMax); + Assert.Equal((ushort)50, options.ConsumerPrefetchCount); + Assert.Equal("ee9f7a44-a5d6-4603-ba22-c5e0803fc834", options.DiagnosticPayloadCorrelationId); + Assert.Equal("http://rabbit-01.example.net:15672", options.Management.BaseUrl); + Assert.Equal(5, options.Management.RequestTimeoutSeconds); + } + + [Fact] + public void Validate_Succeeds_ForValidHosts() + { + var result = new NntpdRabbitMqOptionsValidator().Validate(null, CreateValid()); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validate_Fails_WhenHostsMissing() + { + var result = new NntpdRabbitMqOptionsValidator().Validate(null, new RabbitMqOptions()); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Hosts", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenHostEntryEmpty() + { + var options = CreateValid(); + options.Hosts = [" "]; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenHostIncludesCredentials() + { + var options = CreateValid(); + options.Hosts = ["user:pass@broker.example.net"]; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("credentials", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenUsernameConfiguredWithoutPassword() + { + var options = CreateValid(); + options.Username = "nntparticles"; + options.Password = null; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Password", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(0)] + [InlineData(65536)] + public void Validate_Fails_WhenPortOutOfRange(int port) + { + var options = CreateValid(); + options.Port = port; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenMinConnectionsExceedMaxConnections() + { + var options = CreateValid(); + options.MinConnections = 8; + options.MaxConnections = 2; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenLeaseTimeoutLessThanRpcTimeout() + { + var options = CreateValid(); + options.ChannelLeaseTimeoutSeconds = 10; + options.RpcTimeoutSeconds = 30; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void ToRuntimeOptions_ProjectsValidatedSnapshot() + { + var options = CreateValid(); + options.Username = "nntparticles"; + options.Password = "unit-test-password-not-real"; + options.EnableSsl = false; + + var runtime = options.ToRuntimeOptions(); + + Assert.Equal(["127.0.0.1"], runtime.Hosts); + Assert.Equal(5672, runtime.Port); + Assert.Equal("nntparticles", runtime.Username); + Assert.Equal("unit-test-password-not-real", runtime.Password); + Assert.Equal("/", runtime.VirtualHost); + Assert.False(runtime.EnableSsl); + Assert.Equal(60, runtime.RequestedHeartbeatSeconds); + } + + [Fact] + public void ToRuntimeOptions_TrimsDropsBlankAndDeduplicatesHosts() + { + var options = CreateValid(); + options.Hosts = [" broker.example.net ", " ", "BROKER.example.net", "10.0.0.8", ""]; + + var runtime = options.ToRuntimeOptions(); + + Assert.Equal(["broker.example.net", "10.0.0.8"], runtime.Hosts); + } + + [Fact] + public void ToRuntimeOptions_TrimsUsername_AndConvertsBlankToNull() + { + var options = CreateValid(); + options.Username = " nntparticles "; + options.Password = "unit-test-password-not-real"; + + Assert.Equal("nntparticles", options.ToRuntimeOptions().Username); + + options.Username = " \t "; + Assert.Null(options.ToRuntimeOptions().Username); + + options.Username = null; + Assert.Null(options.ToRuntimeOptions().Username); + } + + [Fact] + public void ToRuntimeOptions_TrimsVirtualHost_AndConvertsBlankToSlash() + { + var options = CreateValid(); + options.VirtualHost = " /articles "; + Assert.Equal("/articles", options.ToRuntimeOptions().VirtualHost); + + options.VirtualHost = " \t "; + Assert.Equal("/", options.ToRuntimeOptions().VirtualHost); + + options.VirtualHost = null; + Assert.Equal("/", options.ToRuntimeOptions().VirtualHost); + } + + [Fact] + public void EnvironmentVariables_DocumentedCredentialNames_AreExact() + { + Assert.Equal("VECTOR__RABBITMQ__USERNAME", RabbitMqOptions.UsernameEnvironmentVariable); + Assert.Equal("VECTOR__RABBITMQ__PASSWORD", RabbitMqOptions.PasswordEnvironmentVariable); + Assert.Equal("Username", RabbitMqOptions.UsernameConfigurationKey); + Assert.Equal("Password", RabbitMqOptions.PasswordConfigurationKey); + } + + [Fact] + public void EnvironmentVariables_NntpdPrefixedCredentials_BindToRabbitMqSection() + { + const string username = "nntparticles"; + const string password = "unit-test-password-not-real"; + + var previousUser = Environment.GetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable); + var previousPass = Environment.GetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable); + + try + { + Environment.SetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable, username); + Environment.SetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable, password); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "from-json-should-be-overridden", + ["RabbitMQ:Password"] = "from-json-should-be-overridden", + }) + .AddNntpdPrefixedEnvironmentVariables() + .Build(); + + var options = new RabbitMqOptions(); + configuration.GetSection(RabbitMqOptions.SectionName).Bind(options); + + Assert.Equal(username, options.Username); + Assert.Equal(password, options.Password); + Assert.Equal(username, configuration[$"{RabbitMqOptions.SectionName}:{RabbitMqOptions.UsernameConfigurationKey}"]); + Assert.Equal(password, configuration[$"{RabbitMqOptions.SectionName}:{RabbitMqOptions.PasswordConfigurationKey}"]); + } + finally + { + Environment.SetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable, previousUser); + Environment.SetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable, previousPass); + } + } + + [Fact] + public void Validate_Fails_WhenManagementBaseUrlMissing() + { + var options = CreateValid(); + options.Management.BaseUrl = null; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Management:BaseUrl", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenManagementBaseUrlIncludesApiPath() + { + var options = CreateValid(); + options.Management.BaseUrl = "http://127.0.0.1:15672/api"; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("must not include a path", StringComparison.Ordinal)); + } + + [Fact] + public void GetDefaultConnectionName_UsesNntpdPrefix() + { + Assert.Equal( + "VectorNNTPD.ReaderService:nntpd01.usenet.ninja", + RabbitMqRuntimeOptions.GetDefaultConnectionName("VectorNNTPD.ReaderService", "nntpd01.usenet.ninja")); + } + + internal static RabbitMqOptions CreateValid() => + new() + { + Hosts = ["127.0.0.1"], + EnableSsl = false, + Username = "guest", + Password = "guest", + Management = new RabbitMqManagementOptions + { + BaseUrl = "http://127.0.0.1:15672", + RequestTimeoutSeconds = 5, + }, + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs new file mode 100644 index 00000000..0849980b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs @@ -0,0 +1,322 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class RabbitMqServiceConcurrencyTests +{ + [Fact] + public async Task SimultaneousLossSignals_StartOneReconnect() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + Parallel.For(0, 8, _ => first.SimulateLost()); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + // Eight signals coalesce onto one watch loop. A loss observed during the + // reconnect delay can schedule one follow-up replace after N+1 is installed. + Assert.InRange(factory.ConnectCount, 2, 3); + Assert.True(service.ConnectionGeneration >= 2); + Assert.True(service.IsReady); + Assert.False(service.Execution!.IsCompleted); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StaleConnectionLost_DoesNotAffectCurrentGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + var current = factory.LastConnection!; + + var extra = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = extra; + first.SimulateLost(); + + var extraConnect = extra.Task.WaitAsync(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync(() => extraConnect); + + Assert.Equal(2, factory.ConnectCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.Same(current, handle.Connection); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(0, current.DisposeCount); + + await service.DisposeAsync(); + Assert.Equal(1, current.DisposeCount); + } + + [Fact] + public async Task StaleConnectionDispose_DoesNotDisposeCurrent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + var current = factory.LastConnection!; + + await first.DisposeAsync(); + + Assert.Equal(2, first.DisposeCount); + Assert.Equal(0, current.DisposeCount); + Assert.True(service.IsReady); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.Same(current, handle.Connection); + + await service.DisposeAsync(); + Assert.Equal(1, current.DisposeCount); + } + + [Fact] + public async Task ShutdownDuringReconnect_DoesNotInstallAReplacement() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var connectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.ConnectStarted = connectStarted; + factory.BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await connectStarted.Task.WaitAsync(safety.Token); + + var stop = service.StopAsync(CancellationToken.None); + factory.BlockConnect.TrySetCanceled(); + await stop; + + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, service.ConnectionGeneration); + Assert.False(service.IsReady); + Assert.False(service.TryGetCurrent(out _)); + foreach (var connection in factory.Connections) + { + Assert.Equal(1, connection.DisposeCount); + } + } + + [Fact] + public async Task ConcurrentStopAndLoss_DisposesCurrentOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + + var stop = service.StopAsync(CancellationToken.None); + first.SimulateLost(); + await service.DisposeAsync(); + await stop; + + Assert.Equal(1, first.DisposeCount); + Assert.Equal(1, factory.ConnectCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task CapturedHandle_IsNotCurrentAfterLoss_BeforeConnectionIsDisposed() + { + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 30000; + options.PoolReconnectMaxDelayMs = 30000; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var handle)); + var first = factory.LastConnection!; + Assert.True(handle.IsCurrent); + + first.SimulateLost(); + + Assert.False(handle.IsCurrent); + Assert.False(handle.IsOpen); + Assert.Equal(1, handle.Generation); + Assert.Same(first, handle.Connection); + Assert.Equal(0, first.DisposeCount); + Assert.False(service.TryGetCurrent(out _)); + + await service.DisposeAsync(); + Assert.Equal(1, first.DisposeCount); + } + + [Fact] + public async Task HoldingAHandle_DoesNotPreventDisposeOfThatGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var generationOne)); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.False(generationOne.IsCurrent); + Assert.Equal(1, generationOne.Generation); + Assert.Same(first, generationOne.Connection); + Assert.Equal(1, first.DisposeCount); + Assert.True(service.TryGetCurrent(out var generationTwo)); + Assert.Equal(2, generationTwo.Generation); + Assert.True(generationTwo.IsCurrent); + Assert.NotSame(generationOne.Connection, generationTwo.Connection); + + await service.DisposeAsync(); + } + + [Fact] + public async Task DisposeOfCapturedGeneration_CanOverlapCallerStillHoldingTheHandle() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var handle)); + var first = factory.LastConnection!; + var disposeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.DisposeStarted = disposeStarted; + first.BlockDispose = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await disposeStarted.Task.WaitAsync(safety.Token); + + Assert.False(handle.IsCurrent); + Assert.Same(first, handle.Connection); + Assert.Equal(0, first.DisposeCount); + Assert.False(service.TryGetCurrent(out _)); + + first.BlockDispose.TrySetResult(); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(1, first.DisposeCount); + Assert.False(handle.IsCurrent); + Assert.True(service.TryGetCurrent(out var next)); + Assert.Equal(2, next.Generation); + + await service.DisposeAsync(); + } + + [Fact] + public async Task ConcurrentTryGetCurrent_DuringReplacement_NeverReturnsADisposedCurrentHandle() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + var stop = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var observed = new List(); + var sampler = Task.Run(async () => + { + while (!stop.Task.IsCompleted) + { + if (service.TryGetCurrent(out var handle)) + { + lock (observed) + { + observed.Add(handle); + } + } + + await Task.Yield(); + } + }); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + stop.TrySetResult(); + await sampler.WaitAsync(safety.Token); + + Assert.NotEmpty(observed); + Assert.True(service.TryGetCurrent(out var current)); + Assert.Equal(2, current.Generation); + Assert.Equal(1, first.DisposeCount); + lock (observed) + { + foreach (var handle in observed) + { + if (!handle.IsCurrent) + { + continue; + } + + Assert.Equal(service.ConnectionGeneration, handle.Generation); + Assert.Equal(0, ((FakeRabbitMqConnection)handle.Connection).DisposeCount); + } + } + + await service.DisposeAsync(); + } + + [Fact] + public async Task RepeatedLifecycleCalls_AreIdempotent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await Task.WhenAll( + service.StopAsync(CancellationToken.None), + service.StopAsync(CancellationToken.None), + service.DisposeAsync().AsTask(), + service.DisposeAsync().AsTask()); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, connection.DisposeCount); + Assert.False(service.IsReady); + } + + private static RabbitMqService CreateService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceTests.cs new file mode 100644 index 00000000..a2a993cb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqServiceTests.cs @@ -0,0 +1,549 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class RabbitMqServiceTests +{ + [Fact] + public void ConnectionHandle_DoesNotOwnTheConnection() + { + Assert.DoesNotContain(typeof(IDisposable), typeof(RabbitMqConnectionHandle).GetInterfaces()); + Assert.DoesNotContain(typeof(IAsyncDisposable), typeof(RabbitMqConnectionHandle).GetInterfaces()); + } + + [Fact] + public async Task StartAsync_ConnectsOnce_AndIsReady() + { + var factory = new FakeRabbitMqConnectionFactory(); + var replaced = new List(); + var service = CreateService(factory); + service.ConnectionReplaced += (_, args) => replaced.Add(args); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.NotNull(service.Execution); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.True(handle.IsCurrent); + Assert.Equal(1, handle.Generation); + Assert.Same(factory.LastConnection, handle.Connection); + var generation = Assert.Single(replaced); + Assert.Equal(1, generation.ConnectionGeneration); + Assert.False(generation.IsReplacement); + Assert.Equal("VectorNNTPD.ReaderService:nntpd01.usenet.ninja", factory.LastConnection!.ClientProvidedName); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectThrows() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectException = new InvalidOperationException("broker down"), + }; + var service = CreateService(factory); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal("broker down", ex.Message); + Assert.Equal(0, service.ConnectionCount); + Assert.Equal(0, service.ConnectionGeneration); + Assert.False(service.IsReady); + Assert.Null(service.Execution); + Assert.False(service.TryGetCurrent(out _)); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectionIsNotUsable() + { + var factory = new FakeRabbitMqConnectionFactory { ReturnUnusableConnection = true }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + Assert.Equal(1, factory.LastConnection!.DisposeCount); + } + + [Fact] + public async Task StartAsync_Cancelled_DoesNotLeaveConnection() + { + var factory = new FakeRabbitMqConnectionFactory + { + BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + + var start = service.StartAsync(cts.Token); + cts.Cancel(); + factory.BlockConnect.TrySetCanceled(cts.Token); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + Assert.Null(service.Execution); + } + + [Fact] + public async Task StartAsync_CanRetry_AfterFailedConnect() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectException = new InvalidOperationException("broker down"), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + factory.ConnectException = null; + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsReady); + Assert.Equal(1, service.ConnectionGeneration); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StartAsync_IsIdempotent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + await service.DisposeAsync(); + } + + [Fact] + public async Task ConnectionLost_ReplacesConnection_AndIncrementsGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var replaced = new List(); + var service = CreateService(factory); + service.ConnectionReplaced += (_, args) => replaced.Add(args); + + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var count = await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, count); + Assert.Equal(2, service.ConnectionCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(2, replaced.Count); + Assert.True(replaced[1].IsReplacement); + Assert.Equal(2, replaced[1].ConnectionGeneration); + Assert.True(service.TryGetCurrent(out var current)); + Assert.Same(factory.LastConnection, current.Connection); + + await service.DisposeAsync(); + } + + [Fact] + public async Task ConnectionLost_TryGetCurrent_FailsUntilReplaced() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var before)); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + Assert.False(service.IsReady); + Assert.False(service.TryGetCurrent(out _)); + Assert.False(before.IsCurrent); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + Assert.True(service.IsReady); + Assert.True(service.TryGetCurrent(out var after)); + Assert.Equal(2, after.Generation); + Assert.True(after.IsCurrent); + Assert.False(before.IsCurrent); + + await service.DisposeAsync(); + Assert.False(after.IsCurrent); + } + + [Fact] + public async Task Reconnect_FailsThenSucceeds() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + factory.RemainingConnectFailures = 2; + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = recovered; + + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await recovered.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1 + 2 + 1, factory.AttemptCount); + Assert.Equal(2, factory.ConnectCount); + + await service.DisposeAsync(); + } + + [Fact] + public async Task Reconnect_Continues_BeyondFormerAbandonThreshold() + { + const int formerAbandonThreshold = 5; + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var failures = formerAbandonThreshold + 3; + factory.RemainingConnectFailures = failures; + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = recovered; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await factory.WaitForAttemptsAsync(1 + formerAbandonThreshold + 1, safety.Token); + Assert.False(service.IsReady); + Assert.Equal(1, service.ConnectionGeneration); + Assert.False(service.Execution!.IsCompleted); + + await recovered.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1 + failures + 1, factory.AttemptCount); + Assert.False(service.Execution!.IsCompleted); + + await service.DisposeAsync(); + Assert.True(service.Execution.IsCompleted); + } + + [Fact] + public async Task Reconnect_KeepsTrying_UntilShutdown() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + factory.RemainingConnectFailures = 100; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await factory.WaitForAttemptsAsync(8, safety.Token); + Assert.False(service.IsReady); + Assert.False(service.Execution!.IsCompleted); + + await service.StopAsync(CancellationToken.None); + + var attempts = factory.AttemptCount; + Assert.True(service.Execution.IsCompleted); + Assert.False(service.TryGetCurrent(out _)); + + using var noMore = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync( + () => factory.WaitForAttemptsAsync(attempts + 1, noMore.Token)); + Assert.Equal(attempts, factory.AttemptCount); + } + + [Fact] + public async Task SuccessfulReconnect_ResetsBackoffOnNextLoss() + { + var clock = new RecordingTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 800; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance, + clock); + + await service.StartAsync(CancellationToken.None); + factory.RemainingConnectFailures = 2; + var firstRecovery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = firstRecovery; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await firstRecovery.Task.WaitAsync(safety.Token); + var afterFirstRecovery = clock.Delays.ToArray(); + Assert.Equal( + [TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(200)], + afterFirstRecovery); + + var secondRecovery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondRecovery; + factory.LastConnection!.SimulateLost(); + await secondRecovery.Task.WaitAsync(safety.Token); + + Assert.Equal(3, service.ConnectionGeneration); + Assert.Equal(TimeSpan.FromMilliseconds(50), clock.Delays[^1]); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StopAsync_InterruptsRecoveryDelay() + { + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 30000; + options.PoolReconnectMaxDelayMs = 30000; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.SimulateLost(); + + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, factory.AttemptCount); + Assert.True(service.Execution!.IsCompleted); + Assert.False(service.TryGetCurrent(out _)); + } + + [Fact] + public async Task StopAsync_DisposesConnectionOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.Equal(1, connection.DisposeCount); + Assert.False(service.IsReady); + Assert.True(service.Execution is { IsCompleted: true }); + } + + [Fact] + public async Task DisposeAsync_IsSafe_WhenNeverStarted() + { + var service = CreateService(new FakeRabbitMqConnectionFactory()); + await service.DisposeAsync(); + await service.DisposeAsync(); + Assert.False(service.IsReady); + Assert.Equal(0, service.ConnectionGeneration); + } + + [Fact] + public async Task StopAsync_WhileReconnecting_CancelsAndDisposes() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var connectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.ConnectStarted = connectStarted; + factory.BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await connectStarted.Task.WaitAsync(safety.Token); + + await service.StopAsync(CancellationToken.None); + factory.BlockConnect.TrySetCanceled(); + + Assert.Equal(1, first.DisposeCount); + Assert.False(service.IsReady); + Assert.Equal(1, factory.ConnectCount); + foreach (var connection in factory.Connections) + { + Assert.Equal(1, connection.DisposeCount); + } + } + + [Fact] + public async Task StopAsync_DuringStartupConnect_LeavesNoConnection() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var start = service.StartAsync(cts.Token); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await factory.ConnectStarted.Task.WaitAsync(safety.Token); + cts.Cancel(); + factory.BlockConnect.TrySetCanceled(cts.Token); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task StopAsync_DoesNotReconnect() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + await service.StopAsync(CancellationToken.None); + + first.SimulateLost(); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, factory.AttemptCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task Logs_DoNotContainPassword() + { + const string secret = "unit-test-rabbitmq-password-not-real"; + var logger = new CollectingLogger(); + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "nntparticles"; + options.Password = secret; + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + + var service = new RabbitMqService( + new FakeRabbitMqConnectionFactory(), + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + logger); + + await service.StartAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.NotEmpty(logger.Messages); + Assert.Contains(logger.Messages, static message => message.Contains("Connecting to RabbitMQ", StringComparison.Ordinal)); + Assert.Contains(logger.Messages, static message => message.Contains("RabbitMQ connection established", StringComparison.Ordinal)); + Assert.Contains(logger.Messages, static message => message.Contains("RabbitMQ stopped", StringComparison.Ordinal)); + foreach (var message in logger.Messages) + { + Assert.DoesNotContain(secret, message, StringComparison.Ordinal); + Assert.DoesNotContain("Password=", message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("nntparticles:", message, StringComparison.Ordinal); + } + } + + [Fact] + public void CreateClientFactory_DisablesClientAutomaticRecovery() + { + var runtime = RabbitMqOptionsTests.CreateValid().ToRuntimeOptions(); + var factory = RabbitMqClientConnectionFactory.CreateClientFactory( + runtime, + "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"); + + Assert.False(factory.AutomaticRecoveryEnabled); + Assert.False(factory.TopologyRecoveryEnabled); + Assert.Equal(runtime.Port, factory.Port); + Assert.Equal(runtime.VirtualHost, factory.VirtualHost); + Assert.Equal((ushort)runtime.RequestedChannelMax, factory.RequestedChannelMax); + } + + [Fact] + public void CreateClientFactory_AppliesCredentialsWithoutExposingThemOnTheName() + { + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "nntparticles"; + options.Password = "unit-test-rabbitmq-password-not-real"; + var runtime = options.ToRuntimeOptions(); + const string name = "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"; + + var factory = RabbitMqClientConnectionFactory.CreateClientFactory(runtime, name); + + Assert.Equal("nntparticles", factory.UserName); + Assert.Equal("unit-test-rabbitmq-password-not-real", factory.Password); + Assert.Equal(name, factory.ClientProvidedName); + Assert.DoesNotContain(runtime.Password!, factory.ClientProvidedName, StringComparison.Ordinal); + } + + private static RabbitMqService CreateService( + FakeRabbitMqConnectionFactory factory, + int poolReconnectBaseDelayMs = 50) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = poolReconnectBaseDelayMs; + options.PoolReconnectMaxDelayMs = poolReconnectBaseDelayMs; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private sealed class RecordingTimeProvider : TimeProvider + { + private readonly TimeProvider _inner = TimeProvider.System; + + public List Delays { get; } = []; + + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + if (dueTime > TimeSpan.Zero && dueTime != Timeout.InfiniteTimeSpan) + { + Delays.Add(dueTime); + } + + return _inner.CreateTimer(callback, state, dueTime, period); + } + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs new file mode 100644 index 00000000..81ca4dbd --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class RabbitMqTopologyNamesTests +{ + [Theory] + [InlineData(" BackFiller.Abavia ", "backfiller.abavia")] + [InlineData(" cache.requests ", "cache.requests")] + [InlineData(" cache.requests ", "cache.requests")] + [InlineData("Abavia", "abavia")] + public void Normalize_TrimsAndUsesInvariantLowerCase(string input, string expected) + { + Assert.Equal(expected, RabbitMqTopologyNames.Normalize(input)); + } + + [Fact] + public void CreateFanoutQuorumBinding_NormalizesExchangeQueueAndRoutingKey() + { + var endpoint = RabbitMqArticleRetrievalEndpoints.CreateFanoutQuorumBinding(" BackFiller.Abavia "); + Assert.Equal("backfiller.abavia", endpoint.ExchangeName); + Assert.Equal("backfiller.abavia", endpoint.QueueName); + Assert.Equal("backfiller.abavia", endpoint.RoutingKey); + Assert.Equal(endpoint.ExchangeName, endpoint.QueueName); + Assert.Equal(endpoint.QueueName, endpoint.RoutingKey); + Assert.True(endpoint.QueueDurable); + Assert.False(endpoint.QueueExclusive); + Assert.False(endpoint.QueueAutoDelete); + Assert.True(endpoint.QueueArguments.TryGetValue( + RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(RabbitMqArticleRetrievalEndpoints.QuorumQueueType, queueType); + Assert.Single(endpoint.QueueArguments); + Assert.DoesNotContain("grabbers.", endpoint.ExchangeName, StringComparison.Ordinal); + } + + [Fact] + public void RequiredTopology_HasNoSharedCacheRequestsQuorumQueue() + { + Assert.Empty(ArticleRetrievalTopology.Required); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, CacheRequestsTopology.ExchangeName); + Assert.DoesNotContain("grabbers.", CacheRequestsTopology.ExchangeName, StringComparison.Ordinal); + Assert.DoesNotContain("backfiller.storage", CacheRequestsTopology.ExchangeName, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs new file mode 100644 index 00000000..3f5f2541 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs @@ -0,0 +1,246 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +[Collection(SerilogCollection.Name)] +public sealed class RabbitMqTopologyServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresCacheFanoutExchanges_AndOverviewQueue_WithoutSharedRequestsQueue() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + var channel = Assert.Single(connection.TopologyChannels); + Assert.Equal(1, channel.DisposeCount); + AssertDeclaredTopology(connection, expectedPasses: 1); + + await topology.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DeclareRequiredTopologyAsync_IsIdempotent_AndDoesNotDeleteOrPurge() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await topology.DeclareRequiredTopologyAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Equal(2, connection.TopologyChannels.Count); + Assert.All(connection.TopologyChannels, static channel => Assert.Equal(1, channel.DisposeCount)); + AssertDeclaredTopology(connection, expectedPasses: 2); + var channelMethods = typeof(IRabbitMqTopologyChannel).GetMethods().Select(static m => m.Name).ToHashSet(); + Assert.Contains(nameof(IRabbitMqTopologyChannel.ExchangeDeclareAsync), channelMethods); + Assert.Contains(nameof(IRabbitMqTopologyChannel.QueueDeclareAsync), channelMethods); + Assert.Contains(nameof(IRabbitMqTopologyChannel.QueueBindAsync), channelMethods); + Assert.DoesNotContain("QueueDeleteAsync", channelMethods); + Assert.DoesNotContain("QueuePurgeAsync", channelMethods); + Assert.DoesNotContain("ExchangeDeleteAsync", channelMethods); + Assert.DoesNotContain("BasicPublishAsync", channelMethods); + Assert.DoesNotContain("BasicConsumeAsync", channelMethods); + } + + [Fact] + public async Task StartAsync_SecondCall_DoesNotRedeclare() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Single(connection.TopologyChannels); + } + + [Fact] + public async Task StartAsync_Fails_WhenRabbitMqHasNotStarted() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + var ex = await Assert.ThrowsAsync( + () => topology.StartAsync(CancellationToken.None)); + + Assert.Contains("not ready", ex.Message, StringComparison.Ordinal); + Assert.Null(factory.LastConnection); + } + + [Fact] + public async Task StartAsync_Fails_WhenQueueDeclareIsRejected() + { + var factory = new FakeRabbitMqConnectionFactory + { + QueueDeclareException = new InvalidOperationException("PRECONDITION_FAILED - inequivalent arg 'x-queue-type' for queue 'overviewdb.queue'"), + }; + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + var ex = await Assert.ThrowsAsync( + () => topology.StartAsync(CancellationToken.None)); + + Assert.Contains("x-queue-type", ex.Message, StringComparison.Ordinal); + Assert.Single(factory.LastConnection!.TopologyChannels); + Assert.Equal(1, factory.LastConnection.TopologyChannels[0].DisposeCount); + } + + [Fact] + public async Task ApplicationServiceManager_TopologyFailure_PreventsStartup_AndRollsBackRabbitMq() + { + var factory = new FakeRabbitMqConnectionFactory + { + QueueDeclareException = new InvalidOperationException("classic queue cannot become quorum"), + }; + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + var manager = TestHostFactory.CreateServiceManager([rabbit, topology]); + + var ex = await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + + Assert.Contains("quorum", ex.Message, StringComparison.Ordinal); + Assert.Empty(manager.StartedServices); + Assert.False(rabbit.IsReady); + Assert.False(rabbit.TryGetCurrent(out _)); + } + + [Fact] + public async Task Host_Start_DeclaresArticleRetrievalTopology_BeforeRunning() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + await host.StartAsync(); + + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + var factory = Assert.IsType( + host.Services.GetRequiredService()); + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + AssertDeclaredTopology(connection, expectedPasses: 1); + + await host.StopAsync(); + } + + [Fact] + public void Host_RegistersTopologyServiceOnce_ImmediatelyAfterRabbitMqConnection() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + + // NntpDB and nntpsharedconfig are registered ahead of Cloudflare, Redis, and RabbitMQ. + Assert.Equal(typeof(RabbitMqService), services[4].GetType()); + Assert.Equal(typeof(RabbitMqTopologyService), services[5].GetType()); + Assert.Equal(typeof(StorageServerFleetConsumerService), services[6].GetType()); + Assert.Equal(typeof(StorageArticleLookupService), services[7].GetType()); + Assert.Equal(typeof(VectorNNTPD.ReaderService.RabbitMq.ArticleWork.ArticleWorkRpcService), services[8].GetType()); + Assert.Equal(1, services.Count(static s => s is RabbitMqTopologyService)); + Assert.Same( + host.Services.GetRequiredService(), + services[5]); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static void AssertDeclaredTopology(FakeRabbitMqConnection connection, int expectedPasses) + { + Assert.Empty(ArticleRetrievalTopology.Required); + // Per pass: cache.requests + cache.broadcast exchanges; overviewdb.queue only. + Assert.Equal(2 * expectedPasses, connection.ExchangeDeclarations.Count); + Assert.Equal(1 * expectedPasses, connection.QueueDeclarations.Count); + Assert.Empty(connection.BindingDeclarations); + + for (var pass = 0; pass < expectedPasses; pass++) + { + var exchangeOffset = pass * 2; + var requests = connection.ExchangeDeclarations[exchangeOffset]; + Assert.Equal(CacheRequestsTopology.ExchangeName, requests.Name); + Assert.Equal("fanout", requests.Type); + Assert.True(requests.Durable); + Assert.False(requests.AutoDelete); + + var broadcast = connection.ExchangeDeclarations[exchangeOffset + 1]; + Assert.Equal(CacheBroadcastTopology.ExchangeName, broadcast.Name); + Assert.Equal("fanout", broadcast.Type); + Assert.True(broadcast.Durable); + Assert.False(broadcast.AutoDelete); + + var overview = connection.QueueDeclarations[pass]; + Assert.Equal(OverviewDbTopology.QueueName, overview.Name); + Assert.True(overview.Durable); + Assert.False(overview.Exclusive); + Assert.False(overview.AutoDelete); + AssertQuorumQueueArguments(overview.Arguments); + } + + Assert.DoesNotContain( + connection.QueueDeclarations, + static queue => queue.Name == CacheFleetTopology.RequestsExchangeName); + Assert.DoesNotContain( + connection.QueueDeclarations, + static queue => + queue.Name == CacheFleetTopology.RequestsExchangeName + && queue.Durable + && queue.Arguments is not null + && queue.Arguments.TryGetValue("x-queue-type", out var type) + && Equals(type, "quorum")); + Assert.DoesNotContain( + connection.ExchangeDeclarations, + static exchange => exchange.Name is "backfiller.giganews" or "backfiller.eweka" or "backfiller.abavia"); + } + + private static void AssertQuorumQueueArguments( + IReadOnlyDictionary? arguments) + { + Assert.NotNull(arguments); + Assert.True(arguments.TryGetValue( + RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(RabbitMqArticleRetrievalEndpoints.QuorumQueueType, queueType); + Assert.False(arguments.ContainsKey("x-message-ttl")); + Assert.False(arguments.ContainsKey("x-expires")); + Assert.Single(arguments); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs new file mode 100644 index 00000000..d1cb03e7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs @@ -0,0 +1,49 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.RabbitMq; + +public sealed class StorageArticleRetrievalTopologyTests +{ + [Fact] + public void CacheRequests_IsFanoutExchangeOnly_NotSharedQuorumQueue() + { + Assert.Equal("cache.requests", StorageArticleRetrievalTopology.EntityName); + Assert.Equal("cache.requests", StorageArticleRetrievalTopology.ExchangeName); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, CacheRequestsTopology.ExchangeName); + Assert.Equal("fanout", CacheRequestsTopology.ExchangeTypeName); + Assert.True(CacheRequestsTopology.ExchangeDurable); + Assert.False(CacheRequestsTopology.ExchangeAutoDelete); + Assert.Equal("1000", CacheRequestsTopology.ExpirationMilliseconds); + Assert.Empty(ArticleRetrievalTopology.Required); + Assert.DoesNotContain("backfiller.storage", StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal); + } + + [Fact] + public void StorageEndpoint_IsNotABackFillerProvider() + { + Assert.Equal(12, BackfillArticleRetrievalTopology.Providers.Length); + Assert.DoesNotContain("storage", BackfillArticleRetrievalTopology.Providers, StringComparer.OrdinalIgnoreCase); + Assert.Equal("Storage", StorageArticleRetrievalTopology.Backbone); + Assert.All( + BackfillArticleRetrievalTopology.Definitions, + static definition => + Assert.NotEqual(CacheFleetTopology.RequestsExchangeName, definition.ExchangeName)); + } + + [Fact] + public void StorageServerRequestQueues_ArePerInstance_AndIndependent() + { + var q1 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache01.usenet.ninja"); + var q2 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache02.usenet.ninja"); + var q3 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache03.usenet.ninja"); + Assert.Equal("cache.cache01.usenet.ninja", q1); + Assert.Equal("cache.cache02.usenet.ninja", q2); + Assert.Equal("cache.cache03.usenet.ninja", q3); + Assert.NotEqual(q1, q2); + Assert.NotEqual(q2, q3); + Assert.NotEqual( + CacheFleetTopology.BuildNntpdBroadcastQueueName("nntpd01.usenet.ninja"), + q1); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisOptionsTests.cs new file mode 100644 index 00000000..0a59f9ef --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisOptionsTests.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Redis; + +public sealed class RedisOptionsTests +{ + [Fact] + public void DefaultPort_Is6379() + { + Assert.Equal(6379, RedisOptions.DefaultPort); + Assert.Equal(6379, new RedisOptions().Port); + } + + [Fact] + public void Bind_ReadsHostArrayAndPort() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "redis-01.example.net", + ["Redis:Host:1"] = "redis-02.example.net", + ["Redis:Port"] = "6380", + }) + .Build(); + + var options = new RedisOptions(); + configuration.GetSection(RedisOptions.SectionName).Bind(options); + + Assert.Equal(["redis-01.example.net", "redis-02.example.net"], options.Host); + Assert.Equal(6380, options.Port); + } + + [Fact] + public void Bind_OmitsPort_DefaultsTo6379() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "127.0.0.1", + }) + .Build(); + + var options = new RedisOptions(); + configuration.GetSection(RedisOptions.SectionName).Bind(options); + Assert.Equal(6379, options.Port); + } + + [Fact] + public void Validate_Fails_WhenHostMissing() + { + var result = new RedisOptionsValidator().Validate(null, new RedisOptions()); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenHostEntryEmpty() + { + var result = new RedisOptionsValidator().Validate(null, new RedisOptions { Host = [" "] }); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(0)] + [InlineData(65536)] + public void Validate_Fails_WhenPortOutOfRange(int port) + { + var result = new RedisOptionsValidator().Validate( + null, + new RedisOptions { Host = ["127.0.0.1"], Port = port }); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Succeeds_ForValidHosts() + { + var result = new RedisOptionsValidator().Validate( + null, + new RedisOptions { Host = ["redis-01.example.net", "10.0.0.8"], Port = 6379 }); + Assert.True(result.Succeeded); + } + +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisServiceTests.cs new file mode 100644 index 00000000..a2d0c5de --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Redis/RedisServiceTests.cs @@ -0,0 +1,238 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Redis; + +public sealed class RedisServiceTests +{ + [Fact] + public async Task StartAsync_ConnectsOnce_AndPings() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, factory.LastConnection!.Database.PingCount); + Assert.Same(factory.LastConnection.Database, service.Database); + } + + [Fact] + public async Task StartAsync_DoesNotCreatePerRequestConnections() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + _ = await service.KeyExistsAsync("a"u8.ToArray()); + _ = await service.KeyExistsAsync("b"u8.ToArray()); + _ = await service.GetAsync("c"u8.ToArray()); + await service.SetAsync("c"u8.ToArray(), "d"u8.ToArray(), TimeSpan.FromMinutes(1)); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(2, factory.LastConnection!.Database.KeyExistsCount); + Assert.Equal(1, factory.LastConnection.Database.GetCount); + Assert.Equal(1, factory.LastConnection.Database.SetCount); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectThrows() + { + var factory = new FakeRedisConnectionFactory + { + ConnectException = new RedisUnavailableException("down"), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal(0, service.ConnectionCount); + } + + [Fact] + public async Task StopAsync_DisposesConnectionOnce() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task KeyExistsAsync_PropagatesInfrastructureFailure() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.Database.ExistsException = new RedisUnavailableException("timeout"); + + await Assert.ThrowsAsync( + () => service.KeyExistsAsync("k"u8.ToArray()).AsTask()); + } + + [Fact] + public async Task Circuit_BlocksUntilCooldown_ThenRecovers() + { + var clock = new ManualTimeProvider(); + var factory = new FakeRedisConnectionFactory(); + var service = new RedisService( + factory, + Options.Create(new RedisOptions { Host = ["127.0.0.1"], Port = 6379 }), + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(1)); + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.Database.ExistsException = new RedisUnavailableException("timeout"); + + var history = new VectorNNTPD.ReaderService.History.HistoryDb( + service, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new VectorNNTPD.ReaderService.History.HistoryWriteQueue()); + + Assert.Equal( + VectorNNTPD.ReaderService.History.HistoryLookupResult.Unavailable, + await history.LookupAsync(""u8.ToArray())); + Assert.True(service.IsUnavailable); + var exists = factory.LastConnection.Database.KeyExistsCount; + + Assert.Equal( + VectorNNTPD.ReaderService.History.HistoryLookupResult.Unavailable, + await history.LookupAsync(""u8.ToArray())); + Assert.Equal(exists, factory.LastConnection.Database.KeyExistsCount); + + clock.Advance(TimeSpan.FromSeconds(1)); + factory.LastConnection.Database.ExistsException = null; + Assert.Equal( + VectorNNTPD.ReaderService.History.HistoryLookupResult.Unseen, + await history.LookupAsync(""u8.ToArray())); + Assert.False(service.IsUnavailable); + Assert.Equal(exists + 1, factory.LastConnection.Database.KeyExistsCount); + + await service.DisposeAsync(); + } + + [Fact] + public async Task CancelledLookup_DoesNotOpenCooldown_AndNextLookupHitsRedis() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.LastConnection!.Database.ExistsStarted = started; + factory.LastConnection.Database.BlockExists = block; + + var history = new VectorNNTPD.ReaderService.History.HistoryDb( + service, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new VectorNNTPD.ReaderService.History.HistoryWriteQueue()); + + using var cts = new CancellationTokenSource(); + var lookup = history.LookupAsync(""u8.ToArray(), cts.Token); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await started.Task.WaitAsync(safety.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => lookup.AsTask()); + Assert.False(service.IsUnavailable); + + factory.LastConnection.Database.BlockExists = null; + block.TrySetCanceled(); + Assert.Equal( + VectorNNTPD.ReaderService.History.HistoryLookupResult.Unseen, + await history.LookupAsync(""u8.ToArray())); + Assert.False(service.IsUnavailable); + Assert.Equal(1, factory.LastConnection.Database.KeyExistsCount); + + await service.DisposeAsync(); + } + + [Fact] + public void RecoveryProbe_Alone_CanClearCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var healthyGrant)); + Assert.False(healthyGrant); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out var becameUnavailable, out _); + Assert.True(becameUnavailable); + Assert.True(circuit.IsUnavailable); + Assert.False(circuit.TryBegin(out _)); + + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + Assert.False(circuit.TryBegin(out var other)); + Assert.False(other); + + circuit.Complete(isRecoveryProbe: false, succeeded: true, out _, out var recoveredByOther); + Assert.False(recoveredByOther); + Assert.False(circuit.TryBegin(out _)); + + circuit.Complete(isRecoveryProbe: true, succeeded: true, out _, out var recoveredByProbe); + Assert.True(recoveredByProbe); + Assert.False(circuit.IsUnavailable); + Assert.True(circuit.TryBegin(out var after)); + Assert.False(after); + } + + [Fact] + public void RecoveryProbe_Failure_RestartsCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out _)); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out _, out _); + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + circuit.Complete(isRecoveryProbe: true, succeeded: false, out var becameUnavailable, out var recovered); + Assert.False(recovered); + Assert.False(becameUnavailable); + Assert.True(circuit.IsUnavailable); + Assert.False(circuit.TryBegin(out _)); + } + + [Fact] + public void AbandonProbe_DoesNotOpenCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out _)); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out _, out _); + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + circuit.Abandon(isRecoveryProbe: true); + Assert.False(circuit.IsUnavailable); + Assert.True(circuit.TryBegin(out var next)); + Assert.True(next); + } + + private static RedisService CreateService(FakeRedisConnectionFactory factory) => + new( + factory, + Options.Create(new RedisOptions { Host = ["127.0.0.1"], Port = 6379 }), + NullLogger.Instance); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/ApprovedHeaderParserTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/ApprovedHeaderParserTests.cs new file mode 100644 index 00000000..b047799e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/ApprovedHeaderParserTests.cs @@ -0,0 +1,93 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session.Commands.Posting; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class ApprovedHeaderParserTests +{ + [Theory] + [InlineData("moderator@example.com", "moderator@example.com")] + [InlineData(" Moderator@example.com ", "Moderator@example.com")] + [InlineData("Moderator Name ", "moderator@example.com")] + public void TryParseMailboxList_ExtractsIdentity(string value, string expected) + { + Assert.True(ApprovedHeaderParser.TryParseMailboxList(Encoding.ASCII.GetBytes(value), out var identities, out _)); + Assert.Equal(expected, Assert.Single(identities)); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("")] + [InlineData("not-a-mailbox")] + [InlineData(",moderator@example.com")] + public void TryParseMailboxList_RejectsMalformed(string value) + { + Assert.False(ApprovedHeaderParser.TryParseMailboxList(Encoding.ASCII.GetBytes(value), out _, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidApproved, failure.Category); + } + + [Fact] + public void TryRead_NoApproved_ReturnsEmpty() + { + var article = Parse(Build()); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Empty(identities); + } + + [Fact] + public void TryRead_HeaderNameIsCaseInsensitive() + { + var article = Parse(Build("aPpRoVeD: moderator@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal("moderator@example.com", Assert.Single(identities)); + } + + [Fact] + public void TryRead_IdenticalDuplicates_Collapse() + { + var article = Parse(Build("Approved: moderator@example.com\r\nApproved: Moderator@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal("moderator@example.com", Assert.Single(identities)); + } + + [Fact] + public void TryRead_ConflictingDuplicates_KeepBoth() + { + var article = Parse(Build("Approved: a@example.com\r\nApproved: b@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal(["a@example.com", "b@example.com"], identities); + } + + [Fact] + public void TryRead_MailboxList_CollectsDistinctIdentities() + { + var article = Parse(Build("Approved: a@example.com, b@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal(["a@example.com", "b@example.com"], identities); + } + + [Fact] + public void IdentitiesEqual_IsCaseInsensitive() + { + Assert.True(ApprovedHeaderParser.IdentitiesEqual("Moderator@Example.COM", "moderator@example.com")); + Assert.False(ApprovedHeaderParser.IdentitiesEqual("a@example.com", "b@example.com")); + } + + private static ParsedPostArticle Parse(string article) + { + Assert.True(PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build(string extraHeaders = "") => + "Date: 25 Sep 2026 12:00:00 +0000\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: test\r\n" + + "Message-ID: \r\n" + + extraHeaders + + "\r\n" + + "body\r\n"; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalCommandTests.cs new file mode 100644 index 00000000..34d2ea02 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalCommandTests.cs @@ -0,0 +1,629 @@ +using System.IO.Pipelines; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// ARTICLE/HEAD/BODY/STAT argument-form failure codes and message-id RPC/VATP wiring prerequisites. +/// Successful message-id retrieval is covered by . +/// +public sealed class ArticleRetrievalCommandTests +{ + private static readonly NntpAuthorization Reader = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + public static TheoryData RetrievalVerbs() => + new() + { + "ARTICLE", + "HEAD", + "BODY", + "STAT", + }; + + [Fact] + public async Task ArticleMessageId_InvokesRpc_WithoutVatp_Returns400() + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new RecordingArticleWorkRpcClient(); + var session = duplex.CreateSession(articleWorkRpc: rpc, storageLookup: SilenceLookup()); + await DispatchLineAsync(duplex, session, "ARTICLE <12345@example.invalid>"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + var messageId = Assert.Single(rpc.Lookups); + Assert.Equal("<12345@example.invalid>"u8.ToArray(), messageId.ToArray()); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task MessageId_WithRpcOnly_InvokesRpc_Returns400(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new RecordingArticleWorkRpcClient(); + var session = duplex.CreateSession(articleWorkRpc: rpc, storageLookup: SilenceLookup()); + await DispatchLineAsync(duplex, session, $"{verb} <12345@example.invalid>"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Single(rpc.Lookups); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task MessageId_Returns430_WithoutSelectingGroup(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(); + await DispatchLineAsync(duplex, session, $"{verb} "); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.False(session.HasSelectedGroup); + await AssertNextCommandStillWorksAsync(duplex, session); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Number_WithoutGroup_Returns412(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(); + await DispatchLineAsync(duplex, session, $"{verb} 123456"); + Assert.Equal("412 No newsgroup selected", await duplex.ReadClientLineAsync()); + Assert.False(session.HasSelectedGroup); + await AssertNextCommandStillWorksAsync(duplex, session); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Number_WithGroup_Returns423_AndDoesNotChangeSelection(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.HasSelectedGroup); + var selected = session.SelectedGroupName.ToArray(); + + await DispatchLineAsync(duplex, session, $"{verb} 123456"); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + Assert.True(session.HasSelectedGroup); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + await AssertNextCommandStillWorksAsync(duplex, session); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Current_WithoutGroup_Returns412(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(); + await DispatchLineAsync(duplex, session, verb); + Assert.Equal("412 No newsgroup selected", await duplex.ReadClientLineAsync()); + Assert.False(session.HasSelectedGroup); + await AssertNextCommandStillWorksAsync(duplex, session); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Current_WithGroup_Returns420_BecauseNoCurrentArticleExists(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var selected = session.SelectedGroupName.ToArray(); + + await DispatchLineAsync(duplex, session, verb); + Assert.Equal("420 Current article number is invalid", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + await AssertNextCommandStillWorksAsync(duplex, session); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task FailedNumberLookup_DoesNotEstablishCurrentArticle(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var selected = session.SelectedGroupName.ToArray(); + + await DispatchLineAsync(duplex, session, $"{verb} 123456"); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + + await DispatchLineAsync(duplex, session, verb); + Assert.Equal("420 Current article number is invalid", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task MessageIdVsNumber_AreDistinctNotFoundCodes(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + + await DispatchLineAsync(duplex, session, $"{verb} <123@example.com>"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, $"{verb} 123"); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task MalformedAngleBracket_Is501_NotAnArticleNumber(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, $"{verb} <123@example.com"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual("misc.test"u8)); + } + + [Theory] + [InlineData("1")] + [InlineData("0")] + [InlineData("0001")] + [InlineData("2147483647")] + [InlineData("2147483648")] + [InlineData("9999999999999999")] + public async Task DigitTokenThrough16Digits_IsNumberForm_423WhenGroupSelected(string number) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var selected = session.SelectedGroupName.ToArray(); + + await DispatchLineAsync(duplex, session, "ARTICLE " + number); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + await DispatchLineAsync(duplex, session, "ARTICLE"); + Assert.Equal("420 Current article number is invalid", await duplex.ReadClientLineAsync()); + } + + [Theory] + [InlineData("10000000000000000")] + [InlineData("12345678901234567")] + [InlineData("12a")] + [InlineData("-1")] + public async Task OutsideRfcDigitSyntax_Returns501(string token) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, "ARTICLE " + token); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual("misc.test"u8)); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Error_IsSingleLine_WithoutMultilineTerminator(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + + await DispatchLineAsync(duplex, session, $"{verb} "); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, $"{verb} 123"); + Assert.Equal("412 No newsgroup selected", await duplex.ReadClientLineAsync()); + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, $"{verb} 123"); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await DispatchLineAsync(duplex, session, verb); + Assert.Equal("420 Current article number is invalid", await duplex.ReadClientLineAsync()); + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task SequentialFailedNumberLookups_DoNotChangeGroupOrCurrentArticle() + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(SampleSnapshot()); + await DispatchLineAsync(duplex, session, "GROUP misc.test"); + Assert.StartsWith("211 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var selected = session.SelectedGroupName.ToArray(); + + foreach (var verb in new[] { "ARTICLE", "HEAD", "BODY", "STAT" }) + { + await DispatchLineAsync(duplex, session, $"{verb} 123456"); + Assert.Equal("423 No article with that number", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + } + + await DispatchLineAsync(duplex, session, "STAT"); + Assert.Equal("420 Current article number is invalid", await duplex.ReadClientLineAsync()); + Assert.True(session.SelectedGroupName.SequenceEqual(selected)); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task InvalidArgument_Returns501_NotALookupFailure(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(); + await DispatchLineAsync(duplex, session, $"{verb} not-an-id"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + await AssertNextCommandStillWorksAsync(duplex, session); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task Unauthenticated_StillReturns480(string verb) + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var session = duplex.CreateSession(authorize: false); + await DispatchLineAsync(duplex, session, $"{verb} "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public void Catalog_RequiresAuthenticationAndReader(string verb) + { + var parsed = NntpCommandTestParse.ParseCommand(verb); + Assert.Equal( + NntpCommandAccess.RequiresAuthentication | NntpCommandAccess.RequiresReader, + DefaultNntpCommandCatalog.GetAccess(parsed.Verb, parsed.Qualifier)); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task EmptyRegistry_SkipsStorageRpc_AndStillCallsBackFiller(string verb) + { + var registry = new StorageServerRegistry(); + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var lookup = CreateLookup(rabbit, registry); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new RecordingArticleWorkRpcClient(); + var session = duplex.CreateSession(articleWorkRpc: rpc, storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"{verb} <12345@example.invalid>"); + + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Single(rpc.Lookups); + Assert.Null(factory.LastConnection); + Assert.Null(lookup.CurrentReplyTo); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task EmptyRegistry_BackFillerMiss_Returns430(string verb) + { + var registry = new StorageServerRegistry(); + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var lookup = CreateLookup(rabbit, registry); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new RecordingArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound); + var session = duplex.CreateSession(articleWorkRpc: rpc, storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"{verb} "); + + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.False(session.HasSelectedGroup); + Assert.Single(rpc.Lookups); + Assert.Null(factory.LastConnection); + } + + [Theory] + [MemberData(nameof(RetrievalVerbs))] + public async Task RegisteredStorageServer_PublishesFleetLookup_AndDoesNotCallBackFiller(string verb) + { + var registry = new StorageServerRegistry(); + var now = DateTimeOffset.Parse("2026-10-04T12:00:00Z"); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 1000, 100, 900, now, 1191), + now); + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var lookup = CreateLookup(rabbit, registry, time); + await rabbit.StartAsync(CancellationToken.None); + await lookup.StartAsync(CancellationToken.None); + try + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new RecordingArticleWorkRpcClient(); + var session = duplex.CreateSession(articleWorkRpc: rpc, storageLookup: lookup); + var dispatch = DispatchLineAsync(duplex, session, $"{verb} <12345@example.invalid>"); + var publication = await WaitForFleetPublicationAsync(factory); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, publication.Exchange); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + Assert.NotNull(request); + var consume = factory.LastConnection!.RpcChannels.First(static channel => channel.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + new StorageArticleLookupResponse( + 1, + request!.RequestId, + 1, + "cache01.usenet.ninja", + request.ArticleId, + 1191))); + + await dispatch.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Empty(rpc.Lookups); + } + finally + { + await lookup.StopAsync(CancellationToken.None); + } + } + + private static NewsgroupSnapshot SampleSnapshot() => + NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "General testing", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + ]); + + private static async Task DispatchLineAsync(ArticleDuplex duplex, NntpSession session, string command) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + } + + private static async Task AssertNextCommandStillWorksAsync(ArticleDuplex duplex, NntpSession session) + { + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + private static SilenceStorageLookup SilenceLookup() => new(); + + private sealed class SilenceStorageLookup : IStorageArticleLookupClient + { + public Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) => + Task.FromResult(StorageArticleLookupResult.NotFound( + Guid.NewGuid(), + articleId, + "Storage article lookup timed out with no positive response.")); + } + + private sealed class RecordingArticleWorkRpcClient : IArticleWorkRpcClient + { + private readonly ArticleWorkOutcome _outcome; + + public RecordingArticleWorkRpcClient(ArticleWorkOutcome outcome = ArticleWorkOutcome.Success) + { + _outcome = outcome; + } + + public List> Lookups { get; } = []; + + public Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + Lookups.Add(messageId.ToArray()); + return Task.FromResult(new ArticleWorkRpcResult( + _outcome, + Guid.NewGuid(), + "<12345@example.invalid>", + "Storage", + "backfiller01.usenet.ninja", + 119, + VectorNNTPD.Common.Articles.ArticleId.ParseLowerHex( + "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"), + Error: _outcome == ArticleWorkOutcome.Success ? null : "not found", + "cache.requests")); + } + } + + private sealed class ArticleDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new ArticleDuplex()); + + public NntpSession CreateSession( + NewsgroupSnapshot? snapshot = null, + bool authorize = true, + IArticleWorkRpcClient? articleWorkRpc = null, + IStorageArticleLookupClient? storageLookup = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + var session = new NntpSession( + connection, + NullLogger.Instance, + newsgroupCatalogue: snapshot is null ? null : new StaticNewsgroupCatalogue(snapshot), + articleWorkRpc: articleWorkRpc, + storageArticleLookup: storageLookup); + if (authorize) + { + session.SetAuthorization(Reader); + } + + return session; + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private static StorageArticleLookupService CreateLookup( + RabbitMqService rabbit, + IStorageServerRegistry registry, + TimeProvider? time = null) => + new( + rabbit, + Options.Create(CreateLookupOptions()), + NullLogger.Instance, + time, + registry); + + private static NntpdOptions CreateLookupOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + return options; + } + + private static RabbitMqService CreateRabbitMq(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static async Task WaitForFleetPublicationAsync(FakeRabbitMqConnectionFactory factory) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2); + while (DateTime.UtcNow < deadline) + { + var connection = factory.LastConnection; + if (connection is not null) + { + var publication = connection.RpcChannels + .SelectMany(static channel => channel.Publications) + .FirstOrDefault(static item => item.Exchange == CacheFleetTopology.RequestsExchangeName); + if (publication is not null) + { + return publication; + } + } + + await Task.Delay(10); + } + + throw new TimeoutException("Expected one cache.requests publication."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalVatpCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalVatpCommandTests.cs new file mode 100644 index 00000000..5d25aeed --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/ArticleRetrievalVatpCommandTests.cs @@ -0,0 +1,1848 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Message-id ARTICLE/HEAD/BODY/STAT through StorageServer lookup, then ArticleWork only on silence, then VATP. +/// +public sealed class ArticleRetrievalVatpCommandTests +{ + private static readonly NntpAuthorization Reader = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + private const string MessageId = ""; + private const string BackfillFqdn = "backfiller.test"; + private const int BackfillPort = 119; + private const string StorageFqdn = "cache01.usenet.ninja"; + private const int StoragePort = 563; + + [Theory] + [InlineData("ARTICLE")] + [InlineData("HEAD")] + [InlineData("BODY")] + [InlineData("STAT")] + public async Task MessageId_Success_ServesFromVatp(string verb) + { + var prepared = PrepareArticle(MessageId, body: "hello\r\n.dot\r\n"); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.FromSuccess(prepared.Record, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"{verb} {MessageId}"); + + byte[]? served = null; + switch (verb) + { + case "ARTICLE": + { + var multiline = await duplex.ReadMultilineAsync(); + var wire = Encoding.ASCII.GetString(multiline); + Assert.StartsWith($"220 0 {MessageId}\r\n", wire, StringComparison.Ordinal); + Assert.Contains("Message-ID: " + MessageId, wire, StringComparison.Ordinal); + Assert.Contains("hello\r\n", wire, StringComparison.Ordinal); + Assert.Contains("..dot\r\n", wire, StringComparison.Ordinal); // restuffed + Assert.EndsWith(".\r\n", wire, StringComparison.Ordinal); + served = DestuffCustomerMultiline(multiline); + break; + } + case "HEAD": + { + var multiline = await duplex.ReadMultilineAsync(); + var wire = Encoding.ASCII.GetString(multiline); + Assert.StartsWith($"221 0 {MessageId}\r\n", wire, StringComparison.Ordinal); + Assert.Contains("Message-ID: " + MessageId, wire, StringComparison.Ordinal); + Assert.DoesNotContain("hello", wire, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", wire, StringComparison.Ordinal); + served = DestuffCustomerMultiline(multiline); + break; + } + case "BODY": + { + var wire = Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()); + Assert.StartsWith($"222 0 {MessageId}\r\n", wire, StringComparison.Ordinal); + Assert.DoesNotContain("Message-ID:", wire, StringComparison.Ordinal); + Assert.Contains("hello\r\n", wire, StringComparison.Ordinal); + Assert.Contains("..dot\r\n", wire, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", wire, StringComparison.Ordinal); + break; + } + case "STAT": + Assert.Equal($"223 0 {MessageId}", await duplex.ReadClientLineAsync()); + break; + } + + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, rpc.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(BackfillFqdn, vatp.Fqdn); + Assert.Equal(BackfillPort, vatp.VatpPort); + Assert.Equal(1, queue.AdmitCalls); + var admitted = Assert.Single(queue.Admitted); + Assert.Equal(InboundArticleProducer.BackFiller, admitted.Producer); + var hopped = ArticleRecordFactory.TryCreate( + new NntpArticleParser(NntpdOptions.FormatFqdn(1, "usenet.ninja")), + prepared.Record.ArtData); + Assert.True(hopped.IsAccepted, hopped.MaterializeFailure.ToString()); + Assert.Equal(prepared.Record.ArtId, hopped.Record.ArtId); + Assert.True(admitted.Record.ArtData.Span.SequenceEqual(hopped.Record.ArtData.Span)); + var path = Encoding.ASCII.GetString(admitted.Record.Path); + Assert.StartsWith("nntpd01.usenet.ninja!", path, StringComparison.Ordinal); + Assert.Contains("news.usenet.ninja!", path, StringComparison.Ordinal); + Assert.Contains("!nntpd.test!", path, StringComparison.Ordinal); + Assert.EndsWith("!peer.example", path, StringComparison.Ordinal); + Assert.DoesNotContain("cache01", path, StringComparison.Ordinal); + if (verb == "ARTICLE") + { + Assert.NotNull(served); + Assert.True(served.AsSpan().SequenceEqual(admitted.Record.ArtData.Span)); + Assert.False(served.AsSpan().SequenceEqual(prepared.Record.ArtData.Span)); + } + else if (verb == "HEAD") + { + Assert.NotNull(served); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody( + admitted.Record.ArtData.Span, + out var admittedHeaders, + out _)); + Assert.True(served.AsSpan().SequenceEqual(admittedHeaders)); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody( + prepared.Record.ArtData.Span, + out var fetchedHeaders, + out _)); + Assert.False(served.AsSpan().SequenceEqual(fetchedHeaders)); + } + } + + [Theory] + [InlineData("ARTICLE")] + [InlineData("HEAD")] + public async Task StorageHit_TraversesOnceOnIngress(string verb) + { + var stored = PrepareArticle(MessageId, body: "hello\r\n"); + var entered = ArticleRecordFactory.TryCreate( + new NntpArticleParser(NntpdOptions.FormatFqdn(1, "usenet.ninja")), + stored.Record.ArtData, + ArticlePathMode.Traverse); + Assert.True(entered.IsAccepted, entered.MaterializeFailure.ToString()); + Assert.Equal(stored.Record.ArtId, entered.Record.ArtId); + Assert.False(entered.Record.ArtData.Span.SequenceEqual(stored.Record.ArtData.Span)); + + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var lookup = new StubStorageLookup(StorageFound(stored.Record.ArtId, Guid.NewGuid())); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, stored.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(stored.Record, Guid.NewGuid(), stored.Record.ArtId)); + var session = duplex.CreateSession( + articleWorkRpc: rpc, + vatp: vatp, + ingestion: queue, + storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"{verb} {MessageId}"); + + var served = DestuffCustomerMultiline(await duplex.ReadMultilineAsync()); + var path = Encoding.ASCII.GetString(entered.Record.Path); + Assert.StartsWith("nntpd01.usenet.ninja!", path, StringComparison.Ordinal); + Assert.Contains("news.usenet.ninja!", path, StringComparison.Ordinal); + Assert.Contains("!nntpd.test!", path, StringComparison.Ordinal); + Assert.Equal(1, CountPathToken(path, "news.usenet.ninja")); + Assert.Equal(1, CountPathToken(path, "nntpd01.usenet.ninja")); + if (verb == "ARTICLE") + { + Assert.True(served.AsSpan().SequenceEqual(entered.Record.ArtData.Span)); + Assert.False(served.AsSpan().SequenceEqual(stored.Record.ArtData.Span)); + } + else + { + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody( + entered.Record.ArtData.Span, + out var headers, + out _)); + Assert.True(served.AsSpan().SequenceEqual(headers)); + } + + Assert.Contains("Path: " + path, Encoding.ASCII.GetString(served), StringComparison.Ordinal); + Assert.Equal(0, queue.AdmitCalls); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(StorageFqdn, vatp.Fqdn); + Assert.Equal(StoragePort, vatp.VatpPort); + } + + [Fact] + public async Task Article_Head_Body_Share_SingleFetchPerCommand() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient(VatpFetchResult.FromSuccess(prepared.Record, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"HEAD {MessageId}"); + _ = await duplex.ReadMultilineAsync(); + Assert.Equal(1, vatp.FetchCount); + + await DispatchLineAsync(duplex, session, $"BODY {MessageId}"); + _ = await duplex.ReadMultilineAsync(); + Assert.Equal(2, vatp.FetchCount); + Assert.Equal(2, lookup.LookupCount); + Assert.Equal(2, rpc.LookupCount); + } + + [Fact] + public async Task ArticleWork_NotFound_Returns430_WithoutVatp() + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(ArticleId.FromMessageId(Encoding.ASCII.GetBytes(MessageId)))); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var vatp = new StubVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, rpc.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task Vatp_ConnectionFailure_Returns400() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient(VatpFetchResult.ConnectionFailure("down")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await DispatchLineAsync(duplex, session, "DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Vatp_RemoteFailure_Returns400() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient(VatpFetchResult.RemoteFailure("open", VatpErrorCode.OpenRejected, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"BODY {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task Vatp_Incomplete_NeverServed_Returns400() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient( + VatpFetchResult.IncompleteOrMalformed("truncated", VatpErrorCode.IncompleteTransfer, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task ArticleId_Mismatch_Returns430() + { + var prepared = PrepareArticle(MessageId); + var other = PrepareArticle(""); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + // VATP returns a valid record whose ArtId/MID do not match the OPEN identity. + var vatp = new StubVatpArticleClient(VatpFetchResult.FromSuccess(other.Record, prepared.RequestId, other.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task IngestQueueFull_StillServesArticle() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue { NextResult = ArticleEnqueueResult.Full }; + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient(VatpFetchResult.FromSuccess(prepared.Record, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + var wire = Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()); + Assert.StartsWith($"220 0 {MessageId}\r\n", wire, StringComparison.Ordinal); + Assert.Empty(queue.Admitted); + Assert.Equal(1, queue.AdmitCalls); + Assert.Equal(ArticleEnqueueResult.Full, queue.LastResult); + } + + [Fact] + public async Task SuccessfulAdmit_RemembersHistoryOnce() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var history = new RecordingHistoryDb(); + var queue = new RecordingIngestionQueue(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var vatp = new StubVatpArticleClient(VatpFetchResult.FromSuccess(prepared.Record, prepared.RequestId, prepared.Record.ArtId)); + var session = duplex.CreateSession( + articleWorkRpc: rpc, + vatp: vatp, + ingestion: queue, + historyDb: history, + storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"STAT {MessageId}"); + Assert.Equal($"223 0 {MessageId}", await duplex.ReadClientLineAsync()); + Assert.Equal(MessageId, Encoding.ASCII.GetString(Assert.Single(history.Remembered).Span)); + } + + [Fact] + public async Task Cancellation_DuringVatp_Propagates() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var vatp = new GatedVatpArticleClient(gate.Task); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await vatp.Entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + gate.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.True(vatp.SawCancellation); + } + + [Fact] + public async Task StorageHit_DoesNotCallArticleWork_FetchesStorageEndpoint() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + var wire = Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()); + Assert.StartsWith($"220 0 {MessageId}\r\n", wire, StringComparison.Ordinal); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(StorageFqdn, vatp.Fqdn); + Assert.Equal(StoragePort, vatp.VatpPort); + Assert.NotEqual(BackfillFqdn, vatp.Fqdn); + } + + [Theory] + [InlineData("ARTICLE")] + [InlineData("HEAD")] + [InlineData("BODY")] + [InlineData("STAT")] + public async Task StorageHit_ServesWithoutIngestion(string verb) + { + var prepared = PrepareArticle(MessageId, body: "hello\r\n"); + var requestId = Guid.Parse("11111111-2222-3333-4444-555555555555"); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId)); + var session = duplex.CreateSession( + articleWorkRpc: rpc, + vatp: vatp, + ingestion: queue, + storageLookup: lookup); + + await DispatchLineAsync(duplex, session, $"{verb} {MessageId}"); + + switch (verb) + { + case "ARTICLE": + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + break; + case "HEAD": + Assert.StartsWith($"221 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + break; + case "BODY": + Assert.StartsWith($"222 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + break; + default: + Assert.Equal($"223 0 {MessageId}", await duplex.ReadClientLineAsync()); + break; + } + + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(prepared.Record.ArtId, lookup.LastArticleId); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(StorageFqdn, vatp.Fqdn); + Assert.Equal(StoragePort, vatp.VatpPort); + Assert.Equal(requestId, vatp.RequestId); + Assert.Equal(prepared.Record.ArtId, vatp.ArticleId); + Assert.Equal(0, queue.AdmitCalls); + Assert.Empty(queue.Admitted); + } + + [Fact] + public async Task StorageSilence_ArticleWorkNotFound_Returns430_WithoutVatp() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.Equal(1, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Theory] + [InlineData("Storage article lookup was canceled.")] + [InlineData("Lookup could not be registered.")] + public async Task StorageUnavailable_Returns400_WithoutArticleWork(string error) + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageNotFound(prepared.Record.ArtId, error)); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task StorageThrows_Returns400_WithoutArticleWork() + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new ThrowingStorageLookup(); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task StorageSilence_ArticleWorkThrows_Returns400_WithoutRetryingStorage() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new ThrowingArticleWorkRpcClient(); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(1, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task Cancellation_DuringStorageLookup_PropagatesWithoutReply() + { + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lookup = new GatedStorageLookup(gate.Task); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await lookup.Entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + gate.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.True(lookup.SawCancellation); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(0, vatp.FetchCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + + [Fact] + public async Task StorageHit_OpenRejected_Returns400_WithoutRetry() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.RemoteFailure("open", VatpErrorCode.OpenRejected, requestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(requestId, vatp.RequestId); + Assert.Equal(0, queue.AdmitCalls); + } + + [Fact] + public async Task StorageHit_IdentityMismatch_Returns430_WithoutRetry() + { + var prepared = PrepareArticle(MessageId); + var other = PrepareArticle(""); + var requestId = Guid.NewGuid(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(other.Record, requestId, other.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, queue.AdmitCalls); + } + + [Fact] + public async Task StorageHit_ClientDisconnectDuringServe_DoesNotAdmit() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + using var cts = new CancellationTokenSource(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid())); + var vatp = new CancelOnReturnVatp( + VatpFetchResult.FromSuccess(prepared.Record, prepared.RequestId, prepared.Record.ArtId), + cts); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(0, queue.AdmitCalls); + Assert.Empty(queue.Admitted); + } + + [Fact] + public async Task StorageHit_IncompleteTransfer_Returns400_WithoutRetry() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.IncompleteOrMalformed( + "truncated", + VatpErrorCode.IncompleteTransfer, + requestId, + prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + } + + [Fact] + public async Task StorageHit_CancellationDuringVatp_PropagatesWithoutReply() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid())); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var vatp = new GatedVatpArticleClient(gate.Task); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await vatp.Entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + gate.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.True(vatp.SawCancellation); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + + [Fact] + public async Task TwoSessions_SameMessageId_LookupIndependently() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + await using var firstDuplex = await ArticleDuplex.CreateAsync(); + await using var secondDuplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId)); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId)); + var first = firstDuplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + var second = secondDuplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + + await DispatchLineAsync(firstDuplex, first, $"ARTICLE {MessageId}"); + await DispatchLineAsync(secondDuplex, second, $"ARTICLE {MessageId}"); + + Assert.StartsWith( + $"220 0 {MessageId}\r\n", + Encoding.ASCII.GetString(await firstDuplex.ReadMultilineAsync()), + StringComparison.Ordinal); + Assert.StartsWith( + $"220 0 {MessageId}\r\n", + Encoding.ASCII.GetString(await secondDuplex.ReadMultilineAsync()), + StringComparison.Ordinal); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(2, lookup.LookupCount); + Assert.Equal(2, vatp.FetchCount); + } + + [Fact] + public async Task StorageHit_OneCandidate_StartsFetchWithoutWaitingForAlternate() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + var alternates = new ManualAlternates(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId) with { Alternates = alternates }); + var vatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId)); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(requestId, vatp.RequestId); + Assert.Equal(0, alternates.WaitCount); + } + + [Theory] + [InlineData("ARTICLE", "220")] + [InlineData("HEAD", "221")] + [InlineData("BODY", "222")] + [InlineData("STAT", "223")] + public async Task StorageFailover_PreDataFailure_UsesSecondCandidate(string verb, string code) + { + var prepared = PrepareArticle(MessageId, body: "hello\r\n"); + var requestId = Guid.NewGuid(); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate(2, "cache02.usenet.ninja", 564)); + await using var duplex = await ArticleDuplex.CreateAsync(); + var queue = new RecordingIngestionQueue(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [ + VatpFetchResult.ConnectionFailure("down", requestId, prepared.Record.ArtId, 0), + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId), + ]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, ingestion: queue, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"{verb} {MessageId}"); + if (verb == "STAT") + { + Assert.Equal($"{code} 0 {MessageId}", await duplex.ReadClientLineAsync()); + } + else + { + Assert.StartsWith($"{code} 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + } + + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(2, vatp.FetchCount); + Assert.Equal([(StorageFqdn, StoragePort), ("cache02.usenet.ninja", 564)], vatp.Endpoints); + Assert.All(vatp.RequestIds, id => Assert.Equal(requestId, id)); + Assert.Equal(0, queue.AdmitCalls); + Assert.Empty(queue.Admitted); + } + + [Fact] + public async Task StorageFailover_OpenRejectedBeforeData_UsesSecondCandidate() + { + await AssertSecondCandidateUsedAsync( + VatpFetchResult.RemoteFailure("open", VatpErrorCode.OpenRejected, Guid.Empty, default, 0)); + } + + [Fact] + public async Task StorageFailover_ProtocolFailureBeforeData_UsesSecondCandidate() + { + await AssertSecondCandidateUsedAsync(VatpFetchResult.ProtocolFailure("local", Guid.Empty, null, 0)); + } + + [Fact] + public async Task StorageFailover_ConnectionFailureAfterData_DoesNotUseSecondCandidate() + { + await AssertNoSecondCandidateAsync( + VatpFetchResult.ConnectionFailure("reset", Guid.Empty, default, 1024), + "400 Service temporarily unavailable"); + } + + [Fact] + public async Task StorageFailover_RemoteFailureAfterData_DoesNotUseSecondCandidate() + { + await AssertNoSecondCandidateAsync( + VatpFetchResult.RemoteFailure("fail", VatpErrorCode.OpenRejected, Guid.Empty, default, 64), + "400 Service temporarily unavailable"); + } + + [Fact] + public async Task StorageFailover_IncompleteBeforeData_DoesNotUseSecondCandidate() + { + await AssertNoSecondCandidateAsync( + VatpFetchResult.IncompleteOrMalformed("meta", VatpErrorCode.InvalidMeta, Guid.Empty, default, 0), + "400 Service temporarily unavailable"); + } + + [Fact] + public async Task StorageFailover_IncompleteAfterData_DoesNotUseSecondCandidate() + { + await AssertNoSecondCandidateAsync( + VatpFetchResult.IncompleteOrMalformed("truncated", VatpErrorCode.ArtSizeMismatch, Guid.Empty, default, 32), + "400 Service temporarily unavailable"); + } + + [Fact] + public async Task StorageFailover_IdentityMismatch_Returns430_WithoutSecondCandidate() + { + var prepared = PrepareArticle(MessageId); + var other = PrepareArticle(""); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate(2, "cache02.usenet.ninja", 564)); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [VatpFetchResult.FromSuccess(other.Record, Guid.NewGuid(), other.Record.ArtId)]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, alternates.WaitCount); + } + + [Fact] + public async Task StorageFailover_EligibleWithoutAlternate_PreservesFirstFailure() + { + var prepared = PrepareArticle(MessageId); + var alternates = new ManualAlternates(); + alternates.Publish(null); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [VatpFetchResult.ConnectionFailure("down", Guid.Empty, default, 0)]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(1, alternates.WaitCount); + } + + [Fact] + public async Task StorageFailover_SecondCandidateFails_DoesNotAttemptThird() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate( + 2, + "cache02.usenet.ninja", + 564)); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [ + VatpFetchResult.ConnectionFailure("down", requestId, prepared.Record.ArtId, 0), + VatpFetchResult.RemoteFailure("open", VatpErrorCode.OpenRejected, requestId, prepared.Record.ArtId, 0), + ]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(2, vatp.FetchCount); + Assert.Equal(requestId, vatp.RequestIds[0]); + Assert.Equal(requestId, vatp.RequestIds[1]); + } + + [Fact] + public async Task StorageFailover_AlternateArrivesDuringFirstAttempt() + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + const int secondPort = 564; + var alternates = new ManualAlternates(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [ + VatpFetchResult.ConnectionFailure("down", requestId, prepared.Record.ArtId, 0), + VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId), + ], + entered, + release.Task); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(1, vatp.FetchCount); + alternates.Publish(new StorageArticleCandidate(2, "cache02.usenet.ninja", secondPort)); + release.TrySetResult(); + await dispatch; + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(2, vatp.FetchCount); + Assert.Equal(("cache02.usenet.ninja", secondPort), vatp.Endpoints[1]); + } + + [Fact] + public async Task StorageFailover_CancellationDuringFirstAttempt_DoesNotUseSecond() + { + var prepared = PrepareArticle(MessageId); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate( + 2, + "cache02.usenet.ninja", + 564)); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [VatpFetchResult.ConnectionFailure("down", Guid.Empty, default, 0)], + entered, + release.Task); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + release.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, alternates.WaitCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + + [Fact] + public async Task StorageFailover_CancellationWhileWaitingForAlternate_Propagates() + { + var prepared = PrepareArticle(MessageId); + var alternates = new ManualAlternates(); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [VatpFetchResult.ConnectionFailure("down", Guid.Empty, default, 0)]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await alternates.Entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, vatp.FetchCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + + [Fact] + public async Task StorageFailover_CancellationBeforeAlternateDecision_DoesNotFetchSecond() + { + var prepared = PrepareArticle(MessageId); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate( + 2, + "cache02.usenet.ninja", + 564)); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [ + VatpFetchResult.ConnectionFailure("down", Guid.Empty, default, 0), + VatpFetchResult.FromSuccess(prepared.Record, Guid.NewGuid(), prepared.Record.ArtId), + ], + entered, + release.Task, + returnDespiteCancellation: true); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + using var cts = new CancellationTokenSource(); + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + release.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, alternates.WaitCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + + [Fact] + public async Task StorageFailover_CancellationClosesOpenLookupWindow_DoesNotBecome400() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateStorageLookupRabbit(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateStorageLookupOptions()), + NullLogger.Instance, + time); + var window = new CancelClosedLookupWindow(service); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + window.ReleaseProductionWait = release.Task; + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var vatp = new SequenceVatpArticleClient( + [VatpFetchResult.ConnectionFailure("down", Guid.Empty, default, 0)]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: window); + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(); + try + { + var dispatch = DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}", cts.Token); + var publication = await WaitForStoragePublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static channel => channel.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + CreateStorageLookupResponse(request!, 1, "cache01.usenet.ninja"))); + + await window.Entered.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(1, service.OutstandingCorrelations); + Assert.Equal(1, vatp.FetchCount); + await cts.CancelAsync(); + await window.WindowClosed!.WaitAsync(TimeSpan.FromSeconds(2)); + release.TrySetResult(); + await Assert.ThrowsAnyAsync(async () => await dispatch); + Assert.Equal(1, vatp.FetchCount); + Assert.Null(await duplex.TryReadClientLineAsync(TimeSpan.FromMilliseconds(200))); + } + finally + { + release.TrySetResult(); + await service.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task StorageFailover_ConcurrentCommands_DoNotShareCandidates() + { + var prepared = PrepareArticle(MessageId); + var requestA = Guid.NewGuid(); + var requestB = Guid.NewGuid(); + var alternatesA = new ManualAlternates(); + var alternatesB = new ManualAlternates(); + alternatesA.Publish(new StorageArticleCandidate(2, "cache02.usenet.ninja", 564)); + alternatesB.Publish(new StorageArticleCandidate(3, "cache03.usenet.ninja", 565)); + await using var firstDuplex = await ArticleDuplex.CreateAsync(); + await using var secondDuplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new PairedStorageLookup( + StorageFound(prepared.Record.ArtId, requestA) with { Alternates = alternatesA }, + StorageFound(prepared.Record.ArtId, requestB, "cache09.usenet.ninja", 569) with { Alternates = alternatesB }); + var vatp = new SequenceVatpArticleClient( + [ + VatpFetchResult.ConnectionFailure("a", requestA, prepared.Record.ArtId, 0), + VatpFetchResult.FromSuccess(prepared.Record, requestA, prepared.Record.ArtId), + VatpFetchResult.ProtocolFailure("b", requestB, prepared.Record.ArtId, 0), + VatpFetchResult.FromSuccess(prepared.Record, requestB, prepared.Record.ArtId), + ]); + var first = firstDuplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + var second = secondDuplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(firstDuplex, first, $"ARTICLE {MessageId}"); + await DispatchLineAsync(secondDuplex, second, $"ARTICLE {MessageId}"); + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await firstDuplex.ReadMultilineAsync()), StringComparison.Ordinal); + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await secondDuplex.ReadMultilineAsync()), StringComparison.Ordinal); + Assert.Equal(2, lookup.LookupCount); + Assert.Equal( + [ + (StorageFqdn, StoragePort), + ("cache02.usenet.ninja", 564), + ("cache09.usenet.ninja", 569), + ("cache03.usenet.ninja", 565), + ], + vatp.Endpoints); + Assert.Equal([requestA, requestA, requestB, requestB], vatp.RequestIds); + Assert.Equal(0, rpc.LookupCount); + } + + [Fact] + public async Task StorageMalformedEndpoint_Returns400_WithoutArticleWork() + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid(), "not-a-cache-uri", 563)); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task StorageInvalidArticleId_Returns400_WithoutArticleWork() + { + var prepared = PrepareArticle(MessageId); + var other = PrepareArticle(""); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, prepared.Record.ArtId, BackfillFqdn, BackfillPort); + var lookup = new StubStorageLookup(StorageFound(other.Record.ArtId, Guid.NewGuid())); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(0, rpc.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Theory] + [InlineData(ArticleWorkOutcome.ArticleNotFound)] + [InlineData(ArticleWorkOutcome.InvalidArticle)] + public async Task StorageSilence_ArticleWorkTerminalMiss_Returns430_WithoutVatp(ArticleWorkOutcome outcome) + { + var prepared = PrepareArticle(MessageId); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(outcome, articleId: null); + var lookup = new StubStorageLookup(StorageSilence(prepared.Record.ArtId)); + var vatp = new RecordingVatpArticleClient(VatpFetchResult.ConnectionFailure("unused")); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal("430 No article with that message-id", await duplex.ReadClientLineAsync()); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(1, rpc.LookupCount); + Assert.Equal(0, vatp.FetchCount); + } + + [Fact] + public async Task ConcurrentCommands_DoNotShareSourceSelection() + { + var stored = PrepareArticle(MessageId); + var filled = PrepareArticle(""); + await using var storedDuplex = await ArticleDuplex.CreateAsync(); + await using var filledDuplex = await ArticleDuplex.CreateAsync(); + var storedLookup = new StubStorageLookup(StorageFound(stored.Record.ArtId, Guid.NewGuid())); + var filledLookup = new StubStorageLookup(StorageSilence(filled.Record.ArtId)); + var storedRpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, filled.Record.ArtId, BackfillFqdn, BackfillPort); + var filledRpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.Success, filled.Record.ArtId, BackfillFqdn, BackfillPort); + var storedVatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(stored.Record, Guid.NewGuid(), stored.Record.ArtId)); + var filledVatp = new RecordingVatpArticleClient( + VatpFetchResult.FromSuccess(filled.Record, Guid.NewGuid(), filled.Record.ArtId)); + var storedQueue = new RecordingIngestionQueue(); + var filledQueue = new RecordingIngestionQueue(); + var storedSession = storedDuplex.CreateSession( + articleWorkRpc: storedRpc, + vatp: storedVatp, + ingestion: storedQueue, + storageLookup: storedLookup); + var filledSession = filledDuplex.CreateSession( + articleWorkRpc: filledRpc, + vatp: filledVatp, + ingestion: filledQueue, + storageLookup: filledLookup); + + await Task.WhenAll( + DispatchLineAsync(storedDuplex, storedSession, $"ARTICLE {MessageId}"), + DispatchLineAsync(filledDuplex, filledSession, "ARTICLE ")); + + Assert.StartsWith( + $"220 0 {MessageId}\r\n", + Encoding.ASCII.GetString(await storedDuplex.ReadMultilineAsync()), + StringComparison.Ordinal); + Assert.StartsWith( + "220 0 \r\n", + Encoding.ASCII.GetString(await filledDuplex.ReadMultilineAsync()), + StringComparison.Ordinal); + Assert.Equal(0, storedRpc.LookupCount); + Assert.Equal(1, filledRpc.LookupCount); + Assert.Equal(StorageFqdn, storedVatp.Fqdn); + Assert.Equal(StoragePort, storedVatp.VatpPort); + Assert.Equal(stored.Record.ArtId, storedVatp.ArticleId); + Assert.Equal(BackfillFqdn, filledVatp.Fqdn); + Assert.Equal(BackfillPort, filledVatp.VatpPort); + Assert.Equal(filled.Record.ArtId, filledVatp.ArticleId); + Assert.Equal(1, storedLookup.LookupCount); + Assert.Equal(stored.Record.ArtId, storedLookup.LastArticleId); + Assert.Equal(1, filledLookup.LookupCount); + Assert.Equal(filled.Record.ArtId, filledLookup.LastArticleId); + Assert.Equal(0, storedQueue.AdmitCalls); + Assert.Empty(storedQueue.Admitted); + Assert.Equal(InboundArticleProducer.BackFiller, Assert.Single(filledQueue.Admitted).Producer); + Assert.Equal(filled.Record.ArtId, filledQueue.Admitted[0].Record.ArtId); + } + + private static async Task AssertSecondCandidateUsedAsync(VatpFetchResult firstFailure) + { + var prepared = PrepareArticle(MessageId); + var requestId = Guid.NewGuid(); + const int secondPort = 564; + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate(2, "cache02.usenet.ninja", secondPort)); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, requestId) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient( + [firstFailure, VatpFetchResult.FromSuccess(prepared.Record, requestId, prepared.Record.ArtId)]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.StartsWith($"220 0 {MessageId}\r\n", Encoding.ASCII.GetString(await duplex.ReadMultilineAsync()), StringComparison.Ordinal); + Assert.Equal(1, lookup.LookupCount); + Assert.Equal(2, vatp.FetchCount); + Assert.Equal(("cache02.usenet.ninja", secondPort), vatp.Endpoints[1]); + Assert.All(vatp.RequestIds, id => Assert.Equal(requestId, id)); + } + + private static async Task AssertNoSecondCandidateAsync(VatpFetchResult firstFailure, string expectedLine) + { + var prepared = PrepareArticle(MessageId); + var alternates = new ManualAlternates(); + alternates.Publish(new StorageArticleCandidate( + 2, + "cache02.usenet.ninja", + 564)); + await using var duplex = await ArticleDuplex.CreateAsync(); + var rpc = new StubArticleWorkRpcClient(ArticleWorkOutcome.ArticleNotFound, articleId: null); + var lookup = new StubStorageLookup(StorageFound(prepared.Record.ArtId, Guid.NewGuid()) with { Alternates = alternates }); + var vatp = new SequenceVatpArticleClient([firstFailure]); + var session = duplex.CreateSession(articleWorkRpc: rpc, vatp: vatp, storageLookup: lookup); + await DispatchLineAsync(duplex, session, $"ARTICLE {MessageId}"); + Assert.Equal(expectedLine, await duplex.ReadClientLineAsync()); + Assert.Equal(1, vatp.FetchCount); + Assert.Equal(0, alternates.WaitCount); + } + + private static StorageArticleLookupResult StorageFound( + ArticleId articleId, + Guid requestId, + string fqdn = StorageFqdn, + int vatpPort = StoragePort) => + new( + StorageArticleLookupOutcome.Found, + requestId, + articleId, + 1, + fqdn, + vatpPort, + Error: null); + + private static StorageArticleLookupResult StorageSilence(ArticleId articleId) => + StorageNotFound(articleId, "Storage article lookup timed out with no positive response."); + + private static StorageArticleLookupResult StorageNotFound(ArticleId articleId, string error) => + new( + StorageArticleLookupOutcome.NotFound, + Guid.NewGuid(), + articleId, + null, + null, + null, + error); + + private static int CountPathToken(string path, string token) => + path.Split('!').Count(part => part.Equals(token, StringComparison.OrdinalIgnoreCase)); + + private static byte[] DestuffCustomerMultiline(byte[] multiline) + { + var span = multiline.AsSpan(); + var firstCrlf = span.IndexOf("\r\n"u8); + Assert.True(firstCrlf >= 0); + var stuffed = span[(firstCrlf + 2)..]; + Assert.True(stuffed.EndsWith(".\r\n"u8)); + stuffed = stuffed[..^3]; + Assert.True(NntpArticleDestuffer.TryDestuffStuffedWire( + stuffed, + NntpdOptions.DefaultMaxArticleSize, + out var destuffed)); + return destuffed; + } + + private static Prepared PrepareArticle(string messageId, string body = "body\r\n") + { + var parser = new NntpArticleParser("nntpd.test"); + var created = ArticleRecordFactory.TryCreate( + parser, + Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed(messageId, body))); + Assert.True(created.IsAccepted, created.ParseFailure.ToString()); + return new Prepared(Guid.NewGuid(), created.Record); + } + + private static async Task DispatchLineAsync( + ArticleDuplex duplex, + NntpSession session, + string command, + CancellationToken cancellationToken = default) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command, cancellationToken); + } + + private readonly record struct Prepared(Guid RequestId, ArticleRecord Record); + + private sealed class StubArticleWorkRpcClient( + ArticleWorkOutcome outcome, + ArticleId? articleId, + string? fqdn = null, + int? vatpPort = null) : IArticleWorkRpcClient + { + public int LookupCount { get; private set; } + + public Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + LookupCount++; + var mid = Encoding.ASCII.GetString(messageId.Span); + return Task.FromResult(new ArticleWorkRpcResult( + outcome, + Guid.NewGuid(), + mid, + Backbone: outcome == ArticleWorkOutcome.Success ? "Storage" : null, + Fqdn: fqdn, + VatpPort: vatpPort, + ArticleId: articleId, + Error: outcome == ArticleWorkOutcome.Success ? null : "missing", + SourceExchange: outcome == ArticleWorkOutcome.Success ? "cache.requests" : null)); + } + } + + private sealed class StubVatpArticleClient(VatpFetchResult result) : IVatpArticleClient + { + public int FetchCount { get; private set; } + + public Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + FetchCount++; + return Task.FromResult(result); + } + } + + /// + /// Returns a successful fetch and then cancels the command token, so the serve write + /// observes the disconnect after the record exists. + /// + private sealed class CancelOnReturnVatp(VatpFetchResult result, CancellationTokenSource cancel) : IVatpArticleClient + { + public int FetchCount { get; private set; } + + public Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + FetchCount++; + cancel.Cancel(); + return Task.FromResult(result); + } + } + + private sealed class RecordingVatpArticleClient(VatpFetchResult result) : IVatpArticleClient + { + public int FetchCount { get; private set; } + + public string? Fqdn { get; private set; } + + public int VatpPort { get; private set; } + + public Guid RequestId { get; private set; } + + public ArticleId ArticleId { get; private set; } + + public Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + FetchCount++; + Fqdn = fqdn; + VatpPort = vatpPort; + RequestId = requestId; + ArticleId = articleId; + return Task.FromResult(result); + } + } + + private sealed class ThrowingArticleWorkRpcClient : IArticleWorkRpcClient + { + public int LookupCount { get; private set; } + + public Task LookupByMessageIdAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken) + { + LookupCount++; + throw new InvalidOperationException("article-work unavailable"); + } + } + + private sealed class StubStorageLookup(StorageArticleLookupResult result) : IStorageArticleLookupClient + { + public int LookupCount { get; private set; } + + public ArticleId LastArticleId { get; private set; } + + public Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) + { + LookupCount++; + LastArticleId = articleId; + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(result); + } + } + + private sealed class ThrowingStorageLookup : IStorageArticleLookupClient + { + public int LookupCount { get; private set; } + + public Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) + { + LookupCount++; + throw new InvalidOperationException("storage lookup unavailable"); + } + } + + private sealed class GatedStorageLookup(Task release) : IStorageArticleLookupClient + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int LookupCount { get; private set; } + + public bool SawCancellation { get; private set; } + + public async Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) + { + LookupCount++; + Entered.TrySetResult(); + try + { + await release.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + SawCancellation = true; + throw; + } + + return StorageArticleLookupResult.NotFound(Guid.NewGuid(), articleId, "unexpected"); + } + } + + private sealed class GatedVatpArticleClient(Task release) : IVatpArticleClient + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool SawCancellation { get; private set; } + + public async Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + Entered.TrySetResult(); + try + { + await release.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + SawCancellation = true; + throw; + } + + return VatpFetchResult.ConnectionFailure("unexpected"); + } + } + + private sealed class ManualAlternates : IStorageArticleAlternateSource + { + private readonly TaskCompletionSource _next = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int WaitCount { get; private set; } + + public Task WhenClosed => Task.CompletedTask; + + public void Publish(StorageArticleCandidate? candidate) => _next.TrySetResult(candidate); + + public async ValueTask WaitForAlternateAsync(CancellationToken cancellationToken) + { + WaitCount++; + Entered.TrySetResult(); + return await _next.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private sealed class SequenceVatpArticleClient : IVatpArticleClient + { + private readonly Queue _results; + private readonly TaskCompletionSource? _firstEntered; + private readonly Task? _firstRelease; + + private readonly bool _returnDespiteCancellation; + + public SequenceVatpArticleClient( + IEnumerable results, + TaskCompletionSource? firstEntered = null, + Task? firstRelease = null, + bool returnDespiteCancellation = false) + { + _results = new Queue(results); + _firstEntered = firstEntered; + _firstRelease = firstRelease; + _returnDespiteCancellation = returnDespiteCancellation; + } + + public List<(string Fqdn, int VatpPort)> Endpoints { get; } = []; + + public List RequestIds { get; } = []; + + public int FetchCount => Endpoints.Count; + + public async Task FetchArticleAsync( + string fqdn, + int vatpPort, + Guid requestId, + ArticleId articleId, + CancellationToken cancellationToken) + { + Endpoints.Add((fqdn, vatpPort)); + RequestIds.Add(requestId); + if (Endpoints.Count == 1 && _firstEntered is not null) + { + _firstEntered.TrySetResult(); + if (_firstRelease is not null) + { + if (_returnDespiteCancellation) + { + await _firstRelease.ConfigureAwait(false); + } + else + { + await _firstRelease.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + if (!_returnDespiteCancellation) + { + cancellationToken.ThrowIfCancellationRequested(); + } + + return _results.Dequeue(); + } + } + + /// + /// Production lookup whose alternate wait is held until the test has cancelled. + /// The inner wait then observes the window that cancellation already closed and + /// returns null instead of throwing, which is the MapFetch-to-400 race. + /// + private sealed class CancelClosedLookupWindow(StorageArticleLookupService service) : IStorageArticleLookupClient + { + private WindowClosedAlternate? _alternate; + + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task? ReleaseProductionWait { get; set; } + + public Task? WindowClosed => _alternate?.WhenClosed; + + public async Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) + { + var result = await service.LookupAsync(articleId, cancellationToken).ConfigureAwait(false); + if (result.Alternates is null) + { + return result; + } + + _alternate = new WindowClosedAlternate(result.Alternates, Entered, ReleaseProductionWait!); + return result with { Alternates = _alternate }; + } + + private sealed class WindowClosedAlternate( + IStorageArticleAlternateSource inner, + TaskCompletionSource entered, + Task release) : IStorageArticleAlternateSource + { + public Task WhenClosed => inner.WhenClosed; + + public async ValueTask WaitForAlternateAsync(CancellationToken cancellationToken) + { + entered.TrySetResult(); + await release.ConfigureAwait(false); + return await inner.WaitForAlternateAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + private static RabbitMqService CreateStorageLookupRabbit(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static NntpdOptions CreateStorageLookupOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + return options; + } + + private static StorageArticleLookupResponse CreateStorageLookupResponse( + StorageArticleLookupRequest request, + int serverId, + string fqdn) => + new( + 1, + request.RequestId, + serverId, + fqdn, + request.ArticleId, + 1191); + + private static async Task WaitForStoragePublicationAsync(FakeRabbitMqConnectionFactory factory) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2); + while (DateTime.UtcNow < deadline) + { + var connection = factory.LastConnection; + if (connection is not null) + { + var publication = connection.RpcChannels + .SelectMany(static channel => channel.Publications) + .FirstOrDefault(static candidate => candidate.Exchange == CacheFleetTopology.RequestsExchangeName); + if (publication is not null) + { + return publication; + } + } + + await Task.Delay(10); + } + + throw new TimeoutException("Expected a cache.requests publication."); + } + + private sealed class PairedStorageLookup(StorageArticleLookupResult first, StorageArticleLookupResult second) + : IStorageArticleLookupClient + { + private readonly Queue _results = new([first, second]); + + public int LookupCount { get; private set; } + + public Task LookupAsync(ArticleId articleId, CancellationToken cancellationToken) + { + LookupCount++; + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(_results.Dequeue()); + } + } + + private sealed class RecordingIngestionQueue : IArticleIngestionQueue + { + public List Admitted { get; } = []; + + public ArticleEnqueueResult NextResult { get; set; } = ArticleEnqueueResult.Accepted; + + public ArticleEnqueueResult LastResult { get; private set; } + + public long MemoryLimitBytes => 64 * 1024 * 1024; + + public long QueuedBytes => 0; + + public long PeakQueuedBytes => 0; + + public int MaxArticleBytes => 4 * 1024 * 1024; + + public int Count => Admitted.Count; + + public int PeakCount => Admitted.Count; + + public bool IsAccepting => true; + + public int AdmitCalls { get; private set; } + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + AdmitCalls++; + LastResult = NextResult; + if (NextResult == ArticleEnqueueResult.Accepted) + { + Admitted.Add(article); + } + + return NextResult; + } + + public bool TryEnqueue(InboundArticle article) => TryAdmit(article) == ArticleEnqueueResult.Accepted; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) => + new(TryAdmit(article)); + + public bool TryProbeCapacity() => true; + + public void Complete() + { + } + + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + ValueTask.FromResult(null); + } + + private sealed class RecordingHistoryDb : IHistoryDb + { + public List> Remembered { get; } = []; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public void Remember(ReadOnlyMemory messageId) => Remembered.Add(messageId.ToArray()); + + public bool ContainsLocal(in HistoryDigest digest) => false; + } + + private sealed class ArticleDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new ArticleDuplex()); + + public NntpSession CreateSession( + IArticleWorkRpcClient? articleWorkRpc = null, + IVatpArticleClient? vatp = null, + IArticleIngestionQueue? ingestion = null, + IHistoryDb? historyDb = null, + IStorageArticleLookupClient? storageLookup = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: ingestion, + historyDb: historyDb, + articleWorkRpc: articleWorkRpc, + vatpArticleClient: vatp, + storageArticleLookup: storageLookup); + session.SetAuthorization(Reader); + return session; + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task TryReadClientLineAsync(TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + try + { + return await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + } + catch (OperationCanceledException) + { + return null; + } + } + + public async Task ReadMultilineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var buffer = new System.Buffers.ArrayBufferWriter(); + while (true) + { + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + var ascii = Encoding.ASCII.GetBytes(line + "\r\n"); + ascii.CopyTo(buffer.GetSpan(ascii.Length)); + buffer.Advance(ascii.Length); + if (line == ".") + { + return buffer.WrittenSpan.ToArray(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs new file mode 100644 index 00000000..e2e3efa4 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs @@ -0,0 +1,65 @@ +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Session.Authentication; + +public sealed class NewsmasterNntpAuthenticationProviderTests +{ + [Fact] + public void Create_WithoutCredentials_IsDenyAll() + { + var provider = NewsmasterNntpAuthenticationProvider.Create(TestHostFactory.CreateValidOptions()); + Assert.Same(DenyAllNntpAuthenticationProvider.Instance, provider); + } + + [Fact] + public async Task Authenticate_MatchingCredentials_GrantsControlCancel() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("newsmaster", "unit-test-newsmaster-password"); + Assert.True(result.Succeeded); + Assert.NotNull(result.Authorization); + Assert.True(result.Authorization!.ControlCancelPermitted); + Assert.True(result.Authorization.PostingPermitted); + Assert.False(result.Authorization.AuthorizedTransit); + Assert.True(result.Authorization.With(isAuthenticated: true).ControlCancelPermitted); + } + + [Fact] + public async Task Authenticate_WrongPassword_FailsWithoutPrivileges() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("newsmaster", "wrong"); + Assert.False(result.Succeeded); + Assert.Null(result.Authorization); + } + + [Fact] + public async Task Authenticate_WrongUser_Fails() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("other", "unit-test-newsmaster-password"); + Assert.False(result.Succeeded); + } + + [Fact] + public void Unauthenticated_DoesNotPermitControlCancel() + { + Assert.False(NntpAuthorization.Unauthenticated.ControlCancelPermitted); + Assert.False(NntpAuthorization.TrustedTransitPeer.ControlCancelPermitted); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/BenchItCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/BenchItCommandTests.cs new file mode 100644 index 00000000..e492998e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/BenchItCommandTests.cs @@ -0,0 +1,211 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// Internal BENCHIT benchmark facility contracts (not a production protocol feature). +public sealed class BenchItCommandTests +{ + [Fact] + public void StaticPayload_IsExactTargetSize_AndWireIsReusable() + { + Assert.Equal(750 * 1024, BenchIt.TargetArticleContentBytes); + Assert.Equal(BenchIt.TargetArticleContentBytes, BenchIt.ArticleContentBytes); + Assert.True(BenchIt.WireResponseBytes > BenchIt.ArticleContentBytes); + Assert.True(BenchIt.WireResponse.Span.StartsWith("220 0 \r\n"u8)); + Assert.True(BenchIt.WireResponse.Span.EndsWith("\r\n.\r\n"u8)); + Assert.True(BenchIt.WireResponse.Span.IndexOf("..DOTSTUFF-CHECK\r\n"u8) >= 0); + } + + [Fact] + public void Registry_ResolvesBenchIt_AsPublic() + { + var parsed = NntpCommandTestParse.ParseCommand("BENCHIT"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.BenchIt, parsed.Verb); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(parsed.Verb, parsed.Qualifier)); + Assert.Equal("BENCHIT", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + } + + [Fact] + public async Task Handle_WritesPrecomputedWire_WithoutAuth() + { + await using var duplex = await BenchDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + // Consume concurrently: production pipe pause threshold would otherwise stall a 750 KiB write. + var readTask = duplex.ReadExactClientBytesAsync(BenchIt.WireResponseBytes); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "BENCHIT"); + var received = await readTask; + + Assert.Equal(BenchIt.WireResponse.ToArray(), received); + } + + [Fact] + public async Task Capabilities_DoesNotAdvertiseBenchIt() + { + await using var duplex = await BenchDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadUntilTerminatorAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + var text = Encoding.ASCII.GetString(await readTask); + Assert.DoesNotContain("BENCHIT", text, StringComparison.OrdinalIgnoreCase); + } + + private sealed class BenchDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new BenchDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadExactClientBytesAsync(int count) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var buffer = new byte[count]; + var filled = 0; + while (filled < count) + { + var result = await _serverToClient.Reader.ReadAsync(cts.Token).ConfigureAwait(false); + var readable = result.Buffer; + if (readable.Length == 0 && result.IsCompleted) + { + throw new EndOfStreamException("Pipe completed before expected BENCHIT wire bytes arrived."); + } + + var toCopy = (int)Math.Min(readable.Length, count - filled); + readable.Slice(0, toCopy).CopyTo(buffer.AsSpan(filled)); + filled += toCopy; + var consumed = readable.GetPosition(toCopy); + _serverToClient.Reader.AdvanceTo(consumed, readable.End); + } + + return buffer; + } + + public async Task ReadUntilTerminatorAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var ms = new System.IO.MemoryStream(); + while (true) + { + var result = await _serverToClient.Reader.ReadAsync(cts.Token); + foreach (var segment in result.Buffer) + { + ms.Write(segment.Span); + } + + _serverToClient.Reader.AdvanceTo(result.Buffer.End); + var bytes = ms.GetBuffer().AsSpan(0, (int)ms.Length); + if (bytes.IndexOf("\r\n.\r\n"u8) >= 0) + { + return ms.ToArray(); + } + + if (result.IsCompleted) + { + return ms.ToArray(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs new file mode 100644 index 00000000..9fd1d0f3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs @@ -0,0 +1,287 @@ +using System.Text; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class CatalogueNewsgroupPostingPolicyTests +{ + [Fact] + public void Allowed_IsAccepted() + { + var evaluation = Evaluate(["misc.test"], Status("misc.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Null(evaluation.Failure); + } + + [Fact] + public void Prohibited_IsRejected() + { + AssertRejected(["no.post"], "posting prohibited", Status("no.post", NewsgroupPostingStatus.Prohibited)); + } + + [Fact] + public void Moderated_IsRequiresModeration_ApprovedIsNotPolicyInput() + { + var policy = Policy(Status("mod.test", NewsgroupPostingStatus.Moderated)); + var evaluation = policy.Evaluate(["mod.test"]); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Null(evaluation.Failure); + Assert.Equal(["mod.test"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Closed_X_IsRejected() + { + AssertRejected(["closed.test"], "newsgroup closed", Status("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)); + } + + [Fact] + public void PeerOnly_J_IsRejected() + { + AssertRejected(["peer.only"], "local posting not accepted", Status("peer.only", NewsgroupPostingStatus.PeerOnly)); + } + + [Fact] + public void UnknownGroup_IsRejected() + { + AssertRejected(["unknown.group"], "unknown newsgroup", Status("misc.test", NewsgroupPostingStatus.Allowed)); + } + + [Fact] + public void Multiple_AllAllowed_IsAccepted() + { + var evaluation = Evaluate( + ["misc.test", "comp.test"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + } + + [Fact] + public void Multiple_OneUnknown_IsRejected() + { + AssertRejected( + ["misc.test", "unknown.group"], + "unknown newsgroup", + Status("misc.test", NewsgroupPostingStatus.Allowed)); + } + + [Fact] + public void Multiple_OneProhibited_IsRejected() + { + AssertRejected( + ["misc.test", "no.post"], + "posting prohibited", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("no.post", NewsgroupPostingStatus.Prohibited)); + } + + [Fact] + public void Multiple_OneModerated_IsRequiresModeration() + { + var evaluation = Evaluate( + ["misc.test", "mod.test"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("mod.test", NewsgroupPostingStatus.Moderated)); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Equal(["mod.test"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Multiple_TwoModerated_PreservesNewsgroupsOrder() + { + var evaluation = Evaluate( + ["mod.b", "misc.test", "mod.a"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("mod.a", NewsgroupPostingStatus.Moderated), + Status("mod.b", NewsgroupPostingStatus.Moderated)); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Equal(["mod.b", "mod.a"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Multiple_OneClosed_IsRejected() + { + AssertRejected( + ["misc.test", "closed.test"], + "newsgroup closed", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)); + } + + [Fact] + public void Multiple_OnePeerOnly_IsRejected() + { + AssertRejected( + ["misc.test", "peer.only"], + "local posting not accepted", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("peer.only", NewsgroupPostingStatus.PeerOnly)); + } + + [Fact] + public void MixedCase_MatchesSnapshotIgnoreCase() + { + var evaluation = Evaluate(["MISC.TEST"], Status("misc.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + } + + [Fact] + public void CaseVariantDuplicateTokens_AreBothLookedUpOnOneSnapshot() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var evaluation = policy.Evaluate(["misc.test", "Misc.Test"]); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Equal(1, catalogue.CurrentReadCount); + } + + [Fact] + public void Evaluate_CapturesCurrentOnce() + { + var first = Snapshot( + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + var catalogue = new PublishOnReadCatalogue(first, NewsgroupSnapshot.Empty); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var evaluation = policy.Evaluate(["misc.test", "comp.test"]); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Equal(1, catalogue.Reads); + } + + [Fact] + public void EmptyGroupList_IsRejected() + { + var evaluation = Policy(Status("misc.test", NewsgroupPostingStatus.Allowed)) + .Evaluate([]); + Assert.Equal(NewsgroupCatalogStatus.Rejected, evaluation.Status); + Assert.True(evaluation.Failure.HasValue); + var emptyFailure = evaluation.Failure.GetValueOrDefault(); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, emptyFailure.Category); + } + + [Fact] + public void TryValidate_MalformedNewsgroups_DoesNotConsultCatalogue() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("misc.test,,comp.test")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_EmptyNewsgroups_DoesNotConsultCatalogue() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_WhitespaceSeparatedGroups_AreAcceptedWhenBothAllowed() + { + var policy = Policy( + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + var parsed = Parse(Build("misc.test, comp.test")); + Assert.True(PostArticleValidator.TryValidate(parsed, Now, policy, out _)); + Assert.Equal(["misc.test", "comp.test"], parsed.Newsgroups); + } + + [Fact] + public void TryValidate_InvalidCharacters_AreRejectedBeforeLookup() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("misc test")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_UnknownGroup_IsPolicyRejected() + { + var policy = Policy(Status("misc.test", NewsgroupPostingStatus.Allowed)); + var parsed = Parse(Build("unknown.group")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.PolicyRejected, failure.Category); + Assert.Equal("unknown newsgroup", failure.Detail); + } + + private static readonly DateTimeOffset Now = new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + + private static void AssertRejected(string[] groups, string detail, params NewsgroupDefinition[] definitions) + { + var evaluation = Evaluate(groups, definitions); + Assert.Equal(NewsgroupCatalogStatus.Rejected, evaluation.Status); + Assert.True(evaluation.Failure.HasValue); + var failure = evaluation.Failure.GetValueOrDefault(); + Assert.Equal(PostingFailureCategory.PolicyRejected, failure.Category); + Assert.Equal(detail, failure.Detail); + } + + private static NewsgroupPostingEvaluation Evaluate(string[] groups, params NewsgroupDefinition[] definitions) => + Policy(definitions).Evaluate(groups); + + private static CatalogueNewsgroupPostingPolicy Policy(params NewsgroupDefinition[] definitions) => + new(new StaticNewsgroupCatalogue(Snapshot(definitions))); + + private static NewsgroupSnapshot Snapshot(params NewsgroupDefinition[] definitions) => + NewsgroupSnapshot.Create(definitions); + + private static NewsgroupDefinition Status(string name, NewsgroupPostingStatus status) => + new(name, string.Empty, 2, 1, status); + + private static ParsedPostArticle Parse(string article) + { + Assert.True( + PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), + failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build(string newsgroups) + { + return + "Date: 25 Sep 2026 12:00:00 +0000\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "Subject: test\r\n" + + "Message-ID: \r\n" + + "\r\n" + + "body\r\n"; + } + + private sealed class PublishOnReadCatalogue : INewsgroupCatalogue + { + private readonly StaticNewsgroupCatalogue _inner; + private readonly NewsgroupSnapshot _replacement; + + public PublishOnReadCatalogue(NewsgroupSnapshot first, NewsgroupSnapshot replacement) + { + _inner = new StaticNewsgroupCatalogue(first); + _replacement = replacement; + } + + public int Reads => _inner.CurrentReadCount; + + public NewsgroupSnapshot Current + { + get + { + var snapshot = _inner.Current; + _inner.Publish(_replacement); + return snapshot; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs new file mode 100644 index 00000000..af31bd48 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs @@ -0,0 +1,451 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// +/// ARTICLE/BODY wire shapes on the shared TX path from externally supplied destuffed bytes. +/// Command handlers return RFC lookup-failure codes; storage/catalog is still out of scope. +/// +public sealed class NntpArticleBodyTxIntegrationTests +{ + private static readonly string IncomingRoot = @"C:\Temp\Incoming"; + + [Fact] + public async Task Article_Success_FramingAndRestuff() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\nhello\r\n"); + await harness.Writer.WriteCustomerArticleAsync(stored, ""); + var wire = await harness.DrainAllAsync(); + Assert.Equal(BuildArticleOracle(stored, "", 0), wire); + Assert.StartsWith("220 0 \r\n", Encoding.ASCII.GetString(wire), StringComparison.Ordinal); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(wire), StringComparison.Ordinal); + } + + [Fact] + public async Task Article_WithArticleNumber_InStatusLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: x\r\n\r\nbody\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(stored, "", articleNumber: 3000234); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.StartsWith("220 3000234 \r\n", wire, StringComparison.Ordinal); + } + + [Fact] + public async Task Head_Success_HeadersOnly() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\nFrom: a@b\r\n\r\nbody-line\r\n"); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out var headers, out _)); + await harness.Writer.WriteCustomerHeadAsync(headers.ToArray(), ""); + var text = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.StartsWith("221 0 \r\n", text, StringComparison.Ordinal); + Assert.Contains("Subject: t\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("body-line", text, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task Body_Success_BodyOnly_NoHeaders() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\nFrom: a@b\r\n\r\nbody-line\r\n"); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + var wire = await harness.DrainAllAsync(); + var expected = BuildBodyOracle("body-line\r\n"u8, "", 0); + Assert.Equal(expected, wire); + var text = Encoding.ASCII.GetString(wire); + Assert.StartsWith("222 0 \r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("Subject:", text, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task Body_EmptyBody_WhenNoBlankLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: only-headers\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(ReadOnlySpan.Empty, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_EmptyBody_AfterBlankLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: t\r\n\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(ReadOnlySpan.Empty, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_DotLeadingLines_Restuffed() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\n.example\r\n..two\r\n"); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(".example\r\n..two\r\n"u8, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_FromSeparatedBody_MatchesSplitPath() + { + await using var a = await TxHarness.CreateAsync(); + await using var b = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("H: 1\r\n\r\n.body\r\n"); + await a.Writer.WriteCustomerBodyAsync(stored, "", 42); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out _, out var body)); + await b.Writer.WriteCustomerBodyFromBodyAsync(body.ToArray(), "", 42); + Assert.Equal(await a.DrainAllAsync(), await b.DrainAllAsync()); + } + + [Fact] + public async Task ArticleAndBody_PreserveMessageId() + { + await using var harness = await TxHarness.CreateAsync(); + const string mid = ""; + var stored = "Subject: t\r\n\r\nx\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(stored, mid); + await harness.Writer.WriteCustomerBodyAsync(stored, mid, 7); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.Contains("220 0 " + mid, wire, StringComparison.Ordinal); + Assert.Contains("222 7 " + mid, wire, StringComparison.Ordinal); + } + + [Theory] + [InlineData(NntpArticleTxChunkBudget.DefaultBytes)] + [InlineData(128 * 1024)] + [InlineData(NntpArticleTxChunkBudget.MaxBytes)] + public async Task Article_Large_ChunkBudgets_ByteIdentical(int chunkBytes) + { + await using var harness = await TxHarness.CreateAsync(); + var stored = BuildStoredArticle(740 * 1024, leadingDotEvery: 40); + var framing = NntpArticleTxFraming.CustomerArticle("", 0); + await harness.Writer.WriteArticleAsync(stored, framing, chunkBytes); + var actual = await harness.DrainAllAsync(); + Assert.Equal(BuildArticleOracle(stored, "", 0), actual); + Assert.True(harness.Writer.ChannelEnqueueCount < 100); + Assert.True(harness.Writer.ChannelEnqueueCount >= 2); + } + + [Fact] + public async Task Body_Large_UsesChunkedPath_NotPerLine() + { + await using var harness = await TxHarness.CreateAsync(); + var body = BuildStoredArticle(750 * 1024, leadingDotEvery: 50); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\n").Concat(body).ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + _ = await harness.DrainAllAsync(); + // ~750 KiB body → O(10) chunks, not tens of thousands of lines + Assert.InRange(harness.Writer.ChannelEnqueueCount, 2, 64); + Assert.InRange(harness.Writer.PipeFlushCount, 2, 128); + } + + [Fact] + public async Task Article_TwoMiB_IfSynthetic() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = BuildStoredArticle(2 * 1024 * 1024, leadingDotEvery: 80); + await harness.Writer.WriteCustomerArticleAsync(stored, "<2m@id>"); + Assert.Equal(BuildArticleOracle(stored, "<2m@id>", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task ConsecutiveArticles_RemainOrdered() + { + await using var harness = await TxHarness.CreateAsync(); + var a = "Subject: a\r\n\r\nA\r\n"u8.ToArray(); + var b = "Subject: b\r\n\r\n.B\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(a, "", 1); + await harness.Writer.WriteCustomerArticleAsync(b, "", 2); + var expected = Concat( + BuildArticleOracle(a, "", 1), + BuildArticleOracle(b, "", 2)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task ConsecutiveBodies_RemainOrdered() + { + await using var harness = await TxHarness.CreateAsync(); + var a = "H: a\r\n\r\na\r\n"u8.ToArray(); + var b = "H: b\r\n\r\n.b\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(a, ""); + await harness.Writer.WriteCustomerBodyAsync(b, "", 9); + var expected = Concat( + BuildBodyOracle("a\r\n"u8, "", 0), + BuildBodyOracle(".b\r\n"u8, "", 9)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task Cancellation_DuringLargeArticle() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 4 * 1024, + resumeWriterThreshold: 2 * 1024, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var stored = BuildStoredArticle(300 * 1024, 0); + using var cts = new CancellationTokenSource(); + var task = writer.WriteCustomerArticleAsync(stored, "", cancellationToken: cts.Token); + await Task.Delay(20); + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => await task); + pipe.Reader.CancelPendingRead(); + await pipe.Reader.CompleteAsync(); + await pipe.Writer.CompleteAsync(); + } + + [Fact] + public async Task Corpus_ArticleAndBody_IfPresent() + { + if (!Directory.Exists(IncomingRoot)) + { + return; + } + + var file = Directory.EnumerateFiles(IncomingRoot, "*", SearchOption.AllDirectories) + .Select(static f => new FileInfo(f)) + .Where(static f => f.Exists && f.Length is > 1024 and < 2_000_000) + .OrderBy(static f => Math.Abs(f.Length - 750 * 1024)) + .Select(static f => f.FullName) + .FirstOrDefault(); + if (file is null) + { + return; + } + + var stored = await File.ReadAllBytesAsync(file); + await using var articleHarness = await TxHarness.CreateAsync(); + await articleHarness.Writer.WriteCustomerArticleAsync(stored, ""); + Assert.Equal(BuildArticleOracle(stored, "", 0), await articleHarness.DrainAllAsync()); + + await using var bodyHarness = await TxHarness.CreateAsync(); + await bodyHarness.Writer.WriteCustomerBodyAsync(stored, ""); + ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out _, out var body); + Assert.Equal(BuildBodyOracle(body, "", 0), await bodyHarness.DrainAllAsync()); + } + + [Fact] + public void ArticleHandlers_WireMessageIdThroughWriterHelpers() + { + var src = File.ReadAllText( + Path.Join( + FindRepoRoot(), + "src", + "VectorNNTPD.ReaderService", + "Session", + "Commands", + "Article.cs")); + Assert.DoesNotContain("NntpCommandNotImplemented", src, StringComparison.Ordinal); + Assert.Contains("NoArticleWithMessageId", src, StringComparison.Ordinal); + Assert.Contains("NoArticleWithNumber", src, StringComparison.Ordinal); + Assert.Contains("NoNewsgroupSelected", src, StringComparison.Ordinal); + Assert.Contains("CurrentArticleNumberInvalid", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerArticleAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerHeadAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerBodyFromBodyAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerStatAsync", src, StringComparison.Ordinal); + Assert.Contains("IVatpArticleClient", src, StringComparison.Ordinal); + Assert.Contains("InboundArticleProducer.BackFiller", src, StringComparison.Ordinal); + } + + [Fact] + public void SplitHeadersAndBody_FindsBlankLine() + { + var stored = "A: 1\r\nB: 2\r\n\r\n.body\r\n"u8; + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out var headers, out var body)); + Assert.Equal("A: 1\r\nB: 2\r\n\r\n"u8.ToArray(), headers.ToArray()); + Assert.Equal(".body\r\n"u8.ToArray(), body.ToArray()); + } + + private static byte[] BuildArticleOracle(ReadOnlySpan stored, string mid, long number) + { + var framing = NntpArticleTxFraming.CustomerArticle(mid, number); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(stored); + return Concat(header, body); + } + + private static byte[] BuildBodyOracle(ReadOnlySpan storedBody, string mid, long number) + { + var framing = NntpArticleTxFraming.CustomerBody(mid, number); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(storedBody); + return Concat(header, body); + } + + private static byte[] Concat(params byte[][] parts) + { + var len = parts.Sum(static p => p.Length); + var result = new byte[len]; + var o = 0; + foreach (var p in parts) + { + p.CopyTo(result.AsSpan(o)); + o += p.Length; + } + + return result; + } + + private static byte[] Concat(byte[] a, byte[] b) + { + var result = new byte[a.Length + b.Length]; + a.CopyTo(result.AsSpan()); + b.CopyTo(result.AsSpan(a.Length)); + return result; + } + + private static byte[] BuildStoredArticle(int sizeHint, int leadingDotEvery) + { + using var ms = new MemoryStream(sizeHint + 256); + ms.Write("Subject: synthetic\r\n\r\n"u8); + var n = 0; + while (ms.Length < sizeHint) + { + if (leadingDotEvery > 0 && n % leadingDotEvery == 0) + { + ms.Write("."u8); + } + + ms.Write(Encoding.ASCII.GetBytes($"line-{n}-xxxxxxxx\r\n")); + n++; + } + + return ms.ToArray(); + } + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not locate repository root."); + } + + private sealed class TxHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private readonly MemoryStream _capture = new(); + private readonly CancellationTokenSource _cts = new(); + private readonly Task _drainTask; + private int _disposed; + + private TxHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + _drainTask = DrainLoopAsync(_cts.Token); + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new TxHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task DrainAllAsync() + { + Interlocked.Exchange(ref _disposed, 1); + await Writer.DisposeAsync().ConfigureAwait(false); + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + await _drainTask.ConfigureAwait(false); + lock (_capture) + { + return _capture.ToArray(); + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + await Writer.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + await _cts.CancelAsync().ConfigureAwait(false); + try + { + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + try + { + await _drainTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + _cts.Dispose(); + } + + private async Task DrainLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var result = await _pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + foreach (var segment in buffer) + { + lock (_capture) + { + _capture.Write(segment.Span); + } + } + + _pipe.Reader.AdvanceTo(buffer.End); + if (result.IsCompleted) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + await _pipe.Reader.CompleteAsync().ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleTxWriterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleTxWriterTests.cs new file mode 100644 index 00000000..bb5b9651 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpArticleTxWriterTests.cs @@ -0,0 +1,608 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; +using Xunit.Abstractions; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// Phase 1 shared article TX primitive — correctness, ownership, cancellation. +public sealed class NntpArticleTxWriterTests +{ + private readonly ITestOutputHelper _output; + + public NntpArticleTxWriterTests(ITestOutputHelper output) => _output = output; + + private static readonly string IncomingRoot = @"C:\Temp\Incoming"; + + [Fact] + public async Task WriteArticle_Empty_EmitsTerminatorOnlyAfterHeader() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, ReadOnlySpan.Empty); + + await harness.Writer.WriteArticleAsync(ReadOnlyMemory.Empty, framing); + + Assert.Equal(oracle, await harness.DrainAllAsync()); + Assert.Equal(1, harness.Writer.ChannelEnqueueCount); // header+terminator fit one 64KiB chunk + } + + [Fact] + public async Task WriteArticle_Small_CustomerFraming() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "hello\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, stored); + + await harness.Writer.WriteArticleAsync(stored, framing); + + Assert.Equal(oracle, await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_PeerTakeThisFraming() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "body\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.PeerTakeThis(""); + var oracle = BuildOracle(framing, stored); + + await harness.Writer.WriteArticleAsync(stored, framing); + + var wire = await harness.DrainAllAsync(); + Assert.Equal(oracle, wire); + Assert.StartsWith("TAKETHIS \r\n", Encoding.ASCII.GetString(wire)); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(wire)); + } + + [Fact] + public async Task WriteArticle_LeadingDot_Restuffed() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = ".example\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_MultipleLeadingDots() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "..example\r\n.\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_TerminatorPresentAndLoneDotContentDoesNotEndEarly() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = ".\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var wire = BuildOracle(framing, stored); + await harness.Writer.WriteArticleAsync(stored, framing); + var actual = await harness.DrainAllAsync(); + Assert.Equal(wire, actual); + + var header = framing.BuildHeaderBytes(); + var bodyWire = actual.AsMemory(header.Length); + var result = await NntpMultilineDataReader.ReadArticleAsync( + PipeReader.Create(new ReadOnlySequence(bodyWire)), + 1024, + CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(stored, result.Payload.ToArray()); + } + + [Fact] + public async Task WriteArticle_PreservesCrlf_AndBareLfInsideLine() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + // Bare LF is not a line separator for restuff (CRLF-based); preserved as content. + var stored = "Path: host\nnot-for-mail\r\nSubject: x\r\n\r\nbody\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Theory] + [InlineData(8)] + [InlineData(16)] + [InlineData(64)] + [InlineData(100)] + public async Task WriteArticle_ChunkBoundary_InsideBody(int chunkBytes) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes(new string('A', 250) + "\r\n" + new string('B', 250) + "\r\n"); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleForTestsAsync(stored, framing, chunkBytes); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + Assert.True(harness.Writer.ChannelEnqueueCount >= 2); + } + + [Fact] + public async Task WriteArticle_ChunkBoundary_OnCrlfAndDotLine() + { + var stored = Encoding.ASCII.GetBytes( + new string('x', 20) + "\r\n" + + ".dotted\r\n" + + new string('y', 20) + "\r\n"); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, stored); + foreach (var chunk in new[] { 7, 11, 17, 23, 32 }) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + await harness.Writer.WriteArticleForTestsAsync(stored, framing, chunk); + Assert.Equal(oracle, await harness.DrainAllAsync()); + } + } + + [Theory] + [InlineData(NntpArticleTxChunkBudget.DefaultBytes)] + [InlineData(128 * 1024)] + [InlineData(NntpArticleTxChunkBudget.MaxBytes)] + public async Task WriteArticle_ProductionChunkSizes_TypicalSizedArticle(int chunkBytes) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = BuildStoredArticleBytes(sizeHint: 740 * 1024, leadingDotEvery: 50); + var framing = NntpArticleTxFraming.CustomerArticle("<740k@test>"); + await harness.Writer.WriteArticleAsync(stored, framing, chunkBytes); + var actual = await harness.DrainAllAsync(); + Assert.Equal(BuildOracle(framing, stored), actual); + var expectedChunks = (int)Math.Ceiling(actual.Length / (double)chunkBytes); + Assert.Equal(expectedChunks, harness.Writer.ChannelEnqueueCount); + } + + [Fact] + public async Task WriteArticle_TwoMiB_Article() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = BuildStoredArticleBytes(sizeHint: 2 * 1024 * 1024, leadingDotEvery: 100); + var framing = NntpArticleTxFraming.CustomerArticle("<2m@test>"); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_MultipleConsecutive_PreserveOrder() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var a = "aaa\r\n"u8.ToArray(); + var b = ".bbb\r\n"u8.ToArray(); + var c = "ccc\r\n"u8.ToArray(); + var fa = NntpArticleTxFraming.CustomerArticle(""); + var fb = NntpArticleTxFraming.PeerTakeThis(""); + var fc = NntpArticleTxFraming.CustomerArticle(""); + + await harness.Writer.WriteArticleAsync(a, fa); + await harness.Writer.WriteArticleAsync(b, fb); + await harness.Writer.WriteArticleAsync(c, fc); + + var expected = Concat(BuildOracle(fa, a), BuildOracle(fb, b), BuildOracle(fc, c)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_RejectsChunkOutsideProductionRange() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAsync(async () => + await harness.Writer.WriteArticleAsync("a\r\n"u8.ToArray(), framing, 1024)); + await Assert.ThrowsAsync(async () => + await harness.Writer.WriteArticleAsync("a\r\n"u8.ToArray(), framing, 512 * 1024)); + } + + [Fact] + public async Task WriteArticle_Cancellation_BeforeEnqueue_Throws() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAnyAsync(async () => + await harness.Writer.WriteArticleAsync("body\r\n"u8.ToArray(), framing, cts.Token)); + } + + [Fact] + public async Task WriteArticle_Cancellation_WhileQueued_ReleasesWaiters() + { + // Capacity 1 + unread pipe → first chunk flush blocks pump; second enqueue waits; cancel producer. + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 4 * 1024, + resumeWriterThreshold: 2 * 1024, + minimumSegmentSize: 1024, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var stored = BuildStoredArticleBytes(sizeHint: 200 * 1024, leadingDotEvery: 0); + using var cts = new CancellationTokenSource(); + + var writeTask = writer.WriteArticleAsync(stored, framing, cts.Token); + // Allow first chunk to enter channel / pipe. + await Task.Delay(50); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(async () => await writeTask); + pipe.Reader.CancelPendingRead(); + await pipe.Reader.CompleteAsync(); + await pipe.Writer.CompleteAsync(); + } + + [Fact] + public async Task WriteArticle_DisposeWriter_CancelsInFlight() + { + var pipe = new Pipe(); + var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var stored = BuildStoredArticleBytes(180 * 1024, 0); + var writeTask = writer.WriteArticleAsync(stored, framing); + await Task.Delay(30); + await writer.DisposeAsync(); + await Assert.ThrowsAnyAsync(async () => await writeTask); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task WriteArticle_CompletedPipe_SurfacesFailure() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await pipe.Reader.CompleteAsync(); // send path sees completed reader → FlushAsync IsCompleted + await pipe.Writer.CompleteAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAnyAsync(async () => + await writer.WriteArticleAsync("x\r\n"u8.ToArray(), framing)); + } + + [Fact] + public async Task WriteArticle_DoesNotUseSessionMode() + { + // Primitive has no mode parameter; both framings work on the same writer instance. + await using var harness = await ArticleTxHarness.CreateAsync(); + await harness.Writer.WriteArticleAsync("r\r\n"u8.ToArray(), NntpArticleTxFraming.CustomerArticle("")); + await harness.Writer.WriteArticleAsync("s\r\n"u8.ToArray(), NntpArticleTxFraming.PeerTakeThis("")); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.Contains("220 0 ", wire, StringComparison.Ordinal); + Assert.Contains("TAKETHIS ", wire, StringComparison.Ordinal); + } + + [Fact] + public async Task WriteArticle_ChannelItemsLessThanPerLineModel() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var lineCount = 2000; + var sb = new StringBuilder(); + for (var i = 0; i < lineCount; i++) + { + sb.Append("line-").Append(i).Append("\r\n"); + } + + var stored = Encoding.ASCII.GetBytes(sb.ToString()); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + _ = await harness.DrainAllAsync(); + // Per-line model would be ~2002 Channel items; chunked must be far fewer. + Assert.True(harness.Writer.ChannelEnqueueCount < lineCount / 10); + } + + [Fact] + public async Task ProductionRestuff_MatchesRoundTripWithMultilineReader() + { + var originalWire = Encoding.ASCII.GetBytes( + "Subject: t\r\n\r\n..leading\r\n...three\r\nnormal\r\n.\r\n"); + var reader = PipeReader.Create(new ReadOnlySequence(originalWire)); + var destuffed = await NntpMultilineDataReader + .ReadArticleAsync(reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + var reconstructed = ArticleWireReconstructor.RestuffArticle(destuffed.Payload.Span); + Assert.Equal(originalWire, reconstructed); + } + + [Fact] + public async Task Corpus_Oracle_IfPresent() + { + if (!Directory.Exists(IncomingRoot)) + { + return; // environment without corpus — synthetic tests still cover Phase 1 + } + + var samples = PickCorpusSamples(IncomingRoot, maxFiles: 8); + if (samples.Count == 0) + { + return; + } + + foreach (var path in samples) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = await File.ReadAllBytesAsync(path); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + } + + [Fact] + public async Task FocusedPerf_ChunkedVsPerLine_TypicalArticle() + { + var stored = BuildStoredArticleBytes(750 * 1024, leadingDotEvery: 40); + var framing = NntpArticleTxFraming.CustomerArticle(""); + + // Chunked path + await using var chunked = await ArticleTxHarness.CreateAsync(); + PrepareForAllocationSample(); + var before = GC.GetTotalAllocatedBytes(precise: true); + var sw = System.Diagnostics.Stopwatch.StartNew(); + await chunked.Writer.WriteArticleAsync(stored, framing); + sw.Stop(); + var chunkedAlloc = GC.GetTotalAllocatedBytes(precise: true) - before; + var chunkedMs = sw.Elapsed.TotalMilliseconds; + var chunkedEnq = chunked.Writer.ChannelEnqueueCount; + var chunkedFlush = chunked.Writer.PipeFlushCount; + _ = await chunked.DrainAllAsync(); + + // Legacy per-line path + await using var lines = await ArticleTxHarness.CreateAsync(); + PrepareForAllocationSample(); + before = GC.GetTotalAllocatedBytes(precise: true); + sw.Restart(); + await lines.Writer.WriteMultilineStartAsync(220, "0 "); + + // Deterministic line emission matching restuff semantics via oracle body without header: + var body = ArticleWireReconstructor.RestuffArticle(stored); + // Strip terminator for per-line write then end: + var bodyWithoutTerm = body.AsSpan(0, body.Length - 3); // remove .\r\n + var text = Encoding.ASCII.GetString(bodyWithoutTerm); + var parts = text.Split("\r\n"); + // last split empty if body ended with CRLF before terminator + for (var i = 0; i < parts.Length; i++) + { + if (i == parts.Length - 1 && parts[i].Length == 0) + { + break; + } + + await lines.Writer.WriteMultilineDataAsync(parts[i]); + } + + await lines.Writer.WriteMultilineEndAsync(); + sw.Stop(); + var lineAlloc = GC.GetTotalAllocatedBytes(precise: true) - before; + var lineMs = sw.Elapsed.TotalMilliseconds; + var lineEnq = lines.Writer.ChannelEnqueueCount; + var lineFlush = lines.Writer.PipeFlushCount; + + Assert.True(chunkedEnq < lineEnq, $"chunked enq {chunkedEnq} should be < line enq {lineEnq}"); + Assert.True(chunkedFlush < lineFlush, $"chunked flush {chunkedFlush} should be < line flush {lineFlush}"); + // Soft throughput check: chunked should not be dramatically slower on in-memory pipe. + Assert.True(chunkedMs < lineMs * 5 + 50, $"chunked {chunkedMs:F1}ms vs lines {lineMs:F1}ms"); + + _output.WriteLine( + $"PERF chunked: {chunkedMs:F2}ms alloc={chunkedAlloc} enq={chunkedEnq} flush={chunkedFlush}; " + + $"per-line: {lineMs:F2}ms alloc={lineAlloc} enq={lineEnq} flush={lineFlush}"); + Assert.True(chunkedAlloc > 0 && lineAlloc > 0); + } + + private static byte[] BuildOracle(NntpArticleTxFraming framing, ReadOnlySpan stored) + { + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(stored); + var result = new byte[header.Length + body.Length]; + header.CopyTo(result.AsSpan()); + body.CopyTo(result.AsSpan(header.Length)); + return result; + } + + /// Forces a full GC cycle before allocation sampling in the focused perf probe. + private static void PrepareForAllocationSample() + { + // codeql[cs/call-to-gc] Perf probe resets GC before allocation sampling. + GC.Collect(); + GC.WaitForPendingFinalizers(); + // codeql[cs/call-to-gc] Second collect after finalizers for stable allocation baseline. + GC.Collect(); + } + + private static byte[] Concat(params byte[][] parts) + { + var len = parts.Sum(static p => p.Length); + var result = new byte[len]; + var o = 0; + foreach (var p in parts) + { + p.CopyTo(result.AsSpan(o)); + o += p.Length; + } + + return result; + } + + private static byte[] BuildStoredArticleBytes(int sizeHint, int leadingDotEvery) + { + using var ms = new MemoryStream(sizeHint + 1024); + var n = 0; + while (ms.Length < sizeHint) + { + if (leadingDotEvery > 0 && n % leadingDotEvery == 0) + { + ms.Write("."u8); + } + + var line = Encoding.ASCII.GetBytes($"line-{n}-payload-xxxxxxxx\r\n"); + ms.Write(line); + n++; + } + + return ms.ToArray(); + } + + private static List PickCorpusSamples(string root, int maxFiles) + { + var files = Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .Where(static f => !f.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase)) + .Take(2000) + .Select(static f => new FileInfo(f)) + .Where(static f => f.Exists && f.Length > 0) + .OrderBy(static f => f.Length) + .ToList(); + if (files.Count == 0) + { + return []; + } + + var picks = new List(); + void Add(FileInfo? f) + { + if (f is not null && picks.Count < maxFiles && !picks.Contains(f.FullName)) + { + picks.Add(f.FullName); + } + } + + Add(files.First()); + Add(files[files.Count / 2]); + Add(files.FirstOrDefault(static f => Math.Abs(f.Length - 750 * 1024) < 50 * 1024)); + Add(files[(int)(files.Count * 0.95)]); + Add(files[(int)(files.Count * 0.99)]); + Add(files.Last()); + foreach (var f in files.Where(static x => + { + try + { + var head = File.ReadAllBytes(x.FullName).AsSpan(0, Math.Min(4096, (int)x.Length)); + return head.IndexOf("\r\n."u8) >= 0; + } + // codeql[cs/catch-of-all-exceptions] Test process-boundary helper must tolerate any failure while releasing resources. + catch (Exception ex) + { + _ = ex.GetType(); // observed: unreadable sample skipped + return false; + } + }).Take(2)) + { + Add(f); + } + + return picks; + } + + private sealed class ArticleTxHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private readonly MemoryStream _capture = new(); + private readonly CancellationTokenSource _cts = new(); + private readonly Task _drainTask; + private int _disposed; + + private ArticleTxHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + _drainTask = DrainLoopAsync(_cts.Token); + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + return Task.FromResult(new ArticleTxHarness(pipe, writer)); + } + + public async Task DrainAllAsync() + { + Interlocked.Exchange(ref _disposed, 1); + await Writer.DisposeAsync().ConfigureAwait(false); + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + await _drainTask.ConfigureAwait(false); + + lock (_capture) + { + return _capture.ToArray(); + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + await Writer.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + await _cts.CancelAsync().ConfigureAwait(false); + try + { + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Test duplex teardown must complete pipes after any protocol or cancel fault. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + try + { + await _drainTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Test duplex teardown must complete pipes after any protocol or cancel fault. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + _cts.Dispose(); + } + + private async Task DrainLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var result = await _pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + foreach (var segment in buffer) + { + lock (_capture) + { + _capture.Write(segment.Span); + } + } + + _pipe.Reader.AdvanceTo(buffer.End); + if (result.IsCompleted) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + await _pipe.Reader.CompleteAsync().ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpPhaseCResponseTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpPhaseCResponseTests.cs new file mode 100644 index 00000000..ea581a71 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpPhaseCResponseTests.cs @@ -0,0 +1,449 @@ +using System.Buffers; +using System.Globalization; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Session; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// +/// Phase C: byte-native dynamic replies and one-write static multiline bodies. +/// +public sealed class NntpPhaseCResponseTests +{ + private static readonly NntpAuthorization TransitAuth = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Theory] + [InlineData("")] + [InlineData("")] + [InlineData("")] + public void Check_Compose_ExactWire_FromMessageIdBytes(string messageId) + { + var mid = Encoding.ASCII.GetBytes(messageId); + var wire = NntpResponseCompose.Concat( + NntpResponses.CheckPrefix.Span, + mid, + NntpResponses.CheckSuffix.Span); + Assert.Equal( + "238 " + messageId + " send article to be transferred\r\n", + Encoding.ASCII.GetString(wire)); + } + + [Fact] + public void Check_Compose_NearMaximumMessageId() + { + var mid = NearMaxMessageId(); + Assert.Equal(VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.MaxMessageIdLength, mid.Length); + Assert.True(VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.IsValidMessageId(mid)); + + var wire = NntpResponseCompose.Concat( + NntpResponses.CheckPrefix.Span, + mid, + NntpResponses.CheckSuffix.Span); + var expected = "238 " + Encoding.ASCII.GetString(mid) + " send article to be transferred\r\n"; + Assert.Equal(expected, Encoding.ASCII.GetString(wire)); + Assert.Equal(4 + mid.Length + NntpResponses.CheckSuffix.Length, wire.Length); + } + + [Theory] + [InlineData("")] + [InlineData("")] + public void TakeThis_Compose_Exact239And439_FromMessageIdBytes(string messageId) + { + var mid = Encoding.ASCII.GetBytes(messageId); + var ok = NntpResponseCompose.Concat( + NntpResponses.ArticleTransferredOkPrefix.Span, + mid, + NntpResponses.Crlf.Span); + var rejected = NntpResponseCompose.Concat( + NntpResponses.TransferRejectedPrefix.Span, + mid, + NntpResponses.Crlf.Span); + Assert.Equal("239 " + messageId + "\r\n", Encoding.ASCII.GetString(ok)); + Assert.Equal("439 " + messageId + "\r\n", Encoding.ASCII.GetString(rejected)); + } + + [Fact] + public void Date_FormatUtcDateLine_ExactStatusAndStamp() + { + var utc = new DateTime(2026, 9, 24, 1, 2, 3, DateTimeKind.Utc); + var wire = Date.FormatUtcDateLine(utc); + Assert.Equal("111 20260924010203\r\n", Encoding.ASCII.GetString(wire)); + Assert.Equal(20, wire.Length); + Assert.Equal( + utc.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture), + Encoding.ASCII.GetString(wire, 4, 14)); + } + + [Fact] + public void StaticSingleLine_ConnectionClosing_RemainsByteIdentical() + { + Assert.True(NntpResponses.ConnectionClosing.Span.SequenceEqual("205 Connection closing\r\n"u8)); + } + + [Fact] + public async Task Help_Dispatch_OneEnqueue_ExactCompleteWire() + { + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + var expected = ExpectedHelpWire(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Equal(Help.BodyLines.Count + 2, CountCrlf(expected)); + } + + [Fact] + public async Task Capabilities_DefaultSession_OneEnqueue_ExactWire() + { + var expected = Concat( + NntpResponses.CapabilityListFollows, + NntpResponses.CapabilityVersion2, + NntpResponses.CapabilityImplementation, + NntpResponses.CapabilityReader, + NntpResponses.CapabilityList, + NntpResponses.CapabilityModeReader, + NntpResponses.CapabilityAuthinfoUser, + NntpResponses.CapabilityStartTls, + NntpResponses.CapabilityCompressDeflate, + NntpResponses.MultilineTerminator); + + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Contains("VERSION 2", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("IHAVE", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("STREAMING", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("SPEEDTEST", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("NEWNEWS", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("NEWGROUPS", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.True(expected.AsSpan().EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task Capabilities_Authenticated_OmitsAuthinfoAndModeReader_OneEnqueue() + { + var expected = Concat( + NntpResponses.CapabilityListFollows, + NntpResponses.CapabilityVersion2, + NntpResponses.CapabilityImplementation, + NntpResponses.CapabilityReader, + NntpResponses.CapabilityList, + NntpResponses.CapabilityStartTls, + NntpResponses.CapabilityCompressDeflate, + NntpResponses.MultilineTerminator); + + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.ApplySuccessfulAuthentication( + "user", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.DoesNotContain("AUTHINFO", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("MODE-READER", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("IHAVE", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("STREAMING", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("SPEEDTEST", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + } + + [Fact] + public async Task Check_Dispatch_OnReader_Exact502Bytes_OneEnqueue() + { + const string id = ""; + var expected = "502 Command not available on this server\r\n"; + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK " + id); + Assert.Equal(expected, await read); + Assert.Equal(1, response.ChannelEnqueueCount); + } + + [Fact] + public async Task Check_Dispatch_OnReader_NearMaximumMessageId_Exact502Bytes() + { + var mid = Encoding.ASCII.GetString(NearMaxMessageId()); + var expected = "502 Command not available on this server\r\n"; + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK " + mid); + Assert.Equal(expected, await read); + } + + [Fact] + public async Task TakeThis_Dispatch_OnReader_Exact502_AndNothingQueued() + { + const string id = ""; + var expected = "502 Command not available on this server\r\n"; + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + // The Reader role gate answers before any article is read, so no body is written. + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "TAKETHIS " + id); + Assert.Equal(1, response.ChannelEnqueueCount); + await response.FlushCoalescedAsync(); + Assert.Equal(expected, await read); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TakeThis_Dispatch_OnReader_OversizedLimit_Exact502_NotRejection() + { + const string id = ""; + var expected = "502 Command not available on this server\r\n"; + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "TAKETHIS " + id); + Assert.Equal(1, response.ChannelEnqueueCount); + await response.FlushCoalescedAsync(); + Assert.Equal(expected, await read); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task Date_Dispatch_ExactStatusAndDigitStamp_OneEnqueue() + { + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(20); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "DATE"); + var wire = await read; + Assert.Equal(20, wire.Length); + Assert.StartsWith("111 ", wire, StringComparison.Ordinal); + Assert.EndsWith("\r\n", wire, StringComparison.Ordinal); + Assert.All(wire[4..18], static c => Assert.True(char.IsAsciiDigit(c))); + Assert.Equal(1, response.ChannelEnqueueCount); + } + + private static byte[] ExpectedHelpWire() + { + var builder = new StringBuilder(); + builder.Append("100 Help text follows\r\n"); + foreach (var line in Help.BodyLines) + { + builder.Append(line).Append("\r\n"); + } + + builder.Append(".\r\n"); + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + private static byte[] Concat(params ReadOnlyMemory[] parts) => + NntpResponseCompose.Concatenate(parts); + + private static byte[] NearMaxMessageId() + { + var mid = new byte[VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.MaxMessageIdLength]; + mid[0] = (byte)'<'; + for (var i = 1; i < mid.Length - 3; i++) + { + mid[i] = (byte)'a'; + } + + mid[^3] = (byte)'@'; + mid[^2] = (byte)'x'; + mid[^1] = (byte)'>'; + return mid; + } + + private static int CountCrlf(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + + private sealed class PhaseCDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new PhaseCDuplex()); + + public NntpSession CreateSession( + IArticleIngestionQueue? queue = null, + bool allowCleartextAuth = true) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + allowCleartextAuth: allowCleartextAuth); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _serverToClient.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _serverToClient.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs new file mode 100644 index 00000000..a3bc3910 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs @@ -0,0 +1,259 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// +/// Bounded TX coalescing: logical response bytes stay identical; only Pipe flush granularity changes. +/// +public sealed class NntpResponseWriterCoalescingTests +{ + [Fact] + public async Task TxChannel_AllowsSynchronousContinuations() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + Assert.True(writer.ChannelAllowsSynchronousContinuations); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task SingleWriteLine_DeliversExactBytes() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(200, "Hello"); + Assert.Equal("200 Hello\r\n", await harness.ReadExactAsciiAsync("200 Hello\r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task TwoWriteLines_ExactBytesAndOrder() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(200, "one"); + await harness.Writer.WriteLineAsync(201, "two"); + Assert.Equal("200 one\r\n201 two\r\n", await harness.ReadExactAsciiAsync("200 one\r\n201 two\r\n".Length)); + } + + [Fact] + public async Task EightEnqueueLines_OneFlush_ExactBytesAndOrder() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 1; i <= NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + var text = $""; + expected.Append("239 ").Append(text).Append("\r\n"); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, text); + } + + var wire = expected.ToString(); + Assert.Equal(wire, await harness.ReadExactAsciiAsync(wire.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task NineEnqueueLines_FirstEightFlush_NinthOrderedAfterBarrier() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 1; i <= 9; i++) + { + var text = $""; + expected.Append("239 ").Append(text).Append("\r\n"); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, text); + } + + var firstEight = expected.ToString()[..^("239 \r\n".Length)]; + Assert.Equal(firstEight, await harness.ReadExactAsciiAsync(firstEight.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.True(harness.Writer.HasCoalescedUnflushed); + + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal("239 \r\n", await harness.ReadExactAsciiAsync("239 \r\n".Length)); + Assert.Equal(2, harness.Writer.PipeFlushCount); + Assert.Equal(expected.ToString(), firstEight + "239 \r\n"); + } + + [Fact] + public async Task PartialBatch_FlushCoalesced_DeliversHeldLines() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + Assert.Equal(0, harness.Writer.PipeFlushCount); + + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal( + "239 \r\n239 \r\n239 \r\n", + await harness.ReadExactAsciiAsync("239 \r\n239 \r\n239 \r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task WriteLineBarrier_FlushesQueuedEnqueueLinesFirst() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.WriteLineAsync(382, "Continue with TLS negotiation"); + + Assert.Equal( + "239 \r\n239 \r\n382 Continue with TLS negotiation\r\n", + await harness.ReadExactAsciiAsync( + "239 \r\n239 \r\n382 Continue with TLS negotiation\r\n".Length)); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task Cancellation_PendingPartialBatch_DoesNotHang() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + await writer.EnqueueLineAsync(239, ""); + await writer.EnqueueLineAsync(239, ""); + + using var cts = new CancellationTokenSource(); + var flush = writer.FlushCoalescedAsync(cts.Token).AsTask(); + cts.Cancel(); + await writer.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + var ex = await Assert.ThrowsAnyAsync(() => flush); + Assert.True( + ex is OperationCanceledException or ObjectDisposedException, + $"Unexpected {ex.GetType().FullName}: {ex.Message}"); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task Disconnect_DisposeTerminatesPumpWithHeldBatch() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + await writer.EnqueueLineAsync(239, ""); + await writer.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task Backpressure_EnqueueWaitsWhenChannelAndPipeAreSaturated() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 2); + + for (var i = 0; i < NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + await writer.EnqueueLineAsync(239, $""); + } + + // Batch of 8 flushes into a paused pipe. Two more fill the bounded channel; the next waits. + await writer.EnqueueLineAsync(239, ""); + await writer.EnqueueLineAsync(239, ""); + var blocked = writer.EnqueueLineAsync(239, "").AsTask(); + await Assert.ThrowsAsync(() => blocked.WaitAsync(TimeSpan.FromMilliseconds(200))); + + var read = await pipe.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + pipe.Reader.AdvanceTo(read.Buffer.End); + await blocked.WaitAsync(TimeSpan.FromSeconds(5)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task NonTakethisWriteLine_IsNotHeldForBatch() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var write = harness.Writer.WriteLineAsync(111, "Server date-time").AsTask(); + await write.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal("111 Server date-time\r\n", await harness.ReadExactAsciiAsync("111 Server date-time\r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task ExactBytes_CrlfAndFramingPreserved() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(439, ""); + await harness.Writer.FlushCoalescedAsync(); + var bytes = Encoding.ASCII.GetBytes(await harness.ReadExactAsciiAsync("239 \r\n439 \r\n".Length)); + Assert.Equal("239 \r\n439 \r\n"u8.ToArray(), bytes); + } + + [Fact] + public async Task FlushCoalesced_WhenEmpty_IsNoOp() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal(0, harness.Writer.PipeFlushCount); + } + + private sealed class CoalesceHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private int _disposed; + + private CoalesceHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new CoalesceHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _pipe.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _pipe.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await Writer.DisposeAsync(); + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStaticResponseTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStaticResponseTests.cs new file mode 100644 index 00000000..1edfc335 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStaticResponseTests.cs @@ -0,0 +1,310 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Runtime.InteropServices; +using System.Text; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// +/// Phase B: immortal static NNTP replies are encoded once, CRLF-terminated, and written +/// without a per-call string interpolation / ASCII encode. +/// +public sealed class NntpStaticResponseTests +{ + [Theory] + [MemberData(nameof(SingleLineResponses))] + public void StaticResponse_IsAscii_CrlfOnce_AndImmutable(string name, ReadOnlyMemory wire, string expected) + { + _ = name; + var expectedBytes = Encoding.ASCII.GetBytes(expected); + Assert.True(wire.Span.SequenceEqual(expectedBytes), Encoding.ASCII.GetString(wire.Span)); + Assert.True(wire.Span.EndsWith("\r\n"u8)); + Assert.Equal(1, CountCrlf(wire.Span)); + + Assert.True(MemoryMarshal.TryGetArray(wire, out ArraySegment segment)); + Assert.NotNull(segment.Array); + var snapshot = wire.ToArray(); + Assert.True(wire.Span.SequenceEqual(snapshot)); + Assert.True(wire.Span.SequenceEqual(expectedBytes)); + } + + [Fact] + public async Task WriteLine_Static_DeliversExactBytes_Repeatedly() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(NntpResponses.ConnectionClosing); + await harness.Writer.WriteLineAsync(NntpResponses.ConnectionClosing); + var expected = "205 Connection closing\r\n205 Connection closing\r\n"; + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + Assert.Equal(2, harness.Writer.PipeFlushCount); + Assert.True(NntpResponses.ConnectionClosing.Span.SequenceEqual("205 Connection closing\r\n"u8)); + } + + [Fact] + public async Task EnqueueLine_Static_PreservesCoalesceBatchOfEight() + { + await using var harness = await WriterHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 0; i < NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + expected.Append("205 Connection closing\r\n"); + await harness.Writer.EnqueueLineAsync(NntpResponses.ConnectionClosing); + } + + var wire = expected.ToString(); + Assert.Equal(wire, await harness.ReadExactAsciiAsync(wire.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task StaticBarrier_FlushesQueuedEnqueueLinesFirst() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.WriteLineAsync(NntpResponses.ContinueWithTls); + Assert.Equal( + "239 \r\n382 Continue with TLS negotiation\r\n", + await harness.ReadExactAsciiAsync( + "239 \r\n382 Continue with TLS negotiation\r\n".Length)); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task DynamicCheck_StillUsesStringPath_AndIncludesMessageId() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync( + NntpReplyCodes.SendArticleToBeTransferred, + " send article to be transferred"); + Assert.Equal( + "238 send article to be transferred\r\n", + await harness.ReadExactAsciiAsync( + "238 send article to be transferred\r\n".Length)); + } + + [Fact] + public async Task DynamicTakethis_EnqueueLine_StillEncodesMessageIdPerCall() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, ""); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.TransferRejected, ""); + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal( + "239 \r\n439 \r\n", + await harness.ReadExactAsciiAsync("239 \r\n439 \r\n".Length)); + } + + [Fact] + public async Task WriteBytesAndFlush_ReusesCallerMemory_WithoutCopyingIntoNewArray() + { + await using var harness = await WriterHarness.CreateAsync(); + var payload = NntpResponses.CompressionActive; + Assert.True(MemoryMarshal.TryGetArray(payload, out var before)); + await harness.Writer.WriteBytesAndFlushAsync(payload); + Assert.True(MemoryMarshal.TryGetArray(payload, out var after)); + Assert.Same(before.Array, after.Array); + Assert.Equal("206 Compression active\r\n", await harness.ReadExactAsciiAsync("206 Compression active\r\n".Length)); + } + + [Fact] + public void HelpBody_MatchesExistingStringLines_AndNoneNeedDotStuffing() + { + Assert.Equal(Help.BodyLines.Count, NntpResponses.HelpBodyLines.Length); + for (var i = 0; i < Help.BodyLines.Count; i++) + { + Assert.False(Help.BodyLines[i].StartsWith('.')); + var expected = Encoding.ASCII.GetBytes(Help.BodyLines[i] + "\r\n"); + Assert.True(NntpResponses.HelpBodyLines[i].Span.SequenceEqual(expected)); + } + } + + [Fact] + public void HelpComplete_IsExactConcatenationOfStatusBodyAndTerminator() + { + var expected = ExpectedHelpWire(); + Assert.True(NntpResponses.HelpComplete.Span.SequenceEqual(expected)); + Assert.True(NntpResponses.HelpComplete.Span.EndsWith(".\r\n"u8)); + Assert.Equal(Help.BodyLines.Count + 2, CountCrlf(NntpResponses.HelpComplete.Span)); + Assert.False(NntpResponses.HelpComplete.Span.EndsWith("\r\n.\r\n.\r\n"u8)); + } + + [Fact] + public async Task HelpComplete_WriteOnce_OneChannelItem_ExactBytes() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(NntpResponses.HelpComplete); + var expected = Encoding.ASCII.GetString(ExpectedHelpWire()); + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + Assert.Equal(1, harness.Writer.ChannelEnqueueCount); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public void StaticResponseTable_Access_DoesNotAllocate() + { + _ = NntpResponses.ConnectionClosing; + _ = NntpResponses.HelpBodyLines; + var sink = 0; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 256; i++) + { + sink += NntpResponses.ConnectionClosing.Length; + sink += NntpResponses.GreetingPostingProhibited.Length; + sink += NntpResponses.HelpTextFollows.Length; + sink += NntpResponses.HelpBodyLines.Length; + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + Assert.True(sink > 0); + } + + [Fact] + public void ResponseEncoding_StaticTableIsZeroAlloc_StringPathAllocates() + { + const int iterations = 256; + var sink = 0; + + for (var i = 0; i < 8; i++) + { + sink += Encoding.ASCII.GetBytes($"{205} Connection closing\r\n").Length; + sink += NntpResponses.ConnectionClosing.Length; + } + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < iterations; i++) + { + sink += Encoding.ASCII.GetBytes($"{205} Connection closing\r\n").Length; + } + + var stringEncodeAlloc = GC.GetAllocatedBytesForCurrentThread() - before; + + before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < iterations; i++) + { + sink += NntpResponses.ConnectionClosing.Length; + } + + var staticEncodeAlloc = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.Equal(0, staticEncodeAlloc); + Assert.True( + stringEncodeAlloc >= iterations * NntpResponses.ConnectionClosing.Length, + $"string encode allocated {stringEncodeAlloc} B over {iterations} lines."); + Assert.NotEqual(int.MinValue, sink); + } + + public static TheoryData, string> SingleLineResponses() => + new() + { + { "greeting-200", NntpResponses.GreetingPostingPermitted, "200 VectorNNTPD.ReaderService ready, posting permitted\r\n" }, + { "greeting-201", NntpResponses.GreetingPostingProhibited, "201 VectorNNTPD.ReaderService ready, posting prohibited\r\n" }, + { "quit", NntpResponses.ConnectionClosing, "205 Connection closing\r\n" }, + { "compress-206", NntpResponses.CompressionActive, "206 Compression active\r\n" }, + { "auth-281", NntpResponses.AuthenticationAccepted, "281 Authentication accepted\r\n" }, + { "auth-381", NntpResponses.PasswordRequired, "381 Password required\r\n" }, + { "starttls-382", NntpResponses.ContinueWithTls, "382 Continue with TLS negotiation\r\n" }, + { "240", NntpResponses.ArticleReceivedOk, "240 Article received OK\r\n" }, + { "340", NntpResponses.PostSendArticle, "340 Input article; end with .\r\n" }, + { "400", NntpResponses.ServiceTemporarilyUnavailable, "400 Service temporarily unavailable\r\n" }, + { "440", NntpResponses.PostingProhibited, "440 Posting not permitted\r\n" }, + { "441", NntpResponses.PostingFailed, "441 Posting failed\r\n" }, + { "480", NntpResponses.AuthenticationRequired, "480 Authentication required\r\n" }, + { "500-unknown", NntpResponses.UnknownCommand, "500 Unknown command\r\n" }, + { "500-stub", NntpResponses.CommandNotImplemented, "500 Command not implemented\r\n" }, + { "501", NntpResponses.SyntaxError, "501 Syntax error\r\n" }, + { "503-list-data", NntpResponses.ListDataItemNotStored, "503 Data item not stored\r\n" }, + { "215-overview-fmt", NntpResponses.ListOverviewFmtFollows, "215 Order of fields in overview database.\r\n" }, + { "215-headers", NntpResponses.ListHeadersFollows, "215 headers and metadata items supported:\r\n" }, + { "412", NntpResponses.NoNewsgroupSelected, "412 No newsgroup selected\r\n" }, + { "420", NntpResponses.CurrentArticleNumberInvalid, "420 Current article number is invalid\r\n" }, + { "423", NntpResponses.NoArticleWithNumber, "423 No article with that number\r\n" }, + { "430", NntpResponses.NoArticleWithMessageId, "430 No article with that message-id\r\n" }, + { "help-100", NntpResponses.HelpTextFollows, "100 Help text follows\r\n" }, + { "caps-101", NntpResponses.CapabilityListFollows, "101 Capability list:\r\n" }, + { "multiline-end", NntpResponses.MultilineTerminator, ".\r\n" }, + }; + + private static byte[] ExpectedHelpWire() + { + var builder = new StringBuilder(); + builder.Append("100 Help text follows\r\n"); + foreach (var line in Help.BodyLines) + { + builder.Append(line).Append("\r\n"); + } + + builder.Append(".\r\n"); + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + private static int CountCrlf(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + + private sealed class WriterHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private int _disposed; + + private WriterHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new WriterHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _pipe.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _pipe.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await Writer.DisposeAsync(); + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs new file mode 100644 index 00000000..2ab7472e --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs @@ -0,0 +1,839 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Tests.Session.Commands; + +/// +/// Phase 3: bounded STREAM outstanding-depth scheduler (no storage / no catalog). +/// +public sealed class NntpStreamArticleTxSchedulerTests +{ + [Fact] + public void DefaultDepth_IsEight() + { + using var scheduler = new SyncDispose(new NntpStreamArticleTxScheduler()); + Assert.Equal(8, scheduler.Inner.Depth); + Assert.Equal(8, NntpStreamArticleTxScheduler.DefaultDepth); + } + + [Theory] + [InlineData(4)] + [InlineData(8)] + [InlineData(16)] + public void ValidDepths_Accepted(int depth) + { + using var scheduler = new SyncDispose(new NntpStreamArticleTxScheduler(depth)); + Assert.Equal(depth, scheduler.Inner.Depth); + Assert.True(NntpStreamArticleTxScheduler.IsValidDepth(depth)); + } + + [Theory] + [InlineData(3)] + [InlineData(0)] + [InlineData(-1)] + [InlineData(17)] + [InlineData(64)] + public void InvalidDepths_Rejected(int depth) + { + Assert.False(NntpStreamArticleTxScheduler.IsValidDepth(depth)); + Assert.Throws(() => new NntpStreamArticleTxScheduler(depth)); + Assert.Throws(() => + NntpStreamArticleTxScheduler.ThrowIfNotValidDepth(depth)); + } + + [Fact] + public async Task AtMostN_ConcurrentlyAdmitted() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var started = new TaskCompletionSource[depth + 1]; + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + for (var i = 0; i < started.Length; i++) + { + started[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var tasks = new Task[depth + 1]; + for (var i = 0; i < tasks.Length; i++) + { + var index = i; + tasks[i] = scheduler.ExecuteAsync( + async ct => + { + started[index].TrySetResult(); + await release.Task.WaitAsync(ct).ConfigureAwait(false); + }, + CancellationToken.None) + .AsTask(); + } + + var firstWave = started.Take(depth).Select(s => s.Task); + await Task.WhenAll(firstWave).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(depth, scheduler.Outstanding); + Assert.False(started[depth].Task.IsCompleted); + + release.TrySetResult(); + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(5)); + await started[depth].Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CompletedOperation_ReleasesSlot_ForWaiter() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var blockers = new TaskCompletionSource[depth]; + for (var i = 0; i < depth; i++) + { + blockers[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var holders = new Task[depth]; + for (var i = 0; i < depth; i++) + { + var index = i; + holders[i] = scheduler.ExecuteAsync( + async ct => await blockers[index].Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask(); + } + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + var admitted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waiter = scheduler.ExecuteAsync( + _ => + { + admitted.TrySetResult(); + return ValueTask.CompletedTask; + }, + CancellationToken.None) + .AsTask(); + + Assert.False(admitted.Task.IsCompleted); + blockers[0].TrySetResult(); + await admitted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + for (var i = 1; i < depth; i++) + { + blockers[i].TrySetResult(); + } + + await Task.WhenAll(holders.Append(waiter)).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWhileWaiting_DoesNotLeakSlot() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holders = Enumerable.Range(0, depth) + .Select(_ => scheduler.ExecuteAsync( + async ct => await hold.Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask()) + .ToArray(); + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + using var cts = new CancellationTokenSource(); + var waiting = scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, cts.Token).AsTask(); + await WaitUntilAsync(() => scheduler.Outstanding == depth && !waiting.IsCompleted, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => waiting); + Assert.Equal(depth, scheduler.Outstanding); + + hold.TrySetResult(); + await Task.WhenAll(holders).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + + await scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, CancellationToken.None); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelAfterAcquisition_ReleasesSlot() + { + await using var scheduler = new NntpStreamArticleTxScheduler(4); + using var cts = new CancellationTokenSource(); + var acquired = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var task = scheduler.ExecuteAsync( + async ct => + { + acquired.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + }, + cts.Token) + .AsTask(); + + await acquired.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => task); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task ExceptionDuringOperation_ReleasesSlot() + { + await using var scheduler = new NntpStreamArticleTxScheduler(4); + await Assert.ThrowsAsync(() => + scheduler.ExecuteAsync( + _ => throw new InvalidOperationException("boom"), + CancellationToken.None) + .AsTask()); + Assert.Equal(0, scheduler.Outstanding); + + await scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, CancellationToken.None); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task RepeatedAcquireRelease_NeverExceedsBound() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var peak = 0; + var tasks = Enumerable.Range(0, 40) + .Select(_ => scheduler.ExecuteAsync( + _ => + { + var o = scheduler.Outstanding; + while (true) + { + var observed = Volatile.Read(ref peak); + if (o <= observed || Interlocked.CompareExchange(ref peak, o, observed) == observed) + { + break; + } + } + + Assert.True(o <= depth); + return ValueTask.CompletedTask; + }, + CancellationToken.None) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.True(peak <= depth); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWaiting_UnblocksWithoutAcquire() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holders = Enumerable.Range(0, depth) + .Select(_ => scheduler.ExecuteAsync( + async ct => await hold.Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask()) + .ToArray(); + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + using var cts = new CancellationTokenSource(); + var waiting = scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, cts.Token).AsTask(); + await WaitUntilAsync(() => !waiting.IsCompleted, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waiting); + + hold.TrySetResult(); + await Task.WhenAll(holders).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task OverlappingWriteArticle_PreservesFifoWireOrder() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var bodies = new[] + { + "AAA\r\n"u8.ToArray(), + "BBB\r\n"u8.ToArray(), + "CCC\r\n"u8.ToArray(), + "DDD\r\n"u8.ToArray(), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + var framing = NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"); + tasks[index] = scheduler + .WriteArticleAsync(writer, bodies[index], framing, CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + + var buffer = new ArrayBufferWriter(); + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + buffer.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + var wire = Encoding.ASCII.GetString(buffer.WrittenSpan); + var expected = new StringBuilder(); + for (var i = 0; i < bodies.Length; i++) + { + var framing = NntpArticleTxFraming.PeerTakeThis($"<{i}@id>"); + expected.Append(Encoding.ASCII.GetString(framing.BuildHeaderBytes())); + expected.Append(Encoding.ASCII.GetString(ArticleWireReconstructor.RestuffArticle(bodies[i]))); + } + + Assert.Equal(expected.ToString(), wire); + } + + [Fact] + public async Task WriteArticle_FramingRemainsByteCorrect() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var stored = ".dot\r\nbody\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle("", 42); + await scheduler.WriteArticleAsync(writer, stored, framing); + + await writer.DisposeAsync(); + pipe.Writer.Complete(); + + var read = await pipe.Reader.ReadAsync(); + var wire = read.Buffer.ToArray(); + pipe.Reader.AdvanceTo(read.Buffer.End); + + var expected = framing.BuildHeaderBytes() + .Concat(ArticleWireReconstructor.RestuffArticle(stored)) + .ToArray(); + Assert.Equal(expected, wire); + } + + [Fact] + public async Task CancelDuringBackpressuredWrite_ReleasesSlot() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + using var cts = new CancellationTokenSource(); + var large = Encoding.ASCII.GetBytes(new string('x', 256 * 1024) + "\r\n"); + + var write = scheduler + .WriteArticleAsync( + writer, + large, + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await WaitUntilAsync( + () => scheduler.Outstanding > 0 || write.IsCompleted, + TimeSpan.FromSeconds(5)); + + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task ThreeConcurrentArticles_NeverInterleaveChunks() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var bodies = new[] + { + Encoding.ASCII.GetBytes("ONE\r\n" + new string('1', 3000) + "\r\n"), + Encoding.ASCII.GetBytes("TWO\r\n" + new string('2', 3000) + "\r\n"), + Encoding.ASCII.GetBytes("THREE\r\n" + new string('3', 3000) + "\r\n"), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleForTestsAsync( + writer, + bodies[index], + NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"), + chunkBytes: 1024, + CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + var wire = await DrainPipeAsync(pipe, writer); + Assert.Equal(BuildOracle(bodies), wire); + } + + [Fact] + public async Task Preparation_CanOverlap_UnderDepth() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var afterPrepare = new TaskCompletionSource[4]; + var releasePrepare = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + for (var i = 0; i < 4; i++) + { + afterPrepare[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var seen = 0; + scheduler.TestAfterPrepare = async ct => + { + var n = Interlocked.Increment(ref seen); + afterPrepare[n - 1].TrySetResult(); + await releasePrepare.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + var bodies = Enumerable.Range(0, 4) + .Select(i => Encoding.ASCII.GetBytes($"P{i}\r\n")) + .ToArray(); + var tasks = new Task[4]; + for (var i = 0; i < 4; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleAsync(writer, bodies[index], NntpArticleTxFraming.PeerTakeThis($"")) + .AsTask(); + } + + await Task.WhenAll(afterPrepare.Select(t => t.Task)).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(scheduler.PeakPreparing >= 2); + Assert.Equal(4, scheduler.Outstanding); + + releasePrepare.TrySetResult(); + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task Enqueue_RemainsArticleAtomic_UnderSlowConsumer() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 2); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var capture = new ArrayBufferWriter(); + using var drainCts = new CancellationTokenSource(); + var drain = DrainSlowCapturingAsync(pipe, capture, drainCts.Token); + + var bodies = new[] + { + Encoding.ASCII.GetBytes("AAAA\r\n" + new string('a', 20_000) + "\r\n"), + Encoding.ASCII.GetBytes("BBBB\r\n" + new string('b', 20_000) + "\r\n"), + Encoding.ASCII.GetBytes("CCCC\r\n" + new string('c', 20_000) + "\r\n"), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleForTestsAsync( + writer, + bodies[index], + NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"), + chunkBytes: 4096, + CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(30)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + await drainCts.CancelAsync(); + try + { + await drain.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + // Finish draining any remaining bytes after cancel. + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + capture.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + Assert.Equal(0, scheduler.Enqueuing); + Assert.Equal(0, scheduler.Outstanding); + Assert.Equal(BuildOracle(bodies), Encoding.ASCII.GetString(capture.WrittenSpan)); + } + + [Fact] + public async Task SlowConsumer_DoesNotLeakDepthPermits() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var drain = DrainInBackgroundAsync(pipe); + + var body = Encoding.ASCII.GetBytes(new string('z', 80_000) + "\r\n"); + var tasks = Enumerable.Range(0, 8) + .Select(i => scheduler + .WriteArticleForTestsAsync( + writer, + body, + NntpArticleTxFraming.PeerTakeThis($""), + chunkBytes: 8192) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(60)); + Assert.Equal(0, scheduler.Outstanding); + + await writer.DisposeAsync(); + pipe.Writer.Complete(); + await drain; + } + + [Fact] + public async Task CancelDuringPreparation_ReleasesDepthPermit() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var prepared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestAfterPrepare = async ct => + { + prepared.TrySetResult(); + await hold.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + using var cts = new CancellationTokenSource(); + var write = scheduler + .WriteArticleAsync( + writer, + "body\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await prepared.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + hold.TrySetCanceled(); + } + + [Fact] + public async Task CancelWhileWaitingForEnqueueGate_ReleasesDepthPermit() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestWhileHoldingEnqueueGate = async ct => + { + holding.TrySetResult(); + await releaseGate.Task.WaitAsync(CancellationToken.None).ConfigureAwait(false); + }; + + var holder = scheduler + .WriteArticleAsync( + writer, + "hold\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis("")) + .AsTask(); + + await holding.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Enqueuing); + + using var cts = new CancellationTokenSource(); + var waiter = scheduler + .WriteArticleAsync( + writer, + "wait\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await WaitUntilAsync(() => scheduler.Outstanding == 2, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waiter); + Assert.Equal(1, scheduler.Outstanding); + + releaseGate.TrySetResult(); + await holder.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWhileAwaitingTxFlush_ReleasesDepthPermit() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var enteredFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holdFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestBeforeAwaitFlush = async ct => + { + enteredFlush.TrySetResult(); + await holdFlush.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + using var cts = new CancellationTokenSource(); + var large = Encoding.ASCII.GetBytes(new string('q', 128 * 1024) + "\r\n"); + var write = scheduler + .WriteArticleForTestsAsync( + writer, + large, + NntpArticleTxFraming.PeerTakeThis(""), + chunkBytes: 4096, + cts.Token) + .AsTask(); + + // Drain so enqueue can complete and reach the flush-await hook. + var drain = DrainInBackgroundAsync(pipe); + await enteredFlush.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + holdFlush.TrySetCanceled(); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + try + { + await drain; + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + [Fact] + public async Task ExceptionDuringEnqueue_ReleasesDepthAndGate() + { + var pipe = new Pipe(); + var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + await writer.DisposeAsync(); + + await Assert.ThrowsAnyAsync(() => + scheduler + .WriteArticleAsync( + writer, + "x\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis("")) + .AsTask()); + + Assert.Equal(0, scheduler.Outstanding); + Assert.Equal(0, scheduler.Enqueuing); + pipe.Writer.Complete(); + } + + [Fact] + public async Task EnqueueGate_NeverAllowsConcurrentEnqueuers() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var peakEnq = 0; + scheduler.TestWhileHoldingEnqueueGate = _ => + { + var e = scheduler.Enqueuing; + while (true) + { + var observed = Volatile.Read(ref peakEnq); + if (e <= observed || Interlocked.CompareExchange(ref peakEnq, e, observed) == observed) + { + break; + } + } + + Assert.Equal(1, e); + return ValueTask.CompletedTask; + }; + + var tasks = Enumerable.Range(0, 8) + .Select(i => scheduler + .WriteArticleAsync( + writer, + Encoding.ASCII.GetBytes($"E{i}\r\n"), + NntpArticleTxFraming.PeerTakeThis($"")) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(1, peakEnq); + } + + private static string BuildOracle(IReadOnlyList bodies) + { + var expected = new StringBuilder(); + for (var i = 0; i < bodies.Count; i++) + { + var framing = NntpArticleTxFraming.PeerTakeThis($"<{i}@id>"); + expected.Append(Encoding.ASCII.GetString(framing.BuildHeaderBytes())); + expected.Append(Encoding.ASCII.GetString(ArticleWireReconstructor.RestuffArticle(bodies[i]))); + } + + return expected.ToString(); + } + + private static async Task DrainPipeAsync(Pipe pipe, NntpResponseWriter writer) + { + await writer.DisposeAsync(); + pipe.Writer.Complete(); + var buffer = new ArrayBufferWriter(); + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + buffer.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + return Encoding.ASCII.GetString(buffer.WrittenSpan); + } + + private static async Task DrainSlowCapturingAsync( + Pipe pipe, + ArrayBufferWriter capture, + CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + ReadResult read; + try + { + read = await pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (!read.Buffer.IsEmpty) + { + var take = Math.Min(4096, (int)read.Buffer.Length); + var examined = read.Buffer.Slice(0, take); + foreach (var segment in examined) + { + capture.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(examined.End, read.Buffer.End); + } + else + { + pipe.Reader.AdvanceTo(read.Buffer.End); + } + + if (read.IsCompleted) + { + break; + } + + await Task.Yield(); + } + } + + private static async Task DrainInBackgroundAsync(Pipe pipe) + { + try + { + while (true) + { + var read = await pipe.Reader.ReadAsync().ConfigureAwait(false); + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Reader completed. + } + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var start = Environment.TickCount64; + while (!condition()) + { + if (Environment.TickCount64 - start > timeout.TotalMilliseconds) + { + throw new TimeoutException("Condition not met within timeout."); + } + + await Task.Yield(); + } + } + + /// Sync dispose wrapper for constructor-only tests. + private sealed class SyncDispose : IDisposable + { + public SyncDispose(NntpStreamArticleTxScheduler inner) => Inner = inner; + + public NntpStreamArticleTxScheduler Inner { get; } + + public void Dispose() => Inner.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ArticleWireReconstructorTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ArticleWireReconstructorTests.cs new file mode 100644 index 00000000..b04a2619 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ArticleWireReconstructorTests.cs @@ -0,0 +1,146 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +public sealed class ArticleWireReconstructorTests +{ + [Fact] + public void Restuff_NormalLine_Unchanged() + { + var stored = "hello\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("hello\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_LeadingDot_GetsStuffed() + { + var stored = ".example\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("..example\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_MultipleLeadingDots() + { + var stored = "..example\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("...example\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public async Task Restuff_LoneDotContentLine_DoesNotTerminateEarly() + { + var stored = ".\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("..\r\n.\r\n"u8.ToArray(), wire); + + var reader = PipeReader.Create(new ReadOnlySequence(wire)); + var result = await NntpMultilineDataReader + .ReadArticleAsync(reader, 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(".\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public void Restuff_EmptyArticle() + { + var wire = ArticleWireReconstructor.RestuffArticle(ReadOnlySpan.Empty); + Assert.Equal(".\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_WithoutTerminator_MatchesIngestQueueContract() + { + var stored = "Subject: t\r\n\r\n.hidden\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored, includeTerminator: false); + Assert.Equal("Subject: t\r\n\r\n..hidden\r\n"u8.ToArray(), wire); + Assert.False(wire.AsSpan().EndsWith("\r\n.\r\n"u8)); + } + + [Fact] + public async Task RoundTrip_DestuffThenRestuff_MatchesOriginalWire() + { + var originalWire = Encoding.ASCII.GetBytes( + "Subject: t\r\n\r\n..leading\r\n...three\r\nnormal\r\n.\r\n"); + var reader = PipeReader.Create(new ReadOnlySequence(originalWire)); + var destuffed = await NntpMultilineDataReader + .ReadArticleAsync(reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + var reconstructed = ArticleWireReconstructor.RestuffArticle(destuffed.Payload.Span); + Assert.Equal(originalWire, reconstructed); + } + + [Fact] + public void BuildTakeThisTransaction_PrefixesCommand() + { + var stored = "body\r\n"u8.ToArray(); + var txn = FramingWireFactory.BuildTakeThisTransaction("", stored); + Assert.StartsWith("TAKETHIS \r\n", Encoding.ASCII.GetString(txn)); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(txn)); + } + + [Fact] + public async Task ContinuousParser_DoesNotTreatCommandsInsideArticleAsControl() + { + var stored = Encoding.ASCII.GetBytes("QUIT\r\nCHECK x\r\nTAKETHIS y\r\n"); + var txn = FramingWireFactory.BuildTakeThisTransaction("", stored); + var next = FramingWireFactory.BuildTakeThisTransaction("", "x\r\n"u8); + var stream = txn.Concat(next).ToArray(); + var result = await ProductionTakethisStream.ParseAsync(stream); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal(0, result.Quits); + Assert.Equal(0, result.Checks); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } + + [Fact] + public void Restuff_IsDeterministic_AndMatchesByteEstimate() + { + var stored = Encoding.ASCII.GetBytes(".a\r\nbb\r\n..c\r\n"); + var first = ArticleWireReconstructor.RestuffArticle(stored); + var second = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal(first, second); + Assert.Equal(first.Length, ArticleWireReconstructor.EstimateRestuffedWireBytes(stored)); + } +} + +public sealed class MessageIdDigestTests +{ + [Fact] + public void Blake3_WithAngleBrackets_MatchesKnownCorpusFilename() + { + const string mid = "<3810724$5fcf702$2a675b8@82ecaa5ff0.e67ff>"; + const string expected = "00024a9fbd6824b3529c34dabdb6af28505b5be684936bdca64279279736e7ad"; + Assert.Equal(expected, MessageIdDigest.ComputeHexLower(mid)); + Assert.Equal( + Path.Join("00", "02", expected), + MessageIdDigest.BuildRelativePath(expected)); + } + + [Fact] + public void WithoutAngleBrackets_DoesNotMatch() + { + const string mid = "<3810724$5fcf702$2a675b8@82ecaa5ff0.e67ff>"; + const string expected = "00024a9fbd6824b3529c34dabdb6af28505b5be684936bdca64279279736e7ad"; + Assert.NotEqual(expected, MessageIdDigest.ComputeHexLower(mid.Trim('<', '>'))); + } + + [Fact] + public void ExtractMessageId_AcceptsBareLfAfterPath() + { + var stored = Encoding.ASCII.GetBytes( + "Path: host!not-for-mail\n" + + "Message-ID: \r\n" + + "Subject: x\r\n" + + "\r\n" + + "body\r\n"); + Assert.True(StoredArticleReader.TryExtractMessageId(stored, out var mid)); + Assert.Equal("", mid); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/BulkMultilineFramerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/BulkMultilineFramerTests.cs new file mode 100644 index 00000000..454f2949 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/BulkMultilineFramerTests.cs @@ -0,0 +1,167 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// +/// Correctness tests for production destuff/terminator rules. +/// +public sealed class BulkMultilineFramerTests +{ + [Fact] + public async Task EmptyArticle() + { + var (status, payload) = await ReadRecording(FramingWireFactory.EmptyArticle, segmentSize: 1); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal(0, payload.Length); + } + + [Fact] + public async Task FiveByteDelimiterIsDetected() + { + var (status, payload) = await ReadRecording(FramingWireFactory.DataThenDelimiter(), segmentSize: 64); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal("data\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task HelloDotIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("Hello.\r\nSomething else.\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 8); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal("Hello.\r\nSomething else.\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task DotStuffedLineIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("..foo\r\nbar\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 3); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal(".foo\r\nbar\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task TerminatorAtEndOfSegment() + { + var wire = FramingWireFactory.WithTerminator("abc\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 5); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("abc\r\n", Encoding.ASCII.GetString(result.Payload.Span)); + } + + [Fact] + public async Task TerminatorSplitAcrossSegments() + { + var wire = FramingWireFactory.WithTerminator("split-term\r\n"); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBackBytes: 2); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("split-term\r\n", Encoding.ASCII.GetString(result.Payload.Span)); + } + + [Fact] + public async Task TerminatorSplitAcrossReads() + { + var wire = FramingWireFactory.WithTerminator("across-reads\r\n"); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(FramingWireFactory.ExpectedDestuffedLength(wire), result.Payload.Length); + } + + [Fact] + public async Task MultiSegmentArticle() + { + var wire = FramingWireFactory.SmallArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 17); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task PipelinedSecondArticleRemainsUnread() + { + var wire = FramingWireFactory.PipelinedTwoArticles(); + var pipe = new System.IO.Pipelines.Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.Equal("article-one\r\n", Encoding.ASCII.GetString(first.Payload.Span)); + Assert.Equal("article-two\r\n", Encoding.ASCII.GetString(second.Payload.Span)); + } + + [Fact] + public async Task LargeArticle() + { + var wire = FramingWireFactory.Large768KiBArticle(); + var expectedLen = FramingWireFactory.ExpectedDestuffedLength(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 64); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(expectedLen, result.Payload.Length); + } + + [Fact] + public async Task MultipleCRLFs() + { + var wire = FramingWireFactory.ManyCrlfArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 11); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task MultipleDotStuffedLines() + { + var wire = FramingWireFactory.ManyDotStuffedLinesArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 9); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Contains(".stuffed-line-0\r\n"u8, payload.Span); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task TrailingPeriodsDoNotTerminate() + { + var wire = FramingWireFactory.TrailingPeriodLinesArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 13); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + Assert.EndsWith("period.\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task TerminatorNotDeliveredToSink() + { + var wire = FramingWireFactory.WithTerminator("keep-me\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 2); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.False(payload.Span.EndsWith(".\r\n"u8)); + Assert.Equal("keep-me\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task PipeReader_ChunkedFeed_MatchesExpected() + { + var wire = FramingWireFactory.MediumArticle(); + var expected = FramingWireFactory.ExpectedDestuffedLength(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 32); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(expected, result.Payload.Length); + } + + private static async Task<(NntpMultilineReadStatus Status, ReadOnlyMemory Payload)> ReadRecording( + byte[] wire, + int segmentSize) + { + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + return (result.Status, result.Payload); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ContinuousTakethisParserTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ContinuousTakethisParserTests.cs new file mode 100644 index 00000000..b65d687c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/ContinuousTakethisParserTests.cs @@ -0,0 +1,113 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +public sealed class ContinuousTakethisParserTests +{ + [Fact] + public async Task ParsesMultipleTakethis_Contiguous() + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(articleBytes: 128, articleCount: 5); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(5, result.Articles); + Assert.Equal(5 * 128, result.ArticleBytes); + } + + [Fact] + public async Task ParsesAcrossSmallSegments() + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(256, 3); + var result = await ProductionTakethisStream.ParseAsync(wire, chunkSize: 3); + Assert.True(result.IsComplete); + Assert.Equal(3, result.Articles); + } + + [Fact] + public async Task DotStuffedAndTrailingPeriod_NotTerminator() + { + var body = Encoding.ASCII.GetBytes("Hello.\r\n..foo\r\n...\r\n"); + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + ms.Write("TAKETHIS \r\n"u8); + ms.Write("x\r\n"u8); + ms.Write(".\r\n"u8); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + var destuffedFirst = FramingWireFactory.DestuffPayload(body); + Assert.Equal(destuffedFirst, result.Payloads[0].ToArray()); + Assert.Equal("x\r\n"u8.ToArray(), result.Payloads[1].ToArray()); + Assert.Equal(destuffedFirst.Length + result.Payloads[1].Length, result.ArticleBytes); + } + + [Fact] + public async Task CheckAndQuit_EscapeToControlPlane() + { + using var ms = new MemoryStream(); + ms.Write("CHECK \r\n"u8); + ms.Write("TAKETHIS \r\n"u8); + ms.Write("hi\r\n"u8); + ms.Write(".\r\n"u8); + ms.Write("QUIT\r\n"u8); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal(1, result.Checks); + Assert.Equal(1, result.Quits); + } + + [Fact] + public async Task EmptyArticle() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n.\r\n"); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal(0, result.ArticleBytes); + } + + [Fact] + public async Task IncompleteArticle_ReturnsIncomplete() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nno-terminator"); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.False(result.IsComplete); + } + + [Theory] + [InlineData(1)] + [InlineData(17)] + public async Task ProductionReader_AgreesOnMultiArticleStream_AcrossChunkSizes(int chunkSize) + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(512, 8); + var result = await ProductionTakethisStream.ParseAsync(wire, chunkSize: chunkSize); + Assert.True(result.IsComplete); + Assert.Equal(8, result.Articles); + Assert.Equal(8 * 512, result.ArticleBytes); + + var stuffed = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\nbar\r\n"); + var segmented = await FramingPipe.ReadSegmentedAsync(stuffed, chunkSize); + Assert.Equal(NntpMultilineReadStatus.Completed, segmented.Status); + Assert.Equal(FramingWireFactory.ExpectedDestuffed(stuffed), segmented.Payload.ToArray()); + } + + [Fact] + public async Task InnWire7_SingleArticleStream() + { + var article = InnArticleCorpus.ReadAllBytes("wire-7"); + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(article); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal( + FramingWireFactory.DestuffCompleteWire(article).Length, + result.ArticleBytes); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/DataPlanePipelineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/DataPlanePipelineTests.cs new file mode 100644 index 00000000..b639eb6b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/DataPlanePipelineTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +public sealed class DataPlanePipelineTests +{ + [Fact] + public async Task Pipeline_Synthetic_CheckQuitAndPayloadCommands() + { + var aStored = Encoding.ASCII.GetBytes("CHECK ignored\r\nQUIT ignored\r\nTAKETHIS ignored\r\n"); + var cStored = Encoding.ASCII.GetBytes("hello\r\n"); + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", aStored)); + ms.Write("CHECK \r\n"u8); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", cStored)); + ms.Write("QUIT\r\n"u8); + ms.Write("TAKETHIS \r\nbody\r\n.\r\n"u8); + + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray(), stopAfterQuit: true); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal(1, result.Checks); + Assert.Equal(1, result.Quits); + Assert.Equal("", result.MessageIds[0]); + Assert.Equal("", result.MessageIds[1]); + Assert.Equal(aStored, result.Payloads[0].ToArray()); + Assert.Equal(cStored, result.Payloads[1].ToArray()); + } + + [Fact] + public async Task Pipeline_ControlPlane_CheckAndQuit() + { + var aStored = Encoding.ASCII.GetBytes("CHECK ignored\r\nQUIT ignored\r\nTAKETHIS ignored\r\n"); + var cStored = Encoding.ASCII.GetBytes("hello\r\n"); + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", aStored)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", cStored)); + + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal("", result.MessageIds[0]); + Assert.Equal("", result.MessageIds[1]); + } + + [Fact] + public async Task Pipeline_CommandsInsideArticle_ArePayload() + { + var stored = Encoding.ASCII.GetBytes("line\r\nCHECK x\r\nQUIT\r\nTAKETHIS y\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(0, result.Checks); + Assert.Equal(0, result.Quits); + Assert.Equal(1, result.Articles); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } + + [Fact] + public async Task Pipeline_ResponseOrdering_Preserved() + { + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<1@t>", "a\r\n"u8)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<2@t>", "b\r\n"u8)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<3@t>", "c\r\n"u8)); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(3, result.MessageIds.Count); + Assert.Equal("<1@t>", result.MessageIds[0]); + Assert.Equal("<2@t>", result.MessageIds[1]); + Assert.Equal("<3@t>", result.MessageIds[2]); + } + + [Fact] + public async Task Pipeline_RoundTrip_StoredEqualsDestuffedCurrent() + { + var stored = Encoding.ASCII.GetBytes(".leading\r\n..two\r\nnormal\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(stored.Length, result.ArticleBytes); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/FramingTestSupport.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/FramingTestSupport.cs new file mode 100644 index 00000000..8c673a0d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/FramingTestSupport.cs @@ -0,0 +1,665 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Blake3; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// Test-only helpers for production / restuff coverage. +internal static class FramingPipe +{ + public static async Task ReadOneAsync( + byte[] wire, + int maxArticleBytes = 8 * 1024 * 1024, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(wire, cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + + public static async Task ReadSegmentedAsync( + byte[] wire, + int segmentSize, + int maxArticleBytes = 8 * 1024 * 1024, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, segmentSize), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + + var read = NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, maxArticleBytes, cancellationToken) + .AsTask(); + await WriteChunkedAsync(pipe.Writer, wire, segmentSize, cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await read.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken).ConfigureAwait(false); + } + + public static async Task WriteChunkedAsync( + PipeWriter writer, + ReadOnlyMemory wire, + int chunkSize, + CancellationToken cancellationToken = default) + { + var offset = 0; + while (offset < wire.Length) + { + var n = Math.Min(chunkSize, wire.Length - offset); + await writer.WriteAsync(wire.Slice(offset, n), cancellationToken).ConfigureAwait(false); + await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + offset += n; + } + } + + public static async Task ReadSplitTerminatorAsync( + byte[] wire, + int holdBackBytes = 1, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(holdBackBytes, 1); + if (wire.Length <= holdBackBytes) + { + throw new ArgumentException("Wire is shorter than the split.", nameof(wire)); + } + + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: 8, + pauseWriterThreshold: 1024 * 1024, + resumeWriterThreshold: 1024 * 1024, + useSynchronizationContext: false)); + var read = NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, 8 * 1024 * 1024, cancellationToken) + .AsTask(); + await pipe.Writer.WriteAsync(wire.AsMemory()[..^holdBackBytes], cancellationToken).ConfigureAwait(false); + await pipe.Writer.FlushAsync(cancellationToken).ConfigureAwait(false); + await pipe.Writer.WriteAsync(wire.AsMemory()[^holdBackBytes..], cancellationToken).ConfigureAwait(false); + await pipe.Writer.FlushAsync(cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await read.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); + } +} + +/// Synthetic complete-wire articles for production destuff tests. +internal static class FramingWireFactory +{ + public static byte[] EmptyArticle { get; } = ".\r\n"u8.ToArray(); + + public static byte[] WithTerminator(string bodyWithoutTerminator) => + Encoding.ASCII.GetBytes(bodyWithoutTerminator + ".\r\n"); + + public static byte[] DataThenDelimiter() => WithTerminator("data\r\n"); + + public static byte[] SmallArticle() + { + var sb = new StringBuilder(1600); + for (var i = 0; i < 40; i++) + { + sb.Append("Subject: line-").Append(i).Append("\r\n"); + } + + sb.Append("\r\n"); + for (var i = 0; i < 20; i++) + { + sb.Append("body text line ").Append(i).Append(" with some padding abcdefghijklmnop\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] MediumArticle() => BuildFilledArticle(128 * 1024, 64); + + public static byte[] Large768KiBArticle() => BuildFilledArticle(768 * 1024, 128); + + public static byte[] ManyCrlfArticle() + { + var sb = new StringBuilder(70_000); + for (var i = 0; i < 8000; i++) + { + sb.Append('x').Append(i % 10).Append("\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] ManyDotStuffedLinesArticle() + { + var sb = new StringBuilder(40_000); + for (var i = 0; i < 2000; i++) + { + sb.Append("..stuffed-line-").Append(i).Append("\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] TrailingPeriodLinesArticle() + { + var sb = new StringBuilder(40_000); + sb.Append("Hello.\r\n"); + sb.Append("Something else.\r\n"); + sb.Append("This line has a trailing period.\r\n"); + for (var i = 0; i < 1000; i++) + { + sb.Append("Line ").Append(i).Append(" ends with a period.\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] ManyDotsHostileArticle() + { + var sb = new StringBuilder(280_000); + for (var i = 0; i < 4000; i++) + { + sb.Append("Hello.\r\n"); + sb.Append("....not-a-terminator.\r\n"); + sb.Append("..\r\n"); + sb.Append("x.y.z.\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] PipelinedTwoArticles() + { + var a1 = WithTerminator("article-one\r\n"); + var a2 = WithTerminator("article-two\r\n"); + var combined = new byte[a1.Length + a2.Length]; + a1.CopyTo(combined, 0); + a2.CopyTo(combined, a1.Length); + return combined; + } + + public static byte[] ExpectedDestuffed(ReadOnlySpan completeWire) => + DestuffCompleteWire(completeWire); + + public static int ExpectedDestuffedLength(ReadOnlySpan completeWire) => + DestuffCompleteWire(completeWire).Length; + + public static byte[] DestuffCompleteWire(ReadOnlySpan completeWire) + { + if (completeWire.Length == 3 + && completeWire[0] == (byte)'.' + && completeWire[1] == (byte)'\r' + && completeWire[2] == (byte)'\n') + { + return []; + } + + if (completeWire.Length < 3 + || completeWire[^3] != (byte)'.' + || completeWire[^2] != (byte)'\r' + || completeWire[^1] != (byte)'\n') + { + throw new ArgumentException("Expected a complete wire article ending in .\\r\\n.", nameof(completeWire)); + } + + return DestuffPayload(completeWire[..^3]); + } + + public static byte[] DestuffPayload(ReadOnlySpan wirePayloadWithoutTerminator) + { + if (wirePayloadWithoutTerminator.IsEmpty) + { + return []; + } + + var output = new ArrayBufferWriter(wirePayloadWithoutTerminator.Length); + var offset = 0; + while (offset < wirePayloadWithoutTerminator.Length) + { + var remaining = wirePayloadWithoutTerminator[offset..]; + var crlf = remaining.IndexOf("\r\n"u8); + if (crlf < 0) + { + throw new InvalidOperationException("Expected CRLF-framed wire payload."); + } + + var line = remaining[..crlf]; + if (line.Length > 0 && line[0] == (byte)'.') + { + line = line[1..]; + } + + var span = output.GetSpan(line.Length + 2); + line.CopyTo(span); + span[line.Length] = (byte)'\r'; + span[line.Length + 1] = (byte)'\n'; + output.Advance(line.Length + 2); + offset += crlf + 2; + } + + return output.WrittenSpan.ToArray(); + } + + public static byte[] BuildTakeThisTransaction(string messageId, ReadOnlySpan storedDestuffed) + { + var header = Encoding.ASCII.GetBytes("TAKETHIS " + messageId + "\r\n"); + var body = ArticleWireReconstructor.RestuffArticle(storedDestuffed); + var wire = new byte[header.Length + body.Length]; + header.CopyTo(wire, 0); + body.CopyTo(wire, header.Length); + return wire; + } + + public static byte[] BuildFixedSizeTakethisStream(int articleBytes, int articleCount, int seed = 42) + { + var rng = new Random(seed); + using var ms = new MemoryStream(); + var body = new byte[articleBytes]; + for (var i = 0; i < articleCount; i++) + { + ms.Write(Encoding.ASCII.GetBytes($"TAKETHIS \r\n")); + for (var b = 0; b < body.Length; b++) + { + if (b > 0 && b % 256 == 255) + { + body[b] = (byte)'\n'; + body[b - 1] = (byte)'\r'; + } + else + { + var v = (byte)rng.Next(1, 250); + body[b] = v == (byte)'.' ? (byte)'x' : v; + } + } + + if (body.Length >= 2) + { + body[^2] = (byte)'\r'; + body[^1] = (byte)'\n'; + } + + if (body.Length > 16) + { + body[8] = (byte)(i & 0xFF); + body[9] = (byte)((i >> 8) & 0xFF); + } + + ms.Write(body); + ms.Write(".\r\n"u8); + } + + return ms.ToArray(); + } + + private static byte[] BuildFilledArticle(int targetBodyBytes, int lineLength) + { + var sb = new StringBuilder(targetBodyBytes + 64); + var n = 0; + while (sb.Length < targetBodyBytes) + { + var pad = new string('a', Math.Max(1, lineLength - 8)); + sb.Append(n).Append(' ').Append(pad).Append("\r\n"); + n++; + } + + return WithTerminator(sb.ToString()); + } +} + +/// Production command-line + multiline reader over a TAKETHIS/CHECK/QUIT stream. +internal static class ProductionTakethisStream +{ + public static async Task ParseAsync( + byte[] wire, + bool stopAfterQuit = false, + int? chunkSize = null, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, chunkSize ?? 4096), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + var parse = ParseFromReaderAsync(pipe.Reader, stopAfterQuit, cancellationToken); + if (chunkSize is int size) + { + await FramingPipe.WriteChunkedAsync(pipe.Writer, wire, size, cancellationToken).ConfigureAwait(false); + } + else + { + await pipe.Writer.WriteAsync(wire, cancellationToken).ConfigureAwait(false); + } + + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + + return await parse.ConfigureAwait(false); + } + + private static async Task ParseFromReaderAsync( + PipeReader reader, + bool stopAfterQuit, + CancellationToken cancellationToken) + { + var articles = 0; + var articleBytes = 0L; + var checks = 0; + var quits = 0; + var payloads = new List>(); + var messageIds = new List(); + + while (true) + { + var line = await NntpCommandLineReader + .ReadLineAsync(reader, cancellationToken) + .ConfigureAwait(false); + if (line is null) + { + break; + } + + if (line.StartsWith("TAKETHIS ", StringComparison.OrdinalIgnoreCase)) + { + var id = line["TAKETHIS ".Length..]; + var article = await NntpMultilineDataReader + .ReadArticleAsync(reader, 8 * 1024 * 1024, cancellationToken) + .ConfigureAwait(false); + if (article.Status != NntpMultilineReadStatus.Completed) + { + return new TakethisStreamResult( + IsComplete: false, + articles, + articleBytes, + checks, + quits, + payloads, + messageIds); + } + + articles++; + articleBytes += article.Payload.Length; + payloads.Add(article.Payload); + messageIds.Add(id); + continue; + } + + if (line.StartsWith("CHECK ", StringComparison.OrdinalIgnoreCase)) + { + checks++; + continue; + } + + if (line.Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + quits++; + if (stopAfterQuit) + { + break; + } + } + } + + return new TakethisStreamResult(true, articles, articleBytes, checks, quits, payloads, messageIds); + } +} + +internal readonly record struct TakethisStreamResult( + bool IsComplete, + int Articles, + long ArticleBytes, + int Checks, + int Quits, + IReadOnlyList> Payloads, + IReadOnlyList MessageIds); + +/// INN fixture loader for the corpus already stored under tests/Data/InnArticles. +internal static class InnArticleCorpus +{ + public const string UpstreamCommit = "6843efceb8a15774d23e4e2892d08daf54574314"; + + private static readonly object Gate = new(); + private static InnArticleManifest? _manifest; + private static string? _root; + + public static string ResolveRoot() + { + if (_root is not null) + { + return _root; + } + + lock (Gate) + { + if (_root is not null) + { + return _root; + } + + var candidates = new[] + { + Path.Join(AppContext.BaseDirectory, "Data", "InnArticles"), + Path.GetFullPath(Path.Join(AppContext.BaseDirectory, "..", "..", "..", "Data", "InnArticles")), + }; + + foreach (var candidate in candidates) + { + if (File.Exists(Path.Join(candidate, "inn-articles.manifest.json"))) + { + _root = candidate; + return _root; + } + } + + throw new DirectoryNotFoundException( + "INN article corpus not found. Expected Data/InnArticles/inn-articles.manifest.json."); + } + } + + public static InnArticleManifest LoadManifest() + { + if (_manifest is not null) + { + return _manifest; + } + + lock (Gate) + { + if (_manifest is not null) + { + return _manifest; + } + + var json = File.ReadAllText(Path.Join(ResolveRoot(), "inn-articles.manifest.json")); + var manifest = JsonSerializer.Deserialize(json, InnArticleManifestJsonContext.Default.InnArticleManifest) + ?? throw new InvalidOperationException("Failed to deserialize inn-articles.manifest.json."); + _manifest = manifest; + return _manifest; + } + } + + public static byte[] ReadAllBytes(string relativePath) + { + var full = Path.Join(ResolveRoot(), relativePath); + if (!File.Exists(full)) + { + throw new FileNotFoundException($"INN fixture missing: {relativePath}", full); + } + + return File.ReadAllBytes(full); + } + + public static void VerifySha256(InnArticleManifestEntry entry) + { + var bytes = ReadAllBytes(entry.RelativePath); + var hash = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + // Hash the raw file bytes. InnArticles is marked binary in .gitattributes so + // core.autocrlf cannot rewrite LF fixtures before this comparison. + if (!string.Equals(hash, entry.Sha256, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"SHA-256 mismatch for {entry.RelativePath}: disk={hash}, manifest={entry.Sha256}."); + } + + if (bytes.Length != entry.Length) + { + throw new InvalidOperationException( + $"Length mismatch for {entry.RelativePath}: disk={bytes.Length}, manifest={entry.Length}."); + } + } + + public static IReadOnlyList CompleteWireArticles() => + LoadManifest().Files + .Where(static f => f.FramingClassification == "complete_multiline_wire") + .ToArray(); + + public static bool ContainsFiveByteDelimiter(ReadOnlySpan data) => + data.IndexOf("\r\n.\r\n"u8) >= 0; +} + +internal sealed class InnArticleManifest +{ + [JsonPropertyName("repository")] + public string Repository { get; set; } = ""; + + [JsonPropertyName("sourcePath")] + public string SourcePath { get; set; } = ""; + + [JsonPropertyName("upstreamCommit")] + public string UpstreamCommit { get; set; } = ""; + + [JsonPropertyName("fileCount")] + public int FileCount { get; set; } + + [JsonPropertyName("totalBytes")] + public long TotalBytes { get; set; } + + [JsonPropertyName("files")] + public List Files { get; set; } = []; +} + +internal sealed class InnArticleManifestEntry +{ + [JsonPropertyName("relativePath")] + public string RelativePath { get; set; } = ""; + + [JsonPropertyName("length")] + public int Length { get; set; } + + [JsonPropertyName("sha256")] + public string Sha256 { get; set; } = ""; + + [JsonPropertyName("framingClassification")] + public string FramingClassification { get; set; } = ""; + + [JsonPropertyName("containsLeadingDotLines")] + public bool ContainsLeadingDotLines { get; set; } +} + +[JsonSerializable(typeof(InnArticleManifest))] +[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] +internal partial class InnArticleManifestJsonContext : JsonSerializerContext; + +internal static class MessageIdDigest +{ + public static string ComputeHexLower(string messageIdWithBrackets) + { + var utf8 = Encoding.UTF8.GetBytes(messageIdWithBrackets); + var hash = Hasher.Hash(utf8); + return Convert.ToHexString(hash.AsSpan()).ToLowerInvariant(); + } + + public static string BuildRelativePath(string digestHexLower) => + Path.Join(digestHexLower[..2], digestHexLower.Substring(2, 2), digestHexLower); +} + +internal static class StoredArticleReader +{ + public static bool TryExtractMessageId(ReadOnlySpan storedHead, out string messageIdWithBrackets) + { + messageIdWithBrackets = string.Empty; + var offset = 0; + while (offset < storedHead.Length) + { + var remaining = storedHead[offset..]; + var eol = remaining.IndexOf((byte)'\n'); + ReadOnlySpan physical; + if (eol < 0) + { + physical = remaining; + offset = storedHead.Length; + } + else + { + physical = remaining[..eol]; + offset += eol + 1; + } + + if (!physical.IsEmpty && physical[^1] == (byte)'\r') + { + physical = physical[..^1]; + } + + if (physical.Length == 0) + { + break; + } + + if (!StartsWithIgnoreCase(physical, "Message-ID:"u8)) + { + continue; + } + + var colon = physical.IndexOf((byte)':'); + var value = physical[(colon + 1)..]; + while (!value.IsEmpty && value[0] is (byte)' ' or (byte)'\t') + { + value = value[1..]; + } + + var lt = value.IndexOf((byte)'<'); + var gt = value.LastIndexOf((byte)'>'); + if (lt >= 0 && gt > lt) + { + messageIdWithBrackets = Encoding.ASCII.GetString(value[lt..(gt + 1)]); + return true; + } + + return false; + } + + return false; + } + + private static bool StartsWithIgnoreCase(ReadOnlySpan line, ReadOnlySpan asciiPrefix) + { + if (line.Length < asciiPrefix.Length) + { + return false; + } + + for (var i = 0; i < asciiPrefix.Length; i++) + { + var a = line[i]; + var b = asciiPrefix[i]; + if (a is >= (byte)'A' and <= (byte)'Z') + { + a = (byte)(a + 32); + } + + if (b is >= (byte)'A' and <= (byte)'Z') + { + b = (byte)(b + 32); + } + + if (a != b) + { + return false; + } + } + + return true; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/IHaveArticleReaderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/IHaveArticleReaderTests.cs new file mode 100644 index 00000000..1eaedcdf --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/IHaveArticleReaderTests.cs @@ -0,0 +1,218 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +public sealed class IHaveArticleReaderTests +{ + private static Pipe CreatePipe() => + new(new PipeOptions(pauseWriterThreshold: 4 * 1024 * 1024, resumeWriterThreshold: 1024 * 1024, useSynchronizationContext: false)); + + [Fact] + public async Task OrdinaryLines_SurviveByteForByte_AndLeaveNextCommand() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: hi\r\n\r\nhello\r\n.\r\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: hi\r\n\r\nhello\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal(result.Payload.Length, result.Metrics.ArticleSize); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task DotStuffedLines_RemainStuffed_InOwnedPayload() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: d\r\n\r\n..foo\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: d\r\n\r\n..foo\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.DoesNotContain("\r\n.foo\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task DotStuffing_WithinDestuffedLimit_IsRetainedWhenStuffedWireExceedsIt() + { + var pipe = CreatePipe(); + var stuffed = string.Concat(Enumerable.Repeat("..\r\n", 10)); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffed + ".\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 35, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(40, result.Payload.Length); + Assert.Equal(stuffed, Encoding.ASCII.GetString(result.Payload.Span)); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task DotStuffing_OverDestuffedLimit_ReturnsTooLarge_AndLeavesNextCommand() + { + var pipe = CreatePipe(); + var stuffed = string.Concat(Enumerable.Repeat("..\r\n", 10)); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffed + ".\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 29, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.TooLarge, result.Status); + Assert.Equal(0, result.Payload.Length); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task Terminator_IsNotStored() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("From: a\r\n\r\nbody\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal("From: a\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.False(result.Payload.Span.EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task TerminatorAndNextCommand_SameBuffer_LeavesCommand() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: n\r\n\r\nx\r\n.\r\nQUIT\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("QUIT\r\n", await ReadLeftoverAsync(pipe.Reader)); + Assert.DoesNotContain("QUIT"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task TerminatorSplitAcrossReads_Completes() + { + var pipe = CreatePipe(); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await pipe.Writer.WriteAsync("Subject: s\r\n\r\nbody\r\n.\r"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: s\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task PartialPipeSegments_ReassembleWire() + { + var pipe = CreatePipe(); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await pipe.Writer.WriteAsync("Sub"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("ject: split\r\n\r\nbo"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("dy\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: split\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.True(result.Metrics.PipeReads >= 2); + } + + [Fact] + public async Task PrematureEof_IsIncomplete() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: x\r\n\r\nno-terminator"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + } + + [Fact] + public async Task TooLarge_ThenTerminator_ReturnsTooLarge_AndLeavesCommand() + { + var pipe = CreatePipe(); + var body = new string('A', 64); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"Subject: big\r\n\r\n{body}\r\n.\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 16, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.TooLarge, result.Status); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task LargeArticle_IsOwnedAfterAdvance() + { + var body = new string('B', 120 * 1024); + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes("Subject: big\r\n\r\n" + body + "\r\n.\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 2 * 1024 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var leftover = await ReadLeftoverAsync(pipe.Reader); + Assert.Equal("DATE\r\n", leftover); + Assert.StartsWith("Subject: big\r\n\r\n", Encoding.ASCII.GetString(result.Payload.Span), StringComparison.Ordinal); + Assert.True(result.Payload.Length >= 120 * 1024); + _ = result.Payload.Span[0]; + _ = result.Payload.Span[^1]; + } + + [Fact] + public async Task StreamingFourKiBChunks_LeaveNextCommandIntact() + { + var body = new string('S', 12 * 1024); + var framed = Encoding.ASCII.GetBytes($"Subject: stream\r\n\r\n{body}\r\n.\r\nDATE\r\n"); + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 8 * 1024, + resumeWriterThreshold: 4 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false)); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await Task.Yield(); + for (var offset = 0; offset < framed.Length; offset += 4096) + { + var n = Math.Min(4096, framed.Length - offset); + await pipe.Writer.WriteAsync(framed.AsMemory(offset, n)); + await pipe.Writer.FlushAsync(); + } + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + Assert.True(result.Metrics.PipeReads >= 2); + } + + [Fact] + public async Task DeclaredSizes_DoNotConsumePastTerminator() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Content-Length: 999999\r\n\r\nshort\r\n.\r\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Content-Length: 999999\r\n\r\nshort\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + private static async Task ReadLeftoverAsync(PipeReader reader) + { + var result = await reader.ReadAsync(); + var text = Encoding.ASCII.GetString(result.Buffer.ToArray()); + reader.AdvanceTo(result.Buffer.End); + return text; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs new file mode 100644 index 00000000..c064002b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs @@ -0,0 +1,237 @@ +using System.IO.Pipelines; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// +/// Correctness coverage for the INN tests/data/articles corpus against production +/// destuff/terminator rules. +/// +public sealed class InnArticleCorpusFramerTests +{ + public static IEnumerable AllFixtureNames() + { + foreach (var entry in InnArticleCorpus.LoadManifest().Files) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable CompleteWireFixtureNames() + { + foreach (var entry in InnArticleCorpus.CompleteWireArticles()) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable SegmentSizes() + { + foreach (var size in new[] { 1, 2, 4, 16, 4096 }) + { + yield return [size]; + } + } + + [Fact] + public void Manifest_MatchesPinnedCommit_AndFixtureCount() + { + var manifest = InnArticleCorpus.LoadManifest(); + Assert.Equal(InnArticleCorpus.UpstreamCommit, manifest.UpstreamCommit); + Assert.Equal("InterNetNews/inn", manifest.Repository); + Assert.Equal("tests/data/articles", manifest.SourcePath); + Assert.Equal(25, manifest.FileCount); + Assert.Equal(25, manifest.Files.Count); + Assert.Equal(19544, manifest.TotalBytes); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public void EveryFixture_Sha256MatchesManifest(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + InnArticleCorpus.VerifySha256(entry); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public async Task EveryFixture_BulkFramer_StatusMatchesClassification(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 7); + + if (entry.FramingClassification == "complete_multiline_wire") + { + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.Equal(expected.Length, result.Payload.Length); + Assert.NotEqual(wire.Length, result.Payload.Length); + } + else + { + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + // Production does not emit an incomplete payload (retired bulk prototype forwarded the buffer). + Assert.True(result.Payload.IsEmpty); + Assert.Equal(wire, InnArticleCorpus.ReadAllBytes(relativePath)); + } + } + + [Theory] + [MemberData(nameof(CompleteWireFixtureNames))] + public async Task CompleteWire_PreservesDotStuffing_AndExcludesTerminator(string relativePath) + { + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + Assert.True(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + var stuffedWithoutTerminator = wire.AsSpan()[..^3]; + Assert.Contains("..\r\n"u8, stuffedWithoutTerminator); + + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().StartsWith("."u8)); + Assert.Contains("..\r\n"u8, expected); + Assert.NotEqual(wire.Length, expected.Length); + Assert.False(wire.AsSpan()[..^3].SequenceEqual(expected)); + + var result = await FramingPipe.ReadSegmentedAsync(wire, 3); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.True(result.Payload.Span.StartsWith("."u8)); + } + + [Theory] + [MemberData(nameof(SegmentSizes))] + public async Task Wire7_IdenticalPayload_AcrossSegmentSizes(int segmentSize) + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task Wire7_FiveByteDelimiter_SplitAtEveryBoundary() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(wire.AsSpan().EndsWith("\r\n.\r\n"u8)); + + var whole = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, whole.Status); + Assert.True(expected.AsSpan().SequenceEqual(whole.Payload.Span)); + + for (var holdBack = 1; holdBack <= 5; holdBack++) + { + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBack); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + } + + [Fact] + public async Task Wire7_DelimiterSplitAcrossReads() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task Wire7_HelloDotStyleTrailingPeriod_IsNotTerminator() + { + // Upstream wire-7 body includes lines that end with '.' before CRLF inside content; + // terminator is only the final .\r\n line. Prove payload retains those periods. + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.Contains(".\r\n"u8, expected); + + var result = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.Contains("parser.\r\n"u8, result.Payload.Span); + } + + [Fact] + public async Task Wire7_CurrentUnstuffs_BulkPreservesWire() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var stuffedWithoutTerminator = wire[..^3]; + var destuffed = FramingWireFactory.DestuffCompleteWire(wire); + + var current = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, current.Status); + Assert.True(destuffed.AsSpan().SequenceEqual(current.Payload.Span)); + + // Retired bulk prototype kept stuffed wire; production destuffs. + Assert.NotEqual(current.Payload.Length, stuffedWithoutTerminator.Length); + Assert.True(current.Payload.Span.SequenceEqual(FramingWireFactory.DestuffPayload(stuffedWithoutTerminator))); + } + + [Fact] + public async Task WireTruncated_IsIncomplete() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-truncated"); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + var result = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + } + + [Fact] + public async Task WireStrange_IsIncomplete_PreservesExactBytesIncludingNul() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-strange"); + Assert.Contains((byte)0, wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, 1); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + // Fixture integrity: production does not echo incomplete bytes, but the NUL fixture is unchanged. + Assert.Contains((byte)0, InnArticleCorpus.ReadAllBytes("wire-strange")); + Assert.Equal(477, wire.Length); + } + + [Theory] + [InlineData("5")] + [InlineData("6")] + [InlineData("7")] + public async Task LeadingDotSpoolFixtures_DoNotFalseComplete(string name) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == name); + Assert.True(entry.ContainsLeadingDotLines); + Assert.Equal("spool_or_non_wire_no_nntp_terminator", entry.FramingClassification); + var wire = InnArticleCorpus.ReadAllBytes(name); + var result = await FramingPipe.ReadSegmentedAsync(wire, 2); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + } + + [Fact] + public async Task Pipelined_TwoWire7_SecondRemains() + { + var one = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(one); + var combined = new byte[one.Length * 2]; + one.CopyTo(combined.AsSpan()); + one.CopyTo(combined.AsSpan(one.Length)); + + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(combined); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var leftover = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.True(expected.AsSpan().SequenceEqual(first.Payload.Span)); + Assert.True(expected.AsSpan().SequenceEqual(second.Payload.Span)); + Assert.Equal(NntpMultilineReadStatus.Incomplete, leftover.Status); + Assert.True(leftover.Payload.IsEmpty); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpContinuousRxTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpContinuousRxTests.cs new file mode 100644 index 00000000..f3d4b798 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpContinuousRxTests.cs @@ -0,0 +1,677 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// +/// Continuous STREAM scanner: command/article units, ownership, splits, and control-plane rules. +/// TAKETHIS payload is framed wire bytes (terminator omitted, no destuff). +/// +public sealed class NntpContinuousRxTests +{ + [Fact] + public async Task OneTakeThis_CopiesFramedWireAndOmitsTerminator() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n..foo\r\nbar\r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("..foo\r\nbar\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task ManyTakeThis_SamePipeBuffer_NoExtraTransportRead() + { + var wire = Encoding.ASCII.GetBytes( + "TAKETHIS \r\none\r\n.\r\nTAKETHIS \r\ntwo\r\n.\r\nTAKETHIS \r\nthree\r\n.\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(3, units.Count); + Assert.All(units, u => Assert.Equal(NntpContinuousRxKind.TakeThis, u.Kind)); + Assert.Equal(["", "", ""], units.Select(u => u.MessageId)); + Assert.Equal("one\r\n"u8.ToArray(), units[0].Article.Payload.ToArray()); + Assert.Equal("two\r\n"u8.ToArray(), units[1].Article.Payload.ToArray()); + Assert.Equal("three\r\n"u8.ToArray(), units[2].Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(5)] + [InlineData(17)] + [InlineData(64)] + public async Task ArbitrarySegmentBoundaries_MatchFramedWireOracle(int segmentSize) + { + var article = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\n...\r\n"); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var expected = article[..^3]; + var unit = await ReadOneSegmentedAsync(wire, segmentSize, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(expected, unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public async Task CommandCrlfSplit_StillRecognizesTakeThis(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nbody\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + Assert.Equal("body\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(5)] + public async Task ArticleTerminatorSplit_Completes(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nkeep\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("keep\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task CommandLikeTextInsideArticle_IsPayload() + { + var stored = Encoding.ASCII.GetBytes("QUIT\r\nCHECK x\r\nTAKETHIS y\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true); + Assert.Single(units); + Assert.Equal(NntpContinuousRxKind.TakeThis, units[0].Kind); + Assert.Equal(stored, units[0].Article.Payload.ToArray()); + } + + [Fact] + public async Task CheckAndQuit_AreControlCommands() + { + var wire = Encoding.ASCII.GetBytes( + "CHECK \r\nTAKETHIS \r\nhi\r\n.\r\nQUIT\r\nTAKETHIS \r\nz\r\n.\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true, stopAfterQuit: true); + Assert.Equal(3, units.Count); + Assert.Equal(NntpContinuousRxKind.Command, units[0].Kind); + Assert.Equal("CHECK ", units[0].CommandLine); + Assert.Equal(NntpContinuousRxKind.TakeThis, units[1].Kind); + Assert.Equal("", units[1].MessageId); + Assert.Equal(NntpContinuousRxKind.Command, units[2].Kind); + Assert.Equal("QUIT", units[2].CommandLine); + } + + [Fact] + public async Task IncompleteCommand_NeedMoreThenCompletes() + { + var parser = new NntpContinuousRxParser(); + var partial = new ReadOnlySequence("TAKETH"u8.ToArray()); + Assert.Equal(NntpContinuousRxKind.NeedMore, parser.TryConsume(ref partial, true, 1024).Kind); + + var unit = await ReadOneAsync("TAKETHIS \r\nok\r\n.\r\n"u8.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + } + + [Fact] + public async Task IncompleteArticle_Eof_IsIncomplete() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nno-terminator"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Incomplete, unit.Article.Status); + Assert.True(unit.Article.Payload.IsEmpty); + } + + [Fact] + public async Task EmptyArticle_IsCompletedEmptyPayload() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(0, unit.Article.Payload.Length); + } + + [Fact] + public async Task Eof_EmptyPipe_IsNeedMore() + { + var unit = await ReadOneAsync([], consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.NeedMore, unit.Kind); + } + + [Fact] + public async Task Cancellation_Throws() + { + var pipe = new Pipe(); + var parser = new NntpContinuousRxParser(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => + NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, true, 1024, cts.Token) + .AsTask()); + } + + [Fact] + public async Task LargeArticle_MatchesOracle() + { + var article = FramingWireFactory.Large768KiBArticle(); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var expected = article[..^3]; + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true, maxArticleBytes: 4 * 1024 * 1024); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(expected.Length, unit.Article.Payload.Length); + Assert.True(expected.AsSpan().SequenceEqual(unit.Article.Payload.Span)); + } + + [Fact] + public async Task BinaryNulArticle_PreservedInFramedPayload() + { + var body = new byte[] { (byte)'a', 0, (byte)'b', (byte)'\r', (byte)'\n' }; + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + var unit = await ReadOneAsync(ms.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(body, unit.Article.Payload.ToArray()); + Assert.Contains((byte)0, unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task TooLarge_ConsumesTerminator_EmptyPayload() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n0123456789\r\n.\r\nQUIT\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true, maxArticleBytes: 4); + Assert.Equal(2, units.Count); + Assert.Equal(NntpMultilineReadStatus.TooLarge, units[0].Article.Status); + Assert.True(units[0].Article.Payload.IsEmpty); + Assert.Equal("QUIT", units[1].CommandLine); + } + + [Fact] + public async Task WithoutConsumeArticle_TakeThisIsCommandLineOnly() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nQUIT\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: false); + Assert.Equal(2, units.Count); + Assert.Equal(NntpContinuousRxKind.Command, units[0].Kind); + Assert.Equal("TAKETHIS ", units[0].CommandLine); + Assert.Equal("QUIT", units[1].CommandLine); + } + + [Fact] + public async Task OwnedPayload_SurvivesSubsequentAdvance() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nkeep-me\r\n.\r\nTAKETHIS \r\nx\r\n.\r\n"); + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var first = await NntpContinuousRxReader.ReadUnitAsync(pipe.Reader, parser, true, 64 * 1024, CancellationToken.None); + var second = await NntpContinuousRxReader.ReadUnitAsync(pipe.Reader, parser, true, 64 * 1024, CancellationToken.None); + Assert.Equal("keep-me\r\n"u8.ToArray(), first.Article.Payload.ToArray()); + Assert.Equal("x\r\n"u8.ToArray(), second.Article.Payload.ToArray()); + } + + [Fact] + public async Task WriterComplete_ThenReaderComplete_DoesNotReadAfterCompletion() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + await pipe.Writer.WriteAsync("DATE\r\n"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var date = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + Assert.Equal(NntpVerb.Date, date.Command.Verb); + + var eof = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + Assert.Equal(NntpContinuousRxKind.NeedMore, eof.Kind); + + await pipe.Reader.CompleteAsync(); + var completed = false; + try + { + _ = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task CancelPendingRead_ThenCompleteWriterAndReader_DoesNotReadAfterCompletion() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + using var cts = new CancellationTokenSource(); + var pending = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle: false, 1024, cts.Token) + .AsTask(); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => pending); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + + var completed = false; + try + { + _ = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task StreamScanner_PreservesStuffedWire_WhileMultilineReaderDestuffs() + { + var article = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\nbar\r\n"); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var stream = await ReadOneAsync(wire, consumeTakeThisArticle: true); + var reader = await FramingPipe.ReadOneAsync(article); + Assert.Equal(NntpMultilineReadStatus.Completed, stream.Article.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, reader.Status); + Assert.Equal("Hello.\r\n..foo\r\nbar\r\n"u8.ToArray(), stream.Article.Payload.ToArray()); + Assert.Equal("Hello.\r\n.foo\r\nbar\r\n"u8.ToArray(), reader.Payload.ToArray()); + Assert.NotEqual(reader.Payload.ToArray(), stream.Article.Payload.ToArray()); + } + + [Fact] + public async Task ManyCrlfLines_PreserveExactPayload() + { + var body = new StringBuilder(); + for (var i = 0; i < 200; i++) + { + body.Append("line-").Append(i).Append("\r\n"); + } + + var stored = Encoding.ASCII.GetBytes(body.ToString()); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(stored, unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task StuffedDotLines_ArePayloadNotTerminator() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n..one\r\n..two\r\n...three\r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal("..one\r\n..two\r\n...three\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + public async Task FiveByteTerminator_SplitAcrossEveryByteBoundary(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nbody-line\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("body-line\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task BinaryCrLfInsideLine_DoesNotPrematurelyTerminate() + { + // RFC 3977 forbids isolated CR/LF in a line; STREAM still treats only \r\n.\r\n as the end. + var body = new byte[] { (byte)'a', (byte)'\r', (byte)'b', (byte)'\n', (byte)'c', (byte)'\r', (byte)'\n' }; + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + var unit = await ReadOneAsync(ms.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(body, unit.Article.Payload.ToArray()); + } + + [Fact] + public void TryConsume_MultipleUnitsFromOneSequence() + { + var wire = Encoding.ASCII.GetBytes( + "TAKETHIS <1@t>\r\na\r\n.\r\nTAKETHIS <2@t>\r\nb\r\n.\r\n"); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + var first = Capture(parser, parser.TryConsume(ref buffer, true, 1024)); + var second = Capture(parser, parser.TryConsume(ref buffer, true, 1024)); + Assert.Equal(NntpContinuousRxKind.TakeThis, first.Kind); + Assert.Equal(NntpContinuousRxKind.TakeThis, second.Kind); + Assert.Equal("<1@t>", first.MessageId); + Assert.Equal("<2@t>", second.MessageId); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_ExactlyMaxCommandLine_ParsesTheWholeLine() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes); + var wire = WithCrlf(line); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + var unit = Capture(parser, parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024)); + + Assert.Equal(NntpContinuousRxKind.Command, unit.Kind); + Assert.True(unit.Unit.Command.IsValid); + Assert.Equal(NntpVerb.Group, unit.Unit.Command.Verb); + Assert.Equal(NntpCommandLineReader.MaxCommandLineBytes, parser.CurrentCommandLine.Length); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongLine_IsSyntaxErrorAndDoesNotParseThePrefix() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + + Assert.Equal(NntpContinuousRxKind.Command, unit.Kind); + Assert.False(unit.Command.IsValid); + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.Equal(NntpVerb.None, unit.Command.Verb); + Assert.Equal(0, parser.CurrentCommandLine.Length); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_ValidPrefixPlusExtraPastLimit_IsNotDispatchedAsThePrefix() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 6]; + "STAT "u8.CopyTo(line); + line.AsSpan("STAT "u8.Length, NntpCommandLineReader.MaxCommandLineBytes - "STAT "u8.Length) + .Fill((byte)' '); + " extra"u8.CopyTo(line.AsSpan(NntpCommandLineReader.MaxCommandLineBytes)); + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.NotEqual(NntpVerb.Stat, unit.Command.Verb); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongInvalidPrefix_IsTooLongNotAParsedVerb() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan(0, NntpCommandLineReader.MaxCommandLineBytes).Fill((byte)'!'); + line[^1] = (byte)'x'; + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.NotEqual(NntpParseStatus.UnknownVerb, unit.Command.Status); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongTakeThisThenQuit_ConsumesTheLineAndParsesTheNextCommand() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 8]; + "TAKETHIS "u8.CopyTo(line); + line.AsSpan("TAKETHIS "u8.Length).Fill((byte)'x'); + var wire = new byte[line.Length + 2 + "QUIT\r\n"u8.Length]; + WithCrlf(line).CopyTo(wire); + "QUIT\r\n"u8.CopyTo(wire.AsSpan(line.Length + 2)); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + + var first = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + var second = Capture(parser, parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024)); + + Assert.Equal(NntpContinuousRxKind.Command, first.Kind); + Assert.Equal(NntpParseStatus.CommandLineTooLong, first.Command.Status); + Assert.NotEqual(NntpContinuousRxKind.TakeThis, first.Kind); + Assert.Equal(NntpVerb.Quit, second.Unit.Command.Verb); + Assert.True(second.Unit.Command.IsValid); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_EmbeddedLfIsNotTheDelimiter() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan().Fill((byte)'a'); + line[10] = (byte)'\n'; + line[20] = (byte)'\r'; + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_NoCrlf_StaysNeedMore() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 100]; + line.AsSpan().Fill((byte)'a'); + var buffer = new ReadOnlySequence(line); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpContinuousRxKind.NeedMore, unit.Kind); + Assert.Equal(line.Length, buffer.Length); + } + + [Fact] + public async Task TryConsume_OverlongSplitAcrossReads_RejectsOnceTheCrlfArrives() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = WithCrlf(line); + var pipe = new Pipe(); + var parser = new NntpContinuousRxParser(); + await pipe.Writer.WriteAsync(wire.AsMemory(0, 1000)); + var first = await ReadAvailableAsync(pipe.Reader, parser); + Assert.Equal(NntpContinuousRxKind.NeedMore, first.Kind); + + await pipe.Writer.WriteAsync(wire.AsMemory(1000)); + var second = await ReadAvailableAsync(pipe.Reader, parser); + Assert.Equal(NntpParseStatus.CommandLineTooLong, second.Command.Status); + Assert.Equal(NntpVerb.None, second.Command.Verb); + } + + private static byte[] GroupLine(int length) + { + var line = new byte[length]; + "GROUP "u8.CopyTo(line); + line.AsSpan("GROUP "u8.Length).Fill((byte)'a'); + return line; + } + + private static byte[] WithCrlf(byte[] line) + { + var wire = new byte[line.Length + 2]; + line.CopyTo(wire); + wire[^2] = (byte)'\r'; + wire[^1] = (byte)'\n'; + return wire; + } + + private static async Task ReadAvailableAsync(PipeReader reader, NntpContinuousRxParser parser) + { + var read = await reader.ReadAsync(); + var buffer = read.Buffer; + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + reader.AdvanceTo(buffer.Start, buffer.End); + return unit; + } + + private static CapturedRxUnit Capture(NntpContinuousRxParser parser, NntpContinuousRxUnit unit) + { + var line = parser.CurrentCommandLine.Span; + return new CapturedRxUnit( + unit, + Encoding.ASCII.GetString(line), + Encoding.ASCII.GetString(unit.Command.ArgumentSpan(line))); + } + + private static async Task ReadOneAsync( + byte[] wire, + bool consumeTakeThisArticle, + int maxArticleBytes = 8 * 1024 * 1024) + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + if (wire.Length > 0) + { + await pipe.Writer.WriteAsync(wire); + } + + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var unit = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle, + maxArticleBytes, + CancellationToken.None); + return Capture(parser, unit); + } + + private static async Task ReadOneSegmentedAsync( + byte[] wire, + int segmentSize, + bool consumeTakeThisArticle) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, segmentSize), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + var read = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle, 8 * 1024 * 1024, CancellationToken.None) + .AsTask(); + await FramingPipe.WriteChunkedAsync(pipe.Writer, wire, segmentSize, CancellationToken.None); + await pipe.Writer.CompleteAsync(); + return Capture(parser, await read.WaitAsync(TimeSpan.FromSeconds(10))); + } + + private static async Task ReadSplitAsync( + byte[] wire, + int holdBack, + bool consumeTakeThisArticle) + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + var read = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle, 8 * 1024 * 1024, CancellationToken.None) + .AsTask(); + await pipe.Writer.WriteAsync(wire.AsMemory()[..^holdBack]); + await pipe.Writer.FlushAsync(); + await pipe.Writer.WriteAsync(wire.AsMemory()[^holdBack..]); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + return Capture(parser, await read.WaitAsync(TimeSpan.FromSeconds(5))); + } + + private static async Task> ReadAllAsync( + byte[] wire, + bool consumeTakeThisArticle, + bool stopAfterQuit = false, + int maxArticleBytes = 8 * 1024 * 1024) + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var units = new List(); + while (true) + { + var unit = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle, + maxArticleBytes, + CancellationToken.None); + if (unit.Kind == NntpContinuousRxKind.NeedMore) + { + break; + } + + var captured = Capture(parser, unit); + units.Add(captured); + if (stopAfterQuit + && captured.Kind == NntpContinuousRxKind.Command + && captured.Unit.Command.Verb == NntpVerb.Quit) + { + break; + } + } + + return units; + } + + private readonly struct CapturedRxUnit + { + public CapturedRxUnit(NntpContinuousRxUnit unit, string commandLine, string argument) + { + Unit = unit; + CommandLine = commandLine; + Argument = argument; + } + + public NntpContinuousRxUnit Unit { get; } + + public string CommandLine { get; } + + public string Argument { get; } + + public string MessageId => Argument; + + public NntpContinuousRxKind Kind => Unit.Kind; + + public NntpMultilineReadResult Article => Unit.Article; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpDelimiterSearchTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpDelimiterSearchTests.cs new file mode 100644 index 00000000..a12cde09 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/NntpDelimiterSearchTests.cs @@ -0,0 +1,100 @@ +using System.Buffers; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// +/// STREAM article terminator location: one \r\n.\r\n search, not per-CRLF line walks. +/// +public sealed class NntpDelimiterSearchTests +{ + [Fact] + public void EmptyArticle_AtStart_IsThreeByteTerminator() + { + var buffer = new ReadOnlySequence(".\r\nQUIT\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out var consumed)); + Assert.Equal(0, payload); + Assert.Equal(3, consumed); + } + + [Fact] + public void EmptyArticle_NotAtStart_IsNotThreeByteTerminator() + { + var buffer = new ReadOnlySequence(".\r\n"u8.ToArray()); + Assert.False(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: false, out _, out _)); + } + + [Fact] + public void FiveByteTerminator_PayloadIncludesLeadingCrlf() + { + var buffer = new ReadOnlySequence("data\r\n.\r\nNEXT\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: false, out var payload, out var consumed)); + Assert.Equal("data\r\n"u8.Length, payload); + Assert.Equal("data\r\n.\r\n"u8.Length, consumed); + } + + [Fact] + public void HelloDot_IsNotTerminator() + { + var buffer = new ReadOnlySequence("Hello.\r\nSomething else.\r\n.\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out _)); + Assert.Equal("Hello.\r\nSomething else.\r\n"u8.Length, payload); + } + + [Fact] + public void StuffedDotLine_IsNotTerminator() + { + var buffer = new ReadOnlySequence("..foo\r\nbar\r\n.\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out _)); + Assert.Equal("..foo\r\nbar\r\n"u8.Length, payload); + } + + [Fact] + public void IncompleteFiveByte_ReturnsFalse() + { + var buffer = new ReadOnlySequence("keep\r\n.\r"u8.ToArray()); + Assert.False(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out _, out _)); + } + + [Fact] + public void MultiSegment_FindsTerminatorSpanningSegments() + { + var first = "keep\r\n."u8.ToArray(); + var second = "\r\nNEXT"u8.ToArray(); + var segment1 = new SequenceSegment(first); + var segment2 = new SequenceSegment(second); + segment1.SetNext(segment2); + var buffer = new ReadOnlySequence(segment1, 0, segment2, second.Length); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out var consumed)); + Assert.Equal("keep\r\n"u8.Length, payload); + Assert.Equal("keep\r\n.\r\n"u8.Length, consumed); + } + + [Fact] + public void CrlfSplitAcrossSegments_StillFindsLine() + { + var first = "DATE\r"u8.ToArray(); + var second = "\n"u8.ToArray(); + var segment1 = new SequenceSegment(first); + var segment2 = new SequenceSegment(second); + segment1.SetNext(segment2); + var buffer = new ReadOnlySequence(segment1, 0, segment2, second.Length); + Assert.True(NntpDelimiterSearch.TryReadLine(ref buffer, out var line)); + Assert.Equal("DATE"u8.ToArray(), line.ToArray()); + Assert.True(buffer.IsEmpty); + } + + private sealed class SequenceSegment : ReadOnlySequenceSegment + { + public SequenceSegment(byte[] memory) + { + Memory = memory; + } + + public void SetNext(SequenceSegment next) + { + Next = next; + next.RunningIndex = RunningIndex + Memory.Length; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs new file mode 100644 index 00000000..f510abb6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs @@ -0,0 +1,159 @@ +using VectorNNTPD.ReaderService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.ReaderService.Tests.Session.Framing; + +/// +/// Segment-boundary destuff coverage for production . +/// +public sealed class SimdBulkMultilineFramerTests +{ + public static IEnumerable AllFixtureNames() + { + foreach (var entry in InnArticleCorpus.LoadManifest().Files) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable SegmentSizes() + { + foreach (var size in new[] { 1, 2, 4, 16, 4096 }) + { + yield return [size]; + } + } + + [Fact] + public async Task EmptyArticle() + { + var result = await FramingPipe.ReadSegmentedAsync(FramingWireFactory.EmptyArticle, 1); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(0, result.Payload.Length); + } + + [Fact] + public async Task FiveByteDelimiterIsDetected() + { + var result = await FramingPipe.ReadSegmentedAsync(FramingWireFactory.DataThenDelimiter(), 64); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("data\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task HelloDotIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("Hello.\r\nSomething else.\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, 8); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Hello.\r\nSomething else.\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task DotStuffedLineIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("..foo\r\nbar\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, 3); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(".foo\r\nbar\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + public async Task DelimiterSplitAtEveryBoundary(int split) + { + var wire = FramingWireFactory.WithTerminator("split-term\r\n"); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var holdBack = split == 0 ? 5 : Math.Max(1, 5 - split); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBack); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Theory] + [MemberData(nameof(SegmentSizes))] + public async Task Wire7_IdenticalAcrossSegmentSizes(int segmentSize) + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public async Task EveryInnFixture_MatchesBulkSemantics(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + var result = await FramingPipe.ReadOneAsync(wire); + + if (entry.FramingClassification == "complete_multiline_wire") + { + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + else + { + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + } + } + + [Fact] + public async Task ManyDotsHostile_DoesNotFalseTerminate() + { + var wire = FramingWireFactory.ManyDotsHostileArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + Assert.True(expected.AsSpan().IndexOf("Hello.\r\n"u8) >= 0); + + var result = await FramingPipe.ReadSegmentedAsync(wire, 17); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task IndexOfFiveByteDelimiter_RejectsTrailingPeriodLines() + { + var content = "Hello.\r\nSomething else.\r\n"; + var withoutTerm = await FramingPipe.ReadOneAsync(System.Text.Encoding.ASCII.GetBytes(content)); + Assert.Equal(NntpMultilineReadStatus.Incomplete, withoutTerm.Status); + + var withTerm = FramingWireFactory.WithTerminator(content); + var complete = await FramingPipe.ReadOneAsync(withTerm); + Assert.Equal(NntpMultilineReadStatus.Completed, complete.Status); + Assert.Equal(content, System.Text.Encoding.ASCII.GetString(complete.Payload.Span)); + } + + [Fact] + public async Task DelimiterSplitAcrossReads() + { + var wire = FramingWireFactory.WithTerminator("across-reads\r\n"); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task PipelinedSecondArticleRemains() + { + var wire = FramingWireFactory.PipelinedTwoArticles(); + var pipe = new System.IO.Pipelines.Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.Equal("article-one\r\n"u8.ToArray(), first.Payload.ToArray()); + Assert.Equal("article-two\r\n"u8.ToArray(), second.Payload.ToArray()); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/GroupCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/GroupCommandTests.cs new file mode 100644 index 00000000..721a6c01 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/GroupCommandTests.cs @@ -0,0 +1,273 @@ +using System.Buffers; +using System.IO.Pipelines; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class GroupCommandTests +{ + private static readonly NntpAuthorization Reader = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + [Fact] + public async Task Group_Existing_WritesPrecomputed211_AndSelectsGroup() + { + var snapshot = SampleSnapshot(); + Assert.True(snapshot.TryGet("misc.test"u8, out var group)); + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync(duplex, session, "GROUP misc.test", group.GroupSelectedLine); + Assert.True(session.HasSelectedGroup); + Assert.True(session.SelectedGroupName.SequenceEqual("misc.test"u8)); + } + + [Fact] + public async Task Group_IsCaseInsensitive_PreservesOriginalNameOnWire() + { + var snapshot = SampleSnapshot(); + Assert.True(snapshot.TryGet("misc.test"u8, out var group)); + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync(duplex, session, "GROUP MISC.TEST", group.GroupSelectedLine); + Assert.True(group.GroupSelectedLine.Span.SequenceEqual("211 2089 3000234 3002322 misc.test\r\n"u8)); + } + + [Fact] + public async Task Group_XAndJ_RemainDistinctFromProhibited() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("group.closed", "", 100, 1, NewsgroupPostingStatus.NoPostingOrPeerArticles), + new NewsgroupDefinition("group.junk", "", 50, 2, NewsgroupPostingStatus.PeerOnly), + new NewsgroupDefinition("group.n", "", 100, 1, NewsgroupPostingStatus.Prohibited), + ]); + Assert.True(snapshot.TryGet("group.closed"u8, out var closed)); + Assert.True(snapshot.TryGet("group.junk"u8, out var junk)); + Assert.True(snapshot.TryGet("group.n"u8, out var prohibited)); + Assert.Equal(NewsgroupPostingStatus.NoPostingOrPeerArticles, closed.PostingStatus); + Assert.Equal(NewsgroupPostingStatus.PeerOnly, junk.PostingStatus); + Assert.Equal(NewsgroupPostingStatus.Prohibited, prohibited.PostingStatus); + Assert.NotEqual(closed.PostingStatus, prohibited.PostingStatus); + Assert.NotEqual(junk.PostingStatus, prohibited.PostingStatus); + Assert.True(closed.ListActiveLine.Span.SequenceEqual("group.closed 100 1 x\r\n"u8)); + Assert.True(junk.ListActiveLine.Span.SequenceEqual("group.junk 50 2 j\r\n"u8)); + Assert.True(prohibited.ListActiveLine.Span.SequenceEqual("group.n 100 1 n\r\n"u8)); + + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync(duplex, session, "GROUP group.closed", closed.GroupSelectedLine); + Assert.True(closed.GroupSelectedLine.Span.SequenceEqual("211 100 1 100 group.closed\r\n"u8)); + await AssertExactAsync(duplex, session, "GROUP group.junk", junk.GroupSelectedLine); + Assert.True(junk.GroupSelectedLine.Span.SequenceEqual("211 49 2 50 group.junk\r\n"u8)); + } + + [Fact] + public async Task Group_Missing_Returns411_AndDoesNotChangeSelection() + { + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, "GROUP no.such.group", "411 No such newsgroup"); + Assert.False(session.HasSelectedGroup); + } + + [Fact] + public async Task ListGroup_IsRegisteredPlaceholder_Returns500() + { + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, "LISTGROUP misc.test", "500 Command not implemented"); + Assert.False(session.HasSelectedGroup); + } + + [Fact] + public async Task Group_DoesNotQueryMySql() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("misc.test", "Testing", 10, 2, (byte)'y')], + }; + var db = new NntpDbService( + factory, + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }), + NullLogger.Instance); + await db.StartAsync(CancellationToken.None); + await using var catalogue = new NewsgroupCatalogueService( + db, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + await catalogue.StartAsync(CancellationToken.None); + var queriesAfterLoad = factory.QueryNewsgroupsCount; + + await using var duplex = await GroupDuplex.CreateAsync(); + var session = duplex.CreateSession(catalogue); + await AssertLinePrefixAsync(duplex, session, "GROUP misc.test", "211 "); + Assert.Equal(queriesAfterLoad, factory.QueryNewsgroupsCount); + await catalogue.StopAsync(CancellationToken.None); + } + + private static NewsgroupSnapshot SampleSnapshot() => + NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "General testing", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + ]); + + private static async Task AssertExactAsync( + GroupDuplex duplex, + NntpSession session, + string command, + ReadOnlyMemory expected) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + var read = duplex.ReadExactAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.True((await read).AsSpan().SequenceEqual(expected.Span)); + } + + private static async Task AssertLineAsync(GroupDuplex duplex, NntpSession session, string command, string expected) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.Equal(expected, await duplex.ReadClientLineAsync()); + } + + private static async Task AssertLinePrefixAsync(GroupDuplex duplex, NntpSession session, string command, string prefix) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.StartsWith(prefix, await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + private sealed class GroupDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new GroupDuplex()); + + public NntpSession CreateSession(INewsgroupCatalogue catalogue) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + var session = new NntpSession( + connection, + NullLogger.Instance, + newsgroupCatalogue: catalogue); + session.SetAuthorization(Reader); + return session; + } + + public async Task ReadExactAsync(int length) + { + var buffer = new byte[length]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < length) + { + var result = await _serverToClient.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, length - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _serverToClient.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < length) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {length} bytes."); + } + } + + return buffer; + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task WaitForOutboundDeliveryAndPauseReadsAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync(VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/HelpCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/HelpCommandTests.cs new file mode 100644 index 00000000..0a5d3d48 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/HelpCommandTests.cs @@ -0,0 +1,304 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// HELP command — static syntax listing (RFC 3977 §7.2) with ABNF-consistent notation. +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class HelpCommandTests +{ + [Fact] + public async Task Help_Returns100_AndStaticSyntaxLines() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadMultilineResponseAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + var (status, body) = await readTask; + + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Equal("100 Help text follows", status); + Assert.Equal(Help.BodyLines, body); + Assert.DoesNotContain(body, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain("NEWGROUPS", body); + Assert.DoesNotContain("NEWNEWS", body); + AssertUnsupportedListVariantsAbsent(body); + AssertHelpBodyStructure(body); + } + + [Fact] + public async Task Help_IsIdenticalAcrossSessionStates() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + session.SetMode(NntpSessionMode.Reader); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadMultilineResponseAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + var (_, body) = await readTask; + + Assert.Equal(Help.BodyLines, body); + } + + [Fact] + public async Task Help_TrailingArgs_Returns501() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP MORE"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task Help_SessionRemainsOpen_ForSubsequentCommand() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); // greeting + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("100 Help text follows", await duplex.ReadClientLineAsync()); + var body = await duplex.ReadMultilineBodyAsync(); + Assert.Equal(Help.BodyLines, body); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + Assert.Equal(14, date.Length - 4); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void SyntaxLines_AreDeterministicAndUseAbnfConsistentNotation() + { + Assert.Equal(Help.SyntaxLines, Help.SyntaxLines.ToArray()); + Assert.Equal(Help.BodyLines, Help.BodyLines.ToArray()); + Assert.Contains("HELP", Help.SyntaxLines); + Assert.Contains("COMPRESS DEFLATE", Help.SyntaxLines); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + AssertUnsupportedListVariantsAbsent(Help.SyntaxLines); + Assert.Contains("LIST", Help.SyntaxLines); + Assert.Contains("LIST ACTIVE [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST COUNTS [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST HEADERS [MSGID / RANGE]", Help.SyntaxLines); + Assert.Contains("LIST NEWSGROUPS [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST OVERVIEW.FMT", Help.SyntaxLines); + Assert.Contains("LIST MOTD", Help.SyntaxLines); + + // ABNF-consistent: optional args use [], alternatives use /; no {…} or |. + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('{', StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('}', StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('|', StringComparison.Ordinal)); + Assert.Contains("ARTICLE [message-id / article-number]", Help.SyntaxLines); + Assert.Contains("AUTHINFO USER username", Help.SyntaxLines); + Assert.Contains("AUTHINFO PASS password", Help.SyntaxLines); + Assert.Contains("AUTHINFO SASL mechanism [initial-response]", Help.SyntaxLines); + Assert.Contains("GROUP newsgroup", Help.SyntaxLines); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Equals("GROUP [newsgroup]", StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.StartsWith("GROUP [", StringComparison.Ordinal)); + Assert.Contains("HDR header [range / message-id]", Help.SyntaxLines); + Assert.Contains("CHECK message-id", Help.SyntaxLines); + Assert.Contains("IHAVE message-id", Help.SyntaxLines); + Assert.Contains("TAKETHIS message-id", Help.SyntaxLines); + Assert.Contains("SPEEDTEST ", Help.SyntaxLines); + Assert.Contains("LISTGROUP [newsgroup [range]]", Help.SyntaxLines); + Assert.Contains("XOVER [range]", Help.SyntaxLines); + + Assert.Equal("ARTICLE [message-id / article-number]", Help.SyntaxLines[0]); + Assert.Equal("XOVER [range]", Help.SyntaxLines[^1]); + AssertHelpBodyStructure(Help.BodyLines); + } + + private static void AssertHelpBodyStructure(IReadOnlyList body) + { + Assert.Equal(Help.BodyLines, body); + + var syntaxEnd = Help.SyntaxLines.Count; + Assert.Equal(string.Empty, body[syntaxEnd]); + Assert.Equal("Range formats:", body[syntaxEnd + 1]); + Assert.Equal(Help.RangeHelpLines, body.Skip(syntaxEnd + 1).Take(Help.RangeHelpLines.Count).ToArray()); + + var afterRangeBlank = syntaxEnd + 1 + Help.RangeHelpLines.Count; + Assert.Equal(string.Empty, body[afterRangeBlank]); + Assert.Equal("Wildmat formats:", body[afterRangeBlank + 1]); + Assert.Equal(Help.WildmatHelpLines, body.Skip(afterRangeBlank + 1).Take(Help.WildmatHelpLines.Count).ToArray()); + + Assert.Contains("123 a single article number", body); + Assert.Contains("123-456 articles from 123 through 456 inclusive", body); + Assert.Contains("123- article 123 and all following article numbers", body); + Assert.Contains("* matches zero or more characters", body); + Assert.Contains("? matches exactly one character", body); + Assert.Contains("Examples: a* ; a*,!*b ; *.recovery", body); + Assert.DoesNotContain(body, l => l.Contains("regex", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(body, l => l.Contains("[^", StringComparison.Ordinal)); + } + + private static void AssertUnsupportedListVariantsAbsent(IReadOnlyList lines) + { + Assert.DoesNotContain(lines, l => l.Contains("ACTIVE.TIMES", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("DISTRIB.PATS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("DISTRIBUTIONS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("MODERATORS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("SUBSCRIPTIONS", StringComparison.Ordinal)); + } + + private sealed class HelpDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new HelpDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async Task<(string Status, List Body)> ReadMultilineResponseAsync() + { + // Caller must start this before DispatchAsync so FlushAsync backpressure can drain. + var status = await ReadClientLineAsync().ConfigureAwait(false); + var body = await ReadMultilineBodyAsync().ConfigureAwait(false); + return (status, body); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/IHaveCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/IHaveCommandTests.cs new file mode 100644 index 00000000..31fcf553 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/IHaveCommandTests.cs @@ -0,0 +1,598 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class IHaveCommandTests +{ + private static NntpAuthorization TransitAuth { get; } = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + // IHAVE is a Transit-only command. A Reader session answers it with the role gate before the message-id is + // checked against history, before the ingestion queue/budget is consulted, and before any article is read, so + // none of the Transit IHAVE outcomes (335/235/435/436/437) can occur. Each test keeps the dependency + // configuration of the former Transit IHAVE scenario and proves it is never touched. + private const string CommandNotAvailable = "502 Command not available on this server"; + + [Fact] + public async Task ValidIhave_Returns502_AndEnqueuesNothing() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await AssertStillFramedAsync(duplex); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.QueuedBytes); + } + + [Fact] + public async Task ValidIhave_UnknownNewsgroup_WantTrash_Returns502_AndWritesNoNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession( + queue, + newsLog: news, + transit: new TransitOptions { WantTrash = true, LogTrash = true }); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await AssertStillFramedAsync(duplex); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + Assert.Empty(news.Events); + } + + [Fact] + public async Task ValidIhave_DoesNotEvaluatePostFilter() + { + var filter = new CountingPostFilter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, postFilter: filter); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, filter.EvaluateCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task HistoryHit_Returns502_NotArticleNotWanted() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + history.Remember(""u8.ToArray()); + var writes = history.Writes.Count; + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 2 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, history, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Empty(news.Events); + await AssertStillFramedAsync(duplex); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + Assert.Equal(writes, history.Writes.Count); + } + + [Fact] + public async Task RedisUnavailable_Returns502_NotTransferDeferral() + { + var redis = new FakeRedisService { IsUnavailable = true }; + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 2 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task MalformedMessageId_Is501() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE not-an-id"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task QueueUnavailable_Returns502_NotTransferDeferral() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(DisabledArticleIngestionQueue.Instance); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ArticleLargerThanQueueBudget_Returns502_AndWritesNoRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { MaxArticleBytes = 1024 }, + transitQueueMemoryLimit: 16); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.QueuedBytes); + Assert.Empty(news.Events); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task BudgetExhausted_Returns502Immediately_AndDoesNotWait() + { + var held = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + Assert.False(queue.TryProbeCapacity()); + + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var response = duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await response.WaitAsync(TimeSpan.FromSeconds(2))); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task QueueCompleted_Returns502_AndLeavesAccounting() + { + var sample = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: sample.Payload.Length); + queue.Complete(); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void BudgetExhausted_WarningIsRateLimited() + { + IHaveAdmissionLog.ResetForTests(); + IHaveAdmissionLog.WarningInterval = TimeSpan.FromHours(1); + var recording = new RecordingLogger(); + var held = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + + IHaveAdmissionLog.BudgetExhausted(recording, queue); + IHaveAdmissionLog.BudgetExhausted(recording, queue); + var first = Assert.Single(recording.Warnings); + Assert.Contains("TransitQueueMemoryLimit exhausted", first, StringComparison.Ordinal); + Assert.Contains($"{held.Payload.Length}/{held.Payload.Length}", first, StringComparison.Ordinal); + Assert.DoesNotContain("Subject:", first, StringComparison.Ordinal); + + IHaveAdmissionLog.ResetForTests(); + IHaveAdmissionLog.BudgetExhausted(recording, queue); + Assert.Equal(2, recording.Warnings.Count); + IHaveAdmissionLog.ResetForTests(); + } + + [Fact] + public async Task TooLargeConfiguration_Returns502_AndWritesNoRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = 2, MaxArticleBytes = 16 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await AssertStillFramedAsync(duplex); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + Assert.Empty(news.Events); + } + + [Fact] + public async Task NewsWriter_IsNeverCalled_ForUnavailableIhave() + { + var news = new ThrowingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(0, news.WriteCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Capabilities_DoesNotAdvertiseIhave() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("CAPABILITIES"); + var lines = new List(); + string line; + do + { + line = await duplex.ReadClientLineAsync(); + lines.Add(line); + } + while (line != "."); + + Assert.DoesNotContain("IHAVE", lines); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task WantTrashFalse_UnknownGroup_Returns502_AndWritesNoRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession( + queue, + newsLog: news, + transit: new TransitOptions { WantTrash = false, LogTrash = false }, + catalogue: CarriedCatalogue("alt.test")); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Empty(news.Events); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Theory] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + public async Task IngressRejectionScenarios_Return502_AndWriteNoRejectedNews(string commandId) + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true, + transitPeerName: "BlueWorldHosting")); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE " + commandId); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await AssertStillFramedAsync(duplex); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + Assert.Empty(news.Events); + } + + private static async Task AssertStillFramedAsync(IHaveDuplex duplex) + { + // No 335 was sent, so no article body is expected; the next command is parsed as a command. + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + private static InboundArticle FillArticle(string messageId) => + CanonicalArticleText.CreateQueued(messageId, InboundArticleProducer.IHave); + + private static INewsgroupCatalogue CarriedCatalogue(string group) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition(group, string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)])); + + private sealed class RecordingLogger : ILogger + { + public List Warnings { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) + { + Warnings.Add(formatter(state, exception)); + } + } + + private sealed class NullScope : IDisposable + { + public static NullScope Instance { get; } = new(); + + public void Dispose() + { + } + } + } + + private sealed class IHaveDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + + public static Task CreateAsync() => Task.FromResult(new IHaveDuplex()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IHistoryDb? history = null, + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null, + TransitOptions? transit = null, + INewsgroupCatalogue? catalogue = null) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: history, + postFilter: postFilter, + newsgroupCatalogue: catalogue, + newsLog: newsLog, + transit: transit); + } + + public async Task WriteClientLineAsync(string line) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(line + "\r\n")); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/ListCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/ListCommandTests.cs new file mode 100644 index 00000000..5c05149f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/ListCommandTests.cs @@ -0,0 +1,679 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class ListCommandTests +{ + private static readonly NntpAuthorization Reader = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + [Fact] + public async Task List_AndListActive_WritePrecomputedBytes() + { + var snapshot = SampleSnapshot(); + var catalogue = new StaticNewsgroupCatalogue(snapshot); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(catalogue); + + await AssertExactAsync(duplex, session, "LIST", snapshot.ListActiveComplete); + await AssertExactAsync(duplex, session, "LIST ACTIVE", snapshot.ListActiveComplete); + Assert.Equal(2, catalogue.CurrentReadCount); + } + + [Fact] + public async Task ListActive_Wildmat_MatchesGroupNameOnly() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "alt.rfc-writers.recovery 4 1 y\r\n"u8.ToArray(), + "tx.natives.recovery 89 56 n\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST ACTIVE *.recovery", expected); + } + + [Fact] + public async Task ListActive_EmitsExactXAndJStatusBytes() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("group.closed", "", 100, 1, NewsgroupPostingStatus.NoPostingOrPeerArticles), + new NewsgroupDefinition("group.junk", "", 100, 1, NewsgroupPostingStatus.PeerOnly), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "group.closed 100 1 x\r\n"u8.ToArray(), + "group.junk 100 1 j\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST ACTIVE", expected); + Assert.DoesNotContain("=", Encoding.ASCII.GetString(expected)); + } + + [Fact] + public async Task ListActive_Wildmat_PreservesXAndJStatusBytes() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("alt.closed", "", 10, 1, NewsgroupPostingStatus.NoPostingOrPeerArticles), + new NewsgroupDefinition("alt.junk", "", 20, 2, NewsgroupPostingStatus.PeerOnly), + new NewsgroupDefinition("misc.test", "", 3, 1, NewsgroupPostingStatus.Allowed), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "alt.closed 10 1 x\r\n"u8.ToArray(), + "alt.junk 20 2 j\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST ACTIVE alt.*", expected); + } + + [Fact] + public async Task ListActive_CommaWildmat_UsesRightmostMatch() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("aaa", "", 3, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("abb", "", 4, 1, NewsgroupPostingStatus.Prohibited), + new NewsgroupDefinition("ccb", "", 5, 1, NewsgroupPostingStatus.Moderated), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "aaa 3 1 y\r\n"u8.ToArray(), + "ccb 5 1 m\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST ACTIVE a*,!*b,*c*", expected); + } + + [Fact] + public async Task ListCounts_WritesPrecomputedFiveFieldLines() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync(duplex, session, "LIST COUNTS", snapshot.ListCountsComplete); + + var wire = Encoding.ASCII.GetString(snapshot.ListCountsComplete.Span); + Assert.StartsWith("215 list of newsgroups follows\r\n", wire, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", wire, StringComparison.Ordinal); + Assert.Contains("misc.test 3002322 3000234 2089 y\r\n", wire, StringComparison.Ordinal); + Assert.DoesNotContain('\t', wire); + } + + [Fact] + public async Task ListCounts_Estimate_HighGreaterThanLow() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync( + duplex, + session, + "LIST COUNTS", + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test 3002322 3000234 2089 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator)); + Assert.Equal(2089UL, Assert.Single(snapshot.Groups).EstimatedCount); + } + + [Fact] + public async Task ListCounts_Estimate_HighEqualsLow_UsesGroupSemantics() + { + var emptyZero = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("local.empty-zero", "", 0, 0, NewsgroupPostingStatus.Allowed), + ]); + var singleArticle = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("local.one", "", 5, 5, NewsgroupPostingStatus.Allowed), + ]); + Assert.Equal(0UL, Assert.Single(emptyZero.Groups).EstimatedCount); + Assert.Equal(1UL, Assert.Single(singleArticle.Groups).EstimatedCount); + + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(emptyZero)); + await AssertExactAsync( + duplex, + session, + "LIST COUNTS", + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "local.empty-zero 0 0 0 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator)); + session = duplex.CreateSession(new StaticNewsgroupCatalogue(singleArticle)); + await AssertExactAsync( + duplex, + session, + "LIST COUNTS", + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "local.one 5 5 1 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator)); + } + + [Fact] + public async Task ListCounts_Estimate_HighLessThanLow_IsZero() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("local.empty", "", 7, 8, NewsgroupPostingStatus.Allowed), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync( + duplex, + session, + "LIST COUNTS", + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "local.empty 7 8 0 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator)); + Assert.Equal(0UL, Assert.Single(snapshot.Groups).EstimatedCount); + } + + [Fact] + public async Task ListCounts_EmitsExactStatusOctets() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("group.y", "", 10, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("group.n", "", 10, 1, NewsgroupPostingStatus.Prohibited), + new NewsgroupDefinition("group.m", "", 10, 1, NewsgroupPostingStatus.Moderated), + new NewsgroupDefinition("group.x", "", 10, 1, NewsgroupPostingStatus.NoPostingOrPeerArticles), + new NewsgroupDefinition("group.j", "", 10, 1, NewsgroupPostingStatus.PeerOnly), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync( + duplex, + session, + "LIST COUNTS", + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "group.j 10 1 10 j\r\n"u8.ToArray(), + "group.m 10 1 10 m\r\n"u8.ToArray(), + "group.n 10 1 10 n\r\n"u8.ToArray(), + "group.x 10 1 10 x\r\n"u8.ToArray(), + "group.y 10 1 10 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator)); + } + + [Fact] + public async Task ListCounts_Wildmat_MatchesGroupNameOnly() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "alt.rfc-writers.recovery 4 1 4 y\r\n"u8.ToArray(), + "tx.natives.recovery 89 56 34 n\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST COUNTS *.recovery", expected); + } + + [Fact] + public async Task ListCounts_CommaWildmat_UsesRightmostMatch() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("aaa", "", 3, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("abb", "", 4, 1, NewsgroupPostingStatus.Prohibited), + new NewsgroupDefinition("ccb", "", 5, 1, NewsgroupPostingStatus.Moderated), + ]); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "aaa 3 1 3 y\r\n"u8.ToArray(), + "ccb 5 1 5 m\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST COUNTS a*,!*b,*c*", expected); + } + + [Fact] + public async Task ListCounts_NoMatch_ReturnsEmptyMultiline215() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + var expected = Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator); + await AssertExactAsync(duplex, session, "LIST COUNTS no.such.*", expected); + } + + [Fact] + public async Task ListCounts_CapturesExactlyOneSnapshot() + { + var first = SampleSnapshot(); + var second = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("only.new", "n", 2, 1, NewsgroupPostingStatus.Allowed), + ]); + var catalogue = new SwapOnSecondReadCatalogue(first, second); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(catalogue); + + await AssertExactAsync(duplex, session, "LIST COUNTS", first.ListCountsComplete); + Assert.Equal(1, catalogue.Reads); + } + + [Fact] + public async Task ListActive_MalformedWildmat_Returns501() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, "LIST ACTIVE u[ks].*", "501 Syntax error"); + await AssertLineAsync(duplex, session, "LIST COUNTS u[ks].*", "501 Syntax error"); + } + + [Fact] + public async Task ListActive_NoMatch_ReturnsEmptyMultiline215() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + var expected = Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator); + await AssertExactAsync(duplex, session, "LIST ACTIVE no.such.*", expected); + } + + [Fact] + public async Task ListNewsgroups_WritesPrecomputedUtf8Lines() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + await AssertExactAsync(duplex, session, "LIST NEWSGROUPS", snapshot.ListNewsgroupsComplete); + } + + [Fact] + public async Task ListNewsgroups_Wildmat_MatchesGroupName_WritesPrecomputedLine() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test General testing\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator); + + await AssertExactAsync(duplex, session, "LIST NEWSGROUPS misc.*", expected); + } + + [Fact] + public async Task ListNewsgroups_Wildmat_DoesNotMatchDescription() + { + var snapshot = SampleSnapshot(); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(snapshot)); + var expected = Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator); + await AssertExactAsync(duplex, session, "LIST NEWSGROUPS *Testing*", expected); + } + + [Fact] + public async Task ListNewsgroups_NoMatch_ReturnsEmptyMultiline215() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + var expected = Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator); + await AssertExactAsync(duplex, session, "LIST NEWSGROUPS zzz.*", expected); + } + + [Fact] + public async Task List_CapturesExactlyOneSnapshot() + { + var first = SampleSnapshot(); + var second = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("only.new", "n", 2, 1, NewsgroupPostingStatus.Allowed), + ]); + var catalogue = new SwapOnSecondReadCatalogue(first, second); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(catalogue); + + await AssertExactAsync(duplex, session, "LIST", first.ListActiveComplete); + Assert.Equal(1, catalogue.Reads); + } + + [Fact] + public async Task List_AfterRefresh_NewCommandsSeeNewSnapshot() + { + var first = SampleSnapshot(); + var second = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("only.new", "n", 2, 1, NewsgroupPostingStatus.Allowed), + ]); + var catalogue = new StaticNewsgroupCatalogue(first); + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(catalogue); + await AssertExactAsync(duplex, session, "LIST", first.ListActiveComplete); + catalogue.Publish(second); + await AssertExactAsync(duplex, session, "LIST NEWSGROUPS", second.ListNewsgroupsComplete); + } + + [Fact] + public async Task ListOverviewFmt_WritesPrecomputedStandardFields() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertExactAsync(duplex, session, "LIST OVERVIEW.FMT", NntpResponses.ListOverviewFmtComplete); + + var wire = Encoding.ASCII.GetString(NntpResponses.ListOverviewFmtComplete.Span); + Assert.Equal( + "215 Order of fields in overview database.\r\n" + + "Subject:\r\nFrom:\r\nDate:\r\nMessage-ID:\r\nReferences:\r\nBytes:\r\nLines:\r\n.\r\n", + wire); + Assert.Contains("Bytes:\r\n", wire, StringComparison.Ordinal); + Assert.Contains("Lines:\r\n", wire, StringComparison.Ordinal); + Assert.DoesNotContain(":bytes", wire, StringComparison.Ordinal); + Assert.DoesNotContain(":lines", wire, StringComparison.Ordinal); + Assert.True(NntpResponses.ListOverviewFmtComplete.Span.EndsWith(".\r\n"u8)); + } + + [Theory] + [InlineData("LIST HEADERS")] + [InlineData("LIST HEADERS MSGID")] + [InlineData("LIST HEADERS RANGE")] + [InlineData("list headers msgid")] + public async Task ListHeaders_WritesSamePrecomputedFieldList(string command) + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertExactAsync(duplex, session, command, NntpResponses.ListHeadersComplete); + + var wire = Encoding.ASCII.GetString(NntpResponses.ListHeadersComplete.Span); + Assert.Equal( + "215 headers and metadata items supported:\r\n" + + "Subject\r\nFrom\r\nDate\r\nMessage-ID\r\nReferences\r\n:bytes\r\n:lines\r\n.\r\n", + wire); + Assert.DoesNotContain("Subject:", wire, StringComparison.Ordinal); + Assert.DoesNotContain("Bytes:", wire, StringComparison.Ordinal); + Assert.DoesNotContain("Lines:", wire, StringComparison.Ordinal); + Assert.True(NntpResponses.ListHeadersComplete.Span.EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task ListMotd_RemainsRecognizedWithoutStoredData() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, "LIST MOTD", "503 Data item not stored"); + } + + [Theory] + [InlineData("LIST ACTIVE.TIMES")] + [InlineData("LIST DISTRIBUTIONS")] + [InlineData("LIST DISTRIB.PATS")] + [InlineData("LIST MODERATORS")] + [InlineData("LIST SUBSCRIPTIONS")] + [InlineData("LIST SOMETHING-THAT-DOES-NOT-EXIST")] + public async Task UnknownListKeyword_Returns501UnknownCommandVariant(string command) + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, command, "501 Unknown command variant"); + } + + [Theory] + [InlineData("LIST ACTIVE foo bar")] + [InlineData("LIST COUNTS foo bar")] + [InlineData("LIST MOTD extra")] + [InlineData("LIST OVERVIEW.FMT extra")] + [InlineData("LIST HEADERS MSGID extra")] + public async Task MalformedList_Returns501SyntaxError(string command) + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, command, "501 Syntax error"); + } + + [Fact] + public async Task ListHeaders_InvalidArgument_Returns501SyntaxError() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + await AssertLineAsync(duplex, session, "LIST HEADERS FOO", "501 Syntax error"); + } + + [Fact] + public async Task Capabilities_AdvertisesOnlyImplementedListKeywords() + { + await using var duplex = await ListDuplex.CreateAsync(); + var session = duplex.CreateSession(new StaticNewsgroupCatalogue(SampleSnapshot())); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + var body = await duplex.ReadUntilTerminatorAsync(); + var lines = body.Split('\n'); + Assert.Equal("101 Capability list:", lines[0]); + Assert.Equal("VERSION 2", lines[1]); + Assert.Contains("LIST ACTIVE COUNTS HEADERS NEWSGROUPS OVERVIEW.FMT", body, StringComparison.Ordinal); + Assert.DoesNotContain("ACTIVE.TIMES", body, StringComparison.Ordinal); + Assert.DoesNotContain("MOTD", body, StringComparison.Ordinal); + Assert.DoesNotContain("DISTRIB", body, StringComparison.Ordinal); + Assert.DoesNotContain("MODERATORS", body, StringComparison.Ordinal); + Assert.DoesNotContain("SUBSCRIPTIONS", body, StringComparison.Ordinal); + Assert.DoesNotContain("\nOVER\n", body, StringComparison.Ordinal); + Assert.DoesNotContain("\nHDR\n", body, StringComparison.Ordinal); + Assert.DoesNotContain("NEWNEWS", body, StringComparison.Ordinal); + Assert.DoesNotContain("NEWGROUPS", body, StringComparison.Ordinal); + Assert.Equal(".", lines[^1]); + } + + private static NewsgroupSnapshot SampleSnapshot() => + NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("alt.rfc-writers.recovery", "RFC Writers Recovery", 4, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("misc.test", "General testing", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("tx.natives.recovery", "Texas Natives Recovery", 89, 56, NewsgroupPostingStatus.Prohibited), + ]); + + private static async Task AssertExactAsync( + ListDuplex duplex, + NntpSession session, + string command, + ReadOnlyMemory expected) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + var read = duplex.ReadExactAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.True((await read).AsSpan().SequenceEqual(expected.Span)); + } + + private static async Task AssertLineAsync(ListDuplex duplex, NntpSession session, string command, string expected) + { + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.Equal(expected, await duplex.ReadClientLineAsync()); + } + + private static byte[] Concat(params ReadOnlyMemory[] parts) + { + var total = parts.Sum(p => p.Length); + var result = new byte[total]; + var offset = 0; + foreach (var part in parts) + { + part.Span.CopyTo(result.AsSpan(offset)); + offset += part.Length; + } + + return result; + } + + private sealed class SwapOnSecondReadCatalogue : INewsgroupCatalogue + { + private readonly NewsgroupSnapshot _first; + private readonly NewsgroupSnapshot _second; + + public SwapOnSecondReadCatalogue(NewsgroupSnapshot first, NewsgroupSnapshot second) + { + _first = first; + _second = second; + } + + public int Reads { get; private set; } + + public NewsgroupSnapshot Current + { + get + { + Reads++; + return Reads == 1 ? _first : _second; + } + } + } + + private sealed class ListDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new ListDuplex()); + + public NntpSession CreateSession(INewsgroupCatalogue catalogue) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + var session = new NntpSession( + connection, + NullLogger.Instance, + newsgroupCatalogue: catalogue); + session.SetAuthorization(Reader); + return session; + } + + public async Task ReadExactAsync(int length) + { + var buffer = new byte[length]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < length) + { + var result = await _serverToClient.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, length - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _serverToClient.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < length) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {length} bytes."); + } + } + + return buffer; + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task ReadUntilTerminatorAsync() + { + var lines = new List(); + string line; + do + { + line = await ReadClientLineAsync(); + lines.Add(line); + } + while (line != "."); + + return string.Join('\n', lines); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task WaitForOutboundDeliveryAndPauseReadsAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync(VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/ModeratedPostCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/ModeratedPostCommandTests.cs new file mode 100644 index 00000000..6b9b94e3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/ModeratedPostCommandTests.cs @@ -0,0 +1,1462 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Email; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Email; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Moderation; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class ModeratedPostCommandTests +{ + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(20); + + [Fact] + public async Task Y_OrdinaryUser_NoApproved_Returns240() + { + var outcome = await PostAsync("group.y", extraHeaders: "", user: MapNntpAuthenticationProvider.NormalUser); + AssertAccepted(outcome); + } + + [Theory] + [InlineData("group.n")] + [InlineData("group.x")] + [InlineData("group.j")] + public async Task Njx_NoApproved_Returns441(string group) + { + var outcome = await PostAsync(group, extraHeaders: "", user: MapNntpAuthenticationProvider.NormalUser); + AssertRejected(outcome); + } + + [Fact] + public async Task M_OrdinaryUser_NoApproved_TakesModerationPath() + { + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Equal(0, outcome.History.PeekCalls); + Assert.Equal(0, outcome.History.RememberCalls); + var recorded = Assert.Single(submission.Submissions); + Assert.Equal("group.a", recorded.TargetModeratedGroup); + Assert.Equal("moderator-a@example.com", recorded.ModeratorAddress); + Assert.Equal(MapNntpAuthenticationProvider.NormalUser, recorded.AuthenticatedUsername); + Assert.Equal("", recorded.MessageId); + var text = Encoding.ASCII.GetString(recorded.ProtoArticle.Span); + Assert.Contains("Message-ID: ", text, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", text, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Date:", text, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", text, StringComparison.Ordinal); + Assert.DoesNotContain("Path: .POSTED", text, StringComparison.Ordinal); + } + + [Fact] + public async Task M_OrdinaryUser_DoesNotEvaluatePostFilter() + { + var filter = new CountingPostFilter(); + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + postFilter: filter); + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(0, filter.EvaluateCalls); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Single(submission.Submissions); + } + + [Fact] + public async Task M_OrdinaryUser_WritesModeratedNews_NotAccepted() + { + var news = new RecordingNewsLogWriter(); + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + newsLog: news); + Assert.Equal("240 Article received OK", outcome.Response); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Moderated, evt.Disposition); + Assert.NotEqual(NewsLogDisposition.Accepted, evt.Disposition); + Assert.NotEqual((byte)'?', (byte)evt.Disposition); + Assert.True(evt.MessageId.Span.SequenceEqual(""u8)); + var timestamp = evt.Timestamp == default + ? new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero) + : evt.Timestamp; + var buffer = new byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + var line = Encoding.ASCII.GetString(buffer.AsSpan(0, written)); + Assert.Contains(" m ? ", line, StringComparison.Ordinal); + Assert.DoesNotContain(" + ", line, StringComparison.Ordinal); + Assert.DoesNotContain(" " + evt.Size + " ?", line, StringComparison.Ordinal); + } + + [Fact] + public async Task M_Unauthenticated_Approved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: null); + AssertRejected(outcome); + } + + [Fact] + public async Task M_NormalUser_Approved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.NormalUser); + AssertRejected(outcome); + } + + [Fact] + public async Task M_CorrectModerator_CorrectApproved_Returns240() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertAccepted(outcome); + var text = Encoding.ASCII.GetString(outcome.Admitted!.Payload.Span); + Assert.Contains("Approved: moderator-a@example.com", text, StringComparison.Ordinal); + Assert.Contains("Injection-Info:", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date:", text, StringComparison.Ordinal); + } + + [Fact] + public async Task M_CorrectModerator_WrongApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-b@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task M_WrongModerator_CorrectApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorB); + AssertRejected(outcome); + } + + [Fact] + public async Task M_WrongModerator_WrongApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: other@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorB); + AssertRejected(outcome); + } + + [Fact] + public async Task M_MalformedApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: 1\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task M_ConflictingApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\nApproved: moderator-b@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task M_IdenticalApprovedDuplicates_AreAccepted() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\nApproved: Moderator-A@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertAccepted(outcome); + } + + [Fact] + public async Task M_ApprovedHeaderNameCase_IsIgnored() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "aPpRoVeD: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertAccepted(outcome); + } + + [Fact] + public async Task M_ApprovedWhitespace_IsIgnored() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com \r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertAccepted(outcome); + } + + [Fact] + public async Task CrossPost_YY_Returns240() + { + var outcome = await PostAsync( + "group.y,group.y2", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser); + AssertAccepted(outcome); + } + + [Fact] + public async Task CrossPost_YM_NoApproved_ForwardsLeftmostModerated() + { + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.y,group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + var recorded = Assert.Single(submission.Submissions); + Assert.Equal("group.a", recorded.TargetModeratedGroup); + Assert.Equal("moderator-a@example.com", recorded.ModeratorAddress); + } + + [Fact] + public async Task CrossPost_YM_AuthorizedModerator_Returns240() + { + var outcome = await PostAsync( + "group.y,group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertAccepted(outcome); + } + + [Fact] + public async Task CrossPost_MM_SameModerator_Returns240() + { + var authorization = ModeratorTestSnapshot.Create( + ("group.a", "shared@example.com", MapNntpAuthenticationProvider.ModeratorA), + ("group.shared", "shared@example.com", MapNntpAuthenticationProvider.ModeratorA)); + var snapshot = Snapshot( + Allowed("group.y"), + Group("group.a", NewsgroupPostingStatus.Moderated), + Group("group.shared", NewsgroupPostingStatus.Moderated)); + var outcome = await PostAsync( + "group.a,group.shared", + extraHeaders: "Approved: shared@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA, + snapshot: snapshot, + authorization: authorization); + AssertAccepted(outcome); + } + + [Fact] + public async Task CrossPost_MM_DifferentModerators_OneApproved_Returns441() + { + var outcome = await PostAsync( + "group.a,group.b", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task CrossPost_MM_DifferentModerators_NoApproved_ForwardsLeftmostOnly() + { + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.a,group.b", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + Assert.Equal("240 Article received OK", outcome.Response); + var recorded = Assert.Single(submission.Submissions); + Assert.Equal("group.a", recorded.TargetModeratedGroup); + Assert.Equal("moderator-a@example.com", recorded.ModeratorAddress); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + } + + [Theory] + [InlineData("group.a,group.x")] + [InlineData("group.a,group.j")] + [InlineData("group.a,group.n")] + public async Task CrossPost_MWithRejectedStatus_Returns441(string groups) + { + var outcome = await PostAsync(groups, extraHeaders: "", user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Theory] + [InlineData("group.a,group.n")] + [InlineData("group.a,group.x")] + [InlineData("group.a,group.j")] + public async Task CrossPost_ApprovedDoesNotBypassRejectedStatus(string groups) + { + var outcome = await PostAsync( + groups, + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task UnresolvedModerator_Returns441_WithoutSubmission() + { + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.orphan", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: Snapshot(Group("group.orphan", NewsgroupPostingStatus.Moderated)), + submission: submission); + AssertRejected(outcome); + Assert.Empty(submission.Submissions); + } + + [Fact] + public async Task ModerationDeliveryFailure_Returns441() + { + var submission = new RecordingModerationSubmissionService(ModerationSubmissionStatus.Failed, "smtp failed"); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + AssertRejected(outcome); + Assert.Single(submission.Submissions); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Equal(0, outcome.History.RememberCalls); + } + + [Fact] + public async Task UnavailableSubmission_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: UnavailableModerationSubmissionService.Instance); + AssertRejected(outcome); + } + + [Fact] + public async Task ModeratorReinject_WithoutApproved_IsModerationPathNotInjection() + { + var submission = new RecordingModerationSubmissionService(); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.ModeratorA, + submission: submission); + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Single(submission.Submissions); + } + + [Fact] + public async Task ModeratorReinject_WrongApproved_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-b@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task ApprovedValueSplitAcrossPipeSegments_IsAuthorized() + { + var article = Article("group.a", "Approved: moderator-a@example.com\r\n") + ".\r\n"; + var outcome = await PostRawAsync( + article, + user: MapNntpAuthenticationProvider.ModeratorA, + chunkSize: 3); + AssertAccepted(outcome); + } + + [Fact] + public async Task ApprovedHeaderNameSplitAcrossPipeSegments_IsAuthorized() + { + var article = Article("group.a", "Approved: moderator-a@example.com\r\n") + ".\r\n"; + var outcome = await PostRawAsync( + article, + user: MapNntpAuthenticationProvider.ModeratorA, + chunkSize: 1); + AssertAccepted(outcome); + } + + [Fact] + public async Task LargeModeratedArticle_DrainsAndSubmits() + { + var submission = new RecordingModerationSubmissionService(); + var body = new string('Z', 8000) + "\r\n"; + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + body: body); + Assert.Equal("240 Article received OK", outcome.Response); + var recorded = Assert.Single(submission.Submissions); + Assert.Contains("ZZZZ", Encoding.ASCII.GetString(recorded.ProtoArticle.Span), StringComparison.Ordinal); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + } + + [Fact] + public async Task DotStuffedModeratedArticle_PreservesStuffingOnSubmission() + { + var submission = new RecordingModerationSubmissionService(); + _ = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + body: "..dot\r\n"); + var recorded = Assert.Single(submission.Submissions); + var proto = Encoding.ASCII.GetString(recorded.ProtoArticle.Span); + Assert.Contains("..dot\r\n", proto, StringComparison.Ordinal); + var destuffed = Encoding.ASCII.GetString( + ModerationEmailComposer.DestuffProtoArticle(recorded.ProtoArticle.Span)); + Assert.Contains(".dot\r\n", destuffed, StringComparison.Ordinal); + } + + [Fact] + public async Task CancellationDuringModerationPath_DoesNotWriteFinalResponse() + { + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession( + NewQueue(), + new RecordingHistoryDb(), + new StaticNewsgroupCatalogue(DefaultSnapshot()), + StandardAuthorization(), + new RecordingModerationSubmissionService(), + MapNntpAuthenticationProvider.CreateStandard()); + using var cts = new CancellationTokenSource(); + var run = session.RunAsync(cts.Token); + _ = await duplex.ReadClientLineAsync(); + await AuthenticateAsync(duplex, MapNntpAuthenticationProvider.NormalUser, MapNntpAuthenticationProvider.NormalUserPassword); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article("group.a")); + await cts.CancelAsync(); + await run.WaitAsync(Safety); + } + + [Theory] + [InlineData("group.n")] + [InlineData("group.x")] + [InlineData("group.j")] + public async Task Njx_ApprovedDoesNotBypassProhibitedStatus(string group) + { + var outcome = await PostAsync( + group, + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task FailedAuthinfoPass_DoesNotLeaveModeratorIdentity_ApprovedReturns441() + { + await using var duplex = new PostDuplex(); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + var session = duplex.CreateSession( + queue, + history, + new StaticNewsgroupCatalogue(DefaultSnapshot()), + StandardAuthorization(), + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + MapNntpAuthenticationProvider.CreateStandard()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER " + MapNntpAuthenticationProvider.ModeratorA); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS not-the-moderator-password"); + Assert.StartsWith("481 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + Assert.False(session.Authorization.PostingPermitted); + + // Existing seam: posting privilege without an AUTHINFO principal (same as unauthenticated POST tests). + session.SetAuthorization(MapNntpAuthenticationProvider.PosterPrivileges); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.TryAdmitCalls); + Assert.Equal(0, history.PeekCalls); + Assert.Equal(0, history.RememberCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AuthenticatedModerator_FailedReAuthinfo_IsRejectedAndIdentityPreserved() + { + await using var duplex = new PostDuplex(); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + var session = duplex.CreateSession( + queue, + history, + new StaticNewsgroupCatalogue(DefaultSnapshot()), + StandardAuthorization(), + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + MapNntpAuthenticationProvider.CreateStandard()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await AuthenticateAsync( + duplex, + MapNntpAuthenticationProvider.ModeratorA, + MapNntpAuthenticationProvider.ModeratorAPassword); + + await duplex.WriteClientLineAsync("AUTHINFO USER " + MapNntpAuthenticationProvider.ModeratorA); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS not-the-moderator-password"); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal(MapNntpAuthenticationProvider.ModeratorA, session.Authentication.Username); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(1, history.PeekCalls); + Assert.Equal(1, history.RememberCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task MessageId_LookingLikeModerator_DoesNotAuthorize() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: string.Empty, + user: MapNntpAuthenticationProvider.NormalUser, + messageId: ""); + AssertRejected(outcome); + } + + [Fact] + public async Task ModeratorA_CannotApproveModeratorBGroup() + { + var outcome = await PostAsync( + "group.b", + extraHeaders: "Approved: moderator-b@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA); + AssertRejected(outcome); + } + + [Fact] + public async Task AuthenticatedModerator_DoesNotGrantControlCancel() + { + await using var duplex = new PostDuplex(); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + var session = duplex.CreateSession( + queue, + history, + new StaticNewsgroupCatalogue(DefaultSnapshot()), + StandardAuthorization(), + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + MapNntpAuthenticationProvider.CreateStandard()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await AuthenticateAsync(duplex, MapNntpAuthenticationProvider.ModeratorA, PasswordFor(MapNntpAuthenticationProvider.ModeratorA)); + Assert.False(session.Authorization.ControlCancelPermitted); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + Article( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\nControl: cancel \r\n") + + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.TryAdmitCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Post_DoesNotQueryModeratorRepository() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "group.a", + ModeratorAddress = "moderator-a@example.com", + AccountName = MapNntpAuthenticationProvider.ModeratorA, + }); + await using var catalogue = new ModeratorCatalogueService( + repository, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + await catalogue.StartAsync(CancellationToken.None); + Assert.Equal(1, repository.QueryCount); + + await using var duplex = new PostDuplex(); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + var session = duplex.CreateSession( + queue, + history, + new StaticNewsgroupCatalogue(DefaultSnapshot()), + catalogue.Current, + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Accepted), + MapNntpAuthenticationProvider.CreateStandard(), + catalogue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await AuthenticateAsync(duplex, MapNntpAuthenticationProvider.ModeratorA, PasswordFor(MapNntpAuthenticationProvider.ModeratorA)); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n", messageId: "") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, repository.QueryCount); + + await catalogue.StopAsync(CancellationToken.None); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task From_CannotImpersonateModerator_NormalUser_Returns441() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.NormalUser, + from: "moderator-a@example.com"); + AssertRejected(outcome); + } + + [Fact] + public async Task From_MatchingMailbox_SucceedsOnlyForAuthenticatedModerator() + { + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA, + from: "moderator-a@example.com"); + AssertAccepted(outcome); + } + + [Fact] + public async Task PgpAuthorities_DoNotAuthorizeOrdinaryModeratedPost() + { + var control = new ControlOptions + { + PgpAuthorities = new PgpAuthoritiesOptions + { + Authorities = + [ + new PgpAuthorityOptions + { + Name = "TEST-HIERARCHY", + Contact = "moderator-a@example.com", + Authorizations = + [ + new PgpAuthorityAuthorizationOptions + { + From = "moderator-a@example.com", + Message = "newgroup", + Newsgroups = "group.a", + VerificationIdentity = "moderator-a@example.com", + }, + ], + }, + ], + }, + }; + Assert.Equal("moderator-a@example.com", control.PgpAuthorities.Authorities[0].Authorizations[0].From); + Assert.True(new ControlOptionsValidator().Validate(null, control).Succeeded); + + var authorization = ModeratorTestSnapshot.Create( + ("group.a", "moderator-a@example.com", MapNntpAuthenticationProvider.ModeratorA)); + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.NormalUser, + authorization: authorization); + AssertRejected(outcome); + } + + [Fact] + public async Task SessionAuthentication_IsIsolated_ModeratorThenNormal() + { + await AssertIsolatedSessionsAsync( + firstUser: MapNntpAuthenticationProvider.ModeratorA, + firstExpected: "240 Article received OK", + secondUser: MapNntpAuthenticationProvider.NormalUser, + secondExpected: "441 Posting failed"); + } + + [Fact] + public async Task SessionAuthentication_IsIsolated_NormalThenModerator() + { + await AssertIsolatedSessionsAsync( + firstUser: MapNntpAuthenticationProvider.NormalUser, + firstExpected: "441 Posting failed", + secondUser: MapNntpAuthenticationProvider.ModeratorA, + secondExpected: "240 Article received OK"); + } + + [Fact] + public async Task MessageId_UnapprovedPreserved_ThenReinjectsSameId() + { + const string originalId = ""; + var submission = new RecordingModerationSubmissionService(); + var unapproved = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + messageId: originalId); + Assert.Equal("240 Article received OK", unapproved.Response); + Assert.Equal(0, unapproved.Queue.TryAdmitCalls); + Assert.Equal(0, unapproved.History.PeekCalls); + var recorded = Assert.Single(submission.Submissions); + Assert.Equal(originalId, recorded.MessageId); + var proto = Encoding.ASCII.GetString(recorded.ProtoArticle.Span); + Assert.Equal(1, CountMessageIdHeaders(proto)); + Assert.Contains("Message-ID: " + originalId, proto, StringComparison.Ordinal); + + var reinject = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA, + messageId: originalId); + AssertAccepted(reinject); + Assert.Equal(originalId, reinject.Admitted!.MessageId); + Assert.Equal([originalId], reinject.History.PeekedMessageIds); + Assert.Equal([originalId], reinject.History.RememberedMessageIds); + var injected = Encoding.ASCII.GetString(reinject.Admitted.Payload.Span); + Assert.Equal(1, CountMessageIdHeaders(injected)); + Assert.Contains("Message-ID: " + originalId, injected, StringComparison.Ordinal); + } + + [Fact] + public async Task InnRoutingTemplate_UnapprovedUsesExpandedAddress_DoesNotAuthorize() + { + var authorization = ModeratorTestSnapshot.Create( + ("fido7.*", "%s@fido7.org", string.Empty), + ("perl.*", "news-moderator-%s@perl.org", string.Empty), + ("*", "%s@moderators.isc.org", string.Empty)); + var snapshot = Snapshot( + Group("fido7.some.group", NewsgroupPostingStatus.Moderated), + Group("perl.foo.bar", NewsgroupPostingStatus.Moderated), + Group("comp.test", NewsgroupPostingStatus.Moderated)); + var submission = new RecordingModerationSubmissionService(); + var unapproved = await PostAsync( + "fido7.some.group", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: snapshot, + authorization: authorization, + submission: submission); + Assert.Equal("240 Article received OK", unapproved.Response); + var recorded = Assert.Single(submission.Submissions); + Assert.Equal("fido7-some-group@fido7.org", recorded.ModeratorAddress); + Assert.Equal(0, unapproved.Queue.TryAdmitCalls); + + var perlSubmission = new RecordingModerationSubmissionService(); + var perl = await PostAsync( + "perl.foo.bar", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: snapshot, + authorization: authorization, + submission: perlSubmission); + Assert.Equal("240 Article received OK", perl.Response); + Assert.Equal("news-moderator-perl-foo-bar@perl.org", Assert.Single(perlSubmission.Submissions).ModeratorAddress); + + var catchAll = new RecordingModerationSubmissionService(); + _ = await PostAsync( + "comp.test", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: snapshot, + authorization: authorization, + submission: catchAll); + Assert.Equal("comp-test@moderators.isc.org", Assert.Single(catchAll.Submissions).ModeratorAddress); + + var approved = await PostAsync( + "fido7.some.group", + extraHeaders: "Approved: fido7-some-group@fido7.org\r\n", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: snapshot, + authorization: authorization); + AssertRejected(approved); + } + + [Fact] + public async Task EmailQueue_UnapprovedInnRoute_Returns240WithoutSmtpOnPost() + { + using var harness = new EmailSpoolHarness(); + var submission = EmailModerationSubmissionServiceTests.Create(harness, enabled: true); + var authorization = ModeratorTestSnapshot.Create(("fido7.*", "%s@fido7.org", string.Empty)); + var snapshot = Snapshot(Group("fido7.some.group", NewsgroupPostingStatus.Moderated)); + var outcome = await PostAsync( + "fido7.some.group", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + snapshot: snapshot, + authorization: authorization, + submission: submission); + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Single(harness.EmlFiles()); + var bytes = await File.ReadAllBytesAsync(harness.EmlFiles()[0]); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("fido7-some-group@fido7.org", item!.Recipients[0].Address); + var encoded = Encoding.ASCII.GetString(item.EncodedMessage.Span); + Assert.Contains("application/news-transmission; usage=moderate", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Date:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", encoded, StringComparison.Ordinal); + } + + [Fact] + public async Task EmailDisabled_Unapproved_Returns441() + { + using var harness = new EmailSpoolHarness(enabled: false); + var submission = EmailModerationSubmissionServiceTests.Create(harness, enabled: false); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + AssertRejected(outcome); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task EmailEnabled_AuthorizedReinject_DoesNotEnqueue() + { + using var harness = new EmailSpoolHarness(); + var submission = EmailModerationSubmissionServiceTests.Create(harness, enabled: true); + var outcome = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA, + submission: submission); + AssertAccepted(outcome); + Assert.Empty(harness.EmlFiles()); + Assert.Equal(1, outcome.Queue.TryAdmitCalls); + } + + [Fact] + public async Task EmailSpoolWriteFailure_Unapproved_Returns441() + { + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-post-blocker-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + var options = EmailModerationSubmissionServiceTests.EnabledOptions(); + options.Spool.Directory = blocker; + var spool = new FilesystemEmailSpool( + Microsoft.Extensions.Options.Options.Create(options), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var submission = new EmailModerationSubmissionService( + new EmailService( + spool, + new Rfc5322MessageEncoder(), + Microsoft.Extensions.Options.Options.Create(options), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance), + new ModerationEmailComposer(Microsoft.Extensions.Options.Options.Create(options)), + Microsoft.Extensions.Options.Options.Create(options)); + var outcome = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission); + AssertRejected(outcome); + } + finally + { + File.Delete(blocker); + } + } + + [Fact] + public async Task MessageId_SynthesizedOnce_ThenReinjectPreservesSameId() + { + var submission = new RecordingModerationSubmissionService(); + var unapproved = await PostAsync( + "group.a", + extraHeaders: "", + user: MapNntpAuthenticationProvider.NormalUser, + submission: submission, + messageId: null); + Assert.Equal("240 Article received OK", unapproved.Response); + var recorded = Assert.Single(submission.Submissions); + Assert.False(string.IsNullOrEmpty(recorded.MessageId)); + Assert.StartsWith("<", recorded.MessageId, StringComparison.Ordinal); + Assert.EndsWith("@usenet.ninja>", recorded.MessageId, StringComparison.Ordinal); + var proto = Encoding.ASCII.GetString(recorded.ProtoArticle.Span); + Assert.Equal(1, CountMessageIdHeaders(proto)); + Assert.Contains("Message-ID: " + recorded.MessageId, proto, StringComparison.Ordinal); + + var reinject = await PostAsync( + "group.a", + extraHeaders: "Approved: moderator-a@example.com\r\n", + user: MapNntpAuthenticationProvider.ModeratorA, + messageId: recorded.MessageId); + AssertAccepted(reinject); + Assert.Equal(recorded.MessageId, reinject.Admitted!.MessageId); + Assert.Equal([recorded.MessageId], reinject.History.PeekedMessageIds); + var injected = Encoding.ASCII.GetString(reinject.Admitted.Payload.Span); + Assert.Equal(1, CountMessageIdHeaders(injected)); + Assert.Contains("Message-ID: " + recorded.MessageId, injected, StringComparison.Ordinal); + } + + private static void AssertAccepted(PostOutcome outcome) + { + Assert.Equal("240 Article received OK", outcome.Response); + Assert.Equal(1, outcome.Queue.TryAdmitCalls); + Assert.Equal(1, outcome.History.PeekCalls); + Assert.Equal(1, outcome.History.RememberCalls); + Assert.NotNull(outcome.Admitted); + } + + private static void AssertRejected(PostOutcome outcome) + { + Assert.Equal("441 Posting failed", outcome.Response); + Assert.Equal(0, outcome.Queue.TryAdmitCalls); + Assert.Equal(0, outcome.History.PeekCalls); + Assert.Equal(0, outcome.History.RememberCalls); + Assert.Null(outcome.Admitted); + } + + private static async Task AssertIsolatedSessionsAsync( + string firstUser, + string firstExpected, + string secondUser, + string secondExpected) + { + var authorization = StandardAuthorization(); + var catalogue = new StaticNewsgroupCatalogue(DefaultSnapshot()); + var provider = MapNntpAuthenticationProvider.CreateStandard(); + var firstQueue = new RecordingIngestionQueue(NewQueue()); + var secondQueue = new RecordingIngestionQueue(NewQueue()); + var firstHistory = new RecordingHistoryDb(); + var secondHistory = new RecordingHistoryDb(); + await using var firstDuplex = new PostDuplex(); + await using var secondDuplex = new PostDuplex(); + var firstSession = firstDuplex.CreateSession( + firstQueue, + firstHistory, + catalogue, + authorization, + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + provider); + var secondSession = secondDuplex.CreateSession( + secondQueue, + secondHistory, + catalogue, + authorization, + new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + provider); + var firstRun = firstSession.RunAsync(); + var secondRun = secondSession.RunAsync(); + _ = await firstDuplex.ReadClientLineAsync(); + _ = await secondDuplex.ReadClientLineAsync(); + await AuthenticateAsync(firstDuplex, firstUser, PasswordFor(firstUser)); + await AuthenticateAsync(secondDuplex, secondUser, PasswordFor(secondUser)); + Assert.Equal(firstUser, firstSession.Authentication.Username); + Assert.Equal(secondUser, secondSession.Authentication.Username); + + await firstDuplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await firstDuplex.ReadClientLineAsync()); + await firstDuplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n") + ".\r\n"); + Assert.Equal(firstExpected, await firstDuplex.ReadClientLineAsync()); + + await secondDuplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await secondDuplex.ReadClientLineAsync()); + await secondDuplex.WriteClientAsync( + Article("group.a", extraHeaders: "Approved: moderator-a@example.com\r\n") + ".\r\n"); + Assert.Equal(secondExpected, await secondDuplex.ReadClientLineAsync()); + + Assert.NotEqual(firstSession.Authentication.Username, secondSession.Authentication.Username); + if (firstExpected.StartsWith("240", StringComparison.Ordinal)) + { + Assert.Equal(1, firstQueue.TryAdmitCalls); + } + else + { + Assert.Equal(0, firstQueue.TryAdmitCalls); + Assert.Equal(0, firstHistory.PeekCalls); + Assert.Equal(0, firstHistory.RememberCalls); + } + + if (secondExpected.StartsWith("240", StringComparison.Ordinal)) + { + Assert.Equal(1, secondQueue.TryAdmitCalls); + } + else + { + Assert.Equal(0, secondQueue.TryAdmitCalls); + Assert.Equal(0, secondHistory.PeekCalls); + Assert.Equal(0, secondHistory.RememberCalls); + } + + await firstDuplex.WriteClientLineAsync("QUIT"); + await secondDuplex.WriteClientLineAsync("QUIT"); + _ = await firstDuplex.ReadClientLineAsync(); + _ = await secondDuplex.ReadClientLineAsync(); + await firstRun; + await secondRun; + } + + private static async Task PostAsync( + string newsgroups, + string extraHeaders, + string? user, + IModerationSubmissionService? submission = null, + NewsgroupSnapshot? snapshot = null, + IModeratorAuthorization? authorization = null, + string body = "body\r\n", + string from = "poster@example.com", + string? messageId = "", + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null) => + await PostRawAsync( + Article(newsgroups, extraHeaders, body, from, messageId) + ".\r\n", + user, + submission, + snapshot, + authorization, + postFilter: postFilter, + newsLog: newsLog); + + private static async Task PostRawAsync( + string stuffedArticle, + string? user, + IModerationSubmissionService? submission = null, + NewsgroupSnapshot? snapshot = null, + IModeratorAuthorization? authorization = null, + int? chunkSize = null, + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null) + { + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession( + queue, + history, + new StaticNewsgroupCatalogue(snapshot ?? DefaultSnapshot()), + authorization ?? StandardAuthorization(), + submission ?? new RecordingModerationSubmissionService(ModerationSubmissionStatus.Unavailable, "unused"), + user is null ? null : MapNntpAuthenticationProvider.CreateStandard(), + postFilter: postFilter, + newsLog: newsLog); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + if (user is not null) + { + await AuthenticateAsync(duplex, user, PasswordFor(user)); + } + else + { + session.SetAuthorization(MapNntpAuthenticationProvider.PosterPrivileges); + } + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + if (chunkSize is int size) + { + await duplex.WriteClientChunkedAsync(stuffedArticle, size); + } + else + { + await duplex.WriteClientAsync(stuffedArticle); + } + + var response = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + return new PostOutcome(response, queue, history, queue.Admitted.Count == 0 ? null : queue.Admitted[0]); + } + + private static async Task AuthenticateAsync(PostDuplex duplex, string username, string password) + { + await duplex.WriteClientLineAsync("AUTHINFO USER " + username); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS " + password); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + private static string PasswordFor(string username) => username switch + { + MapNntpAuthenticationProvider.NormalUser => MapNntpAuthenticationProvider.NormalUserPassword, + MapNntpAuthenticationProvider.ModeratorA => MapNntpAuthenticationProvider.ModeratorAPassword, + MapNntpAuthenticationProvider.ModeratorB => MapNntpAuthenticationProvider.ModeratorBPassword, + _ => throw new ArgumentOutOfRangeException(nameof(username)), + }; + + private static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = NntpdOptions.DefaultMaxArticleSize }); + + private static IModeratorAuthorization StandardAuthorization() => + ModeratorTestSnapshot.Create( + ("group.a", "moderator-a@example.com", MapNntpAuthenticationProvider.ModeratorA), + ("group.b", "moderator-b@example.com", MapNntpAuthenticationProvider.ModeratorB)); + + private static NewsgroupSnapshot DefaultSnapshot() => + Snapshot( + Allowed("group.y"), + Allowed("group.y2"), + Group("group.n", NewsgroupPostingStatus.Prohibited), + Group("group.x", NewsgroupPostingStatus.NoPostingOrPeerArticles), + Group("group.j", NewsgroupPostingStatus.PeerOnly), + Group("group.a", NewsgroupPostingStatus.Moderated), + Group("group.b", NewsgroupPostingStatus.Moderated), + Group("group.orphan", NewsgroupPostingStatus.Moderated), + Group("group.shared", NewsgroupPostingStatus.Moderated)); + + private static NewsgroupSnapshot Snapshot(params NewsgroupDefinition[] definitions) => + NewsgroupSnapshot.Create(definitions); + + private static NewsgroupDefinition Allowed(string name) => Group(name, NewsgroupPostingStatus.Allowed); + + private static NewsgroupDefinition Group(string name, NewsgroupPostingStatus status) => + new(name, string.Empty, 2, 1, status); + + private static string Article( + string newsgroups, + string extraHeaders = "", + string body = "body\r\n", + string from = "poster@example.com", + string? messageId = "") + { + var messageIdLine = messageId is null ? string.Empty : "Message-ID: " + messageId + "\r\n"; + return + "Date: " + PostRfcDate.Format(DateTimeOffset.UtcNow) + "\r\n" + + "From: " + from + "\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "Subject: test\r\n" + + messageIdLine + + extraHeaders + + "\r\n" + + body; + } + + private static int CountMessageIdHeaders(string article) + { + var count = 0; + var index = 0; + while (index < article.Length) + { + var found = article.IndexOf("Message-ID:", index, StringComparison.OrdinalIgnoreCase); + if (found < 0) + { + break; + } + + count++; + index = found + "Message-ID:".Length; + } + + return count; + } + + private sealed record PostOutcome( + string Response, + RecordingIngestionQueue Queue, + RecordingHistoryDb History, + InboundArticle? Admitted); + + private sealed class RecordingHistoryDb : IHistoryDb + { + public int PeekCalls { get; private set; } + + public int RememberCalls { get; private set; } + + public List PeekedMessageIds { get; } = []; + + public List RememberedMessageIds { get; } = []; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + PeekCalls++; + PeekedMessageIds.Add(Encoding.ASCII.GetString(messageId.Span)); + return new(HistoryLookupResult.Unseen); + } + + public void Remember(ReadOnlyMemory messageId) + { + RememberCalls++; + RememberedMessageIds.Add(Encoding.ASCII.GetString(messageId.Span)); + } + + public bool ContainsLocal(in HistoryDigest digest) => false; + } + + private sealed class RecordingIngestionQueue : IArticleIngestionQueue + { + private readonly IArticleIngestionQueue _inner; + + public RecordingIngestionQueue(IArticleIngestionQueue inner) + { + _inner = inner; + } + + public int TryAdmitCalls { get; private set; } + + public List Admitted { get; } = []; + + public long MemoryLimitBytes => _inner.MemoryLimitBytes; + + public long QueuedBytes => _inner.QueuedBytes; + + public long PeakQueuedBytes => _inner.PeakQueuedBytes; + + public int MaxArticleBytes => _inner.MaxArticleBytes; + + public int Count => _inner.Count; + + public int PeakCount => _inner.PeakCount; + + public bool IsAccepting => _inner.IsAccepting; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) => + _inner.EnqueueAsync(article, cancellationToken); + + public bool TryProbeCapacity() => _inner.TryProbeCapacity(); + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + TryAdmitCalls++; + var result = _inner.TryAdmit(article); + if (result == ArticleEnqueueResult.Accepted) + { + Admitted.Add(article); + } + + return result; + } + + public bool TryEnqueue(InboundArticle article) => _inner.TryEnqueue(article); + + public void Complete() => _inner.Complete(); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + _inner.DequeueAsync(cancellationToken); + } + + private sealed class PostDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IHistoryDb historyDb, + INewsgroupCatalogue catalogue, + IModeratorAuthorization authorization, + IModerationSubmissionService submission, + INntpAuthenticationProvider? authenticationProvider, + IModeratorCatalogue? moderatorCatalogue = null, + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: authenticationProvider, + articleIngestion: queue, + historyDb: historyDb, + postingTraceProtector: AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }), + newsgroupCatalogue: catalogue, + moderatorCatalogue: moderatorCatalogue, + moderatorAuthorization: authorization, + moderationSubmission: submission, + postFilter: postFilter, + newsLog: newsLog); + } + + public async Task WriteClientLineAsync(string line) => await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientChunkedAsync(string payload, int chunkSize) + { + var bytes = Encoding.ASCII.GetBytes(payload); + for (var i = 0; i < bytes.Length; i += chunkSize) + { + var length = Math.Min(chunkSize, bytes.Length - i); + await _clientToServer.Writer.WriteAsync(bytes.AsMemory(i, length)); + await _clientToServer.Writer.FlushAsync(); + } + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs new file mode 100644 index 00000000..744d6b5d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs @@ -0,0 +1,243 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// NEWNEWS and NEWGROUPS are intentionally unregistered. They must take the generic +/// unknown-command path and must not appear as capability tokens. +/// +public sealed class NewNewsAndNewGroupsRemovedTests +{ + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + [InlineData("newnews")] + [InlineData("NewGroups")] + [InlineData("NEWNEWS * 19990624 000000 GMT")] + [InlineData("NEWGROUPS 19990624 000000 GMT")] + public void Parser_TreatsCommandsAsUnknownVerb(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.False(parsed.IsValid); + Assert.Equal(NntpVerb.Unknown, parsed.Verb); + Assert.Equal(NntpParseStatus.UnknownVerb, parsed.Status); + } + + [Fact] + public void Catalog_DoesNotRegisterCommands() + { + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.GetRegisteredKeys()); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.GetRegisteredKeys()); + Assert.Contains("LIST NEWSGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + } + + [Fact] + public void ListNewsgroups_RemainsARegisteredListKeyword() + { + var parsed = NntpCommandTestParse.ParseCommand("LIST NEWSGROUPS"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.List, parsed.Verb); + Assert.Equal(NntpVerb.Newsgroups, parsed.Qualifier); + Assert.Equal("LIST NEWSGROUPS", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + } + + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + [InlineData("NEWNEWS * 19990624 000000 GMT")] + [InlineData("NEWGROUPS 19990624 000000 GMT")] + public async Task Dispatch_Unauthenticated_ReturnsGenericUnknownCommand(string commandLine) + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + var line = await duplex.ReadClientLineAsync(); + + Assert.Equal("500 Unknown command", line); + Assert.DoesNotContain("Command not implemented", line, StringComparison.Ordinal); + Assert.DoesNotContain("480", line, StringComparison.Ordinal); + } + + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + public async Task Dispatch_FullyAuthorized_StillReturnsGenericUnknownCommand(string commandLine) + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + var line = await duplex.ReadClientLineAsync(); + + Assert.Equal("500 Unknown command", line); + Assert.DoesNotContain("Command not implemented", line, StringComparison.Ordinal); + } + + [Fact] + public async Task Capabilities_DoesNotAdvertiseEitherCommand() + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var read = duplex.ReadUntilTerminatorAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + var body = await read; + var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal) + .Split('\n', StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal("101 Capability list:", lines[0]); + Assert.Contains("LIST ACTIVE COUNTS HEADERS NEWSGROUPS OVERVIEW.FMT", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWNEWS", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWGROUPS", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWNEWS", body, StringComparison.Ordinal); + Assert.DoesNotContain("NEWGROUPS", body, StringComparison.Ordinal); + } + + [Fact] + public async Task ListNewsgroups_Unauthenticated_StillRequiresAuthentication() + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST NEWSGROUPS"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + } + + private sealed class RemovedDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new RemovedDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task ReadUntilTerminatorAsync() + { + var builder = new StringBuilder(); + while (true) + { + var line = await ReadClientLineAsync(); + builder.Append(line); + builder.Append('\n'); + if (line == ".") + { + return builder.ToString(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpArticleNumberTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpArticleNumberTests.cs new file mode 100644 index 00000000..60087059 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpArticleNumberTests.cs @@ -0,0 +1,51 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// RFC 3977 §9.8 article-number = 1*16DIGIT. No CLR integer parse exists. +/// +public sealed class NntpArticleNumberTests +{ + [Theory] + [InlineData("1")] + [InlineData("0")] + [InlineData("0001")] + [InlineData("2147483647")] + [InlineData("2147483648")] + [InlineData("9999999999999999")] + public void Syntax_AcceptsRfcDigitTokensThrough16Digits(string token) + { + Assert.True(NntpArticleNumber.IsSyntax(Encoding.ASCII.GetBytes(token))); + Assert.True(NntpCommandTestParse.ParseCommand("ARTICLE " + token).IsValid); + } + + [Theory] + [InlineData("")] + [InlineData("10000000000000000")] + [InlineData("12345678901234567")] + [InlineData("12a")] + [InlineData("-1")] + [InlineData("+1")] + [InlineData("1.0")] + public void Syntax_RejectsEmptySeventeenDigitsAndNonDigits(string token) + { + Assert.False(NntpArticleNumber.IsSyntax(Encoding.ASCII.GetBytes(token))); + if (token.Length == 0) + { + return; + } + + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandTestParse.ParseCommand("ARTICLE " + token).Status); + } + + [Fact] + public void MaxDigits_IsRfc16() + { + Assert.Equal(16, NntpArticleNumber.MaxDigits); + Assert.True(NntpArticleNumber.IsSyntax("9999999999999999"u8)); + Assert.False(NntpArticleNumber.IsSyntax("10000000000000000"u8)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpAuthinfoTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpAuthinfoTests.cs new file mode 100644 index 00000000..61ba636a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpAuthinfoTests.cs @@ -0,0 +1,686 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpAuthinfoTests +{ + [Fact] + public async Task AuthinfoUser_WithoutArgument_Returns501() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER"); + Assert.StartsWith("501 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.PendingAuthUsername); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextAllowed_AuthinfoUserPass_SucceedsWithoutTls() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal("fred", session.PendingAuthUsername); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("fred", session.Authentication.Username); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoUser_Repeated_ReplacesPendingUsername() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO USER barney"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal("barney", session.PendingAuthUsername); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_WithoutUser_Returns482() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("482 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_WrongPassword_Returns481_LeavesUnauthenticated() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.StartsWith("481 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.StreamingPermitted); + Assert.Equal("fred", session.PendingAuthUsername); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Success_AuthenticatedWithoutImpliedPrivileges() + { + var provider = new ScriptedAuthenticationProvider( + (user, pass) => user == "fred" && pass == "flintstone" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("fred", session.Authentication.Username); + Assert.Null(session.PendingAuthUsername); + Assert.True(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.StreamingPermitted); + Assert.False(session.Authorization.ControlCancelPermitted); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Newsmaster_GrantsControlCancelOnly() + { + var options = new NntpdOptions + { + NewsmasterUser = "newsmaster", + NewsmasterPassword = "unit-test-newsmaster-password", + }; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER newsmaster"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS unit-test-newsmaster-password"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.PostingPermitted); + Assert.True(session.Authorization.ControlCancelPermitted); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.StreamingPermitted); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Success_AppliesProviderAuthorizationIndependently() + { + var provider = new ScriptedAuthenticationProvider( + (user, pass) => user == "reader1" && pass == "pw" + ? NntpAuthenticationResult.Success( + "reader1", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER reader1"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS pw"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authorization.AuthorizedReader); + Assert.True(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.StreamingPermitted); + Assert.False(session.Authorization.ControlCancelPermitted); + + await duplex.WriteClientLineAsync("ARTICLE"); + Assert.StartsWith("412 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Authinfo_AfterSuccess_Returns502_AndCapabilitiesOmitAuthinfo() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("AUTHINFO USER other"); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.DoesNotContain(caps, static c => c.StartsWith("AUTHINFO", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Equals("MODE-READER", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithoutTls_AuthinfoUser_Returns483_DoesNotCallProvider() + { + var provider = new CountingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("483 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Null(session.PendingAuthUsername); + Assert.Equal(0, provider.CallCount); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithoutTls_AuthinfoPass_Returns483_DoesNotCallProvider() + { + var provider = new CountingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(allowCleartextAuth: false); + // Simulate a pending USER that somehow existed before policy change — PASS must still gate first. + session.SetPendingAuthUsername("fred"); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("483 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal(0, provider.CallCount); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithTls_AuthinfoUserPass_Succeeds() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider(), isTls: true); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextAllowed_AdvertisesAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO USER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextDisabled_WithoutTls_DoesNotAdvertiseAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO", caps); + Assert.DoesNotContain(caps, static c => c.Equals("AUTHINFO USER", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextDisabled_WithTls_AdvertisesAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider(), isTls: true); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO USER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_DoesNotLogPassword_OnHandlerFailure() + { + var recording = new RecordingLoggerFactory(); + var provider = new ThrowingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(provider, allowCleartextAuth: true, loggerFactory: recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS super-secret-password-xyz"); + Assert.StartsWith("403 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("super-secret-password-xyz", joined, StringComparison.Ordinal); + Assert.Contains(recording.Messages, m => m.Contains("AUTHINFO PASS", StringComparison.Ordinal)); + Assert.Contains( + recording.Categories, + c => c == "VectorNNTPD.ReaderService.Session.Commands.AuthInfo"); + } + + private static ScriptedAuthenticationProvider CreateAcceptingProvider() => + new( + (user, pass) => user == "fred" && pass == "flintstone" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + private sealed class ScriptedAuthenticationProvider : INntpAuthenticationProvider + { + private readonly Func _authenticate; + + public ScriptedAuthenticationProvider(Func authenticate) + { + _authenticate = authenticate; + } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(_authenticate(username, password)); + } + + private sealed class CountingAuthenticationProvider : INntpAuthenticationProvider + { + public int CallCount { get; private set; } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + CallCount++; + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class ThrowingAuthenticationProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + throw new InvalidOperationException("provider boom"); + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + if (exception is not null) + { + _messages.Add(exception.ToString()); + } + } + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + if (exception is not null) + { + Messages.Add(exception.ToString()); + } + } + } + + private sealed class AuthinfoDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly INntpAuthenticationProvider _provider; + private readonly bool _isTls; + + private AuthinfoDuplex(INntpAuthenticationProvider provider, bool isTls) + { + _provider = provider; + _isTls = isTls; + } + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync( + INntpAuthenticationProvider provider, + bool isTls = false) => + Task.FromResult(new AuthinfoDuplex(provider, isTls)); + + public NntpSession CreateSession( + INntpAuthenticationProvider? provider = null, + bool allowCleartextAuth = true, + ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + isTls: _isTls); + return new NntpSession( + connection, + loggerFactory?.CreateLogger() ?? NullLogger.Instance, + authenticationProvider: provider ?? _provider, + allowCleartextAuth: allowCleartextAuth, + loggerFactory: loggerFactory); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("201 ", line, StringComparison.Ordinal); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection( + PipeReader input, + PipeWriter output, + ConnectionClientIdentity identity, + bool isTls = false) + { + Input = input; + Output = output; + ClientIdentity = identity; + IsTls = isTls; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls { get; } + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandInventoryTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandInventoryTests.cs new file mode 100644 index 00000000..c585bc93 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandInventoryTests.cs @@ -0,0 +1,341 @@ +using System.IO.Pipelines; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class NntpCommandInventoryTests +{ + [Fact] + public void Catalog_RegistersCompleteInventory() + { + var keys = DefaultNntpCommandCatalog.GetRegisteredKeys(); + + // Production inventory remains complete; BENCHIT is an internal extra (not in InventoryKeys). + foreach (var key in DefaultNntpCommandCatalog.InventoryKeys) + { + Assert.Contains(key, keys); + } + + Assert.Contains("BENCHIT", keys); + Assert.DoesNotContain("BENCHIT", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", keys); + Assert.DoesNotContain("NEWGROUPS", keys); + Assert.Equal(DefaultNntpCommandCatalog.InventoryKeys.Count + 1, keys.Count); + } + + [Theory] + [InlineData("CAPABILITIES")] + [InlineData("DATE")] + [InlineData("HELP")] + [InlineData("QUIT")] + [InlineData("STARTTLS")] + [InlineData("LIST")] + [InlineData("LIST ACTIVE")] + [InlineData("LIST COUNTS")] + [InlineData("LIST COUNTS misc.*")] + [InlineData("LIST HEADERS")] + [InlineData("LIST HEADERS MSGID")] + [InlineData("LIST HEADERS RANGE")] + [InlineData("LIST MOTD")] + [InlineData("LIST NEWSGROUPS")] + [InlineData("LIST OVERVIEW.FMT")] + [InlineData("GROUP misc.test")] + [InlineData("LISTGROUP")] + [InlineData("LISTGROUP misc.test")] + [InlineData("ARTICLE")] + [InlineData("COMPRESS DEFLATE")] + [InlineData("MODE READER")] + [InlineData("MODE STREAM")] + [InlineData("AUTHINFO USER x")] + [InlineData("AUTHINFO PASS y")] + [InlineData("AUTHINFO SASL PLAIN")] + [InlineData("IHAVE ")] + [InlineData("CHECK ")] + [InlineData("TAKETHIS ")] + [InlineData("SPEEDTEST GIGANEWS")] + public void Parser_AcceptsInventoryCommands(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.True(parsed.IsValid); + Assert.NotEqual(NntpVerb.Unknown, parsed.Verb); + Assert.NotEqual(NntpVerb.None, parsed.Verb); + } + + [Theory] + [InlineData("LIST ACTIVE.TIMES")] + [InlineData("LIST DISTRIB.PATS")] + [InlineData("LIST DISTRIBUTIONS")] + [InlineData("LIST MODERATORS")] + [InlineData("LIST SUBSCRIPTIONS")] + [InlineData("LIST SUBSCRIPTIONS *")] + public void Parser_RejectsUnsupportedListVariants(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.Equal(NntpVerb.List, parsed.Verb); + Assert.Equal(NntpParseStatus.UnknownQualifier, parsed.Status); + Assert.False(parsed.IsValid); + } + + [Fact] + public void Inventory_ExcludesUnsupportedListVariants() + { + var keys = DefaultNntpCommandCatalog.InventoryKeys; + Assert.Contains("LIST", keys); + Assert.Contains("LIST ACTIVE", keys); + Assert.Contains("LIST COUNTS", keys); + Assert.DoesNotContain("LIST ACTIVE.TIMES", keys); + Assert.DoesNotContain("LIST DISTRIB.PATS", keys); + Assert.DoesNotContain("LIST DISTRIBUTIONS", keys); + Assert.DoesNotContain("LIST MODERATORS", keys); + Assert.DoesNotContain("LIST SUBSCRIPTIONS", keys); + + var registered = DefaultNntpCommandCatalog.GetRegisteredKeys(); + Assert.DoesNotContain("LIST ACTIVE.TIMES", registered); + Assert.Contains("LIST COUNTS", registered); + Assert.DoesNotContain("LIST DISTRIB.PATS", registered); + Assert.DoesNotContain("LIST DISTRIBUTIONS", registered); + Assert.DoesNotContain("LIST MODERATORS", registered); + Assert.DoesNotContain("LIST SUBSCRIPTIONS", registered); + } + + [Fact] + public void Registry_ExactMatch_RejectsPrefixConfusion() + { + var articleX = NntpCommandTestParse.ParseCommand("ARTICLEX"); + Assert.Equal(NntpVerb.Unknown, articleX.Verb); + Assert.Equal(NntpParseStatus.UnknownVerb, articleX.Status); + + var userXyz = NntpCommandTestParse.ParseCommand("AUTHINFO USERXYZ"); + Assert.Equal(NntpVerb.AuthInfo, userXyz.Verb); + Assert.Equal(NntpParseStatus.UnknownQualifier, userXyz.Status); + } + + [Fact] + public void Parser_Date_IsValidPublicCommand() + { + var parsed = NntpCommandTestParse.ParseCommand("DATE"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.Date, parsed.Verb); + Assert.Equal(0, parsed.TokenCount); + Assert.Equal("DATE", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(parsed.Verb, parsed.Qualifier)); + } + + [Fact] + public async Task List_WithoutCatalogue_Returns500AfterAuthz() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST"); + Assert.Contains("500 Command not implemented", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Placeholder_List_WithoutAuth_StillReturns480() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST"); + Assert.Contains("480 Authentication required", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + /// + /// Unknown LIST keywords stay UnknownQualifier / 501 Unknown command variant, + /// matching the old registry (not a new parser invention). + /// + [Theory] + [InlineData("LIST ACTIVE.TIMES")] + [InlineData("LIST DISTRIB.PATS")] + [InlineData("LIST DISTRIBUTIONS")] + [InlineData("LIST MODERATORS")] + [InlineData("LIST SUBSCRIPTIONS")] + [InlineData("LIST SUBSCRIPTIONS *")] + public async Task UnsupportedListVariants_Return501UnknownCommandVariant(string commandLine) + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.Equal("501 Unknown command variant", await duplex.ReadClientLineAsync()); + } + + [Theory] + [InlineData("LIST HEADERS")] + [InlineData("LIST OVERVIEW.FMT")] + public async Task ImplementedStaticListKeywords_Return215AfterAuthzWithoutCatalogue(string commandLine) + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.StartsWith("215 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task ListMotd_RemainsRecognizedWithoutStoredData_Return503AfterAuthz() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST MOTD"); + Assert.Equal("503 Data item not stored", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task CompressDeflate_IsRegisteredPublic_Returns206() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Contains("206 Compression active", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Connection.IsCompressed); + } + + private sealed class InventoryDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new InventoryDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLineLengthTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLineLengthTests.cs new file mode 100644 index 00000000..1bf853fe --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLineLengthTests.cs @@ -0,0 +1,182 @@ +using System.IO.Pipelines; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Reader-mode command lines longer than +/// are consumed through CRLF and rejected as 501 Syntax error without parsing the prefix. +/// +public sealed class NntpCommandLineLengthTests +{ + [Fact] + public async Task ExactlyMaxBytes_IsCopiedAndParses() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.MaxCommandLineBytes, length); + var command = NntpCommandParser.Parse(scratch.AsSpan(0, length)); + Assert.True(command.IsValid); + Assert.Equal(NntpVerb.Group, command.Verb); + } + + [Fact] + public async Task OneByteOverMax_IsConsumedAndNotCopiedAsACommand() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.True((await pipe.Reader.ReadAsync()).Buffer.IsEmpty); + } + + [Fact] + public async Task ValidPrefixPlusExtra_DoesNotReturnThePrefix() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 6]; + "STAT "u8.CopyTo(line); + line.AsSpan("STAT "u8.Length, NntpCommandLineReader.MaxCommandLineBytes - "STAT "u8.Length) + .Fill((byte)' '); + " extra"u8.CopyTo(line.AsSpan(NntpCommandLineReader.MaxCommandLineBytes)); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + scratch.AsSpan().Fill((byte)'Z'); + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.Equal((byte)'Z', scratch[0]); + } + + [Fact] + public async Task OverlongInvalidPrefix_IsTheSameSentinel() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan(0, NntpCommandLineReader.MaxCommandLineBytes).Fill((byte)'!'); + line[^1] = (byte)'x'; + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + } + + [Fact] + public async Task OverlongLineThenQuit_SecondLineParses() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = new byte[line.Length + 2 + "QUIT\r\n"u8.Length]; + WithCrlf(line).CopyTo(wire); + "QUIT\r\n"u8.CopyTo(wire.AsSpan(line.Length + 2)); + await using var pipe = await CommandPipe.CreateAsync(wire); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var first = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + var second = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, first); + Assert.Equal(4, second); + Assert.True(NntpCommandParser.Parse(scratch.AsSpan(0, second)).IsValid); + Assert.Equal(NntpVerb.Quit, NntpCommandParser.Parse(scratch.AsSpan(0, second)).Verb); + } + + [Fact] + public async Task OverlongSplitAcrossReads_StillOverlong() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = WithCrlf(line); + var pipe = new Pipe(NntpPipeOptions.Create()); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + var read = NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None).AsTask(); + + await pipe.Writer.WriteAsync(wire.AsMemory(0, 1000)); + await pipe.Writer.WriteAsync(wire.AsMemory(1000)); + var length = await read.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task EmbeddedLf_IsNotADelimiter() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan().Fill((byte)'a'); + line[10] = (byte)'\n'; + line[20] = (byte)'\r'; + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.True((await pipe.Reader.ReadAsync()).Buffer.IsEmpty); + } + + [Fact] + public async Task NoCrlfBeforeClose_ReturnsEof() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 100]; + line.AsSpan().Fill((byte)'a'); + var pipe = new Pipe(NntpPipeOptions.Create()); + await pipe.Writer.WriteAsync(line); + await pipe.Writer.CompleteAsync(); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(-1, length); + await pipe.Reader.CompleteAsync(); + } + + private static byte[] GroupLine(int length) + { + var line = new byte[length]; + "GROUP "u8.CopyTo(line); + line.AsSpan("GROUP "u8.Length).Fill((byte)'a'); + return line; + } + + private static byte[] WithCrlf(byte[] line) + { + var wire = new byte[line.Length + 2]; + line.CopyTo(wire); + wire[^2] = (byte)'\r'; + wire[^1] = (byte)'\n'; + return wire; + } + + private sealed class CommandPipe : IAsyncDisposable + { + private readonly Pipe _pipe; + + private CommandPipe(Pipe pipe) => _pipe = pipe; + + public PipeReader Reader => _pipe.Reader; + + public static async Task CreateAsync(byte[] wire) + { + var pipe = new Pipe(NntpPipeOptions.Create()); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + return new CommandPipe(pipe); + } + + public async ValueTask DisposeAsync() + { + await _pipe.Reader.CompleteAsync(); + await _pipe.Writer.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggerCollection.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggerCollection.cs new file mode 100644 index 00000000..5c6a7fe5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggerCollection.cs @@ -0,0 +1,8 @@ +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Serializes tests that configure the process-wide NntpCommandLoggers factory +/// so recording loggers are not cleared by concurrent catalog creation. +/// +[CollectionDefinition(nameof(NntpCommandLoggerCollection), DisableParallelization = true)] +public sealed class NntpCommandLoggerCollection; diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggingTests.cs new file mode 100644 index 00000000..a14c81c5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandLoggingTests.cs @@ -0,0 +1,928 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// Command RX/TX logging contract (ownership, redaction, timing shape). +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpCommandLoggingTests +{ + private static readonly Regex TxPattern = new( + @"TX:\s+(?.+?)(?:\s+\[(?[^\]]*)\])?\s+executed in (?\d+\.\d{3})s", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + [Fact] + public async Task Capabilities_ProducesExactlyOneRxAndOneTx_UnderCapabilitiesLogger() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); // greeting + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + _ = await duplex.ReadMultilineBodyAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var rx = recording.Messages.Where(m => m.Contains("RX:", StringComparison.Ordinal)).ToArray(); + var tx = recording.Messages.Where(m => m.Contains("TX:", StringComparison.Ordinal)).ToArray(); + + Assert.Contains(rx, m => m.Contains("RX: CAPABILITIES", StringComparison.Ordinal)); + Assert.Equal(1, tx.Count(m => m.Contains("TX: CAPABILITIES [101 Capability list:] executed in", StringComparison.Ordinal))); + Assert.Contains(tx, m => TxPattern.IsMatch(m) && m.Contains("CAPABILITIES", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("VERSION 2", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("IMPLEMENTATION", StringComparison.Ordinal)); + Assert.Contains( + recording.Categories, + c => c == typeof(Capabilities).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command CAPABILITIES failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task Help_ProducesExactlyOneRxAndOneTx_UnderHelpLogger() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("100 Help text follows", await duplex.ReadClientLineAsync()); + var body = await duplex.ReadMultilineBodyAsync(); + Assert.Equal(Help.BodyLines, body); + Assert.DoesNotContain(body, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var rx = recording.Messages.Where(m => m.Contains("RX:", StringComparison.Ordinal)).ToArray(); + var tx = recording.Messages.Where(m => m.Contains("TX:", StringComparison.Ordinal)).ToArray(); + + Assert.Contains(rx, m => m.Contains("RX: HELP", StringComparison.Ordinal)); + Assert.Equal(1, tx.Count(m => m.Contains("TX: HELP [100 Help text follows] executed in", StringComparison.Ordinal))); + Assert.DoesNotContain(tx, m => m.Contains("ARTICLE [message-id / article-number]", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("CHECK message-id", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(Help).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command HELP failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task StartTls_CompletionOwnedByStartTlsLogger_NotDispatcher() + { + var recording = new RecordingLoggerFactory(); + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var session = new NntpSession( + server, + recording.CreateLogger(), + certificateProvider: host.CertificateProvider, + loggerFactory: recording); + + var run = session.RunAsync(); + _ = await ReadPlainLineAsync(clientSocket); + + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await server.CompleteAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: STARTTLS", StringComparison.Ordinal)); + var startTlsTx = recording.Messages + .Where(m => m.Contains("TX: STARTTLS [382 Continue with TLS negotiation] executed in", StringComparison.Ordinal)) + .ToArray(); + Assert.Single(startTlsTx); + Assert.Matches(TxPattern, startTlsTx[0]); + Assert.Contains("TlsVersion=", startTlsTx[0], StringComparison.Ordinal); + Assert.Contains("Cipher=", startTlsTx[0], StringComparison.Ordinal); + Assert.DoesNotContain("[failed]", startTlsTx[0], StringComparison.Ordinal); + + Assert.True(server.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)); + Assert.Contains($"TlsVersion={tlsVersion}", startTlsTx[0], StringComparison.Ordinal); + Assert.Contains($"Cipher={cipher}", startTlsTx[0], StringComparison.Ordinal); + + Assert.Contains(recording.Categories, c => c == typeof(StartTls).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command STARTTLS failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task Post_Stub_ProducesExactlyOneTxCompletion() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientBytesAsync( + Encoding.ASCII.GetBytes( + "Date: " + DateTimeOffset.UtcNow.ToString("dd MMM yyyy HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture) + " +0000\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: logging\r\n" + + "Message-ID: \r\n" + + "\r\nbody\r\n.\r\n")); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: POST [441 Posting failed] executed in", StringComparison.Ordinal))); + Assert.Contains(recording.Messages, m => m.Contains("[posting failed]", StringComparison.Ordinal)); + } + + [Fact] + public async Task Quit_ProducesExactlyOneTxCompletion() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: QUIT [205 Connection closing] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task AuthinfoPass_RxIsRedacted_AndPasswordNeverLogged() + { + var recording = new RecordingLoggerFactory(); + var provider = new AcceptFredProvider(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording, authenticationProvider: provider, allowCleartextAuth: true); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS super-secret-password-xyz"); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("super-secret-password-xyz", joined, StringComparison.Ordinal); + Assert.Contains(recording.Messages, m => m.Contains("RX: AUTHINFO PASS ", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: AUTHINFO PASS [281 Authentication accepted] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task EveryInventoryCommand_ProducesRxWhenIssued() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var commands = new (string Line, string RxNeedle)[] + { + ("CAPABILITIES", "RX: CAPABILITIES"), + ("DATE", "RX: DATE"), + ("HELP", "RX: HELP"), + ("MODE READER", "RX: MODE READER"), + ("LIST", "RX: LIST"), + ("GROUP", "RX: GROUP"), + ("LISTGROUP", "RX: LISTGROUP"), + ("ARTICLE", "RX: ARTICLE"), + ("HEAD", "RX: HEAD"), + ("BODY", "RX: BODY"), + ("STAT", "RX: STAT"), + ("LAST", "RX: LAST"), + ("NEXT", "RX: NEXT"), + ("OVER", "RX: OVER"), + ("HDR", "RX: HDR"), + ("POST", "RX: POST"), + ("IHAVE", "RX: IHAVE"), + ("CHECK", "RX: CHECK"), + ("MODE STREAM", "RX: MODE STREAM"), + ("COMPRESS DEFLATE", "RX: COMPRESS DEFLATE"), + ("AUTHINFO SASL", "RX: AUTHINFO SASL"), + }; + + foreach (var (line, _) in commands) + { + await duplex.WriteClientLineAsync(line); + await DrainResponseAsync(duplex); + if (line == "POST") + { + await duplex.WriteClientBytesAsync(".\r\n"u8.ToArray()); + await DrainResponseAsync(duplex); + } + } + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + foreach (var (_, needle) in commands) + { + Assert.Contains(recording.Messages, m => m.Contains(needle, StringComparison.Ordinal)); + } + } + + [Fact] + public async Task TakeThis_OnReader_Logs502_WithoutIngesting() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true), + articleIngestion: queue); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Contains( + recording.Messages, + m => m.Contains("TX: TAKETHIS [502 Command not available on this server] executed in", StringComparison.Ordinal)); + Assert.Contains(recording.Messages, m => m.Contains("RX: DATE", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: DATE [111 ", StringComparison.Ordinal) + && m.Contains("] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task Date_StillProducesRxAndTx_WhenTakeThisSuppressed() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Contains(recording.Messages, m => m.Contains("RX: DATE", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: DATE [111 ", StringComparison.Ordinal) + && m.Contains("] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task StartTls_HandshakeFailure_TxHasFailed_WithoutFakeTlsDetails() + { + var recording = new RecordingLoggerFactory(); + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var session = new NntpSession( + server, + recording.CreateLogger(), + certificateProvider: host.CertificateProvider, + loggerFactory: recording); + + var run = session.RunAsync(); + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: STARTTLS [382 Continue with TLS negotiation] executed in", StringComparison.Ordinal)); + Assert.Contains("[failed]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("TlsVersion=", tx, StringComparison.Ordinal); + Assert.DoesNotContain("Cipher=", tx, StringComparison.Ordinal); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command STARTTLS failed", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(StartTls).FullName); + } + + [Fact] + public async Task Check_OnReader_Logs502_RegardlessOfHistoryWant() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Contains("TX: CHECK [502 Command not available on this server] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("238", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Check_OnReader_Logs502_RegardlessOfHistorySeen() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + historyDb: new SeenHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Contains("TX: CHECK [502 Command not available on this server] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("438", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Check_OnReader_Logs502_RegardlessOfHistoryUnavailable() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + historyDb: new UnavailableHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Contains("TX: CHECK [502 Command not available on this server] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("431", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task TakeThis_OnReader_Logs502_WithoutReadingArticle() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + articleIngestion: queue); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: TAKETHIS", StringComparison.Ordinal)); + Assert.Contains("TX: TAKETHIS [502 Command not available on this server] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("439", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task TakeThis_OnReader_Logs502_RegardlessOfHistoryUnavailable() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + articleIngestion: queue, + historyDb: new UnavailableHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: TAKETHIS", StringComparison.Ordinal)); + Assert.Contains( + "TX: TAKETHIS [502 Command not available on this server] executed in", + tx, + StringComparison.Ordinal); + Assert.DoesNotContain("400 Service temporarily unavailable", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task List_Logs215StatusLine_FromCapturedSnapshot() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false), + newsgroupCatalogue: new StaticNewsgroupCatalogue(NewsgroupSnapshot.Empty)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("LIST"); + Assert.Equal("215 list of newsgroups follows", await duplex.ReadClientLineAsync()); + Assert.Equal(".", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: LIST", StringComparison.Ordinal)); + Assert.Contains( + "TX: LIST [215 list of newsgroups follows] executed in", + tx, + StringComparison.Ordinal); + } + + [Fact] + public void RedactRxLine_ProtectsPassAndSasl() + { + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxLine("AUTHINFO PASS secret")); + Assert.Equal("AUTHINFO SASL ", NntpCommandLogFormat.RedactRxLine("AUTHINFO SASL PLAIN abc")); + Assert.Equal("CAPABILITIES", NntpCommandLogFormat.RedactRxLine("CAPABILITIES")); + } + + private static NntpAuthorization TransitAuthorization() => + new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + private static async Task DrainResponseAsync(LoggingDuplex duplex) + { + var line = await duplex.ReadClientLineAsync(); + // Multi-line responses (CAPABILITIES 101, HELP 100, LIST 215, …). + if (line.StartsWith("100 ", StringComparison.Ordinal) || + line.StartsWith("101 ", StringComparison.Ordinal) || + line.StartsWith("215 ", StringComparison.Ordinal) || + line.StartsWith("224 ", StringComparison.Ordinal) || + line.StartsWith("225 ", StringComparison.Ordinal) || + line.StartsWith("220 ", StringComparison.Ordinal) || + line.StartsWith("221 ", StringComparison.Ordinal) || + line.StartsWith("222 ", StringComparison.Ordinal) || + line.StartsWith("230 ", StringComparison.Ordinal) || + line.StartsWith("231 ", StringComparison.Ordinal)) + { + _ = await duplex.ReadMultilineBodyAsync(); + } + } + + private static async Task ReadPlainLineAsync(System.Net.Sockets.Socket socket) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = System.Text.Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private static async Task WriteSslLineAsync(System.Net.Security.SslStream ssl, string line) + { + var bytes = System.Text.Encoding.ASCII.GetBytes(line + "\r\n"); + await ssl.WriteAsync(bytes); + } + + private static async Task ReadSslLineAsync(System.Net.Security.SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = System.Text.Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private static System.Net.Security.SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private sealed class AcceptFredProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + new( + username == "fred" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + } + } + + private sealed class LoggingDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new LoggingDuplex()); + + public NntpSession CreateSession( + ILoggerFactory loggerFactory, + INntpAuthenticationProvider? authenticationProvider = null, + bool allowCleartextAuth = true, + NntpAuthorization? authorization = null, + IArticleIngestionQueue? articleIngestion = null, + VectorNNTPD.ReaderService.History.IHistoryDb? historyDb = null, + INewsgroupCatalogue? newsgroupCatalogue = null) + { + var connection = new PipeConnection(_clientToServer.Reader, _serverToClient.Writer); + var session = new NntpSession( + connection, + loggerFactory.CreateLogger(), + authenticationProvider: authenticationProvider, + allowCleartextAuth: allowCleartextAuth, + loggerFactory: loggerFactory, + articleIngestion: articleIngestion, + historyDb: historyDb, + newsgroupCatalogue: newsgroupCatalogue); + if (authorization is not null) + { + session.SetAuthorization(authorization); + } + + return session; + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientBytesAsync(ReadOnlyMemory bytes) + { + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output) + { + Input = input; + Output = output; + ClientIdentity = ConnectionClientIdentity.Direct( + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 49860)); + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } + } + + private sealed class UnavailableHistoryDb : VectorNNTPD.ReaderService.History.IHistoryDb + { + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.ReaderService.History.HistoryLookupResult.Unavailable); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.ReaderService.History.HistoryLookupResult.Unavailable); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in VectorNNTPD.ReaderService.History.HistoryDigest digest) => false; + } + + private sealed class SeenHistoryDb : VectorNNTPD.ReaderService.History.IHistoryDb + { + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.ReaderService.History.HistoryLookupResult.Seen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.ReaderService.History.HistoryLookupResult.Seen); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in VectorNNTPD.ReaderService.History.HistoryDigest digest) => true; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandParserTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandParserTests.cs new file mode 100644 index 00000000..1d8a1474 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandParserTests.cs @@ -0,0 +1,517 @@ +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.Common.Articles.Validation; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Full-surface byte parser contracts: classification, syntax status, and reject-before-handler. +/// +public sealed class NntpCommandParserTests +{ + [Theory] + [InlineData("ARTICLE", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 123", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 0", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 0001", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 9999999999999999", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE ", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE <123@example.com>", NntpVerb.Article, NntpVerb.None)] + [InlineData("article", NntpVerb.Article, NntpVerb.None)] + [InlineData("ArTiClE", NntpVerb.Article, NntpVerb.None)] + [InlineData("BODY", NntpVerb.Body, NntpVerb.None)] + [InlineData("BODY 1", NntpVerb.Body, NntpVerb.None)] + [InlineData("HEAD ", NntpVerb.Head, NntpVerb.None)] + [InlineData("STAT 300256", NntpVerb.Stat, NntpVerb.None)] + [InlineData("CAPABILITIES", NntpVerb.Capabilities, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("check ", NntpVerb.Check, NntpVerb.None)] + [InlineData("cHeCk ", NntpVerb.Check, NntpVerb.None)] + [InlineData("COMPRESS DEFLATE", NntpVerb.Compress, NntpVerb.None)] + [InlineData("DATE", NntpVerb.Date, NntpVerb.None)] + [InlineData("GROUP misc.test", NntpVerb.Group, NntpVerb.None)] + [InlineData("HDR", NntpVerb.Hdr, NntpVerb.None)] + [InlineData("HEAD", NntpVerb.Head, NntpVerb.None)] + [InlineData("HELP", NntpVerb.Help, NntpVerb.None)] + [InlineData("IHAVE ", NntpVerb.Ihave, NntpVerb.None)] + [InlineData("LAST", NntpVerb.Last, NntpVerb.None)] + [InlineData("LIST", NntpVerb.List, NntpVerb.None)] + [InlineData("LIST ACTIVE", NntpVerb.List, NntpVerb.Active)] + [InlineData("LIST COUNTS", NntpVerb.List, NntpVerb.Counts)] + [InlineData("LIST COUNTS misc.*", NntpVerb.List, NntpVerb.Counts)] + [InlineData("LIST HEADERS", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST HEADERS MSGID", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST HEADERS RANGE", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST MOTD", NntpVerb.List, NntpVerb.Motd)] + [InlineData("LIST NEWSGROUPS", NntpVerb.List, NntpVerb.Newsgroups)] + [InlineData("LIST OVERVIEW.FMT", NntpVerb.List, NntpVerb.OverviewFmt)] + [InlineData("LISTGROUP", NntpVerb.ListGroup, NntpVerb.None)] + [InlineData("MODE READER", NntpVerb.Mode, NntpVerb.Reader)] + [InlineData("MODE STREAM", NntpVerb.Mode, NntpVerb.Stream)] + [InlineData("mode reader", NntpVerb.Mode, NntpVerb.Reader)] + [InlineData("mode stream", NntpVerb.Mode, NntpVerb.Stream)] + [InlineData("NEXT", NntpVerb.Next, NntpVerb.None)] + [InlineData("OVER", NntpVerb.Over, NntpVerb.None)] + [InlineData("POST", NntpVerb.Post, NntpVerb.None)] + [InlineData("QUIT", NntpVerb.Quit, NntpVerb.None)] + [InlineData("STARTTLS", NntpVerb.StartTls, NntpVerb.None)] + [InlineData("STAT", NntpVerb.Stat, NntpVerb.None)] + [InlineData("TAKETHIS ", NntpVerb.TakeThis, NntpVerb.None)] + [InlineData("takethis ", NntpVerb.TakeThis, NntpVerb.None)] + [InlineData("AUTHINFO USER alice", NntpVerb.AuthInfo, NntpVerb.User)] + [InlineData("AUTHINFO PASS secret", NntpVerb.AuthInfo, NntpVerb.Pass)] + [InlineData("AUTHINFO SASL PLAIN", NntpVerb.AuthInfo, NntpVerb.Sasl)] + [InlineData("authinfo user alice", NntpVerb.AuthInfo, NntpVerb.User)] + [InlineData("BENCHIT", NntpVerb.BenchIt, NntpVerb.None)] + [InlineData("SPEEDTEST GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("SPEEDTEST usenet-ninja", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("speedtest GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("SpEeDtEsT GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + public void Parse_SupportedCommands_AreValid(string text, NntpVerb verb, NntpVerb qualifier) + { + var (command, _) = NntpCommandTestParse.Parse(text); + Assert.True(command.IsValid); + Assert.Equal(NntpParseStatus.Ok, command.Status); + Assert.Equal(verb, command.Verb); + Assert.Equal(qualifier, command.Qualifier); + } + + /// RFC 3977 §3.1: tokens are separated by one or more SP or TAB. + [Theory] + [InlineData("CHECK\t")] + [InlineData("MODE\tSTREAM")] + [InlineData("AUTHINFO\tUSER\talice")] + [InlineData("QUIT\t")] + [InlineData(" DATE ")] + [InlineData("\tHELP")] + public void Parse_SpAndTabWhitespace_IsAccepted(string text) + { + Assert.True(NntpCommandTestParse.ParseCommand(text).IsValid); + } + + [Theory] + [InlineData("", NntpParseStatus.Empty)] + [InlineData(" ", NntpParseStatus.Empty)] + [InlineData("\t", NntpParseStatus.Empty)] + [InlineData(" ", NntpParseStatus.Empty)] + [InlineData("NOSUCHCMD", NntpParseStatus.UnknownVerb)] + [InlineData("NEWNEWS", NntpParseStatus.UnknownVerb)] + [InlineData("NEWGROUPS", NntpParseStatus.UnknownVerb)] + [InlineData("newnews * 19990624 000000 GMT", NntpParseStatus.UnknownVerb)] + [InlineData("NEWGROUPS 19990624 000000 GMT", NntpParseStatus.UnknownVerb)] + [InlineData("ARTICLEX", NntpParseStatus.UnknownVerb)] + [InlineData("MODE", NntpParseStatus.UnknownQualifier)] + [InlineData("MODE UNKNOWN", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO GENERIC x", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO USERXYZ", NntpParseStatus.UnknownQualifier)] + [InlineData("ARTICLE foo", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 12a", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE -1", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 10000000000000000", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE <123@example.com", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 123 extra", NntpParseStatus.ExtraArgument)] + [InlineData("HEAD not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("HEAD 123 extra", NntpParseStatus.ExtraArgument)] + [InlineData("BODY 12345678901234567", NntpParseStatus.InvalidArgument)] + [InlineData("BODY extra", NntpParseStatus.ExtraArgument)] + [InlineData("GROUP", NntpParseStatus.MissingArgument)] + [InlineData("GROUP a b", NntpParseStatus.ExtraArgument)] + [InlineData("LIST ACTIVE u[ks].*", NntpParseStatus.InvalidArgument)] + [InlineData("LIST COUNTS u[ks].*", NntpParseStatus.InvalidArgument)] + [InlineData("LIST ACTIVE foo bar", NntpParseStatus.ExtraArgument)] + [InlineData("LIST COUNTS foo bar", NntpParseStatus.ExtraArgument)] + [InlineData("LIST extra", NntpParseStatus.UnknownQualifier)] + [InlineData("LIST SOMETHING-THAT-DOES-NOT-EXIST", NntpParseStatus.UnknownQualifier)] + [InlineData("LIST MOTD extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST OVERVIEW.FMT extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST HEADERS FOO", NntpParseStatus.InvalidArgument)] + [InlineData("LIST HEADERS MSGID extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST ACTIVE.TIMES", NntpParseStatus.UnknownQualifier)] + [InlineData("CHECK", NntpParseStatus.MissingArgument)] + [InlineData("CHECK ", NntpParseStatus.MissingArgument)] + [InlineData("TAKETHIS", NntpParseStatus.MissingArgument)] + [InlineData("IHAVE", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO USER", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO PASS", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO SASL", NntpParseStatus.MissingArgument)] + [InlineData("COMPRESS", NntpParseStatus.MissingArgument)] + [InlineData("CHECK not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK x>", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK x<", NntpParseStatus.InvalidArgument)] + [InlineData("TAKETHIS not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("IHAVE not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("COMPRESS deflate", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK extra", NntpParseStatus.ExtraArgument)] + [InlineData("TAKETHIS extra", NntpParseStatus.ExtraArgument)] + [InlineData("IHAVE extra", NntpParseStatus.ExtraArgument)] + [InlineData("QUIT extra", NntpParseStatus.ExtraArgument)] + [InlineData("HELP MORE", NntpParseStatus.ExtraArgument)] + [InlineData("COMPRESS DEFLATE EXTRA", NntpParseStatus.ExtraArgument)] + [InlineData("SPEEDTEST", NntpParseStatus.MissingArgument)] + [InlineData("SPEEDTEST ", NntpParseStatus.MissingArgument)] + [InlineData("SPEEDTEST GIGANEWS extra", NntpParseStatus.ExtraArgument)] + [InlineData("SPEEDTEST Usenet Ninja", NntpParseStatus.ExtraArgument)] + public void Parse_InvalidCommands_HaveExpectedStatus(string text, NntpParseStatus status) + { + var command = NntpCommandTestParse.ParseCommand(text); + Assert.False(command.IsValid); + Assert.Equal(status, command.Status); + } + + /// + /// IHAVE without a message-id is parser MissingArgument (501), not the old + /// stub-handler 500. The parser owns required-parameter syntax. + /// + [Fact] + public void Ihave_WithoutMessageId_IsMissingArgument_NotOk() + { + var command = NntpCommandTestParse.ParseCommand("IHAVE"); + Assert.Equal(NntpVerb.Ihave, command.Verb); + Assert.Equal(NntpParseStatus.MissingArgument, command.Status); + Assert.False(command.IsValid); + } + + [Fact] + public void Parse_DoesNotAllocate_OnRepresentativeLines() + { + string[] lines = + [ + "CHECK ", + "TAKETHIS ", + "MODE STREAM", + "MODE READER", + "AUTHINFO USER foo", + "AUTHINFO PASS bar", + "QUIT", + "CHECK not-an-id", + "", + ]; + + foreach (var bytes in lines.Select(static text => Encoding.ASCII.GetBytes(text))) + { + for (var i = 0; i < 8; i++) + { + _ = NntpCommandParser.Parse(bytes); + } + + var before = GC.GetAllocatedBytesForCurrentThread(); + var sink = 0; + for (var i = 0; i < 256; i++) + { + var command = NntpCommandParser.Parse(bytes); + sink += (int)command.Verb + (int)command.Status + command.TokenCount; + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + Assert.NotEqual(int.MinValue, sink); + } + } + + [Theory] + [InlineData("NOSUCHCMD", "500 Unknown command")] + [InlineData("NEWNEWS", "500 Unknown command")] + [InlineData("NEWGROUPS", "500 Unknown command")] + [InlineData("NEWNEWS * 19990624 000000 GMT", "500 Unknown command")] + [InlineData("NEWGROUPS 19990624 000000 GMT", "500 Unknown command")] + [InlineData("MODE UNKNOWN", "501 Unknown command variant")] + [InlineData("LIST DISTRIB.PATS", "501 Unknown command variant")] + [InlineData("ARTICLE foo", "501 Syntax error")] + [InlineData("HEAD 123 extra", "501 Syntax error")] + [InlineData("BODY 12345678901234567", "501 Syntax error")] + [InlineData("STAT extra", "501 Syntax error")] + [InlineData("CHECK", "501 Syntax error")] + [InlineData("CHECK not-an-id", "501 Syntax error")] + [InlineData("CHECK extra", "501 Syntax error")] + [InlineData("IHAVE", "501 Syntax error")] + [InlineData("IHAVE not-an-id", "501 Syntax error")] + [InlineData("AUTHINFO USER", "501 AUTHINFO USER requires a username")] + [InlineData("AUTHINFO PASS", "501 AUTHINFO PASS requires a password")] + [InlineData("AUTHINFO GENERIC x", "501 Unknown command variant")] + [InlineData("COMPRESS", "501 COMPRESS requires a single algorithm argument")] + [InlineData("COMPRESS deflate", "501 Syntactically incorrect compression algorithm")] + [InlineData("QUIT extra", "501 Syntax error")] + [InlineData("HELP MORE", "501 Syntax error")] + [InlineData("SPEEDTEST", "501 Syntax error")] + [InlineData("SPEEDTEST GIGANEWS extra", "501 Syntax error")] + public async Task InvalidCommand_IsRejectedWithProtocolError_AndDoesNotReachHandler(string commandLine, string response) + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, commandLine); + Assert.Equal(response, await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task OverlongCommandLine_Is501_AndTheNextCommandStillRuns() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await dispatcher.DispatchAsync( + session, + NntpCommand.OverlongLine, + ReadOnlyMemory.Empty, + writer, + CancellationToken.None); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task InvalidCheck_DoesNotInvokeHandler_EvenWhenTransitAuthorized() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "CHECK not-an-id"); + var line = await duplex.ReadClientLineAsync(); + Assert.Equal("501 Syntax error", line); + Assert.DoesNotContain("238", line, StringComparison.Ordinal); + } + + [Theory] + [InlineData("")] + [InlineData("NOSUCHCMD")] + [InlineData("MODE")] + [InlineData("AUTHINFO GENERIC x")] + [InlineData("CHECK")] + [InlineData("CHECK not-an-id")] + [InlineData("IHAVE")] + [InlineData("TAKETHIS not-an-id")] + [InlineData("QUIT extra")] + public async Task InvalidCommand_CannotConstructHandlerContext(string commandLine) + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var invalid = NntpCommandTestParse.ParseCommand(commandLine); + Assert.False(invalid.IsValid); + var writer = new NntpResponseWriter(duplex.ServerOutput); + Assert.Throws(() => + new NntpCommandContext(session, invalid, ReadOnlyMemory.Empty, writer)); + } + + [Fact] + public async Task Ihave_ValidMessageId_ParsesValid_AndReaderRoleGateAnswers502() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + var parsed = NntpCommandTestParse.ParseCommand("IHAVE "); + Assert.True(parsed.IsValid); + var dispatch = NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "IHAVE ").AsTask(); + + // A valid IHAVE is parsed, but the Reader role gate answers before the handler (no 335, no article read). + Assert.Contains( + "502 Command not available on this server", + await duplex.ReadClientLineAsync(), + StringComparison.Ordinal); + await dispatch.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void MessageId_CommandArgument_UsesCommonValidator_AndFieldLimit() + { + Assert.True(NntpCommandParser.Parse("CHECK "u8).IsValid); + Assert.False(NntpCommandParser.Parse("CHECK "u8).IsValid); + Assert.False(NntpCommandParser.Parse("CHECK <@>"u8).IsValid); + Assert.True(NntpCommandParser.Parse("ARTICLE "u8).IsValid); + + var exactField = BuildMessageId(NntpMessageIdValidation.MaxMessageIdLength); + Assert.True(NntpMessageIdValidation.IsValidMessageId(exactField)); + Assert.True(NntpCommandParser.Parse(BuildCommand("CHECK ", exactField)).IsValid); + + var overField = BuildMessageId(NntpMessageIdValidation.MaxMessageIdLength + 1); + Assert.False(NntpMessageIdValidation.IsValidMessageId(overField)); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse(BuildCommand("CHECK ", overField)).Status); + + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK <>"u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK "u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK "u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK b@c>"u8).Status); + } + + private static byte[] BuildMessageId(int totalLength) + { + var bytes = new byte[totalLength]; + bytes[0] = (byte)'<'; + bytes[1] = (byte)'a'; + bytes[2] = (byte)'@'; + bytes[^1] = (byte)'>'; + if (totalLength > 4) + { + bytes.AsSpan(3, totalLength - 4).Fill((byte)'b'); + } + + return bytes; + } + + private static byte[] BuildCommand(string prefix, ReadOnlySpan argument) + { + var prefixBytes = Encoding.ASCII.GetBytes(prefix); + var line = new byte[prefixBytes.Length + argument.Length]; + prefixBytes.CopyTo(line); + argument.CopyTo(line.AsSpan(prefixBytes.Length)); + return line; + } + + private sealed class ParserDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new ParserDuplex()); + + public NntpSession CreateSession(IArticleIngestionQueue? queue = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTestParse.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTestParse.cs new file mode 100644 index 00000000..be989155 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTestParse.cs @@ -0,0 +1,28 @@ +using System.Text; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +internal static class NntpCommandTestParse +{ + public static (NntpCommand Command, byte[] Line) Parse(string text) + { + var line = Encoding.ASCII.GetBytes(text); + return (NntpCommandParser.Parse(line), line); + } + + public static NntpCommand ParseCommand(string text) => Parse(text).Command; + + public static ValueTask DispatchAsync( + NntpCommandDispatcher dispatcher, + NntpSession session, + NntpResponseWriter response, + string text, + CancellationToken cancellationToken = default) + { + var (command, line) = Parse(text); + return dispatcher.DispatchAsync(session, command, line, response, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTxStatusLoggingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTxStatusLoggingTests.cs new file mode 100644 index 00000000..57d60267 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCommandTxStatusLoggingTests.cs @@ -0,0 +1,333 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Semantic TX status logging: Debug-gated construction, writer isolation, no wire decode. +/// +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpCommandTxStatusLoggingTests +{ + [Fact] + public void Writer_HasNoLoggingInspectionSurface() + { + var type = typeof(NntpResponseWriter); + Assert.Null(type.GetMethod("TrackCommandResponse", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)); + Assert.Null(type.GetField("CommandResponseScope", BindingFlags.Static | BindingFlags.NonPublic)); + Assert.Null(type.GetNestedType("CommandResponseTrackingScope", BindingFlags.NonPublic)); + Assert.DoesNotContain(type.GetFields(BindingFlags.Instance | BindingFlags.Static | BindingFlags.NonPublic), + f => f.FieldType.IsGenericType && f.FieldType.GetGenericTypeDefinition() == typeof(AsyncLocal<>)); + } + + [Fact] + public void FormatCheck_UsesSemanticResultNotWireDecode() + { + var id = ""u8; + Assert.Equal("238 send article to be transferred", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Unseen, id)); + Assert.Equal("438 ", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Seen, id)); + Assert.Equal("431 ", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Unavailable, id)); + } + + [Fact] + public void FormatTakeThis_UsesSemanticOutcome() + { + var id = ""u8; + Assert.Equal("239 ", NntpCommandStatusText.FormatTakeThisAccepted(id)); + Assert.Equal("439 ", NntpCommandStatusText.FormatTakeThisRejected(id)); + } + + [Fact] + public void StaticStatus_IsNotDecodedFromWireBytes() + { + Assert.Equal("500 Command not implemented", NntpResponseStatus.CommandNotImplemented); + Assert.Equal("412 No newsgroup selected", NntpResponseStatus.NoNewsgroupSelected); + Assert.Equal("420 Current article number is invalid", NntpResponseStatus.CurrentArticleNumberInvalid); + Assert.Equal("423 No article with that number", NntpResponseStatus.NoArticleWithNumber); + Assert.Equal("430 No article with that message-id", NntpResponseStatus.NoArticleWithMessageId); + Assert.Equal("503 Data item not stored", NntpResponseStatus.ListDataItemNotStored); + Assert.Equal("215 Order of fields in overview database.", NntpResponseStatus.ListOverviewFmtFollows); + Assert.Equal("215 headers and metadata items supported:", NntpResponseStatus.ListHeadersFollows); + Assert.Equal("100 Help text follows", NntpResponseStatus.HelpTextFollows); + Assert.Equal("400 Service temporarily unavailable", NntpResponseStatus.ServiceTemporarilyUnavailable); + } + + [Fact] + public async Task WriteCompletion_DisabledLogger_DoesNotFormatClientOrStatus() + { + var logger = new GateLogger(enabled: false); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + + NntpCommandExecution.WriteCompletion( + logger, + session, + "CHECK", + TimeSpan.FromSeconds(0.012), + statusLine: "must-not-be-logged"); + + Assert.Empty(logger.Messages); + Assert.Equal(0, logger.FormatCount); + } + + [Fact] + public async Task RunAsync_DisabledLogger_DoesNotSetStatusLine() + { + var logger = new GateLogger(enabled: false); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("HELP"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + await NntpCommandExecution.RunAsync( + logger, + context, + "HELP", + static (ctx, ct) => NntpCommandReply.WriteAsync( + ctx, + new GateLogger(enabled: false), + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + ct), + CancellationToken.None); + + Assert.Null(context.StatusLine); + Assert.Empty(logger.Messages); + } + + [Fact] + public async Task RunAsync_EnabledLogger_NotesSemanticHelpStatus() + { + var logger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("HELP"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + await NntpCommandExecution.RunAsync( + logger, + context, + "HELP", + static (ctx, ct) => NntpCommandReply.WriteAsync( + ctx, + new GateLogger(enabled: true), + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + ct), + CancellationToken.None); + + Assert.Equal(NntpResponseStatus.HelpTextFollows, context.StatusLine); + var tx = Assert.Single(logger.Messages, m => m.Contains("TX: HELP", StringComparison.Ordinal)); + Assert.Contains("TX: HELP [100 Help text follows] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("ARTICLE [message-id / article-number]", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_HandlerThrowsBeforeResponse_KeepsNoStatusFormat() + { + var logger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("DATE"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + var thrown = await Assert.ThrowsAsync(async () => + await NntpCommandExecution.RunAsync( + logger, + context, + "DATE", + static (_, _) => throw new InvalidOperationException("boom"), + CancellationToken.None)); + Assert.Equal("boom", thrown.Message); + + var tx = Assert.Single(logger.Messages, m => m.Contains("TX: DATE", StringComparison.Ordinal)); + Assert.Contains("TX: DATE executed in", tx, StringComparison.Ordinal); + Assert.Contains("[failed]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("[403", tx, StringComparison.Ordinal); + Assert.DoesNotContain("[no response]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("111 ", tx, StringComparison.Ordinal); + Assert.Null(context.StatusLine); + } + + [Fact] + public async Task ConcurrentContexts_KeepIsolatedStatusLines() + { + var firstLogger = new GateLogger(enabled: true); + var secondLogger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + + var (checkA, lineA) = NntpCommandTestParse.Parse("CHECK "); + var (checkB, lineB) = NntpCommandTestParse.Parse("CHECK "); + var contextA = new NntpCommandContext(session, checkA, lineA, writer); + var contextB = new NntpCommandContext(session, checkB, lineB, writer); + + await Task.WhenAll( + NntpCommandExecution.RunAsync( + firstLogger, + contextA, + "CHECK", + (ctx, _) => + { + ctx.StatusLine = NntpCommandStatusText.FormatCheck( + HistoryLookupResult.Unseen, + ""u8); + return ValueTask.CompletedTask; + }, + CancellationToken.None).AsTask(), + NntpCommandExecution.RunAsync( + secondLogger, + contextB, + "CHECK", + (ctx, _) => + { + ctx.StatusLine = NntpCommandStatusText.FormatCheck( + HistoryLookupResult.Seen, + ""u8); + return ValueTask.CompletedTask; + }, + CancellationToken.None).AsTask()); + + Assert.Equal("238 send article to be transferred", contextA.StatusLine); + Assert.Equal("438 ", contextB.StatusLine); + Assert.Contains(firstLogger.Messages, m => m.Contains("[238 send article to be transferred]", StringComparison.Ordinal)); + Assert.Contains(secondLogger.Messages, m => m.Contains("[438 ]", StringComparison.Ordinal)); + Assert.DoesNotContain(firstLogger.Messages, m => m.Contains("438", StringComparison.Ordinal)); + Assert.DoesNotContain(secondLogger.Messages, m => m.Contains("238", StringComparison.Ordinal)); + } + + private sealed class GateLogger : ILogger + { + private readonly bool _enabled; + + public GateLogger(bool enabled) => _enabled = enabled; + + public ConcurrentBag Messages { get; } = []; + + public int FormatCount => Messages.Count; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => _enabled && logLevel >= LogLevel.Debug; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + Messages.Add(formatter(state, exception)); + } + } + + private sealed class StatusHarness : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public NntpSession CreateSession() + { + var connection = new PipeConnection(_clientToServer.Reader, _serverToClient.Writer); + return new NntpSession(connection, NullLogger.Instance); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output) + { + Input = input; + Output = output; + ClientIdentity = ConnectionClientIdentity.Direct( + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 49860)); + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCompressTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCompressTests.cs new file mode 100644 index 00000000..9a6ce766 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpCompressTests.cs @@ -0,0 +1,753 @@ +using System.Collections.Concurrent; +using System.IO.Compression; +using System.Net.Security; +using System.Net.Sockets; +using System.Runtime.InteropServices; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// Session-level COMPRESS DEFLATE (RFC 8054) regressions over the real transport stack. +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpCompressTests +{ + [Fact] + public async Task Compress_Capabilities_AdvertiseThenStopAfterActivation() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + var before = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.Contains("COMPRESS DEFLATE", before); + Assert.Contains("STARTTLS", before); + Assert.Contains("MODE-READER", before); + Assert.Contains("AUTHINFO USER", before); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + var after = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.True(duplex.Connection.IsCompressed); + Assert.DoesNotContain(after, static c => c.StartsWith("COMPRESS", StringComparison.Ordinal)); + Assert.DoesNotContain("STARTTLS", after); + Assert.DoesNotContain("MODE-READER", after); + Assert.Contains("AUTHINFO", after); + Assert.DoesNotContain("AUTHINFO USER", after); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_Capabilities_AfterModeReader_StillAdvertisesUntilActive() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE READER"); + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + var caps = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.Contains("COMPRESS DEFLATE", caps); + Assert.Contains("READER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Theory] + [InlineData("COMPRESS", "501 ")] + [InlineData("COMPRESS DEFLATE EXTRA", "501 ")] + [InlineData("COMPRESS deflate", "501 ")] + [InlineData("COMPRESS Deflate", "501 ")] + [InlineData("COMPRESS FOO", "503 ")] + [InlineData("COMPRESS GZIP", "503 ")] + public async Task Compress_SyntaxAndUnsupportedAlgorithm(string command, string expectedPrefix) + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.StartsWith(expectedPrefix, await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_Deflate_Exact206Wire_AndIsCompressed() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Equal("206 Compression active", await duplex.ReadClientLineAsync()); + Assert.True(session.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_Deflate_SessionRoundTrip_Repeated_AuthinfoRejected() + { + Assert.True(ProducesRawDeflateNotZlibOrGzip()); + + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var session = new NntpSession( + server, + NullLogger.Instance, + authenticationProvider: provider, + allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.Equal("206 Compression active", await ReadPlainLineAsync(clientSocket, plain)); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await WriteDeflateLineAsync(deflate, "COMPRESS DEFLATE"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await WriteDeflateLineAsync(deflate, "AUTHINFO USER alice"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + await WriteDeflateLineAsync(deflate, "AUTHINFO PASS secret"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxLine("AUTHINFO PASS secret")); + + await WriteDeflateLineAsync(deflate, "QUIT"); + Assert.StartsWith("205 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_Logging_OwnedByCompress_WithRecordingFactory() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = new NntpSession( + duplex.Connection, + recording.CreateLogger(), + loggerFactory: recording); + var sessionTask = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(duplex.Connection.IsCompressed); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: COMPRESS DEFLATE", StringComparison.Ordinal)); + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: COMPRESS [206 Compression active] executed in", StringComparison.Ordinal)); + Assert.Contains("[DEFLATE active]", tx, StringComparison.Ordinal); + Assert.Contains(recording.Categories, c => c == typeof(Compress).FullName); + } + + [Fact] + public async Task Compress_ClientSendsCompressedBytesImmediatelyAfter206_ConsumedByDeflate() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + await WriteDeflateLineAsync(deflate, "DATE"); + + await WaitForServerCompressedAsync(server); + Assert.False(server.IsCompleted); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_AuthinfoBeforeCompress_ThenCompressSucceeds() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + authenticationProvider: provider, + allowCleartextAuth: true) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("AUTHINFO USER alice\r\n"u8.ToArray()); + Assert.StartsWith("381 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await clientSocket.SendAsync("AUTHINFO PASS secret\r\n"u8.ToArray()); + Assert.StartsWith("281 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_AfterStartTls_LayersDeflateAboveTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider, + authenticationProvider: provider, + allowCleartextAuth: false) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var ssl = new SslStream(stream, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "AUTHINFO USER alice"); + Assert.StartsWith("381 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + await WriteSslLineAsync(ssl, "AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await WriteSslLineAsync(ssl, "COMPRESS DEFLATE"); + Assert.Equal("206 Compression active", await ReadSslLineAsync(ssl)); + await WaitForServerCompressedAsync(server); + Assert.True(server.IsTls); + Assert.True(server.IsCompressed); + + await using var deflate = new NntpDeflateStream(ssl); + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_PipelinedPlaintextAfterCommand_IsDiscarded() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\nDATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_ActivationFailureBefore206_Returns403() + { + await using var duplex = await CompressTestDuplex.CreateAsync(pauseReadsFails: true); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Contains("403 Unable to activate compression", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Connection.IsCompressed); + Assert.False(session.Connection.IsCompleted); + } + + [Fact] + public async Task Compress_UpgradeFailureAfter206_TerminatesWithoutSecondaryStatus() + { + await using var duplex = await CompressTestDuplex.CreateAsync(upgradeFails: true); + var sessionTask = duplex.CreateSession().RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(duplex.Connection.IsCompleted); + Assert.False(duplex.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_MalformedCompressedInput_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await clientSocket.SendAsync(new byte[] { 0x00, 0x01, 0x02, 0x03, 0xFF }); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.IsCompleted); + } + + private static bool ProducesRawDeflateNotZlibOrGzip() + { + using var ms = new MemoryStream(); + using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) + { + ds.Write("nntp"u8); + } + + var compressed = ms.ToArray(); + return compressed.Length > 0 + && compressed[0] != 0x78 + && !(compressed.Length >= 2 && compressed[0] == 0x1F && compressed[1] == 0x8B); + } + + private static async Task WaitForServerCompressedAsync(INntpConnection server) + { + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsCompressed) + { + wait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before DEFLATE was published."); + } + + await Task.Delay(10, wait.Token); + } + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + wait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before TLS was published."); + } + + await Task.Delay(10, wait.Token); + } + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static async Task ReadPlainLineAsync(Socket socket, PlainReceiveBuffer? buffer = null) + { + buffer ??= new PlainReceiveBuffer(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + if (buffer.TryReadLine(out var line)) + { + return line; + } + + var scratch = new byte[512]; + var n = await socket.ReceiveAsync(scratch, cts.Token); + Assert.True(n > 0, $"Socket closed while reading plain line (serverCompleted unknown)."); + buffer.Append(scratch.AsSpan(0, n)); + } + } + + /// Preserves leftover octets across plain-line reads before a DEFLATE wrap. + private sealed class PlainReceiveBuffer + { + private readonly List _bytes = []; + + public void Append(ReadOnlySpan data) => _bytes.AddRange(data); + + public ReadOnlyMemory TakeLeftover() + { + if (_bytes.Count == 0) + { + return ReadOnlyMemory.Empty; + } + + var leftover = _bytes.ToArray(); + _bytes.Clear(); + return leftover; + } + + public bool TryReadLine(out string line) + { + line = string.Empty; + var text = Encoding.ASCII.GetString(CollectionsMarshal.AsSpan(_bytes)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx < 0) + { + return false; + } + + line = text[..idx]; + var consumed = idx + 2; + _bytes.RemoveRange(0, consumed); + return true; + } + } + + private static Stream WrapAfterPlain(Socket socket, PlainReceiveBuffer buffer) + { + var network = new NetworkStream(socket, ownsSocket: false); + var leftover = buffer.TakeLeftover(); + return leftover.IsEmpty + ? network + : new PrefixedStream(network, leftover, leaveInnerOpen: false); + } + + private static async Task WriteSslLineAsync(SslStream ssl, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await ssl.WriteAsync(bytes); + await ssl.FlushAsync(); + } + + private static async Task ReadSslLineAsync(SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task WriteDeflateLineAsync(Stream deflate, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await deflate.WriteAsync(bytes); + await deflate.FlushAsync(); + } + + private static async Task ReadDeflateLineAsync(Stream deflate) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await deflate.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task> ReadCapabilitiesFromDuplexAsync(CompressTestDuplex duplex) + { + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = new List(); + while (true) + { + var line = await duplex.ReadClientLineAsync(); + if (line == ".") + { + break; + } + + caps.Add(line); + } + + return caps; + } + + private sealed class FixedAuthProvider : INntpAuthenticationProvider + { + private readonly string _user; + private readonly string _pass; + + public FixedAuthProvider(string user, string pass) + { + _user = user; + _pass = pass; + } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken) + { + if (username == _user && password == _pass) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success( + username, + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false))); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + } + } + + private sealed class CompressTestDuplex : IAsyncDisposable + { + private readonly System.IO.Pipelines.Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly System.IO.Pipelines.Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly PipeConnection _connection; + + private CompressTestDuplex(bool pauseReadsFails, bool upgradeFails) + { + _connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + pauseReadsFails, + upgradeFails); + } + + public System.IO.Pipelines.PipeWriter ServerOutput => _serverToClient.Writer; + public INntpConnection Connection => _connection; + + public static Task CreateAsync( + bool pauseReadsFails = false, + bool upgradeFails = false) => + Task.FromResult(new CompressTestDuplex(pauseReadsFails, upgradeFails)); + + public NntpSession CreateSession() => + new(_connection, NullLogger.Instance); + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private readonly bool _pauseReadsFails; + private readonly bool _upgradeFails; + private int _compressed; + + public PipeConnection( + System.IO.Pipelines.PipeReader input, + System.IO.Pipelines.PipeWriter output, + bool pauseReadsFails, + bool upgradeFails) + { + Input = input; + Output = output; + _pauseReadsFails = pauseReadsFails; + _upgradeFails = upgradeFails; + ClientIdentity = VectorNNTPD.ReaderService.Networking.Proxy.ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + } + + public System.IO.Pipelines.PipeReader Input { get; } + public System.IO.Pipelines.PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public VectorNNTPD.ReaderService.Networking.Proxy.ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + _pauseReadsFails + ? Task.FromException(new InvalidOperationException("simulated pause failure")) + : Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + if (_upgradeFails) + { + return Task.FromException(new InvalidOperationException("simulated upgrade failure")); + } + + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionFoundationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionFoundationTests.cs new file mode 100644 index 00000000..acdf425b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionFoundationTests.cs @@ -0,0 +1,299 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class NntpSessionFoundationTests +{ + [Fact] + public void Authorization_DefaultsDenyTransitAndPosting() + { + var authz = NntpAuthorization.Unauthenticated; + Assert.False(authz.IsAuthenticated); + Assert.False(authz.AuthorizedReader); + Assert.False(authz.AuthorizedTransit); + Assert.False(authz.PostingPermitted); + Assert.False(authz.StreamingPermitted); + } + + [Fact] + public void CommandParser_SplitsVerbAndArguments() + { + var caps = NntpCommandTestParse.ParseCommand("CAPABILITIES"); + Assert.Equal(NntpVerb.Capabilities, caps.Verb); + Assert.Equal(0, caps.TokenCount); + Assert.True(caps.IsValid); + + var (auth, authLine) = NntpCommandTestParse.Parse("AUTHINFO USER fred"); + Assert.Equal(NntpVerb.AuthInfo, auth.Verb); + Assert.Equal(NntpVerb.User, auth.Qualifier); + Assert.Equal("fred", Encoding.ASCII.GetString(auth.ArgumentSpan(authLine))); + + var empty = NntpCommandParser.Parse(" "u8); + Assert.False(empty.IsValid); + Assert.Equal(NntpParseStatus.Empty, empty.Status); + } + + [Fact] + public async Task Dispatcher_UnknownCommand_Returns500() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "NOSUCHCMD"); + Assert.Contains("500 Unknown command", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_UnknownAuthinfoVariant_Returns501() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "AUTHINFO GENERIC x"); + Assert.Contains("501 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_ReaderCommandWithoutAuth_Returns480() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "ARTICLE"); + Assert.Contains("480 Authentication required", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_TransitCommandOnReaderWithoutAuth_Returns502BeforeAuthentication() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK "); + Assert.Contains("502 Command not available on this server", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_TransitCommandOnReaderWhenAuthenticatedButNotAuthorized_Returns502NotAvailable() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "IHAVE "); + Assert.Contains("502 Command not available on this server", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_ModeStreamOnReader_Returns502NotAvailable_EvenWhenStreamingPermitted() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "MODE STREAM"); + Assert.Contains("502 Command not available on this server", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Session_GreetingAndQuit_PublicCommands() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + Assert.Equal(18, date.Length); // "111 " + 14 digit stamp + + await duplex.WriteClientLineAsync("MODE READER"); + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = new List(); + while (true) + { + var line = await duplex.ReadClientLineAsync(); + if (line == ".") + { + break; + } + + caps.Add(line); + } + + Assert.Contains("VERSION 2", caps); + Assert.Contains("READER", caps); + Assert.Contains("AUTHINFO USER", caps); + Assert.Contains("COMPRESS DEFLATE", caps); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask; + } + + [Fact] + public async Task ModeReader_PostingPermitted_Returns200() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(NntpAuthorization.Unauthenticated.With(postingPermitted: true)); + var sessionTask = session.RunAsync(); + + Assert.StartsWith("200 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("MODE READER"); + Assert.StartsWith("200 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask; + } + + private sealed class SessionTestDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + private SessionTestDuplex() + { + } + + public PipeWriter ServerOutput => _serverToClient.Writer; + public PipeReader ServerInput => _clientToServer.Reader; + + public static Task CreateAsync() => Task.FromResult(new SessionTestDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + ServerInput, + ServerOutput, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionIdleTimeoutTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionIdleTimeoutTests.cs new file mode 100644 index 00000000..6fe22cac --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionIdleTimeoutTests.cs @@ -0,0 +1,795 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Framing; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// NNTP command idle timeout: session-owned waiter, ControllableTimeProvider, no 300s sleeps. +/// +public sealed class NntpSessionIdleTimeoutTests +{ + private static readonly TimeSpan Idle = TimeSpan.FromSeconds(5); + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(5); + + private static readonly NntpAuthorization TransitAuth = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Fact] + public void DisconnectReason_IdleTimeout_FormatsForTcpLog() + { + Assert.Equal("IdleTimeout", TcpDisconnectReason.IdleTimeout.ToString()); + } + + [Fact] + public void SessionConstructor_RejectsIdleTimeOutsideValidatedRange() + { + Assert.Throws(() => + new NntpSession( + new StubConnection(), + NullLogger.Instance, + commandIdleTimeout: TimeSpan.Zero)); + Assert.Throws(() => + new NntpSession( + new StubConnection(), + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(NntpdOptions.MaxIdleTime + 1))); + } + + [Fact] + public async Task NewSession_StaysConnectedBeforeIdleTime_ThenDisconnects() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(1)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.True(session.IdleWatchTaskForTests!.IsCompletedSuccessfully); + } + + [Fact] + public async Task CommandActivity_ResetsIdleTimeout() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + // Drain the greeting Delay so the waiter re-samples. T+Idle would have + // disconnected without the DATE reset. + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(2)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task MultipleCommands_TimeoutMeasuredFromMostRecent() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task PartialCommandLine_DoesNotResetIdleTimeout() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + await duplex.WriteClientAsync("DAT"); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task ActiveAuthinfoCommand_LongerThanIdleTime_IsNotKilled() + { + var clock = new ControllableTimeProvider(); + var hold = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var provider = new GatedAuthProvider(hold, entered); + + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: provider); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + await entered.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + clock.Advance(Idle); + await Task.Delay(20); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + + hold.TrySetResult(NntpAuthenticationResult.Failed); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + var afterPass = await duplex.ReadClientLineAsync(); + Assert.StartsWith("481 ", afterPass, StringComparison.Ordinal); + Assert.False(run.IsCompleted); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task Check_OnReader_Returns502_WithoutPipelineWork_ThenIdleTimeout() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.Pipeline!.Occupied); + Assert.False(history.Started.Task.IsCompleted); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task TakeThis_OnReader_Returns502_WithoutPipelineWork_ThenIdleTimeout() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.TakeThisWindow!.Occupied); + Assert.Equal(0, queue.Count); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task Quit_CancelsIdleWatchWithoutLeak() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run.WaitAsync(Safety); + + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + Assert.True(session.IdleWatchTaskForTests!.IsCompleted); + Assert.False(session.IdleWatchTaskForTests.IsFaulted); + Assert.False(session.IdleWatchTaskForTests.IsCanceled); + } + + [Fact] + public async Task CommandWorkConcurrentWithExpiry_DoesNotDisconnectAfterReset() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + session.BeginCommandWork(); + Assert.False(session.IdleCloseCommittedForTests); + hold.TrySetResult(); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.False(session.IdleCloseCommittedForTests); + + session.EndCommandWork(); + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task BeginAndEndDuringIdleCommit_StaleIdleDecisionDoesNotClose() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + session.BeginCommandWork(); + session.EndCommandWork(); + Assert.Equal(0, session.CommandWorkForTests); + Assert.False(session.IdleCloseCommittedForTests); + hold.TrySetResult(); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.False(session.IdleCloseCommittedForTests); + + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task IdleCommitThenBegin_CommandObservesClosingSession() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + hold.TrySetResult(); + await WaitUntilAsync(() => session.IdleCloseCommittedForTests, Safety); + + session.BeginCommandWork(); + Assert.True(session.IdleCloseCommittedForTests); + session.EndCommandWork(); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(0, session.CommandWorkForTests); + } + + [Fact] + public async Task SuccessfulCommand_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task HandlerException_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: new ThrowingAuthProvider()); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + _ = await duplex.ReadClientLineAsync(); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + Assert.False(run.IsCompleted); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task CommandCancellation_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var hold = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: new GatedAuthProvider(hold, entered)); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + await entered.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + } + + [Fact] + public async Task Check_OnReader_DoesNotOccupyPipeline_AndReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.Pipeline!.Occupied); + Assert.Equal(0, session.CommandWorkForTests); + Assert.False(history.Started.Task.IsCompleted); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task Check_OnReader_Immediate502_LeavesPipelineEmpty() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.CommandWorkForTests); + Assert.Equal(0, session.Pipeline!.Occupied); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task Check_OnReader_CloseAfter502_LeavesNoCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.Pipeline!.Occupied); + Assert.Equal(0, session.CommandWorkForTests); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + } + + [Fact] + public async Task TakeThis_OnReader_DoesNotOccupyWindow_AndReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.TakeThisWindow!.Occupied); + Assert.Equal(0, session.CommandWorkForTests); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task TakeThis_OnReader_CloseAfter502_LeavesNoCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.TakeThisWindow!.Occupied); + Assert.Equal(0, session.CommandWorkForTests); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + } + + [Fact] + public async Task ConfiguredIdleTime_IsHonouredBySession() + { + var clock = new ControllableTimeProvider(); + var idle = TimeSpan.FromSeconds(2); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, idle); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(1)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class IdleDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + ControllableTimeProvider clock, + TimeSpan? idle = null, + IHistoryDb? historyDb = null, + IArticleIngestionQueue? articleIngestion = null, + INntpAuthenticationProvider? authenticationProvider = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: authenticationProvider, + articleIngestion: articleIngestion, + historyDb: historyDb, + commandIdleTimeout: idle ?? Idle, + timeProvider: clock); + } + + public async Task WriteClientLineAsync(string line) => + await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class StubConnection : INntpConnection + { + public PipeReader Input => throw new NotSupportedException(); + public PipeWriter Output => throw new NotSupportedException(); + public EndPoint? RemoteEndPoint => null; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => CancellationToken.None; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class GatedAuthProvider( + TaskCompletionSource hold, + TaskCompletionSource entered) : INntpAuthenticationProvider + { + public async ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + entered.TrySetResult(); + return await hold.Task.WaitAsync(cancellationToken); + } + } + + private sealed class ThrowingAuthProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + throw new InvalidOperationException("handler-boom"); + } + + private sealed class GatedHistoryDb : IHistoryDb + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + WaitAsync(cancellationToken); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + WaitAsync(cancellationToken); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in HistoryDigest digest) => false; + + private async ValueTask WaitAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + return await Release.Task.WaitAsync(cancellationToken); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs new file mode 100644 index 00000000..6f46e84a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs @@ -0,0 +1,192 @@ +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// +/// Session RX owns Connection.Input. Teardown must not complete that reader while +/// NntpContinuousRxReader.ReadUnitAsync can still call ReadAsync. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpSessionInputReaderLifecycleTests +{ + [Fact] + public async Task RemoteFin_WhileSessionWaitingForCommand_EndsWithoutReaderCompletedException() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task RemoteFin_AfterDateCommand_EndsWithoutReaderCompletedException() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + await client.SendAsync("DATE\r\n"u8.ToArray()); + Assert.StartsWith("111 ", await ReadLineAsync(client), StringComparison.Ordinal); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task RequestClose_WhileSessionWaitingForCommand_CompletesReaderAfterRxStops() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + Assert.False(server.InputReaderCompletedForTests); + + session.RequestClose(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ConnectionCompleteAsync_WhileSessionWaitingForCommand_DoesNotRaceInputReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + + await server.CompleteAsync().WaitAsync(TimeSpan.FromSeconds(5)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task Quit_CompletesInputReaderOnce_AfterCommandLoopStops() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession(server, NullLogger.Instance); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await ReadLineAsync(client)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + private static async Task ReadLineAsync(Socket socket) + { + var buffer = new byte[256]; + var stored = new List(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + var n = await socket.ReceiveAsync(buffer, SocketFlags.None, timeout.Token); + Assert.True(n > 0); + for (var i = 0; i < n; i++) + { + stored.Add(buffer[i]); + } + + var span = stored.ToArray().AsSpan(); + var crlf = span.IndexOf("\r\n"u8); + if (crlf >= 0) + { + return Encoding.ASCII.GetString(span[..crlf]); + } + } + } + + private sealed class CapturingSessionLogger : ILogger + { + public List Errors { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel < LogLevel.Error) + { + return; + } + + var message = formatter(state, exception); + if (exception is not null) + { + Errors.Add(exception.ToString()); + } + + if (!string.IsNullOrEmpty(message)) + { + Errors.Add(message); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/NntpStartTlsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpStartTlsTests.cs new file mode 100644 index 00000000..d30192b7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/NntpStartTlsTests.cs @@ -0,0 +1,286 @@ +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// Session-level STARTTLS lifecycle regressions (plain → 382 → TLS). +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpStartTlsTests +{ + [Fact] + public async Task StartTls_Success_Stress_NoUnconsumedPlaintextFalsePositive() + { + // Expose the historical ClientHello→Input race under normal AuthenticateAsClient timing. + for (var i = 0; i < 30; i++) + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + } + + [Fact] + public async Task StartTls_Success_ThenDateOverTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + var greeting = await ReadPlainLineAsync(clientSocket); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + Assert.False(server.IsCompleted); + Assert.False(sessionTask.IsCompleted); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + // Tear down without QUIT to avoid racing CompleteAsync against the last status line. + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_CommandFullyConsumed_NoFalsePositiveDrain() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + Assert.False(server.IsCompleted); + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_PipelinedCommand_IsDiscarded_ThenTlsSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\nDATE\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_HandshakeFailure_IsTerminal_NoSecondaryNntpResponse() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.IsCompleted); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.IsTls); + + using var idleCts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + var buffer = new byte[64]; + try + { + var n = await clientSocket.ReceiveAsync(buffer, idleCts.Token); + if (n > 0) + { + var text = Encoding.ASCII.GetString(buffer, 0, n); + Assert.DoesNotContain("403 ", text, StringComparison.Ordinal); + Assert.DoesNotContain("Command failed", text, StringComparison.Ordinal); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (SocketException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + [Fact] + public async Task StartTls_ClientClosesAfter382_SessionEndsCleanly() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + clientSocket.Dispose(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.IsTls); + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var tlsWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + tlsWait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before TLS mode was published."); + } + + await Task.Delay(10, tlsWait.Token); + } + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static async Task ReadPlainLineAsync(Socket socket) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task WriteSslLineAsync(SslStream ssl, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await ssl.WriteAsync(bytes); + await ssl.FlushAsync(); + } + + private static async Task ReadSslLineAsync(SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/PostArticleValidationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/PostArticleValidationTests.cs new file mode 100644 index 00000000..53ea9141 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/PostArticleValidationTests.cs @@ -0,0 +1,489 @@ +using System.Net; +using System.Text; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class PostArticleValidationTests +{ + private static readonly DateTimeOffset Now = + new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public void MissingDate_IsSynthesized() + { + var parsed = Parse(Build(date: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.True(parsed.DateSynthesized); + Assert.Equal(Now, parsed.AuthorDate); + } + + [Fact] + public void OmittedDate_NormalizeWritesGeneratedDateOnce() + { + var parsed = Parse(Build(date: null, extraHeaders: "Path: client.path\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + var generated = PostRfcDate.Format(Now); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + generated + "\r\n", text, StringComparison.Ordinal); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("client.path", text, StringComparison.Ordinal); + } + + [Fact] + public void MissingFrom_IsRejected() + { + AssertRejected(Build(from: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingNewsgroups_IsRejected() + { + AssertRejected(Build(newsgroups: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingSubject_IsRejected() + { + AssertRejected(Build(subject: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingMessageId_IsSynthesized() + { + var parsed = Parse(Build(messageId: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.True(parsed.MessageIdSynthesized); + Assert.StartsWith("<", parsed.MessageId, StringComparison.Ordinal); + Assert.EndsWith("@usenet.ninja>", parsed.MessageId, StringComparison.Ordinal); + Assert.Equal(47, parsed.MessageId!.Length); + } + + [Fact] + public void SynthesizedMessageId_UsesConfiguredDomainAndMd5Length() + { + var parsed = Parse(Build(messageId: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Matches(@"^<[0-9a-f]{32}@usenet\.ninja>$", parsed.MessageId); + } + + [Fact] + public void MalformedMessageId_IsRejected() + { + AssertRejected(Build(messageId: "not-an-id"), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: "b>"), PostingFailureCategory.InvalidMessageId); + } + + [Fact] + public void Rfc5536MessageId_IsAccepted() + { + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: "<\"quoted\"@example.com>"), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + var parsed = Parse(Build(messageId: "")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal("", parsed.MessageId); + + parsed = Parse(Build(messageId: "")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal("", parsed.MessageId); + } + + [Fact] + public void MalformedDate_IsRejected() + { + AssertRejected(Build(date: "yesterday"), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void FutureDateBeyondPolicy_IsRejected() + { + AssertRejected(Build(date: PostRfcDate.Format(Now.AddHours(25))), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void ExcessivelyOldDate_IsRejected() + { + AssertRejected(Build(date: PostRfcDate.Format(Now.AddDays(-15))), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void GmtDate_IsAccepted() + { + var parsed = Parse(Build(date: "25 Sep 2026 12:00:00 GMT")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal(Now, parsed.AuthorDate); + Assert.False(parsed.DateSynthesized); + } + + [Fact] + public void DuplicateSingleton_IsRejected() + { + var extra = "Subject: first\r\nSubject: second\r\n"; + AssertRejected(Build(extraHeaders: extra, subject: null), PostingFailureCategory.DuplicateHeader); + } + + [Fact] + public void MalformedNewsgroup_IsRejected() + { + AssertRejected(Build(newsgroups: "misc..test"), PostingFailureCategory.InvalidNewsgroups); + } + + [Fact] + public void DuplicateNewsgroup_IsRejected() + { + AssertRejected(Build(newsgroups: "misc.test,misc.test"), PostingFailureCategory.InvalidNewsgroups); + } + + [Fact] + public void MultipleValidNewsgroups_AreAccepted() + { + var parsed = Parse(Build(newsgroups: "misc.test, alt.test")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal(["misc.test", "alt.test"], parsed.Newsgroups); + } + + [Fact] + public void FollowupToPoster_IsAccepted() + { + var parsed = Parse(Build(extraHeaders: "Followup-To: poster\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + } + + [Fact] + public void MalformedFollowupTo_IsRejected() + { + AssertRejected(Build(extraHeaders: "Followup-To: ..bad\r\n"), PostingFailureCategory.InvalidFollowupTo); + } + + [Fact] + public void InReplyToNotInReferences_IsRejected() + { + AssertRejected( + Build(extraHeaders: "References: \r\nIn-Reply-To: \r\n"), + PostingFailureCategory.InvalidReferences); + } + + [Fact] + public void ControlHeader_IsRejected() + { + AssertRejected( + Build(extraHeaders: "Control: cancel \r\n"), + PostingFailureCategory.InvalidControl); + } + + [Fact] + public void ControlHeader_AuthorizedCancel_IsAccepted() + { + var parsed = Parse(Build( + messageId: "", + extraHeaders: "Control: cancel \r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _, + controlCancelPermitted: true)); + Assert.Equal("", parsed.MessageId); + } + + [Fact] + public void ControlHeader_Authorized_RejectsOtherVerb() + { + var parsed = Parse(Build(extraHeaders: "Control: newgroup misc.test\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void ControlHeader_Authorized_RejectsMalformedCancel() + { + var parsed = Parse(Build(extraHeaders: "Control: cancel not-an-id\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void ControlHeader_Authorized_RejectsCancelWithExtraArgument() + { + var parsed = Parse(Build(extraHeaders: "Control: cancel extra\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void MalformedApproved_IsRejected() + { + AssertRejected(Build(extraHeaders: "Approved: not-a-mailbox\r\n"), PostingFailureCategory.InvalidApproved); + } + + [Fact] + public void MailRoutingHeader_IsPolicyRejected() + { + AssertRejected(Build(extraHeaders: "Received: from nowhere\r\n"), PostingFailureCategory.PolicyRejected); + } + + [Fact] + public void LeadingFold_IsRejected() + { + Assert.False(PostHeaderParser.TryParse( + Encoding.ASCII.GetBytes(" Subject: folded\r\n\r\nbody\r\n"), + out _, + out var failure)); + Assert.Equal(PostingFailureCategory.MalformedHeader, failure.Category); + } + + [Fact] + public void EmbeddedNul_IsRejected() + { + var bytes = Encoding.ASCII.GetBytes(Build()); + bytes[10] = 0; + Assert.False(PostHeaderParser.TryParse(bytes, out _, out var failure)); + Assert.Equal(PostingFailureCategory.MalformedHeader, failure.Category); + } + + [Fact] + public void NewsgroupPolicy_ReportsCatalogUnavailable() + { + var evaluation = SyntaxOnlyNewsgroupPostingPolicy.Instance.Evaluate(["misc.test"]); + Assert.Equal(NewsgroupCatalogStatus.CatalogUnavailable, evaluation.Status); + Assert.Null(evaluation.Failure); + } + + [Fact] + public void Normalize_ReplacesServerOwnedHeaders_AndPreservesDate() + { + var date = PostRfcDate.Format(Now); + var parsed = Parse(Build( + date: date, + extraHeaders: + "Path: client.path\r\n" + + "Injection-Date: forged\r\n" + + "Injection-Info: forged\r\n" + + "NNTP-Posting-Date: forged\r\n" + + "NNTP-Posting-Host: 1.2.3.4\r\n" + + "X-Trace: forged\r\n" + + "Xref: nntpd01.usenet.ninja misc.test:1\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var peer = IPAddress.Parse("192.0.2.10"); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(peer, 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + Assert.Contains("Date: " + date, text, StringComparison.Ordinal); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date: " + date, text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Date", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + Assert.DoesNotContain("192.0.2.10", text, StringComparison.Ordinal); + Assert.DoesNotContain("1.2.3.4", text, StringComparison.Ordinal); + Assert.DoesNotContain("posting-host", text, StringComparison.Ordinal); + Assert.Contains( + "Injection-Info: nntpd01.usenet.ninja; logging-data=\"\"; mail-complaints-to=\"" + + NntpdOptions.DefaultMailComplaintsTo + "\"", + text, + StringComparison.Ordinal); + var xtraceStart = text.IndexOf("X-Trace: ", StringComparison.Ordinal); + Assert.True(xtraceStart >= 0); + var xtrace = text[(xtraceStart + "X-Trace: ".Length)..text.IndexOf("\r\n", xtraceStart, StringComparison.Ordinal)]; + Assert.StartsWith(AesGcmPostingTraceProtector.TokenPrefix, xtrace, StringComparison.Ordinal); + Assert.NotEqual(Convert.ToBase64String(peer.GetAddressBytes()), xtrace); + Assert.True(protector.TryUnprotect(xtrace, out var recovered)); + Assert.Equal(peer, recovered.Address); + Assert.Equal(119, recovered.Port); + Assert.Equal(Now, recovered.InjectedAtUtc); + Assert.DoesNotContain("Path: client.path", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + Assert.DoesNotContain("forged", text, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(text, "Path:")); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Equal(1, CountOccurrences(text, "Injection-Date:")); + Assert.Equal(1, CountOccurrences(text, "X-Trace:")); + } + + [Fact] + public void Normalize_PreservesClientXPgpSig_AndDoesNotTreatItAsServerOwned() + { + var date = PostRfcDate.Format(Now); + var parsed = Parse(Build( + date: date, + extraHeaders: + "Control: cancel \r\n" + + "X-PGP-Sig: BouncyCastle Subject,Control,Message-ID,Date,From,Sender\r\n" + + "\tiQCVAwUBNhiHnsJdOtO4janBAQGFFwP/SCsq1Isgw8DXHDaRkr0cdkZidVH41N6d\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _, + controlCancelPermitted: true)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("192.0.2.10"), 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + Assert.Contains("X-PGP-Sig: BouncyCastle Subject,Control,Message-ID,Date,From,Sender\r\n", text, StringComparison.Ordinal); + Assert.Contains("\tiQCVAwUBNhiHnsJdOtO4janBAQGFFwP/SCsq1Isgw8DXHDaRkr0cdkZidVH41N6d\r\n", text, StringComparison.Ordinal); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("X-Trace: ", text, StringComparison.Ordinal); + } + + private static void AssertRejected(string article, PostingFailureCategory category) + { + var parsed = Parse(article); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure)); + Assert.Equal(category, failure.Category); + } + + private static ParsedPostArticle Parse(string article) + { + Assert.True(PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build( + string? date = "25 Sep 2026 12:00:00 +0000", + string? from = "poster@example.com", + string? newsgroups = "misc.test", + string? subject = "test", + string? messageId = "", + string extraHeaders = "", + string body = "body\r\n") + { + var sb = new StringBuilder(); + if (date is not null) + { + sb.Append("Date: ").Append(date).Append("\r\n"); + } + + if (from is not null) + { + sb.Append("From: ").Append(from).Append("\r\n"); + } + + if (newsgroups is not null) + { + sb.Append("Newsgroups: ").Append(newsgroups).Append("\r\n"); + } + + if (subject is not null) + { + sb.Append("Subject: ").Append(subject).Append("\r\n"); + } + + if (messageId is not null) + { + sb.Append("Message-ID: ").Append(messageId).Append("\r\n"); + } + + sb.Append(extraHeaders); + sb.Append("\r\n"); + sb.Append(body); + return sb.ToString(); + } + + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private static int CountHeader(string text, string header) + { + var count = text.StartsWith(header, StringComparison.Ordinal) ? 1 : 0; + return count + CountOccurrences(text, "\r\n" + header); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/PostCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/PostCommandTests.cs new file mode 100644 index 00000000..35a078ca --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/PostCommandTests.cs @@ -0,0 +1,1297 @@ +using System.IO.Hashing; +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class PostCommandTests +{ + private static readonly NntpAuthorization Poster = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private static readonly NntpAuthorization ReaderNoPost = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false); + + private static readonly NntpAuthorization Newsmaster = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false, + controlCancelPermitted: true); + + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(20); + private static readonly TimeSpan Idle = TimeSpan.FromSeconds(5); + + [Fact] + public async Task Unauthenticated_Returns480_AndDoesNotReadArticle() + { + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task PostingProhibited_Returns440_AndDoesNotReadArticle() + { + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(authorization: ReaderNoPost); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("440 Posting not permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task OrdinaryPoster_ControlHeader_Returns441_AndDoesNotEnqueue() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle(extraHeaders: "Control: cancel \r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Newsmaster_WellFormedCancel_Returns240_AndKeepsDistinctMessageId() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Newsmaster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle( + messageId: "", + extraHeaders: "Control: cancel \r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var cts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(cts.Token); + Assert.NotNull(inbound); + Assert.Equal("", inbound!.MessageId); + var text = Encoding.ASCII.GetString(inbound.Payload.Span); + Assert.Contains("Control: cancel ", text, StringComparison.Ordinal); + Assert.Contains("Message-ID: ", text, StringComparison.Ordinal); + Assert.DoesNotContain("Message-ID: ", text, StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Newsmaster_OtherControlVerb_Returns441() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Newsmaster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(extraHeaders: "Control: newgroup misc.test\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task AuthenticatedOrdinaryUser_CannotSubmitControl() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, authenticationProvider: new DualAuthProvider()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await AuthenticateAsync(duplex, "poster", "poster-secret"); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle(extraHeaders: "Control: cancel \r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(extraHeaders: "Control: newgroup misc.test\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task AuthenticatedOrdinaryUser_XTraceContainsSessionUsername() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, authenticationProvider: new DualAuthProvider()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await AuthenticateAsync(duplex, "poster+tag/name", "poster-secret"); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle(extraHeaders: "X-Authenticated-User: spoofed\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var timeoutCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(timeoutCts.Token); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + var xtrace = RequireHeader(text, "X-Trace"); + Assert.DoesNotContain("poster+tag/name", text, StringComparison.Ordinal); + Assert.DoesNotContain("spoofed", xtrace, StringComparison.Ordinal); + Assert.True(session.PostingTraceProtector!.TryUnprotect(xtrace, out var recovered)); + Assert.Equal("poster+tag/name", recovered.AuthenticatedUsername); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task AuthenticatedNewsmaster_CancelXTraceContainsUsername_OtherControlRejected() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, authenticationProvider: new DualAuthProvider()); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await AuthenticateAsync(duplex, "newsmaster", "unit-test-newsmaster-password"); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle( + messageId: "", + extraHeaders: "Control: cancel \r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + var xtrace = RequireHeader(text, "X-Trace"); + Assert.DoesNotContain("newsmaster", text.Replace("X-Trace: " + xtrace, string.Empty, StringComparison.Ordinal), StringComparison.Ordinal); + Assert.True(session.PostingTraceProtector!.TryUnprotect(xtrace, out var recovered)); + Assert.Equal("newsmaster", recovered.AuthenticatedUsername); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(extraHeaders: "Control: newgroup misc.test\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(extraHeaders: "Control: cancel not-an-id\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + ValidArticle(extraHeaders: "Control: cancel extra\r\n") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ExtraArgument_Is501() + { + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(authorization: Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST extra"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ValidArticle_Returns240_AndEnqueuesNormalizedArticle() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(messageId: "") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var cts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(cts.Token); + Assert.NotNull(inbound); + Assert.Equal("", inbound!.MessageId); + Assert.Equal(InboundArticleProducer.Post, inbound.Producer); + var text = Encoding.ASCII.GetString(inbound.Payload.Span); + Assert.Contains("Message-ID: ", text, StringComparison.Ordinal); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + Assert.Equal(inbound.Record.ArtData.Length, inbound.Record.ArtSize); + Assert.True(inbound.Record.ArtLines >= 0); + Assert.NotEqual(ArticleType.None, inbound.Record.ArtType); + Assert.True(inbound.Record.MessageId.SequenceEqual(""u8)); + Assert.True(inbound.Record.Fields.MessageId.IsPresent); + Assert.True(inbound.Record.Fields.Newsgroups.IsPresent); + Assert.Equal(ArticleId.FromMessageId(""u8), inbound.Record.ArtId); + Assert.Equal(XxHash3.HashToUInt64(inbound.Record.ArtData.Span), inbound.Record.ArtHash); + Assert.Contains("news.usenet.ninja!", text, StringComparison.Ordinal); + Assert.Contains(".POSTED", text, StringComparison.Ordinal); + Assert.DoesNotContain("Path: client.path", text, StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task RejectedArticle_Returns441_AndDoesNotEnqueue() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(ValidArticle(includeSubject: false) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task CannotBePipelined_SecondCommandIsNotExecutedAsCommand() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("POST\r\nDATE\r\n.\r\n"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ConfigurableLimit_RejectsDuringReceive() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, maxArticleSize: 256); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(PaddedArticle(300) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ReceiveSizeRejection_WritesRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, maxArticleSize: 256, newsLog: news); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(PaddedArticle(300) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(441, evt.ResponseCode); + Assert.Equal(IngressNewsReasons.ArticleTooLarge, Encoding.ASCII.GetString(evt.Reason.Span)); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task NewsWriterFailure_On441_DoesNotChangeRejectionResponse() + { + var news = new ThrowingNewsLogWriter(); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, maxArticleSize: 256, newsLog: news); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(PaddedArticle(300) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(1, news.WriteCalls); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ExactConfiguredLimit_IsAccepted() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, maxArticleSize: 2048); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var destuffed = PaddedArticle(2048); + Assert.Equal(2048, Encoding.ASCII.GetByteCount(destuffed)); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(destuffed + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound!.Record.ParseStatus); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DefaultFiveMibPlusOne_IsRejected() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(PaddedArticle(NntpdOptions.DefaultMaxArticleSize + 1) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DefaultFiveMibExact_IsRejectedByArticleRecordBecauseCompleteArticleExceedsCommonLimit() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var destuffed = PaddedArticle(NntpdOptions.DefaultMaxArticleSize); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(destuffed + ".\r\n"); + // Receive accepts destuffed client size == MaxArticleSize. POST then injects + // server headers and the factory applies Common MaxArticleBytes to the + // destuffed complete article, which exceeds 5 MiB. + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task NearDefaultFiveMib_WithRoomForServerHeaders_IsAccepted() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var destuffed = PaddedArticle(NntpdOptions.DefaultMaxArticleSize - 4096); + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(destuffed + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound!.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task MissingMessageId_IsSynthesizedAndStableOnAcceptedArticle() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(messageId: null) + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Matches(@"^<[0-9a-f]{32}@usenet\.ninja>$", inbound!.MessageId); + var text = Encoding.ASCII.GetString(inbound.Payload.Span); + Assert.Contains("Message-ID: " + inbound.MessageId, text, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(text, "Message-ID:")); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DuplicateMessageId_Returns441_WithoutSecondAdmit() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + history.Remember(""u8.ToArray()); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, historyDb: history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(messageId: "") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ForgedServerHeaders_AreReplaced_AndDateIsPreserved() + { + var queue = NewQueue(); + var clock = new ControllableTimeProvider(); + clock.Advance(new DateTimeOffset(2026, 9, 25, 12, 0, 0, TimeSpan.Zero) - DateTimeOffset.UnixEpoch); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, timeProvider: clock); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var clientDate = "24 Sep 2026 08:00:00 +0000"; + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync( + ValidArticle( + date: clientDate, + extraHeaders: + "Path: evil.path\r\n" + + "Injection-Date: forged-date\r\n" + + "Injection-Info: forged-info\r\n" + + "NNTP-Posting-Date: forged-nntp-date\r\n" + + "NNTP-Posting-Host: 203.0.113.9\r\n" + + "X-Trace: forged-trace\r\n" + + "Xref: other.server group:1\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + var expectedInjection = PostRfcDate.Format(clock.GetUtcNow()); + Assert.Contains("Date: Thu, 24 Sep 2026 08:00:00 +0000", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date: " + expectedInjection, text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Date", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + Assert.DoesNotContain("127.0.0.1", text, StringComparison.Ordinal); + Assert.DoesNotContain("203.0.113.9", text, StringComparison.Ordinal); + Assert.DoesNotContain("posting-host", text, StringComparison.Ordinal); + Assert.Contains( + "Injection-Info: nntpd01.usenet.ninja; logging-data=\"\"; mail-complaints-to=\"abuse@usenet.ninja\"", + text, + StringComparison.Ordinal); + var xtrace = RequireHeader(text, "X-Trace"); + Assert.StartsWith(AesGcmPostingTraceProtector.TokenPrefix, xtrace, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd01.usenet.ninja 20260925120000", text, StringComparison.Ordinal); + Assert.True(session.PostingTraceProtector!.TryUnprotect(xtrace, out var recovered)); + Assert.Equal(IPAddress.Loopback, recovered.Address); + Assert.Equal(119, recovered.Port); + Assert.Equal(clock.GetUtcNow(), recovered.InjectedAtUtc); + Assert.Null(recovered.AuthenticatedUsername); + Assert.Contains("news.usenet.ninja!", text, StringComparison.Ordinal); + Assert.Contains(".POSTED", text, StringComparison.Ordinal); + Assert.DoesNotContain("forged", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Ipv6Peer_IsProtectedInsideXTraceOnly() + { + var queue = NewQueue(); + var peer = IPAddress.Parse("2001:db8::10"); + await using var duplex = new PostDuplex(peer); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + Assert.DoesNotContain("2001:db8::10", text, StringComparison.Ordinal); + Assert.DoesNotContain("posting-host", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + var xtrace = RequireHeader(text, "X-Trace"); + Assert.True(session.PostingTraceProtector!.TryUnprotect(xtrace, out var recovered)); + Assert.Equal(peer, recovered.Address); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TwoPosts_ProduceDistinctXTraceTokens() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(messageId: "") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(messageId: "") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var timeoutCts = new CancellationTokenSource(Safety); + var first = await queue.DequeueAsync(timeoutCts.Token); + var second = await queue.DequeueAsync(timeoutCts.Token); + var left = RequireHeader(Encoding.ASCII.GetString(first!.Payload.Span), "X-Trace"); + var right = RequireHeader(Encoding.ASCII.GetString(second!.Payload.Span), "X-Trace"); + Assert.NotEqual(left, right); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task MissingTraceProtector_Returns441() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, includeTraceProtector: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task EmptyBody_IsAccepted() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(body: string.Empty) + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DotLeadingBody_IsRestuffedOnce_WithoutTerminator() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(body: "..hidden\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound!.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + var text = Encoding.ASCII.GetString(inbound.Record.ArtData.Span); + Assert.Contains("\r\n\r\n..hidden\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("\r\n\r\n.hidden\r\n", text, StringComparison.Ordinal); + Assert.False(inbound.Payload.Span.EndsWith(".\r\n"u8)); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task LinesHeader_DoesNotControlAcceptance() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(extraHeaders: "Lines: 9999\r\n", body: "one\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.True(inbound!.Payload.Length < 2048); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ConfiguredMailComplaintsTo_IsEmittedOnInjectionInfo() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, mailComplaintsTo: "ops@usenet.ninja"); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + Assert.Contains("mail-complaints-to=\"ops@usenet.ninja\"", text, StringComparison.Ordinal); + Assert.DoesNotContain("mail-complaints-to=\"abuse@usenet.ninja\"", text, StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task MultipleDotLeadingLines_AreEachStuffedOnce() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(body: "normal\r\n..\r\n...\r\n....\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound!.Record.ParseStatus); + var text = Encoding.ASCII.GetString(inbound.Record.ArtData.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal("normal\r\n..\r\n...\r\n....\r\n", body); + Assert.False(inbound.Payload.Span.EndsWith("\r\n.\r\n"u8)); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task CompletedQueue_Returns441_Without240() + { + var queue = NewQueue(); + queue.Complete(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DisconnectBeforeAdmission_DoesNotReturn240_AndDoesNotEnqueue() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.CompleteClientAsync(); + await run.WaitAsync(Safety); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task SuccessfulPost_CallsTryAdmitOnce_AndDoesNotCallEnqueueAsync() + { + var inner = NewQueue(); + var queue = new RecordingIngestionQueue(inner); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + Assert.Same(queue, session.ArticleIngestion); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(messageId: "") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(0, queue.EnqueueAsyncCalls); + var admitted = Assert.Single(queue.Admitted); + Assert.Equal("", admitted.MessageId); + Assert.Equal(InboundArticleProducer.Post, admitted.Producer); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DisabledQueue_Returns441_Without240() + { + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(DisabledArticleIngestionQueue.Instance, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task QueueBudgetExhausted_Returns441() + { + var held = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = 64 * 1024 }, + transitQueueMemoryLimit: held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle() + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task QueuedRepresentation_MatchesIhaveStuffedWireContract() + { + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(ValidArticle(body: "plain\r\n..dot\r\n") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + using var dequeueCts = new CancellationTokenSource(Safety); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal(InboundArticleProducer.Post, inbound!.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + Assert.False(inbound.Payload.Span.EndsWith("\r\n.\r\n"u8)); + var text = Encoding.ASCII.GetString(inbound.Record.ArtData.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal("plain\r\n..dot\r\n", body); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ActivePostReceive_IsNotKilledByIdleTimeout() + { + var clock = new ControllableTimeProvider(); + clock.Advance(DateTimeOffset.UtcNow - DateTimeOffset.UnixEpoch); + var queue = NewQueue(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, Poster, timeProvider: clock, idle: Idle); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + Assert.True(session.CommandWorkForTests > 0); + + clock.Advance(Idle); + await Task.Delay(20); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + + await duplex.WriteClientAsync(ValidArticle(date: PostRfcDate.Format(clock.GetUtcNow())) + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task IdleConnection_BeforePost_StillTimesOut() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(authorization: Poster, timeProvider: clock, idle: Idle); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + private static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = NntpdOptions.DefaultMaxArticleSize }); + + private static async Task QuitAsync(PostDuplex duplex, Task run) + { + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private static async Task AuthenticateAsync(PostDuplex duplex, string username, string password) + { + await duplex.WriteClientLineAsync("AUTHINFO USER " + username); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS " + password); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + private sealed class DualAuthProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + _ = cancellationToken; + if (password == "poster-secret" + && (username == "poster" || username == "poster+tag/name")) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, Poster)); + } + + if (username == "newsmaster" && password == "unit-test-newsmaster-password") + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, Newsmaster)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private static string ValidArticle( + string? date = null, + string? messageId = "", + string extraHeaders = "", + string body = "body\r\n", + bool includeDate = true, + bool includeSubject = true) + { + date ??= PostRfcDate.Format(DateTimeOffset.UtcNow); + var sb = new StringBuilder(); + if (includeDate) + { + sb.Append("Date: ").Append(date).Append("\r\n"); + } + sb.Append("From: poster@example.com\r\n"); + sb.Append("Newsgroups: misc.test\r\n"); + if (includeSubject) + { + sb.Append("Subject: test\r\n"); + } + if (messageId is not null) + { + sb.Append("Message-ID: ").Append(messageId).Append("\r\n"); + } + + sb.Append(extraHeaders); + sb.Append("\r\n"); + sb.Append(body); + return sb.ToString(); + } + + private static string PaddedArticle(int destuffedBytes) + { + var prefix = ValidArticle(body: string.Empty); + var needed = destuffedBytes - Encoding.ASCII.GetByteCount(prefix); + Assert.True(needed >= 2); + + // Common parser rejects physical lines over 1024 bytes. Keep body lines + // at 80 bytes so size-limit tests isolate destuffed size, not line length. + const int maxContent = 78; + var sb = new StringBuilder(prefix, destuffedBytes); + var remaining = needed; + while (remaining > 0) + { + var content = Math.Min(maxContent, remaining - 2); + if (remaining > 80 && remaining - (content + 2) == 1) + { + content--; + } + + sb.Append('Z', content); + sb.Append("\r\n"); + remaining -= content + 2; + } + + var article = sb.ToString(); + Assert.Equal(destuffedBytes, Encoding.ASCII.GetByteCount(article)); + return article; + } + + private sealed class RecordingIngestionQueue : IArticleIngestionQueue + { + private readonly IArticleIngestionQueue _inner; + + public RecordingIngestionQueue(IArticleIngestionQueue inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + public int TryAdmitCalls { get; private set; } + + public int EnqueueAsyncCalls { get; private set; } + + public List Admitted { get; } = []; + + public long MemoryLimitBytes => _inner.MemoryLimitBytes; + + public long QueuedBytes => _inner.QueuedBytes; + + public long PeakQueuedBytes => _inner.PeakQueuedBytes; + + public int MaxArticleBytes => _inner.MaxArticleBytes; + + public int Count => _inner.Count; + + public int PeakCount => _inner.PeakCount; + + public bool IsAccepting => _inner.IsAccepting; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + EnqueueAsyncCalls++; + return _inner.EnqueueAsync(article, cancellationToken); + } + + public bool TryProbeCapacity() => _inner.TryProbeCapacity(); + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + TryAdmitCalls++; + var result = _inner.TryAdmit(article); + if (result == ArticleEnqueueResult.Accepted) + { + Admitted.Add(article); + } + + return result; + } + + public bool TryEnqueue(InboundArticle article) => _inner.TryEnqueue(article); + + public void Complete() => _inner.Complete(); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + _inner.DequeueAsync(cancellationToken); + } + + private static string RequireHeader(string article, string name) + { + var prefix = name + ": "; + var start = article.IndexOf(prefix, StringComparison.OrdinalIgnoreCase); + Assert.True(start >= 0, "missing header " + name); + var end = article.IndexOf("\r\n", start, StringComparison.Ordinal); + Assert.True(end > start); + return article[(start + prefix.Length)..end]; + } + + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private sealed class PostDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly IPAddress _address; + + public PostDuplex(IPAddress? address = null) + { + _address = address ?? IPAddress.Loopback; + } + + public NntpSession CreateSession( + IArticleIngestionQueue? queue = null, + NntpAuthorization? authorization = null, + IHistoryDb? historyDb = null, + int? maxArticleSize = null, + TimeProvider? timeProvider = null, + TimeSpan? idle = null, + string? mailComplaintsTo = null, + bool includeTraceProtector = true, + IPostingTraceProtector? postingTraceProtector = null, + INntpAuthenticationProvider? authenticationProvider = null, + INewsLogWriter? newsLog = null, + IPostFilter? postFilter = null) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_address, 119))); + var protector = includeTraceProtector + ? postingTraceProtector ?? AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }) + : null; + var session = new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: authenticationProvider, + articleIngestion: queue, + historyDb: historyDb, + commandIdleTimeout: idle, + timeProvider: timeProvider, + maxArticleSize: maxArticleSize, + mailComplaintsTo: mailComplaintsTo, + postingTraceProtector: protector, + postFilter: postFilter, + newsLog: newsLog); + if (authorization is not null) + { + session.SetAuthorization(authorization); + } + + return session; + } + + public async Task WriteClientLineAsync(string line) + { + await WriteClientAsync(line + "\r\n"); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public Task CompleteClientAsync() => _clientToServer.Writer.CompleteAsync().AsTask(); + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/PostNewsgroupCatalogueTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/PostNewsgroupCatalogueTests.cs new file mode 100644 index 00000000..e534fdff --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/PostNewsgroupCatalogueTests.cs @@ -0,0 +1,567 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Newsgroups; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class PostNewsgroupCatalogueTests +{ + private static readonly NntpAuthorization Poster = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(20); + + [Fact] + public void Session_WithCatalogue_UsesCataloguePolicy() + { + var session = new NntpSession( + new ClosedConnection(), + NullLogger.Instance, + newsgroupCatalogue: new StaticNewsgroupCatalogue(Snapshot(Allowed("misc.test")))); + Assert.IsType(session.NewsgroupPostingPolicy); + } + + [Fact] + public void Session_WithoutCatalogue_KeepsSyntaxOnlyPolicy() + { + var session = new NntpSession(new ClosedConnection(), NullLogger.Instance); + Assert.Same(SyntaxOnlyNewsgroupPostingPolicy.Instance, session.NewsgroupPostingPolicy); + } + + [Fact] + public async Task AllowedGroup_Returns240_AdmitsAndRemembers() + { + var (queue, history, inbound) = await PostAsync( + Snapshot(Allowed("misc.test")), + "misc.test", + expected: "240 Article received OK"); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(1, history.RememberCalls); + Assert.Equal(1, history.PeekCalls); + Assert.NotNull(inbound); + var text = Encoding.ASCII.GetString(inbound!.Payload.Span); + Assert.Contains(".POSTED", text, StringComparison.Ordinal); + Assert.Contains("Injection-Info: nntpd01.usenet.ninja;", text, StringComparison.Ordinal); + Assert.Contains("X-Trace: ", text, StringComparison.Ordinal); + } + + [Fact] + public async Task ProhibitedGroup_Returns441_WithoutAdmitOrRemember() + { + await AssertRejectedAsync(Snapshot(Group("no.post", NewsgroupPostingStatus.Prohibited)), "no.post"); + } + + [Fact] + public async Task ModeratedGroup_Returns441_WithoutAdmitOrRemember() + { + await AssertRejectedAsync(Snapshot(Group("mod.test", NewsgroupPostingStatus.Moderated)), "mod.test"); + } + + [Fact] + public async Task ClosedGroup_Returns441_WithoutAdmitOrRemember() + { + await AssertRejectedAsync( + Snapshot(Group("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)), + "closed.test"); + } + + [Fact] + public async Task PeerOnlyGroup_Returns441_WithoutAdmitOrRemember() + { + await AssertRejectedAsync(Snapshot(Group("peer.only", NewsgroupPostingStatus.PeerOnly)), "peer.only"); + } + + [Fact] + public async Task UnknownGroup_Returns441_WithoutAdmitOrRemember() + { + await AssertRejectedAsync(Snapshot(Allowed("misc.test")), "unknown.group"); + } + + [Fact] + public async Task MultipleAllowed_Returns240() + { + var (queue, history, _) = await PostAsync( + Snapshot(Allowed("misc.test"), Allowed("comp.test")), + "misc.test,comp.test", + expected: "240 Article received OK"); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(1, history.RememberCalls); + } + + [Fact] + public async Task Multiple_OneUnknown_Returns441() + { + await AssertRejectedAsync(Snapshot(Allowed("misc.test")), "misc.test,unknown.group"); + } + + [Fact] + public async Task Multiple_OneProhibited_Returns441() + { + await AssertRejectedAsync( + Snapshot(Allowed("misc.test"), Group("no.post", NewsgroupPostingStatus.Prohibited)), + "misc.test,no.post"); + } + + [Fact] + public async Task Multiple_OneModerated_Returns441() + { + await AssertRejectedAsync( + Snapshot(Allowed("misc.test"), Group("mod.test", NewsgroupPostingStatus.Moderated)), + "misc.test,mod.test"); + } + + [Fact] + public async Task Multiple_OneClosed_Returns441() + { + await AssertRejectedAsync( + Snapshot(Allowed("misc.test"), Group("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)), + "misc.test,closed.test"); + } + + [Fact] + public async Task Multiple_OnePeerOnly_Returns441() + { + await AssertRejectedAsync( + Snapshot(Allowed("misc.test"), Group("peer.only", NewsgroupPostingStatus.PeerOnly)), + "misc.test,peer.only"); + } + + [Fact] + public async Task MixedCaseGroup_Returns240() + { + var (queue, history, _) = await PostAsync( + Snapshot(Allowed("misc.test")), + "MISC.TEST", + expected: "240 Article received OK"); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(1, history.RememberCalls); + } + + [Fact] + public async Task ExactDuplicateNewsgroups_Returns441_WithoutCatalogueLookup() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Allowed("misc.test"))); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, history, catalogue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article("misc.test,misc.test") + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.TryAdmitCalls); + Assert.Equal(0, history.RememberCalls); + Assert.Equal(0, catalogue.CurrentReadCount); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Post_UsesSingleCapturedSnapshot_WhenCatalogueIsReplaced() + { + var first = Snapshot(Allowed("misc.test"), Allowed("comp.test")); + var catalogue = new PublishOnReadCatalogue(first, NewsgroupSnapshot.Empty); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, history, catalogue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article("misc.test,comp.test") + ".\r\n"); + Assert.Equal("240 Article received OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, catalogue.Reads); + Assert.Equal(1, queue.TryAdmitCalls); + Assert.Equal(1, history.RememberCalls); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task EmptyNewsgroups_Returns441_WithoutAdmit() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Allowed("misc.test"))); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, history, catalogue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article(string.Empty) + ".\r\n"); + Assert.Equal("441 Posting failed", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.TryAdmitCalls); + Assert.Equal(0, history.RememberCalls); + + await QuitAsync(duplex, run); + } + + private static async Task AssertRejectedAsync(NewsgroupSnapshot snapshot, string newsgroups) + { + var (queue, history, inbound) = await PostAsync(snapshot, newsgroups, expected: "441 Posting failed"); + Assert.Equal(0, queue.TryAdmitCalls); + Assert.Equal(0, history.RememberCalls); + Assert.Equal(0, history.PeekCalls); + Assert.Null(inbound); + } + + private static async Task<(RecordingIngestionQueue Queue, RecordingHistoryDb History, InboundArticle? Admitted)> + PostAsync(NewsgroupSnapshot snapshot, string newsgroups, string expected) + { + var catalogue = new StaticNewsgroupCatalogue(snapshot); + var queue = new RecordingIngestionQueue(NewQueue()); + var history = new RecordingHistoryDb(); + await using var duplex = new PostDuplex(); + var session = duplex.CreateSession(queue, history, catalogue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("340 Input article; end with .", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article(newsgroups) + ".\r\n"); + Assert.Equal(expected, await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + return (queue, history, queue.Admitted.Count == 0 ? null : queue.Admitted[0]); + } + + private static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 8, MaxArticleBytes = NntpdOptions.DefaultMaxArticleSize }); + + private static async Task QuitAsync(PostDuplex duplex, Task run) + { + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static string Article(string newsgroups) + { + return + "Date: " + PostRfcDate.Format(DateTimeOffset.UtcNow) + "\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "Subject: test\r\n" + + "Message-ID: \r\n" + + "\r\n" + + "body\r\n"; + } + + private static NewsgroupSnapshot Snapshot(params NewsgroupDefinition[] definitions) => + NewsgroupSnapshot.Create(definitions); + + private static NewsgroupDefinition Allowed(string name) => Group(name, NewsgroupPostingStatus.Allowed); + + private static NewsgroupDefinition Group(string name, NewsgroupPostingStatus status) => + new(name, string.Empty, 2, 1, status); + + private sealed class PublishOnReadCatalogue : INewsgroupCatalogue + { + private readonly StaticNewsgroupCatalogue _inner; + private readonly NewsgroupSnapshot _replacement; + + public PublishOnReadCatalogue(NewsgroupSnapshot first, NewsgroupSnapshot replacement) + { + _inner = new StaticNewsgroupCatalogue(first); + _replacement = replacement; + } + + public int Reads => _inner.CurrentReadCount; + + public NewsgroupSnapshot Current + { + get + { + var snapshot = _inner.Current; + _inner.Publish(_replacement); + return snapshot; + } + } + } + + private sealed class RecordingHistoryDb : IHistoryDb + { + public int PeekCalls { get; private set; } + + public int RememberCalls { get; private set; } + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(HistoryLookupResult.Unseen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + PeekCalls++; + return new(HistoryLookupResult.Unseen); + } + + public void Remember(ReadOnlyMemory messageId) => RememberCalls++; + + public bool ContainsLocal(in HistoryDigest digest) => false; + } + + private sealed class RecordingIngestionQueue : IArticleIngestionQueue + { + private readonly IArticleIngestionQueue _inner; + + public RecordingIngestionQueue(IArticleIngestionQueue inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + public int TryAdmitCalls { get; private set; } + + public List Admitted { get; } = []; + + public long MemoryLimitBytes => _inner.MemoryLimitBytes; + + public long QueuedBytes => _inner.QueuedBytes; + + public long PeakQueuedBytes => _inner.PeakQueuedBytes; + + public int MaxArticleBytes => _inner.MaxArticleBytes; + + public int Count => _inner.Count; + + public int PeakCount => _inner.PeakCount; + + public bool IsAccepting => _inner.IsAccepting; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) => + _inner.EnqueueAsync(article, cancellationToken); + + public bool TryProbeCapacity() => _inner.TryProbeCapacity(); + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + TryAdmitCalls++; + var result = _inner.TryAdmit(article); + if (result == ArticleEnqueueResult.Accepted) + { + Admitted.Add(article); + } + + return result; + } + + public bool TryEnqueue(InboundArticle article) => _inner.TryEnqueue(article); + + public void Complete() => _inner.Complete(); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) => + _inner.DequeueAsync(cancellationToken); + } + + private sealed class PostDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IHistoryDb historyDb, + INewsgroupCatalogue catalogue) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: historyDb, + postingTraceProtector: AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }), + newsgroupCatalogue: catalogue); + session.SetAuthorization(Poster); + return session; + } + + public async Task WriteClientLineAsync(string line) => await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class ClosedConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input => throw new InvalidOperationException(); + + public PipeWriter Output => throw new InvalidOperationException(); + + public EndPoint? RemoteEndPoint => null; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } = + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => true; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/PostingTraceProtectorTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/PostingTraceProtectorTests.cs new file mode 100644 index 00000000..282ab73f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/PostingTraceProtectorTests.cs @@ -0,0 +1,224 @@ +using System.Net; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Logging; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class PostingTraceProtectorTests +{ + private static readonly byte[] CurrentKey = Convert.FromHexString(TestHostFactory.TestXTraceKey); + private static readonly byte[] PreviousKey = Convert.FromHexString( + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + [Fact] + public void Protect_RoundTripsTransportIdentity() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("198.18.0.123"), + 1199, + new DateTimeOffset(2026, 9, 25, 12, 0, 0, TimeSpan.Zero), + Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")); + + var token = protector.Protect(payload); + Assert.StartsWith(AesGcmPostingTraceProtector.TokenPrefix, token, StringComparison.Ordinal); + Assert.DoesNotContain("198.18.0.123", token, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToBase64String(payload.Address.GetAddressBytes()), token, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToBase64String(Encoding.ASCII.GetBytes("198.18.0.123")), token, StringComparison.Ordinal); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.InjectedAtUtc, recovered.InjectedAtUtc); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void Protect_RoundTripsAuthenticatedUsername() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("198.18.0.123"), + 1199, + new DateTimeOffset(2026, 9, 25, 12, 0, 0, TimeSpan.Zero), + Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"), + "poster+tag/name"); + + var token = protector.Protect(payload); + Assert.DoesNotContain("poster+tag/name", token, StringComparison.Ordinal); + Assert.DoesNotContain("198.18.0.123", token, StringComparison.Ordinal); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.InjectedAtUtc, recovered.InjectedAtUtc); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Equal("poster+tag/name", recovered.AuthenticatedUsername); + } + + [Fact] + public void Protect_EmptyUsername_IsAbsent() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var token = protector.Protect(new PostingTracePayload( + IPAddress.Loopback, + 119, + DateTimeOffset.UnixEpoch, + Guid.Empty, + " ")); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void LegacyV1Payload_DecryptsWithoutUsername() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("192.0.2.10"), + 119, + DateTimeOffset.UnixEpoch, + Guid.Parse("11111111-2222-3333-4444-555555555555")); + var token = protector.ProtectLegacyV1(payload); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void PreviousKey_CanDecryptUsernamePayload() + { + var oldProtector = new AesGcmPostingTraceProtector(PreviousKey); + var token = oldProtector.Protect(new PostingTracePayload( + IPAddress.Parse("192.0.2.55"), + 563, + DateTimeOffset.UnixEpoch, + Guid.NewGuid(), + "newsmaster")); + var rotated = new AesGcmPostingTraceProtector(CurrentKey, PreviousKey); + Assert.True(rotated.TryUnprotect(token, out var recovered)); + Assert.Equal("newsmaster", recovered.AuthenticatedUsername); + Assert.False(new AesGcmPostingTraceProtector(CurrentKey).TryUnprotect(token, out _)); + } + + [Fact] + public void InvalidToken_IsRejected() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + Assert.False(protector.TryUnprotect("not-a-token", out _)); + Assert.False(protector.TryUnprotect("v1.$$$$", out _)); + Assert.False(protector.TryUnprotect("v2.AAAA", out _)); + } + + [Fact] + public void MalformedInnerPayloads_DoNotDecrypt() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(LegacyWithExtraByte()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(UnknownVersion()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2LengthMismatch()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2InvalidUtf8()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2EmbeddedNul()), out _)); + } + + private static byte[] FixedIPv4Prefix(byte version) + { + var buffer = new byte[32]; + buffer[0] = version; + buffer[1] = 4; + buffer[2] = 127; + buffer[3] = 0; + buffer[4] = 0; + buffer[5] = 1; + buffer[7] = 119; + return buffer; + } + + private static byte[] LegacyWithExtraByte() + { + var prefix = FixedIPv4Prefix(1); + var extra = new byte[prefix.Length + 1]; + prefix.CopyTo(extra, 0); + extra[^1] = 1; + return extra; + } + + private static byte[] UnknownVersion() => FixedIPv4Prefix(99); + + private static byte[] Version2LengthMismatch() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 3]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 5; + raw[34] = (byte)'x'; + return raw; + } + + private static byte[] Version2InvalidUtf8() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 3]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 1; + raw[34] = 0xFF; + return raw; + } + + private static byte[] Version2EmbeddedNul() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 5]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 3; + raw[34] = (byte)'a'; + raw[35] = 0; + raw[36] = (byte)'b'; + return raw; + } + + [Fact] + public void Protect_DoesNotReuseToken() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload(IPAddress.Loopback, 119, DateTimeOffset.UnixEpoch, Guid.Empty); + Assert.NotEqual(protector.Protect(payload), protector.Protect(payload)); + } + + [Fact] + public void TamperedToken_IsRejected() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var token = protector.Protect(new PostingTracePayload(IPAddress.Loopback, 119, DateTimeOffset.UnixEpoch, Guid.NewGuid())); + var chars = token.ToCharArray(); + chars[^1] = chars[^1] == 'A' ? 'B' : 'A'; + Assert.False(protector.TryUnprotect(new string(chars), out _)); + } + + [Fact] + public void PreviousKey_CanDecryptRotatedToken() + { + var oldProtector = new AesGcmPostingTraceProtector(PreviousKey); + var token = oldProtector.Protect(new PostingTracePayload(IPAddress.Parse("192.0.2.55"), 563, DateTimeOffset.UnixEpoch, Guid.NewGuid())); + var rotated = new AesGcmPostingTraceProtector(CurrentKey, PreviousKey); + Assert.True(rotated.TryUnprotect(token, out var recovered)); + Assert.Equal(IPAddress.Parse("192.0.2.55"), recovered.Address); + Assert.False(new AesGcmPostingTraceProtector(CurrentKey).TryUnprotect(token, out _)); + } + + [Fact] + public void Protector_DoesNotAcceptLogger() + { + var ctors = typeof(AesGcmPostingTraceProtector).GetConstructors(BindingFlags.Public | BindingFlags.Instance); + Assert.DoesNotContain(ctors, static ctor => ctor.GetParameters().Any(static p => + typeof(ILogger).IsAssignableFrom(p.ParameterType))); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/QuitCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/QuitCommandTests.cs new file mode 100644 index 00000000..6aa2a884 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/QuitCommandTests.cs @@ -0,0 +1,591 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Networking.Transport; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +/// QUIT command — RFC 3977 §5.4 and peer-disappears-during-205 race. +[Collection(nameof(TransportTestHostCollection))] +public sealed class QuitCommandTests +{ + [Fact] + public async Task Quit_Returns205_AndEndsSession() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(session.Connection.IsCompleted); + } + + [Fact] + public async Task Quit_WithTrailingArgs_Returns501_AndSessionRemainsUsable() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT NOW"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Quit_Logging_RxCentralized_TxOwnedByQuit_NoDuplicate() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: QUIT", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: QUIT [205 Connection closing] executed in", StringComparison.Ordinal))); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && m.Contains("[failed]", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(Quit).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Quit_NoFurtherCommandsProcessed() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("DATE"); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + // DATE after terminal QUIT must not produce a response. + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync( + async () => await duplex.ReadClientLineAsync(cts.Token)); + } + + [Fact] + public async Task Quit_PeerDisconnectDuringWrite_IsQuietAndTerminal() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + // Complete the server→client pipe so FlushAsync of 205 observes a completed writer (peer gone). + await duplex.CompleteServerToClientAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(session.Connection.IsCompleted); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + // May be clean success (enqueue before complete) or peer-disconnected detail. + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + Assert.DoesNotContain("[failed]", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Quit_Socket_NormalCloseAfter205() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var recording = new RecordingLoggerFactory(); + var session = new NntpSession( + server, + recording.CreateLogger(), + loggerFactory: recording); + var run = session.RunAsync(); + + _ = await ReadSocketLineAsync(client); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await ReadSocketLineAsync(client)); + client.Shutdown(SocketShutdown.Both); + client.Close(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && !m.Contains("[failed]", StringComparison.Ordinal)); + } + + [Fact] + public async Task Quit_Socket_ImmediateClientCloseAfterQuit_IsNotApplicationError() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var recording = new RecordingLoggerFactory(); + var session = new NntpSession( + server, + recording.CreateLogger(), + loggerFactory: recording); + var run = session.RunAsync(); + + _ = await ReadSocketLineAsync(client); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + // Do not wait for 205 — close immediately (common client race). + client.LingerState = new LingerOption(true, 0); + client.Close(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP session ended with an error", StringComparison.OrdinalIgnoreCase) + && recording.Messages.Any(x => x.Contains("QUIT failed", StringComparison.Ordinal))); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + // Immediate RST can complete the connection after a successful 205 write; the + // existing RunAsync IsCompleted heuristic may then append [failed]. That is + // not an application error when 205 was selected for TX. + Assert.True( + tx.Contains("[205 Connection closing]", StringComparison.Ordinal) + || !tx.Contains("[failed]", StringComparison.Ordinal), + tx); + } + + [Fact] + public async Task Quit_GenuineNonPeerFailure_IsNotSwallowed() + { + var recording = new RecordingLoggerFactory(); + NntpCommandLoggers.Configure(recording); + + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + await using var connection = new FailOnWaitConnection( + identity, + new IOException("simulated non-peer transport failure")); + var session = new NntpSession( + connection, + recording.CreateLogger(), + loggerFactory: recording); + var response = new NntpResponseWriter(connection.Output); + var (command, line) = NntpCommandTestParse.Parse("QUIT"); + var context = new NntpCommandContext(session, command, line, response); + + try + { + await Quit.HandleAsync(context, CancellationToken.None); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Expected — non-peer failure must propagate. + } + + Assert.Contains( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && m.Contains("[failed]", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(SocketError.ConnectionReset)] + [InlineData(SocketError.ConnectionAborted)] + [InlineData((SocketError)32)] + public void PeerDisconnect_ClassifiesResetLikeSocketErrors(SocketError error) + { + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + using var cts = new CancellationTokenSource(); + var connection = new StaticConnection(identity, cts.Token, completed: false); + var ex = new IOException("send failed", new SocketException((int)error)); + Assert.True(NntpPeerDisconnect.IsPeerDisconnect(ex, connection)); + } + + [Fact] + public void PeerDisconnect_DoesNotClassifyArbitraryIOException() + { + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + using var cts = new CancellationTokenSource(); + var connection = new StaticConnection(identity, cts.Token, completed: false); + Assert.False(NntpPeerDisconnect.IsPeerDisconnect(new IOException("disk full"), connection)); + } + + private static async Task ReadSocketLineAsync(Socket socket) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private sealed class QuitDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new QuitDuplex()); + + public NntpSession CreateSession(ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return loggerFactory is null + ? new NntpSession(connection, NullLogger.Instance) + : new NntpSession( + connection, + loggerFactory.CreateLogger(), + loggerFactory: loggerFactory); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync(CancellationToken cancellationToken = default) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (!cancellationToken.CanBeCanceled) + { + cts.CancelAfter(TimeSpan.FromSeconds(5)); + } + + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task CompleteServerToClientAsync() + { + await _serverToClient.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _completed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => Volatile.Read(ref _completed) == 1; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + if (Interlocked.Exchange(ref _completed, 1) == 0) + { + _cts.Cancel(); + } + + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + /// + /// Connection that accepts the 205 write but fails + /// with a non-peer IOException. + /// + private sealed class FailOnWaitConnection : INntpConnection + { + private readonly Pipe _pipe = new(NntpPipeOptions.Create()); + private readonly CancellationTokenSource _cts = new(); + private readonly Exception _waitFailure; + + public FailOnWaitConnection(ConnectionClientIdentity identity, Exception waitFailure) + { + ClientIdentity = identity; + _waitFailure = waitFailure; + Input = _pipe.Reader; + Output = _pipe.Writer; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.FromException(_waitFailure); + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.FromException(_waitFailure); + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public async ValueTask DisposeAsync() + { + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + _cts.Dispose(); + } + } + + private sealed class StaticConnection : INntpConnection + { + private readonly Pipe _pipe = new(NntpPipeOptions.Create()); + + public StaticConnection(ConnectionClientIdentity identity, CancellationToken closed, bool completed) + { + ClientIdentity = identity; + ConnectionClosed = closed; + IsCompleted = completed; + Input = _pipe.Reader; + Output = _pipe.Writer; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed { get; } + public bool IsCompleted { get; } + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + private readonly object _gate = new(); + public List Messages { get; } = []; + public List Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + lock (_gate) + { + Categories.Add(categoryName); + } + + return new RecordingLogger(categoryName, this); + } + + public void Dispose() + { + } + + public void Add(string message) + { + lock (_gate) + { + Messages.Add(message); + } + } + + private sealed class RecordingLogger(string category, RecordingLoggerFactory owner) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + owner.Add(formatter(state, exception)); + if (exception is not null) + { + owner.Add($"{category}: {exception.GetType().Name}: {exception.Message}"); + } + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/StreamingPostArticleReaderTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/StreamingPostArticleReaderTests.cs new file mode 100644 index 00000000..78cde440 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/StreamingPostArticleReaderTests.cs @@ -0,0 +1,546 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Text; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Session.Commands.Posting; +using VectorNNTPD.ReaderService.Session.Framing; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class StreamingPostArticleReaderTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + private static readonly ConnectionClientIdentity Client = + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + [Fact] + public async Task Streaming_MatchesLegacyMaterialize_ForNormalAndDotBodies() + { + foreach (var stuffed in new[] { "hello\r\n", "..dot\r\n", "...triple\r\n", "....quad\r\n", "a\r\n..\r\n...\r\n" } + .Select(static body => ClientArticle(body: body))) + { + var streaming = await ReadStreamingAsync(stuffed); + var legacy = await LegacyPostMaterialize.ProcessAsync(stuffed, CreateOptions()); + Assert.Equal(StreamingPostReadStatus.Completed, streaming.Status); + Assert.True(streaming.Wire.Span.SequenceEqual(legacy.Span)); + } + } + + [Fact] + public async Task FoldedHeader_IsPreservedOnWire() + { + var stuffed = ClientArticle(extraHeaders: "User-Agent: foo\r\n\tbar\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Contains("User-Agent: foo\r\n\tbar\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task DuplicateSingleton_IsRejected() + { + var stuffed = ClientArticle(extraHeaders: "Subject: again\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.DuplicateHeader, result.Failure.Category); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task MissingRequiredHeaders_AreRejected() + { + var stuffed = "From: poster@example.com\r\nSubject: test\r\n\r\nbody\r\n.\r\n"; + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MissingRequiredHeader, result.Failure.Category); + Assert.Equal("Newsgroups", result.Failure.Detail); + } + + [Theory] + [InlineData("From")] + [InlineData("Newsgroups")] + [InlineData("Subject")] + public async Task MissingMandatoryHeader_IsRejected(string header) + { + var stuffed = header switch + { + "From" => "Newsgroups: misc.test\r\nSubject: test\r\n\r\nbody\r\n.\r\n", + "Newsgroups" => "From: poster@example.com\r\nSubject: test\r\n\r\nbody\r\n.\r\n", + _ => "From: poster@example.com\r\nNewsgroups: misc.test\r\n\r\nbody\r\n.\r\n", + }; + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MissingRequiredHeader, result.Failure.Category); + Assert.Equal(header, result.Failure.Detail); + } + + [Fact] + public async Task MissingDate_IsGeneratedOnce() + { + var result = await ReadStreamingAsync(ClientArticle(date: null)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + PostRfcDate.Format(Now) + "\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task SuppliedDate_IsPreservedOnce() + { + const string date = "25 Sep 2026 11:00:00 +0000"; + var result = await ReadStreamingAsync(ClientArticle(date: date)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + date + "\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MalformedDate_IsRejected() + { + var result = await ReadStreamingAsync(ClientArticle(date: "yesterday")); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidDate, result.Failure.Category); + } + + [Theory] + [InlineData(25, 0)] + [InlineData(0, -15)] + public async Task DateOutsidePolicy_IsRejected(int hours, int days) + { + var date = PostRfcDate.Format(Now.AddHours(hours).AddDays(days)); + var result = await ReadStreamingAsync(ClientArticle(date: date)); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidDate, result.Failure.Category); + } + + [Fact] + public async Task MalformedMessageId_IsRejected() + { + var result = await ReadStreamingAsync(ClientArticle(messageId: "not-an-id")); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidMessageId, result.Failure.Category); + } + + [Fact] + public async Task AbsentPath_IsServerGeneratedOnce() + { + var result = await ReadStreamingAsync(ClientArticle()); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task SuppliedPath_IsDiscardedAndReplacedOnce() + { + var result = await ReadStreamingAsync(ClientArticle(extraHeaders: "Path: evil.path\r\n")); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("evil.path", text, StringComparison.Ordinal); + } + + [Fact] + public async Task ExistingMessageId_IsPreservedOnce() + { + var stuffed = ClientArticle(messageId: ""); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal("", result.MessageId); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountOccurrences(text, "Message-ID:")); + Assert.Contains("Message-ID: ", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MissingMessageId_IsGeneratedOnce() + { + var stuffed = ClientArticle(messageId: null); + var result = await ReadStreamingAsync(stuffed); + Assert.Matches(@"^<[0-9a-f]{32}@usenet\.ninja>$", result.MessageId); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountOccurrences(text, "Message-ID:")); + Assert.Contains("Message-ID: " + result.MessageId, text, StringComparison.Ordinal); + Assert.Contains("logging-data=\"" + result.MessageId + "\"", text, StringComparison.Ordinal); + } + + [Fact] + public async Task ServerOwnedHeaders_AreReplaced_AndClientValuesDiscarded() + { + var stuffed = ClientArticle( + extraHeaders: + "Path: evil.path\r\n" + + "Injection-Date: forged\r\n" + + "Injection-Info: forged\r\n" + + "NNTP-Posting-Date: forged\r\n" + + "NNTP-Posting-Host: 203.0.113.9\r\n" + + "X-Trace: forged\r\n" + + "Xref: other 1\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date: " + PostRfcDate.Format(Now), text, StringComparison.Ordinal); + Assert.Contains( + "Injection-Info: nntpd01.usenet.ninja; logging-data=\"\"; mail-complaints-to=\"abuse@usenet.ninja\"", + text, + StringComparison.Ordinal); + Assert.Contains("X-Trace: v1.test-token\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Date", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + Assert.DoesNotContain("evil.path", text, StringComparison.Ordinal); + Assert.DoesNotContain("forged", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + Assert.DoesNotContain("203.0.113.9", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MailComplaintsTo_IsWrittenOnInjectionInfo() + { + var options = CreateOptions(); + var custom = new StreamingPostReadOptions + { + MaxArticleSize = options.MaxArticleSize, + Time = options.Time, + NewsgroupPolicy = options.NewsgroupPolicy, + InjectionIdentity = options.InjectionIdentity, + ClientIdentity = options.ClientIdentity, + MailComplaintsTo = "ops@usenet.ninja", + TraceProtector = options.TraceProtector, + }; + var result = await ReadStreamingAsync(ClientArticle(), custom); + Assert.Contains( + "mail-complaints-to=\"ops@usenet.ninja\"", + Encoding.ASCII.GetString(result.Wire.Span), + StringComparison.Ordinal); + } + + [Fact] + public async Task BodyDotVariants_AreStoredDestuffed() + { + var stuffed = ClientArticle(body: "plain\r\n..\r\n...\r\n....\r\n"); + var result = await ReadStreamingAsync(stuffed); + var text = Encoding.ASCII.GetString(result.Wire.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal("plain\r\n.\r\n..\r\n...\r\n", body); + } + + [Theory] + [InlineData("hello\r\n", "hello\r\n")] + [InlineData("..hidden\r\n", ".hidden\r\n")] + [InlineData("...hidden\r\n", "..hidden\r\n")] + [InlineData("....hidden\r\n", "...hidden\r\n")] + public async Task Body_RemovesOneStuffingLayer(string stuffedBody, string destuffedBody) + { + var result = await ReadStreamingAsync(ClientArticle(body: stuffedBody)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal(destuffedBody, body); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.False(result.Wire.Span.EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task CrlfAndDotSplitAcrossSegments_ProduceIdenticalWire() + { + var stuffed = ClientArticle(body: "..hello\r\n...more\r\n"); + var expected = await ReadStreamingAsync(stuffed); + foreach (var chunkSize in new[] { 1, 2, 3, 7 }) + { + var segmented = await ReadStreamingAsync(stuffed, CreateOptions(), chunkSize); + Assert.Equal(StreamingPostReadStatus.Completed, segmented.Status); + Assert.True(expected.Wire.Span.SequenceEqual(segmented.Wire.Span), "chunk " + chunkSize); + } + } + + [Fact] + public async Task ExactMaxArticleSize_IsAccepted() + { + var destuffed = PaddedDestuffed(512); + Assert.Equal(512, destuffed.Length); + var stuffed = destuffed + ".\r\n"; + var result = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: 512)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + Assert.Equal(512, result.DestuffedSize); + } + + [Fact] + public async Task MaxArticleSizePlusOne_IsRejected() + { + var destuffed = PaddedDestuffed(513); + var result = await ReadStreamingAsync(destuffed + ".\r\n", CreateOptions(maxArticleSize: 512)); + Assert.Equal(StreamingPostReadStatus.TooLarge, result.Status); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task DotStuffing_DoesNotCountTowardMaxArticleSize() + { + var prefix = ClientArticle(body: string.Empty, includeTerminator: false); + var destuffedPrefix = Encoding.ASCII.GetByteCount(prefix); + var destuffedBody = Encoding.ASCII.GetByteCount(".hello\r\n"); + var limit = destuffedPrefix + destuffedBody; + var stuffed = prefix + "..hello\r\n.\r\n"; + Assert.True(Encoding.ASCII.GetByteCount(stuffed) - 3 > limit); + + var accepted = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: limit)); + Assert.Equal(StreamingPostReadStatus.Completed, accepted.Status); + Assert.Equal(limit, accepted.DestuffedSize); + Assert.Contains(".hello\r\n", Encoding.ASCII.GetString(accepted.Wire.Span), StringComparison.Ordinal); + Assert.DoesNotContain("..hello\r\n", Encoding.ASCII.GetString(accepted.Wire.Span), StringComparison.Ordinal); + + var rejected = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: limit - 1)); + Assert.Equal(StreamingPostReadStatus.TooLarge, rejected.Status); + } + + [Fact] + public async Task Cancellation_Throws_AndDoesNotCompleteWire() + { + var pipe = new Pipe(); + using var cts = new CancellationTokenSource(); + var read = StreamingPostArticleReader.ReadAsync(pipe.Reader, CreateOptions(), cts.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => await read); + } + + [Fact] + public async Task EofBeforeTerminator_IsIncomplete() + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(ClientArticle(includeTerminator: false))); + await pipe.Writer.CompleteAsync(); + var result = await StreamingPostArticleReader.ReadAsync(pipe.Reader, CreateOptions(), CancellationToken.None); + Assert.Equal(StreamingPostReadStatus.Incomplete, result.Status); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task Completed_ExposesParsedMessageIdAndNewsgroups() + { + // Finish()'s Completed path requires a materialized ParsedPostArticle; regressions that + // leave _parsed null must not surface as empty identity fields. + var result = await ReadStreamingAsync(ClientArticle()); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + Assert.False(string.IsNullOrEmpty(result.MessageId)); + Assert.Contains("misc.test", result.Newsgroups); + Assert.False(result.Wire.IsEmpty); + } + + [Fact] + public async Task MalformedArticle_IsRejected() + { + var result = await ReadStreamingAsync("not a header\r\n.\r\n"); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MalformedHeader, result.Failure.Category); + } + + [Fact] + public async Task MissingTraceProtector_IsRejectedAfterHeaders() + { + var options = new StreamingPostReadOptions + { + MaxArticleSize = 64 * 1024, + Time = new FixedTimeProvider(Now), + NewsgroupPolicy = SyntaxOnlyNewsgroupPostingPolicy.Instance, + InjectionIdentity = "nntpd01.usenet.ninja", + ClientIdentity = Client, + MailComplaintsTo = NntpdOptions.DefaultMailComplaintsTo, + TraceProtector = null, + }; + var result = await ReadStreamingAsync(ClientArticle(), options); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.PersistenceFailure, result.Failure.Category); + } + + private static async Task ReadStreamingAsync( + string stuffedWithTerminator, + StreamingPostReadOptions? options = null, + int? chunkSize = null) + { + options ??= CreateOptions(); + var bytes = Encoding.ASCII.GetBytes(stuffedWithTerminator); + if (chunkSize is int size && size > 0) + { + var sequence = SegmentedSequence.FromChunks(bytes, size); + var reader = PipeReader.Create(sequence); + return await StreamingPostArticleReader.ReadAsync(reader, options, CancellationToken.None); + } + + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(bytes); + await pipe.Writer.CompleteAsync(); + return await StreamingPostArticleReader.ReadAsync(pipe.Reader, options, CancellationToken.None); + } + + private static StreamingPostReadOptions CreateOptions(int maxArticleSize = 64 * 1024) => + new() + { + MaxArticleSize = maxArticleSize, + Time = new FixedTimeProvider(Now), + NewsgroupPolicy = SyntaxOnlyNewsgroupPostingPolicy.Instance, + InjectionIdentity = "nntpd01.usenet.ninja", + ClientIdentity = Client, + MailComplaintsTo = NntpdOptions.DefaultMailComplaintsTo, + TraceProtector = DeterministicTraceProtector.Instance, + }; + + private static string ClientArticle( + string? messageId = "", + string extraHeaders = "", + string body = "body\r\n", + bool includeTerminator = true, + string? date = "") + { + var sb = new StringBuilder(); + if (date is not null) + { + sb.Append("Date: ").Append(date.Length == 0 ? PostRfcDate.Format(Now) : date).Append("\r\n"); + } + + sb.Append("From: poster@example.com\r\n"); + sb.Append("Newsgroups: misc.test\r\n"); + sb.Append("Subject: test\r\n"); + if (messageId is not null) + { + sb.Append("Message-ID: ").Append(messageId).Append("\r\n"); + } + + sb.Append(extraHeaders); + sb.Append("\r\n"); + sb.Append(body); + if (includeTerminator) + { + sb.Append(".\r\n"); + } + + return sb.ToString(); + } + + private static string PaddedDestuffed(int destuffedBytes) + { + var prefix = ClientArticle(body: string.Empty, includeTerminator: false); + var needed = destuffedBytes - Encoding.ASCII.GetByteCount(prefix); + Assert.True(needed >= 2); + return prefix + new string('Z', needed - 2) + "\r\n"; + } + + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private static int CountHeader(string text, string header) + { + var count = text.StartsWith(header, StringComparison.Ordinal) ? 1 : 0; + return count + CountOccurrences(text, "\r\n" + header); + } + + private sealed class FixedTimeProvider(DateTimeOffset utc) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => utc; + } + + internal sealed class DeterministicTraceProtector : IPostingTraceProtector + { + public static DeterministicTraceProtector Instance { get; } = new(); + + public string Protect(PostingTracePayload payload) => "v1.test-token"; + + public bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload) + { + payload = default; + return false; + } + } +} + +internal static class LegacyPostMaterialize +{ + public static async Task> ProcessAsync( + string stuffedWithTerminator, + StreamingPostReadOptions options) + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffedWithTerminator)); + await pipe.Writer.CompleteAsync(); + var destuffed = await NntpMultilineDataReader.ReadArticleAsync( + pipe.Reader, + options.MaxArticleSize, + CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + Assert.True(PostHeaderParser.TryParse(destuffed.Payload, out var parsed, out var failure), failure.Detail); + Assert.True( + PostArticleValidator.TryValidate(parsed!, options.Time.GetUtcNow(), options.NewsgroupPolicy, out failure), + failure.Detail); + var normalized = PostHeaderNormalizer.Normalize( + parsed!, + options.Time.GetUtcNow(), + options.InjectionIdentity, + options.ClientIdentity, + options.MailComplaintsTo, + options.TraceProtector!); + return normalized; + } +} + +internal static class SegmentedSequence +{ + public static ReadOnlySequence FromChunks(ReadOnlySpan source, int chunkSize) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkSize, 1); + if (source.IsEmpty) + { + return ReadOnlySequence.Empty; + } + + BufferSegment? first = null; + BufferSegment? last = null; + var offset = 0; + while (offset < source.Length) + { + var take = Math.Min(chunkSize, source.Length - offset); + var memory = source.Slice(offset, take).ToArray(); + var segment = new BufferSegment(memory); + if (first is null) + { + first = segment; + } + else + { + last!.Append(segment); + } + + last = segment; + offset += take; + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(ReadOnlyMemory memory) + { + Memory = memory; + } + + public void Append(BufferSegment next) + { + next.RunningIndex = RunningIndex + Memory.Length; + Next = next; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Session/TakeThisStageProbeTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Session/TakeThisStageProbeTests.cs new file mode 100644 index 00000000..3c23b5b6 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Session/TakeThisStageProbeTests.cs @@ -0,0 +1,87 @@ +using System.Diagnostics; +using VectorNNTPD.ReaderService.Session; + +namespace VectorNNTPD.ReaderService.Tests.Session; + +public sealed class TakeThisStageProbeTests +{ + [Fact] + public void Default_IsDisabled() + { + Assert.False(TakeThisStageProbe.IsEnabled); + Assert.Null(TakeThisStageProbe.CreateSessionLog()); + } + + [Fact] + public void Percentile_IsMonotonic() + { + var values = new double[] { 10, 20, 30, 40, 50 }; + Assert.Equal(10, TakeThisStageReport.Percentile(values, 0)); + Assert.Equal(30, TakeThisStageReport.Percentile(values, 0.5)); + Assert.Equal(50, TakeThisStageReport.Percentile(values, 1)); + Assert.True(TakeThisStageReport.Percentile(values, 0.95) >= TakeThisStageReport.Percentile(values, 0.50)); + } + + [Fact] + public void Intervals_ClassifyReceiveAsDominant_WhenReceiveIsLargest() + { + var origin = Stopwatch.GetTimestamp(); + var freq = Stopwatch.Frequency; + long AtMs(double ms) => origin + (long)(ms / 1000.0 * freq); + + var sample = new TakeThisStageSample( + CommandParsedTs: origin, + PeekStartTs: origin, + PeekCompleteTs: AtMs(0.2), + ReceiveStartTs: origin, + ReceiveCompleteTs: AtMs(1.0), + EmitStartTs: AtMs(1.05), + EnqueueStartTs: AtMs(1.06), + EnqueueAcceptedTs: AtMs(1.08), + RememberStartTs: AtMs(1.08), + RememberCompleteTs: AtMs(1.09), + ReplyEnqueueStartTs: AtMs(1.09), + ReplyEnqueueCompleteTs: AtMs(1.12), + OccupiedAtEmit: 3, + PeakOccupied: 8, + ArticleBytes: 1000, + Outcome: "unseen"); + + var intervals = TakeThisStageReport.Intervals(sample); + Assert.Equal("article receive", intervals.Dominant); + Assert.True(intervals.ReceiveUs > intervals.PeekUs); + Assert.True(intervals.OverlapUs >= intervals.ReceiveUs - 1); + } + + [Fact] + public void Report_IncludesRequestedIntervals() + { + var origin = Stopwatch.GetTimestamp(); + var sample = new TakeThisStageSample( + origin, + origin, + origin + Stopwatch.Frequency / 1000, + origin, + origin + Stopwatch.Frequency / 500, + origin + Stopwatch.Frequency / 400, + origin + Stopwatch.Frequency / 400, + origin + Stopwatch.Frequency / 350, + origin + Stopwatch.Frequency / 350, + origin + Stopwatch.Frequency / 340, + origin + Stopwatch.Frequency / 340, + origin + Stopwatch.Frequency / 300, + OccupiedAtEmit: 2, + PeakOccupied: 4, + ArticleBytes: 64, + Outcome: "unseen"); + + var text = TakeThisStageReport.Format([sample], peakOccupied: 4, "test", csvPath: null); + Assert.Contains("HistoryDB Peek", text, StringComparison.Ordinal); + Assert.Contains("article receive", text, StringComparison.Ordinal); + Assert.Contains("queue EnqueueAsync", text, StringComparison.Ordinal); + Assert.Contains("HistoryDB Remember", text, StringComparison.Ordinal); + Assert.Contains("239 Channel enqueue", text, StringComparison.Ordinal); + Assert.Contains("ready → emit (order)", text, StringComparison.Ordinal); + Assert.Contains("Peak occupied", text, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/AdmissionLeaseLivenessTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/AdmissionLeaseLivenessTests.cs new file mode 100644 index 00000000..c3322eec --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/AdmissionLeaseLivenessTests.cs @@ -0,0 +1,330 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Lease liveness: Redis TTL is crash recovery, not client idle. Renewal is +/// independent of NNTP traffic and aggregated per account. +/// +public sealed class AdmissionLeaseLivenessTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public async Task IdleAuthenticatedSessions_RemainOwnedAcrossMultipleLeaseIntervals() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + var peer = CreateNode("nntpd02", membership, clock); + for (var i = 0; i < 10; i++) + { + Assert.Equal( + SessionAdmissionResult.Success, + await Admit(node, $"s{i}", V4A, sessionLimit: 10, srcIpLimit: 2)); + } + + var previous = Require(membership.SessionExpiry("alice", node.OwnerId)); + for (var i = 0; i < 4; i++) + { + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + var next = Require(membership.SessionExpiry("alice", node.OwnerId)); + Assert.True(next > previous); + previous = next; + } + + Assert.Equal(10, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal(10, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(peer, "extra", V6A, sessionLimit: 10, srcIpLimit: 2)); + } + + [Fact] + public async Task RenewalService_ExtendsIdleOwnershipWithoutClientTraffic() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock, leaseTtl: TimeSpan.FromMinutes(5)); + var peer = CreateNode("nntpd02", membership, clock, leaseTtl: TimeSpan.FromMinutes(5)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 1)); + var previous = Require(membership.SessionExpiry("alice", node.OwnerId)); + var leases = new CountingLeaseManager(node); + var service = new SessionStateService( + leases, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + await service.StartAsync(CancellationToken.None); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (Volatile.Read(ref leases.Renewals) < 3 + || membership.SessionExpiry("alice", node.OwnerId) is not { } current + || current <= previous) + { + wait.Token.ThrowIfCancellationRequested(); + if (!clock.HasScheduledTimers) + { + await Task.Yield(); + continue; + } + + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + } + + Assert.True(membership.SessionExpiry("alice", node.OwnerId) > previous); + Assert.Equal(1, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(peer, "extra", V6A, sessionLimit: 1, srcIpLimit: 1)); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Renewal_MovesExpiryForwardFromEachSuccessfulRenewal() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 1)); + + var admitExpiry = Require(membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(30_000, admitExpiry); + Assert.Equal(admitExpiry, membership.SourceExpiry("alice", "192.0.2.10", node.OwnerId)); + + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + var firstRenew = Require(membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(40_000, firstRenew); + Assert.Equal(firstRenew, membership.SourceExpiry("alice", "192.0.2.10", node.OwnerId)); + + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + var secondRenew = Require(membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(50_000, secondRenew); + Assert.True(secondRenew > firstRenew); + Assert.Equal(1, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task CrashWithoutTeardown_ExpiresOwnershipAndFreesCapacity() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var crashed = CreateNode("nntpd01", membership, clock); + var live = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(crashed, "held", V4A, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(live, "wait", V6A, sessionLimit: 1, srcIpLimit: 1)); + + clock.Advance(TimeSpan.FromSeconds(31)); + Assert.Equal(0, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal(SessionAdmissionResult.Success, await Admit(live, "take", V6A, sessionLimit: 1, srcIpLimit: 1)); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", crashed.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", live.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal(1, crashed.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task RenewalFailure_DoesNotExtendOrRecreateOwnership() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 1)); + var original = Require(membership.SessionExpiry("alice", node.OwnerId)); + + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(original, membership.SessionExpiry("alice", node.OwnerId)); + + clock.Advance(TimeSpan.FromSeconds(21)); + await node.RenewLeasesAsync(); + Assert.Equal(original, membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(0, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal( + SessionAdmissionResult.Unavailable, + await Admit(node, "s2", V4A, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task LostLease_IsNotRecreatedByRenewal() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + var peer = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 1)); + await membership.ReleaseOwnerAsync("alice", node.OwnerId); + Assert.Null(membership.SessionExpiry("alice", node.OwnerId)); + + await node.RenewLeasesAsync(); + Assert.Null(membership.SessionExpiry("alice", node.OwnerId)); + Assert.Null(membership.SourceExpiry("alice", "192.0.2.10", node.OwnerId)); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + Assert.Equal(SessionAdmissionResult.Success, await Admit(peer, "take", V6A, sessionLimit: 1, srcIpLimit: 1)); + } + + [Fact] + public async Task PartialRenewal_ExtendsNoneOfTheRequestedOwnership() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "v4", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "v6", V6A, sessionLimit: 5, srcIpLimit: 2)); + var sessionExpiry = Require(membership.SessionExpiry("alice", node.OwnerId)); + var v4Expiry = Require(membership.SourceExpiry("alice", "192.0.2.10", node.OwnerId)); + var v6Expiry = Require(membership.SourceExpiry("alice", "2001:db8::10", node.OwnerId)); + + membership.BreakSourceGeneration("alice", "2001:db8::10", node.OwnerId); + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + + Assert.Equal(sessionExpiry, membership.SessionExpiry("alice", node.OwnerId)); + Assert.Equal(v4Expiry, membership.SourceExpiry("alice", "192.0.2.10", node.OwnerId)); + Assert.Equal(v6Expiry, membership.SourceExpiry("alice", "2001:db8::10", node.OwnerId)); + } + + [Fact] + public async Task RenewalFaultForOneAccount_DoesNotCorruptAnotherAccount() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a1", V4A, "alice", sessionLimit: 2, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "b1", V6A, "bob", sessionLimit: 2, srcIpLimit: 2)); + var aliceExpiry = Require(membership.SessionExpiry("alice", node.OwnerId)); + var bobExpiry = Require(membership.SessionExpiry("bob", node.OwnerId)); + + membership.FaultRenewAccount = "alice"; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + + Assert.Equal(aliceExpiry, membership.SessionExpiry("alice", node.OwnerId)); + Assert.True(membership.SessionExpiry("bob", node.OwnerId) > bobExpiry); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal(1, membership.OwnerSessionCount("bob", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task NodesRenewIndependently_StoppedNodeExpiresAlone() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership, clock); + var node2 = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "a1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "b1", V6A, sessionLimit: 5, srcIpLimit: 2)); + + clock.Advance(TimeSpan.FromSeconds(10)); + await node1.RenewLeasesAsync(); + await node2.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(10)); + await node2.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(21)); + + var now = clock.GetUtcNow().ToUnixTimeMilliseconds(); + Assert.Equal(0, membership.OwnerSessionCount("alice", node1.OwnerId, now)); + Assert.Equal(1, membership.OwnerSessionCount("alice", node2.OwnerId, now)); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node1.OwnerId, now)); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node2.OwnerId, now)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "b2", V4B, sessionLimit: 5, srcIpLimit: 2)); + } + + [Fact] + public async Task PartialSessionRelease_KeepsRemainingOwnershipOnRenewal() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V6A, sessionLimit: 5, srcIpLimit: 2)); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(2, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(2, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public void EngineRenew_DoesNotRecreateMissingFields() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus("src", "sess", "192.0.2.10", "nntpd01:a", 1, 1, 0, 30_000, 1, 1)); + _ = engine.ReleaseOwner("src", "sess", "nntpd01:a"); + Assert.Equal(0, engine.Renew("src", "sess", "nntpd01:a", 1, 10_000, 30_000, [("192.0.2.10", 1)])); + Assert.False(engine.TryGetOwnership("sess", "nntpd01:a", out _, out _, out _)); + Assert.False(engine.HasSourceOwner("src", "192.0.2.10", "nntpd01:a", 10_000)); + } + + private static long Require(long? expiry) => + expiry ?? throw new InvalidOperationException("Expected ownership expiry."); + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + int sessionLimit = 0, + int srcIpLimit = 0) => + Admit(node, sessionId, ip, "alice", sessionLimit, srcIpLimit); + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + string account, + int sessionLimit = 0, + int srcIpLimit = 0) => + node.TryAdmitAsync(account, sessionId, ip, sessionLimit, srcIpLimit); + + private static DistributedSessionStateTracker CreateNode( + string nodeId, + InMemorySessionStateStore membership, + TimeProvider? time = null, + TimeSpan? leaseTtl = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System, + leaseTtl: leaseTtl); + + private sealed class CountingLeaseManager : ISessionStateLeaseManager + { + private readonly DistributedSessionStateTracker _inner; + + public CountingLeaseManager(DistributedSessionStateTracker inner) + { + _inner = inner; + } + + public int Renewals; + + public ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Renewals); + return _inner.RenewLeasesAsync(cancellationToken); + } + + public ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) => + _inner.ReleaseAllOwnershipAsync(cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/ClusterWideSessionAdmissionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/ClusterWideSessionAdmissionTests.cs new file mode 100644 index 00000000..b6b352f9 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/ClusterWideSessionAdmissionTests.cs @@ -0,0 +1,461 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +public sealed class ClusterWideSessionAdmissionTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V4C = IPAddress.Parse("203.0.113.30"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public async Task NodeB_CannotExceedNodeAClusterSessionLimit() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a2", V4A, sessionLimit: 2)); + Assert.Equal(SessionAdmissionResult.SessionLimitExceeded, await Admit(nodeB, "b1", V6A, sessionLimit: 2)); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(0, nodeB.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task NodeB_CanAdmitAfterNodeAReleasesSlot() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 1)); + Assert.Equal(SessionAdmissionResult.SessionLimitExceeded, await Admit(nodeB, "b1", V6A, sessionLimit: 1)); + await nodeA.ReleaseAsync("alice", "a1"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 1)); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeB.OwnerId, NowMs())); + Assert.Equal(0, membership.OwnerSessionCount("alice", nodeA.OwnerId, NowMs())); + } + + [Fact] + public async Task MultipleNodes_ContributeToSameGlobalSessionCount() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + var nodeC = CreateNode("nntpd03", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a2", V4A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a3", V4A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a4", V4A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b2", V6A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b3", V6A, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeC, "c1", V4B, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeC, "c2", V4B, sessionLimit: 10)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeC, "c3", V4B, sessionLimit: 10)); + Assert.Equal(10, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(SessionAdmissionResult.SessionLimitExceeded, await Admit(nodeA, "a5", V4A, sessionLimit: 10)); + } + + [Fact] + public async Task SessionLimitZero_RemainsUnlimited() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + for (var i = 0; i < 8; i++) + { + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, $"a{i}", V4A, sessionLimit: 0, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, $"b{i}", V6A, sessionLimit: 0, srcIpLimit: 2)); + } + + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(nodeA, "x", V4B, sessionLimit: 0, srcIpLimit: 2)); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task SessionFiveSourceTwo_ThirdIpRejectedAsSessionLimit() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + var nodeC = CreateNode("nntpd03", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a2", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a3", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b2", V6A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(nodeC, "c1", V4B, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(5, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + Assert.False(membership.HasSourceOwner("alice", "198.51.100.20", nodeC.OwnerId, NowMs())); + Assert.Equal(0, nodeC.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task SessionTenSourceTwo_ThirdDistinctIpRejected() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + var nodeC = CreateNode("nntpd03", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a2", V4A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + await Admit(nodeC, "c1", V4B, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + } + + [Fact] + public async Task SameSourceIp_ConsumesManySessionSlotsAndOneSourceSlot() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + Assert.Equal(0, node.LocalHotPathAdmits); + Assert.Equal(3, node.DistributedAdmits); + } + + [Fact] + public async Task Ipv4AndIpv6_AreDistinctSources_MappedIpv6FollowsNormalization() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + var mapped = IPAddress.Parse("::ffff:192.0.2.10"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", mapped, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V6A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + await Admit(node, "s4", V4B, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + } + + [Fact] + public async Task RejectedSessionLimit_DoesNotRegisterSourceOwnership() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.SessionLimitExceeded, await Admit(node, "s2", V6A, sessionLimit: 1, srcIpLimit: 2)); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task RejectedSourceLimit_DoesNotRegisterSessionOwnership() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 10, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s2", V6A, sessionLimit: 10, srcIpLimit: 1)); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.False(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + } + + [Fact] + public async Task Release_DecrementsExactlyOneSessionOnSameNode() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 5)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V4A, sessionLimit: 5)); + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(2, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.Equal(2, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task Release_LastSessionForIpRemovesThisNodesSourceOwnership() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "v4", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "v6", V6A, sessionLimit: 5, srcIpLimit: 2)); + await node.ReleaseAsync("alice", "v4"); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task Release_DoesNotAffectAnotherNodesOwnership() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 5, srcIpLimit: 2)); + await nodeA.ReleaseAsync("alice", "a1"); + Assert.Equal(0, membership.OwnerSessionCount("alice", nodeA.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeB.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", nodeB.OwnerId, NowMs())); + } + + [Fact] + public async Task ExpiredOwnership_FreesSessionAndSourceCapacity() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var crashed = CreateNode("nntpd01", membership, clock); + var live = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(crashed, "dead", V4A, sessionLimit: 1, srcIpLimit: 1)); + clock.Advance(TimeSpan.FromSeconds(31)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(live, "live", V6A, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal(1, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", crashed.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", live.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task ExpiryOfOneNode_DoesNotRemoveAnotherNodesOwnership() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var dying = CreateNode("nntpd01", membership, clock); + var live = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(dying, "d1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(live, "l1", V6A, sessionLimit: 5, srcIpLimit: 2)); + clock.Advance(TimeSpan.FromSeconds(20)); + await live.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(1, membership.OwnerSessionCount("alice", live.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.Equal(0, membership.OwnerSessionCount("alice", dying.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", live.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task StaleReleaseFromOldIncarnation_CannotRemoveNewOwnership() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var oldIncarnation = CreateNode("nntpd01", membership, clock, incarnation: "old"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(oldIncarnation, "old", V4A, sessionLimit: 1, srcIpLimit: 1)); + clock.Advance(TimeSpan.FromSeconds(31)); + var newIncarnation = CreateNode("nntpd01", membership, clock, incarnation: "new"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(newIncarnation, "new", V4A, sessionLimit: 1, srcIpLimit: 1)); + await oldIncarnation.ReleaseAsync("alice", "old"); + Assert.Equal(1, membership.OwnerSessionCount("alice", newIncarnation.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", newIncarnation.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", oldIncarnation.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task RedisUnavailable_FailsClosedWithoutPartialMembership() + { + var membership = new InMemorySessionStateStore { Unavailable = true }; + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s1", V4A, sessionLimit: 2, srcIpLimit: 2)); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.Empty(membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task FiniteSessionLimit_DoesNotUseSourceIpHotPath() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 1)); + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s2", V4A, sessionLimit: 5, srcIpLimit: 1)); + Assert.Equal(0, node.LocalHotPathAdmits); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task RenewalFailure_DoesNotExtendLeases() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 0, srcIpLimit: 1)); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(21)); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s2", V4A, sessionLimit: 0, srcIpLimit: 1)); + } + + [Fact] + public async Task ConcurrentNodes_LastSessionSlot_ExactlyOneSucceeds() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "held", V4A, sessionLimit: 2, srcIpLimit: 4)); + var results = new SessionAdmissionResult[2]; + Parallel.Invoke( + () => results[0] = Admit(nodeA, "a", V4B, sessionLimit: 2, srcIpLimit: 4).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(nodeB, "b", V4C, sessionLimit: 2, srcIpLimit: 4).AsTask().GetAwaiter().GetResult()); + + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.Success)); + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.SessionLimitExceeded)); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task ConcurrentNodes_LastSourceSlot_ExactlyOneSucceeds() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode("nntpd01", membership); + var nodeB = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "held", V4A, sessionLimit: 10, srcIpLimit: 2)); + var results = new SessionAdmissionResult[2]; + Parallel.Invoke( + () => results[0] = Admit(nodeA, "a", V4B, sessionLimit: 10, srcIpLimit: 2).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(nodeB, "b", V6A, sessionLimit: 10, srcIpLimit: 2).AsTask().GetAwaiter().GetResult()); + + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.Success)); + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.SourceAddressLimitExceeded)); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + } + + [Fact] + public async Task RedisStore_OneEvalPerAdmitAndRelease() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var node = new DistributedSessionStateTracker( + store, + NullLogger.Instance, + "nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 3, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 3, srcIpLimit: 2)); + Assert.Equal(2, redis.Database.ScriptEvaluateCount); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(3, redis.Database.ScriptEvaluateCount); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s3", V6A, sessionLimit: 3, srcIpLimit: 1)); + Assert.Equal(4, redis.Database.ScriptEvaluateCount); + } + + [Fact] + public void Engine_RejectedAdmission_LeavesHashesUnchanged() + { + var engine = new SessionStateEngine(); + const string src = "src:alice"; + const string sess = "sess:alice"; + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(src, sess, "192.0.2.10", "nntpd01:a", sessionLimit: 1, srcIpLimit: 2, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.RejectedSessionLimit, + engine.TryAdmitStatus(src, sess, "2001:db8::10", "nntpd02:b", sessionLimit: 1, srcIpLimit: 2, 0, 30_000, 1, 1)); + Assert.Equal(1, engine.ActiveSessionCount(sess, 0)); + Assert.False(engine.HasSourceOwner(src, "2001:db8::10", "nntpd02:b", 0)); + + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(src, sess, "192.0.2.10", "nntpd03:c", sessionLimit: 10, srcIpLimit: 1, 0, 30_000, 1, 1)); + var sessions = engine.ActiveSessionCount(sess, 0); + Assert.Equal( + SessionStateEngine.RejectedSourceLimit, + engine.TryAdmitStatus(src, sess, "198.51.100.20", "nntpd04:d", sessionLimit: 10, srcIpLimit: 1, 0, 30_000, 1, 1)); + Assert.Equal(sessions, engine.ActiveSessionCount(sess, 0)); + Assert.False(engine.HasSourceOwner(src, "198.51.100.20", "nntpd04:d", 0)); + } + + [Fact] + public void Engine_StaleGenerationRelease_DoesNotRemoveNewerOwnership() + { + var engine = new SessionStateEngine(); + const string src = "src:alice"; + const string sess = "sess:alice"; + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(src, sess, "192.0.2.10", "nntpd01:old", 1, 1, 0, 30_000, 1, 1)); + _ = engine.TryAdmitStatus(src, sess, "192.0.2.10", "nntpd01:new", 1, 1, 31_000, 30_000, 2, 2); + _ = engine.Release(src, sess, "192.0.2.10", "nntpd01:old", 1, 1); + Assert.Equal(1, engine.OwnerSessionCount(sess, "nntpd01:new", 31_000)); + Assert.True(engine.HasSourceOwner(src, "192.0.2.10", "nntpd01:new", 31_000)); + } + + [Fact] + public void Engine_SameOwnerNewGenerationAdmit_ReplacesStaleCount() + { + var engine = new SessionStateEngine(); + const string src = "src:alice"; + const string sess = "sess:alice"; + const string owner = "nntpd01:a"; + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(src, sess, "192.0.2.10", owner, 2, 2, 0, 30_000, 1, 1)); + Assert.Equal(1, engine.OwnerSessionCount(sess, owner, 0)); + + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(src, sess, "192.0.2.10", owner, 2, 2, 0, 30_000, 2, 2)); + Assert.Equal(1, engine.OwnerSessionCount(sess, owner, 0)); + Assert.True(engine.TryGetOwnership(sess, owner, out _, out var sessionGeneration, out var sessionCount)); + Assert.Equal(2, sessionGeneration); + Assert.Equal(1, sessionCount); + + _ = engine.Release(src, sess, "192.0.2.10", owner, 2, 2); + Assert.Equal(0, engine.OwnerSessionCount(sess, owner, 0)); + Assert.False(engine.HasSourceOwner(src, "192.0.2.10", owner, 0)); + } + + [Fact] + public async Task RedisUnavailableRelease_ThenReadmit_ReleaseClearsOwnership() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 2, srcIpLimit: 2)); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + + membership.Unavailable = true; + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + + membership.Unavailable = false; + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 2, srcIpLimit: 2)); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + } + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + int sessionLimit = 0, + int srcIpLimit = 0) => + node.TryAdmitAsync("alice", sessionId, ip, sessionLimit, srcIpLimit); + + private static DistributedSessionStateTracker CreateNode( + string nodeId, + InMemorySessionStateStore membership, + TimeProvider? time = null, + string? incarnation = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System, + incarnation: incarnation); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/DistributedSessionStateTrackerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/DistributedSessionStateTrackerTests.cs new file mode 100644 index 00000000..2dd052a5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/DistributedSessionStateTrackerTests.cs @@ -0,0 +1,352 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +public sealed class DistributedSessionStateTrackerTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + private static readonly IPAddress V6B = IPAddress.Parse("2001:db8::11"); + + [Fact] + public async Task SameIpv4Repeated_IsOneDistinctSource() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V4A, srcIpLimit: 1)); + Assert.Equal(3, node.GetLocalIpCount("alice", "192.0.2.10")); + Assert.Equal(3, node.GetLocalSessionCount("alice")); + Assert.Equal(1, node.DistributedAdmits); + Assert.Equal(2, node.LocalHotPathAdmits); + } + + [Fact] + public async Task SameIpv6Repeated_IsOneDistinctSource() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V6A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V6A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s3", V6B, srcIpLimit: 1)); + } + + [Fact] + public async Task CompressedAndExpandedIpv6_AreOneSource() + { + var (node, _) = CreateNode("nntpd01"); + var expanded = IPAddress.Parse("2001:0db8:0000:0000:0000:0000:0000:0010"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V6A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", expanded, srcIpLimit: 1)); + Assert.Equal(2, node.GetLocalIpCount("alice", "2001:db8::10")); + } + + [Fact] + public async Task Ipv4MappedIpv6_FollowsExistingNormalization() + { + var (node, _) = CreateNode("nntpd01"); + var mapped = IPAddress.Parse("::ffff:192.0.2.10"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", mapped, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s3", V4B, srcIpLimit: 1)); + Assert.Equal(2, node.GetLocalIpCount("alice", "192.0.2.10")); + } + + [Fact] + public async Task Ipv4ThenIpv6_WithLimitOne_IsRejected() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s2", V6A, srcIpLimit: 1)); + Assert.Equal(0, node.GetLocalSessionCount("bob")); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task Ipv6ThenIpv4_WithLimitOne_IsRejected() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V6A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s2", V4A, srcIpLimit: 1)); + } + + [Fact] + public async Task MixedIpv4AndIpv6_EachConsumeOneSlot() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V6A, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "s3", V4B, srcIpLimit: 2)); + } + + [Fact] + public async Task SameSourceAcrossNodes_IsOneDistinctSource() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "s2", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node1, "s3", V4B, srcIpLimit: 1)); + } + + [Fact] + public async Task DifferentSourcesAcrossNodes_ObeyLimit() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + var node3 = CreateNode("nntpd03", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "s2", V6A, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node3, "s3", V4B, srcIpLimit: 2)); + } + + [Fact] + public async Task ConcurrentDistributedAdmit_CannotExceedLimit() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + var results = new SessionAdmissionResult[2]; + Parallel.Invoke( + () => results[0] = Admit(node1, "s1", V4A, srcIpLimit: 1).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(node2, "s2", V6A, srcIpLimit: 1).AsTask().GetAwaiter().GetResult()); + + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.Success)); + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.SourceAddressLimitExceeded)); + } + + [Fact] + public async Task ConcurrentSameSourceDifferentNodes_BothSucceed() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + var results = new SessionAdmissionResult[2]; + Parallel.Invoke( + () => results[0] = Admit(node1, "s1", V6A, srcIpLimit: 1).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(node2, "s2", V6A, srcIpLimit: 1).AsTask().GetAwaiter().GetResult()); + + Assert.All(results, static r => Assert.Equal(SessionAdmissionResult.Success, r)); + } + + [Fact] + public async Task LocalReferenceCounts_ReleaseOnlyOnFinalSession() + { + var (node, membership) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "b", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "c", V4A, srcIpLimit: 1)); + await node.ReleaseAsync("alice", "a"); + Assert.Equal(2, node.GetLocalIpCount("alice", "192.0.2.10")); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + await node.ReleaseAsync("alice", "b"); + Assert.Equal(1, node.GetLocalIpCount("alice", "192.0.2.10")); + await node.ReleaseAsync("alice", "c"); + Assert.Equal(0, node.GetLocalIpCount("alice", "192.0.2.10")); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + } + + [Fact] + public async Task OneNodeCannotReleaseAnotherNodesOwnership() + { + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership); + var node2 = CreateNode("nntpd02", membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "s2", V4A, srcIpLimit: 1)); + await node1.ReleaseAsync("alice", "s1"); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node1.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node2.OwnerId, NowMs())); + var node3 = CreateNode("nntpd03", membership); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node3, "s3", V4B, srcIpLimit: 1)); + } + + [Fact] + public async Task ExpiredOwnership_NoLongerConsumesCapacity() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership, clock); + var node2 = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 1)); + clock.Advance(TimeSpan.FromSeconds(31)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node2, "s2", V6A, srcIpLimit: 1)); + } + + [Fact] + public async Task LiveOwnershipRenewal_PreservesCapacity() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node1 = CreateNode("nntpd01", membership, clock); + var node2 = CreateNode("nntpd02", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 1)); + clock.Advance(TimeSpan.FromSeconds(20)); + await node1.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node2, "s2", V6A, srcIpLimit: 1)); + } + + [Fact] + public async Task RedisUnavailable_CannotCreateNewSourceMembership() + { + var (node, membership) = CreateNode("nntpd01"); + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(1, node.UnavailableRejects); + } + + [Fact] + public async Task RedisUnavailable_DoesNotBlockLiveLocalHotPath() + { + var (node, membership) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, srcIpLimit: 1)); + Assert.Equal(1, node.LocalHotPathAdmits); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s3", V6A, srcIpLimit: 2)); + } + + [Fact] + public async Task RedisUnavailableDuringRenewal_DoesNotExtendHotPathPastLease() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode("nntpd01", membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + clock.Advance(TimeSpan.FromSeconds(21)); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s2", V4A, srcIpLimit: 1)); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task SessionLimitAndSourceIpLimit_RemainIndependent() + { + var (node, membership) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 2, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 2, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(node, "s3", V4A, sessionLimit: 2, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(node, "s4", V6A, sessionLimit: 2, srcIpLimit: 1)); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task ZeroSourceLimit_SkipsDistributedMembership() + { + var (node, membership) = CreateNode("nntpd01"); + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 0)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V6A, srcIpLimit: 0)); + Assert.Equal(0, node.DistributedAdmits); + } + + [Fact] + public async Task CancellationDuringDistributedAdmit_DoesNotLeakMembership() + { + var membership = new InMemorySessionStateStore + { + BlockAdmit = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var node = CreateNode("nntpd01", membership); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync("alice", "s1", V4A, 0, 1, cts.Token).AsTask(); + cts.Cancel(); + membership.BlockAdmit.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Empty(membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task FinalReleaseRacingNewLocalSession_ReestablishesOwnership() + { + var (node, membership) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, srcIpLimit: 1)); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.Equal(1, node.GetLocalIpCount("alice", "192.0.2.10")); + } + + [Fact] + public async Task ReAdmitSameSession_IsIdempotent() + { + var (node, _) = CreateNode("nntpd01"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task RedisStore_UsesSharedFakeRedisEngine() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var node1 = new DistributedSessionStateTracker( + store, + NullLogger.Instance, + "nntpd01"); + var node2 = new DistributedSessionStateTracker( + store, + NullLogger.Instance, + "nntpd02"); + + Assert.Equal(SessionAdmissionResult.Success, await Admit(node1, "s1", V4A, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node2, "s2", V6A, srcIpLimit: 1)); + Assert.True(redis.Database.ScriptEvaluateCount >= 2); + } + + [Fact] + public async Task RedisStoreUnavailable_FailsClosed() + { + var redis = new FakeRedisService { IsUnavailable = true }; + var store = new RedisSessionStateStore(redis); + var node = new DistributedSessionStateTracker( + store, + NullLogger.Instance, + "nntpd01"); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s1", V4A, srcIpLimit: 1)); + } + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + int sessionLimit = 0, + int srcIpLimit = 0) => + node.TryAdmitAsync("alice", sessionId, ip, sessionLimit, srcIpLimit); + + private static (DistributedSessionStateTracker Node, InMemorySessionStateStore Membership) CreateNode( + string nodeId) + { + var membership = new InMemorySessionStateStore(); + return (CreateNode(nodeId, membership), membership); + } + + private static DistributedSessionStateTracker CreateNode( + string nodeId, + InMemorySessionStateStore membership, + TimeProvider? time = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateAllocatorInvariantTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateAllocatorInvariantTests.cs new file mode 100644 index 00000000..4123916d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateAllocatorInvariantTests.cs @@ -0,0 +1,279 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState.RateLimiting; + +public sealed class AccountRateAllocatorInvariantTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V4C = IPAddress.Parse("203.0.113.30"); + private const int OneMbpsBps = 1_000_000; + private const int TenMbpsBps = 10_000_000; + + [Fact] + public async Task ThreeNode_LeftoverWouldOvershoot_JoinersAreBlocked() + { + var clock = new ManualClock(); + var membership = new InMemorySessionStateStore(); + var a = CreateNode(membership, "nntpd01", clock); + var b = CreateNode(membership, "nntpd02", clock); + var c = CreateNode(membership, "nntpd03", clock); + + var aCaps = await AdmitMany(a, "alice", "a", V4A, 6, TenMbpsBps); + AssertAggregate(TenMbpsBps, aCaps); + + var b1 = await AdmitOne(b, "alice", "b1", V4B, TenMbpsBps); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, b1.MaxSendBytesPerSecond); + AssertAggregate(TenMbpsBps, aCaps, [b1]); + + var c1 = await AdmitOne(c, "alice", "c1", V4C, TenMbpsBps); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, c1.MaxSendBytesPerSecond); + AssertAggregate(TenMbpsBps, aCaps, [b1], [c1]); + } + + [Fact] + public async Task ThreeNode_UserSequence_NeverExceedsAggregate() + { + var clock = new ManualClock(); + var membership = new InMemorySessionStateStore(); + var a = CreateNode(membership, "nntpd01", clock); + var b = CreateNode(membership, "nntpd02", clock); + var c = CreateNode(membership, "nntpd03", clock); + + var aCaps = await AdmitMany(a, "alice", "a", V4A, 5, TenMbpsBps); + AssertAggregate(TenMbpsBps, aCaps); + + var bCaps = new List(); + for (var i = 0; i < 3; i++) + { + bCaps.Add(await AdmitOne(b, "alice", "b" + i, V4B, TenMbpsBps)); + AssertAggregate(TenMbpsBps, aCaps, bCaps); + } + + var c1 = await AdmitOne(c, "alice", "c1", V4C, TenMbpsBps); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1]); + + var c2 = await AdmitOne(c, "alice", "c2", V4C, TenMbpsBps); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + + b.Rates.Unregister("alice", "b2"); + await b.Tracker.ReleaseAsync("alice", "b2"); + bCaps.RemoveAt(2); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + + await a.Tracker.RenewLeasesAsync(); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + AssertCaps(aCaps, AccountRateFormula.EqualShareCap(TenMbpsBps, 9)); + + await c.Tracker.RenewLeasesAsync(); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + + await b.Tracker.RenewLeasesAsync(); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + + clock.Advance(SessionStateDefaults.LeaseTtl); + a.Rates.ObserveClusterSessionCount("alice", 9); + b.Rates.ObserveClusterSessionCount("alice", 9); + c.Rates.ObserveClusterSessionCount("alice", 9); + AssertAggregate(TenMbpsBps, aCaps, bCaps, [c1, c2]); + Assert.True(c1.MaxSendBytesPerSecond <= 0); + Assert.True(c2.MaxSendBytesPerSecond <= 0); + } + + [Fact] + public void EstablishedEqualShare_IsNotReclassifiedAsJoiner() + { + var clock = new ManualClock(); + var rates = new AccountRateAllocator(clock); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount("alice", 1); + rates.Register("alice", "a1", cap, OneMbpsBps); + Assert.Equal(125_000, cap.MaxSendBytesPerSecond); + + rates.ObserveClusterSessionCount("alice", 400); + Assert.True(cap.MaxSendBytesPerSecond <= AccountRateFormula.EqualShareCap(OneMbpsBps, 400)); + Assert.True(AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond) <= 125_000); + } + + [Theory] + [InlineData(10_000_000, 64)] + [InlineData(1_000_000, 32)] + [InlineData(3_000_000, 16)] + [InlineData(1_000_000, 200)] + public async Task StateMachine_ArbitraryJoinLeaveRenew_NeverExceedsAggregate(int rateBps, int sessionLimit) + { + var clock = new ManualClock(); + var membership = new InMemorySessionStateStore(); + var nodes = new[] + { + CreateNode(membership, "nntpd01", clock), + CreateNode(membership, "nntpd02", clock), + CreateNode(membership, "nntpd03", clock), + }; + var ips = new[] { V4A, V4B, V4C }; + var rng = new Random(rateBps + sessionLimit); + var nextId = 0; + + for (var step = 0; step < 80; step++) + { + var node = nodes[rng.Next(nodes.Length)]; + var roll = rng.Next(6); + if (roll <= 2 && node.Sessions.Count < sessionLimit) + { + var id = "s" + nextId++; + var result = await node.Tracker.TryAdmitAsync( + "alice", + id, + ips[Array.IndexOf(nodes, node)], + sessionLimit, + 0, + rateBps); + if (result == SessionAdmissionResult.Success) + { + var cap = new FakeCap(); + node.Rates.Register("alice", id, cap, rateBps); + node.Sessions[id] = cap; + } + } + else if (roll == 3 && node.Sessions.Count > 0) + { + var id = node.Sessions.Keys.ElementAt(rng.Next(node.Sessions.Count)); + node.Rates.Unregister("alice", id); + await node.Tracker.ReleaseAsync("alice", id); + node.Sessions.Remove(id); + } + else if (roll == 4) + { + await node.Tracker.RenewLeasesAsync(); + } + else + { + clock.Advance(SessionStateDefaults.LeaseTtl); + var live = nodes.Sum(static n => n.Sessions.Count); + if (live > 0) + { + foreach (var liveNode in nodes) + { + await liveNode.Tracker.RenewLeasesAsync(); + liveNode.Rates.ObserveClusterSessionCount("alice", live); + } + } + } + + AssertAggregate(rateBps, nodes.Select(static n => n.Sessions.Values)); + } + } + + [Fact] + public void Allocator_FloorZero_BlocksInsteadOfUnlimited() + { + var clock = new ManualClock(); + var rates = new AccountRateAllocator(clock); + rates.ObserveClusterSessionCount("alice", 200_000); + var cap = new FakeCap(); + rates.Register("alice", "s1", cap, OneMbpsBps); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, cap.MaxSendBytesPerSecond); + + clock.Advance(SessionStateDefaults.LeaseTtl); + rates.ObserveClusterSessionCount("alice", 200_000); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, cap.MaxSendBytesPerSecond); + Assert.Equal(0, AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)); + } + + private static async Task AdmitMany(Node node, string account, string prefix, IPAddress ip, int count, int rateBps) + { + var caps = new FakeCap[count]; + for (var i = 0; i < count; i++) + { + caps[i] = await AdmitOne(node, account, prefix + i, ip, rateBps); + } + + return caps; + } + + private static async Task AdmitOne(Node node, string account, string id, IPAddress ip, int rateBps) + { + Assert.Equal( + SessionAdmissionResult.Success, + await node.Tracker.TryAdmitAsync(account, id, ip, 64, 0, rateBps)); + var cap = new FakeCap(); + node.Rates.Register(account, id, cap, rateBps); + node.Sessions[id] = cap; + return cap; + } + + private static void AssertAggregate(int rateBps, params IEnumerable[] groups) => + AssertAggregate(rateBps, groups.AsEnumerable()); + + private static void AssertAggregate(int rateBps, IEnumerable> groups) + { + var caps = groups.SelectMany(static group => group) + .Select(static cap => cap.MaxSendBytesPerSecond) + .ToArray(); + var allocated = caps.Sum(AccountRateFormula.AllocatedBytesPerSecond); + var limit = AccountRateFormula.AccountBytesPerSecond(rateBps); + Assert.True( + allocated <= limit, + $"allocated {allocated} > {limit}; caps=[{string.Join(",", caps)}]"); + } + + private static void AssertCaps(IEnumerable caps, long expected) + { + foreach (var cap in caps) + { + Assert.Equal(expected, cap.MaxSendBytesPerSecond); + } + } + + private static Node CreateNode(ISessionStateStore membership, string nodeId, TimeProvider clock) + { + var rates = new AccountRateAllocator(clock); + return new Node( + rates, + new DistributedSessionStateTracker( + membership, + NullLogger.Instance, + nodeId, + TimeProvider.System, + rates: rates)); + } + + private sealed class Node + { + public Node(AccountRateAllocator rates, DistributedSessionStateTracker tracker) + { + Rates = rates; + Tracker = tracker; + } + + public AccountRateAllocator Rates { get; } + + public DistributedSessionStateTracker Tracker { get; } + + public Dictionary Sessions { get; } = new(StringComparer.Ordinal); + } + + private sealed class FakeCap : IOutboundRateCap + { + public long MaxSendBytesPerSecond { get; private set; } + + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => MaxSendBytesPerSecond = bytesPerSecond; + } + + private sealed class ManualClock : TimeProvider + { + private DateTimeOffset _utcNow = DateTimeOffset.UnixEpoch; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfNegative(delta.Ticks); + _utcNow += delta; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateFormulaTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateFormulaTests.cs new file mode 100644 index 00000000..1f9eef29 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/AccountRateFormulaTests.cs @@ -0,0 +1,132 @@ +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState.RateLimiting; + +public sealed class AccountRateFormulaTests +{ + private const int OneMbpsBps = 1_000_000; + private const int TenMbpsBps = 10_000_000; + + [Fact] + public void DatabaseRate_240Bps_Is30BytesPerSecond_Not30Million() + { + Assert.Equal(30, AccountRateFormula.AccountBytesPerSecond(240)); + Assert.Equal(30, AccountRateFormula.PerSessionBytesPerSecond(240, 1)); + Assert.Equal(30, AccountRateFormula.EqualShareCap(240, 1)); + Assert.NotEqual(30_000_000, AccountRateFormula.AccountBytesPerSecond(240)); + } + + [Fact] + public void OneMillionBps_IsOneMbps_125000BytesPerSecond() + { + Assert.Equal(8, AccountRateFormula.BitsPerByte); + Assert.Equal(125_000, AccountRateFormula.AccountBytesPerSecond(OneMbpsBps)); + Assert.Equal(1_250_000, AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public void Policy_Database240Bps_MapsToRateLimitBps() + { + var record = MemoryNntpUserRecordStore.Create( + "alice", + "x", + sessionLimit: 1, + srcIpLimit: 1, + rateLimitBps: 240, + byteLimit: 10_000_000_000); + var policy = NntpAccountPolicy.FromRecord(record); + Assert.Equal(240, record.RateLimitBps); + Assert.True(policy.RequiresRateTracking); + Assert.Equal(240, policy.RateLimitBps); + Assert.Equal(30, AccountRateFormula.EqualShareCap(policy.RateLimitBps, 1)); + Assert.NotEqual(30_000_000, AccountRateFormula.AccountBytesPerSecond(policy.RateLimitBps)); + } + + [Theory] + [InlineData(1)] + [InlineData(7)] + public void PositiveSubEightBps_IsBlockedNotUnlimited(int rateBps) + { + Assert.Equal(0, AccountRateFormula.AccountBytesPerSecond(rateBps)); + Assert.Equal(0, AccountRateFormula.PerSessionBytesPerSecond(rateBps, 1)); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, AccountRateFormula.EqualShareCap(rateBps, 1)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void ZeroOrNegativeBps_IsUnlimitedSentinel(int rateBps) + { + Assert.Equal(0, AccountRateFormula.AccountBytesPerSecond(rateBps)); + Assert.Equal(0, AccountRateFormula.PerSessionBytesPerSecond(rateBps, 1)); + Assert.Equal(0, AccountRateFormula.EqualShareCap(rateBps, 1)); + Assert.True(AccountRateFormula.AggregateDoesNotExceed(rateBps, 10)); + } + + [Fact] + public void ZeroActiveSessions_HasNoAllocation() + { + Assert.Equal(0, AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 0)); + Assert.True(AccountRateFormula.AggregateDoesNotExceed(TenMbpsBps, 0)); + } + + [Theory] + [InlineData(10_000_000, 1, 1_250_000)] + [InlineData(10_000_000, 2, 625_000)] + [InlineData(10_000_000, 5, 250_000)] + [InlineData(10_000_000, 10, 125_000)] + [InlineData(10_000_000, 7, 178_571)] + [InlineData(10_000_000, 8, 156_250)] + [InlineData(10_000_000, 3, 416_666)] + [InlineData(1_000, 1, 125)] + [InlineData(8_000, 1, 1_000)] + public void PerSession_FloorsAccountBytesByActiveCount(int rateBps, int sessions, long expected) + { + Assert.Equal(expected, AccountRateFormula.PerSessionBytesPerSecond(rateBps, sessions)); + Assert.True(AccountRateFormula.AggregateDoesNotExceed(rateBps, sessions)); + Assert.True( + expected * sessions <= AccountRateFormula.AccountBytesPerSecond(rateBps)); + } + + [Fact] + public void UsesActiveCount_NotConfiguredSessionLimit() + { + Assert.Equal(625_000, AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, activeSessions: 2)); + Assert.NotEqual(125_000, AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, activeSessions: 2)); + } + + [Fact] + public void AggregateNeverExceeds_ForEverySessionCountThroughLimit() + { + for (var sessions = 1; sessions <= 64; sessions++) + { + Assert.True(AccountRateFormula.AggregateDoesNotExceed(TenMbpsBps, sessions)); + } + } + + [Fact] + public void ConservativeJoin_IsUnusedRemainderAfterPreviousSplit() + { + Assert.Equal(3, AccountRateFormula.ConservativeJoinBytesPerSecond(TenMbpsBps, 7)); + Assert.Equal(0, AccountRateFormula.ConservativeJoinBytesPerSecond(TenMbpsBps, 8)); + Assert.Equal(0, AccountRateFormula.ConservativeJoinBytesPerSecond(TenMbpsBps, 10)); + Assert.Equal(0, AccountRateFormula.ConservativeJoinBytesPerSecond(0, 7)); + } + + [Fact] + public void AccountBytesPerSecond_OverflowSafeForIntMaxBps() + { + Assert.Equal(int.MaxValue / 8, AccountRateFormula.AccountBytesPerSecond(int.MaxValue)); + } + + [Fact] + public void EqualShareCap_FloorZero_IsBlockedNotUnlimited() + { + Assert.Equal(0, AccountRateFormula.PerSessionBytesPerSecond(OneMbpsBps, 200_000)); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, AccountRateFormula.EqualShareCap(OneMbpsBps, 200_000)); + Assert.Equal(0, AccountRateFormula.EqualShareCap(0, 1)); + Assert.Equal(125_000, AccountRateFormula.EqualShareCap(OneMbpsBps, 1)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/OutboundRateLimiterTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/OutboundRateLimiterTests.cs new file mode 100644 index 00000000..79bc2934 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/OutboundRateLimiterTests.cs @@ -0,0 +1,326 @@ +using VectorNNTPD.ReaderService.SessionState.RateLimiting; + +namespace VectorNNTPD.ReaderService.Tests.SessionState.RateLimiting; + +public sealed class OutboundRateLimiterTests +{ + [Fact] + public async Task ZeroCap_IsPassthrough() + { + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 0, leaveInnerOpen: true); + var payload = "hello"u8.ToArray(); + await limiter.WriteAsync(payload); + Assert.Equal(payload, inner.ToArray()); + Assert.Equal(0, limiter.MaxSendBytesPerSecond); + } + + [Fact] + public void Update_IsVisibleImmediately() + { + using var inner = new MemoryStream(); + using var limiter = new OutboundRateLimiter(inner, 100, leaveInnerOpen: true); + limiter.UpdateMaxSendBytesPerSecond(50); + Assert.Equal(50, limiter.MaxSendBytesPerSecond); + limiter.UpdateMaxSendBytesPerSecond(0); + Assert.Equal(0, limiter.MaxSendBytesPerSecond); + } + + [Fact] + public async Task HighRate_WritesLargeBufferWithoutDelay() + { + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, int.MaxValue, leaveInnerOpen: true); + var payload = new byte[64_000]; + Random.Shared.NextBytes(payload); + await limiter.WriteAsync(payload); + Assert.Equal(payload, inner.ToArray()); + } + + [Fact] + public async Task SequentialSmallWrites_StayUnderFixedCap() + { + var clock = new ManualTimeProvider(); + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 4, leaveInnerOpen: true, clock); + await limiter.WriteAsync(new byte[2]); + await limiter.WriteAsync(new byte[2]); + Assert.Equal(4, inner.Length); + + var blocked = limiter.WriteAsync(new byte[1]).AsTask(); + await WaitForPendingTimerAsync(clock, blocked); + Assert.False(blocked.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(blocked.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(25)); + await blocked.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(5, inner.Length); + } + + [Fact] + public async Task NegativeCap_BlocksUntilUpdated() + { + var clock = new ManualTimeProvider(); + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter( + inner, + AccountRateFormula.BlockedBytesPerSecond, + leaveInnerOpen: true, + clock); + var write = limiter.WriteAsync(new byte[4]).AsTask(); + await WaitForPendingTimerAsync(clock, write); + Assert.False(write.IsCompleted); + Assert.Equal(0, inner.Length); + limiter.UpdateMaxSendBytesPerSecond(1_000_000); + clock.Advance(TimeSpan.FromMilliseconds(25)); + await write.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(4, inner.Length); + } + + [Fact] + public async Task DynamicIncrease_UnblocksWaitingWrite() + { + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 1, leaveInnerOpen: true); + await limiter.WriteAsync(new byte[1]); + var write = limiter.WriteAsync(new byte[8]).AsTask(); + Assert.False(write.IsCompleted); + limiter.UpdateMaxSendBytesPerSecond(1_000_000); + await write.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(9, inner.Length); + } + + [Fact] + public async Task DynamicDecrease_IsObservedOnNextSlice() + { + var clock = new ManualTimeProvider(); + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 1_000_000, leaveInnerOpen: true, clock); + await limiter.WriteAsync(new byte[4]); + limiter.UpdateMaxSendBytesPerSecond(4); + var blocked = limiter.WriteAsync(new byte[1]).AsTask(); + Assert.False(blocked.IsCompleted); + clock.Advance(TimeSpan.FromSeconds(1)); + await blocked.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(5, inner.Length); + } + + [Fact] + public async Task Cancellation_AbortsBackoff() + { + var clock = new ManualTimeProvider(); + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 1, leaveInnerOpen: true, clock); + await limiter.WriteAsync(new byte[1]); + using var cts = new CancellationTokenSource(); + var write = limiter.WriteAsync(new byte[8], cts.Token).AsTask(); + await WaitForPendingTimerAsync(clock, write); + Assert.False(write.IsCompleted); + cts.Cancel(); + clock.Advance(TimeSpan.FromMilliseconds(25)); + await Assert.ThrowsAnyAsync(() => write); + } + + [Fact] + public async Task Dispose_RejectsNewWrites() + { + var inner = new MemoryStream(); + var limiter = new OutboundRateLimiter(inner, 0, leaveInnerOpen: true); + await limiter.DisposeAsync(); + await Assert.ThrowsAsync(() => limiter.WriteAsync(new byte[1]).AsTask()); + } + + [Fact] + public async Task Dispose_UnblocksWaitingWrite() + { + var clock = new ManualTimeProvider(); + var inner = new MemoryStream(); + var limiter = new OutboundRateLimiter(inner, 1, leaveInnerOpen: true, clock); + await limiter.WriteAsync(new byte[1]); + var write = limiter.WriteAsync(new byte[8]).AsTask(); + await WaitForPendingTimerAsync(clock, write); + Assert.False(write.IsCompleted); + await limiter.DisposeAsync(); + clock.Advance(TimeSpan.FromMilliseconds(25)); + await Assert.ThrowsAnyAsync(() => write); + } + + [Fact] + public async Task ConcurrentWrites_AreSerialized() + { + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 0, leaveInnerOpen: true); + var first = Enumerable.Repeat((byte)1, 32).ToArray(); + var second = Enumerable.Repeat((byte)2, 32).ToArray(); + await Task.WhenAll(limiter.WriteAsync(first).AsTask(), limiter.WriteAsync(second).AsTask()); + var bytes = inner.ToArray(); + Assert.Equal(64, bytes.Length); + Assert.Equal(32, bytes.Count(static b => b == 1)); + Assert.Equal(32, bytes.Count(static b => b == 2)); + } + + [Fact] + public async Task LeaveInnerOpen_DoesNotDisposeSink() + { + var inner = new MemoryStream(); + var limiter = new OutboundRateLimiter(inner, 0, leaveInnerOpen: true); + await limiter.WriteAsync("x"u8.ToArray()); + await limiter.DisposeAsync(); + inner.WriteByte(2); + Assert.Equal(2, inner.ToArray()[^1]); + await inner.DisposeAsync(); + } + + private static async Task WaitForPendingTimerAsync(ManualTimeProvider clock, Task write) + { + for (var i = 0; i < 50 && !write.IsCompleted && !clock.HasPendingTimer; i++) + { + await Task.Yield(); + } + + Assert.True(clock.HasPendingTimer || write.IsCompleted); + } + + private sealed class ManualTimeProvider : TimeProvider + { + private readonly object _gate = new(); + private DateTimeOffset _utcNow = DateTimeOffset.UnixEpoch; + private readonly List _timers = []; + + public override DateTimeOffset GetUtcNow() + { + lock (_gate) + { + return _utcNow; + } + } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ManualTimer(this, callback, state); + timer.Change(dueTime, period); + return timer; + } + + public bool HasPendingTimer + { + get + { + lock (_gate) + { + return _timers.Exists(static timer => timer.NextDue is not null); + } + } + } + + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfNegative(delta.Ticks); + DateTimeOffset target; + lock (_gate) + { + target = _utcNow + delta; + } + + while (true) + { + ManualTimer? due = null; + DateTimeOffset dueAt = default; + lock (_gate) + { + // codeql[cs/linq/missed-where] Lock-held earliest-due selection; no LINQ allocation. + foreach (var timer in _timers) + { + if (timer.NextDue is { } next && next <= target && (due is null || next < dueAt)) + { + due = timer; + dueAt = next; + } + } + + if (due is null) + { + _utcNow = target; + return; + } + + _utcNow = dueAt; + } + + due.Fire(); + } + } + + internal void Register(ManualTimer timer) + { + lock (_gate) + { + _timers.Add(timer); + } + } + + internal void Unregister(ManualTimer timer) + { + lock (_gate) + { + _timers.Remove(timer); + } + } + + internal DateTimeOffset Snapshot + { + get + { + lock (_gate) + { + return _utcNow; + } + } + } + + internal sealed class ManualTimer : ITimer + { + private readonly ManualTimeProvider _owner; + private readonly TimerCallback _callback; + private readonly object? _state; + private TimeSpan _period = Timeout.InfiniteTimeSpan; + + public ManualTimer(ManualTimeProvider owner, TimerCallback callback, object? state) + { + _owner = owner; + _callback = callback; + _state = state; + owner.Register(this); + } + + public DateTimeOffset? NextDue { get; private set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + _period = period; + NextDue = dueTime == Timeout.InfiniteTimeSpan + ? null + : _owner.Snapshot + (dueTime < TimeSpan.Zero ? TimeSpan.Zero : dueTime); + return true; + } + + public void Fire() + { + NextDue = _period == Timeout.InfiniteTimeSpan ? null : _owner.Snapshot + _period; + _callback(_state); + } + + public void Dispose() + { + NextDue = null; + _owner.Unregister(this); + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingRedisTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingRedisTests.cs new file mode 100644 index 00000000..f51ab8a7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingRedisTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.SessionState.RateLimiting; + +public sealed class SessionStateRateLimitingRedisTests : IClassFixture +{ + private const string IpA = "192.0.2.10"; + private const string IpB = "198.51.100.20"; + private const long Now = 10_000; + private const long Lease = 30_000; + /// Database account_rate_limit for 10 Mbps: 10,000,000 bits/sec. + private const int TenMbpsBps = 10_000_000; + private static readonly IPAddress V4A = IPAddress.Parse(IpA); + private static readonly IPAddress V4B = IPAddress.Parse(IpB); + private readonly SessionStateRedisIntegrationFixture _redis; + + public SessionStateRateLimitingRedisTests(SessionStateRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public async Task AdmitPacksSessionTotal_AndReleaseReturnsRemaining() + { + var account = await _redis.CreateAccountAsync(); + var first = await _redis.TryAdmitAsync(account, IpA, "nntpd01:a", 10, 0, 1, 0, Now, Lease); + Assert.True(first.Accepted); + Assert.Equal(1, first.SessionTotal); + var second = await _redis.TryAdmitAsync(account, IpA, "nntpd02:b", 10, 0, 1, 0, Now, Lease); + Assert.True(second.Accepted); + Assert.Equal(2, second.SessionTotal); + Assert.Equal(1, await _redis.ReleaseAsync(account, IpA, "nntpd02:b", 1, 0)); + Assert.Equal(0, await _redis.ReleaseAsync(account, IpA, "nntpd01:a", 1, 0)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task RateOnlyAdmit_TracksSessionsWhenLimitIsZero() + { + var account = await _redis.CreateAccountAsync(); + var first = await _redis.TryAdmitAsync(account, IpA, "nntpd01:a", 0, 0, 1, 0, Now, Lease, trackSessions: true); + var second = await _redis.TryAdmitAsync(account, IpB, "nntpd02:b", 0, 0, 1, 0, Now, Lease, trackSessions: true); + Assert.Equal(1, first.SessionTotal); + Assert.Equal(2, second.SessionTotal); + Assert.Equal(2, await _redis.SessionCountAsync(account)); + Assert.Equal(1, await _redis.ReleaseAsync(account, IpB, "nntpd02:b", 1, 0)); + Assert.Equal(1, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_ReturnsUnexpiredSessionTotal() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, "nntpd01:a", 10, 0, 1, 0, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpB, "nntpd02:b", 10, 0, 1, 0, Now, Lease)).Accepted); + var renewed = await _redis.RenewAsync(account, "nntpd01:a", 1, Array.Empty<(string, long)>(), 20_000, Lease); + Assert.Equal(SessionStateRenewStatus.Renewed, renewed.Status); + Assert.Equal(2, renewed.SessionTotal); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ConcurrentAdmitAndRelease_KeepsExactCount() + { + var account = await _redis.CreateAccountAsync(); + var admits = Enumerable.Range(0, 10) + .Select(i => _redis.TryAdmitAsync(account, IpA, "node:" + i, 10, 0, 1, 0, Now, Lease).AsTask()); + var results = await Task.WhenAll(admits); + Assert.Equal(10, results.Count(static result => result.Accepted)); + Assert.All( + results.Where(static result => result.Accepted), + static result => Assert.InRange(result.SessionTotal, 1, 10)); + Assert.Equal(10, await _redis.SessionCountAsync(account)); + + var releases = Enumerable.Range(0, 3) + .Select(i => _redis.ReleaseAsync(account, IpA, "node:" + i, 1, 0).AsTask()); + var remaining = await Task.WhenAll(releases); + Assert.Equal(7, await _redis.SessionCountAsync(account)); + Assert.All(remaining, static total => Assert.InRange(total, 7, 9)); + + Assert.True((await _redis.TryAdmitAsync(account, IpA, "node:rejoin", 10, 0, 1, 0, Now, Lease)).Accepted); + Assert.Equal(8, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TrackerNodes_ShareClusterCountAndCaps() + { + var account = await _redis.CreateAccountAsync(); + var ratesA = new AccountRateAllocator(); + var ratesB = new AccountRateAllocator(); + var nodeA = new DistributedSessionStateTracker( + _redis.Store, + NullLogger.Instance, + "nntpd01", + TimeProvider.System, + rates: ratesA); + var nodeB = new DistributedSessionStateTracker( + _redis.Store, + NullLogger.Instance, + "nntpd02", + TimeProvider.System, + rates: ratesB); + + var aCaps = new FakeCap[3]; + for (var i = 0; i < 3; i++) + { + aCaps[i] = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await nodeA.TryAdmitAsync(account, "a" + i, V4A, 10, 0, TenMbpsBps)); + ratesA.Register(account, "a" + i, aCaps[i], TenMbpsBps); + } + + var bCaps = new FakeCap[2]; + for (var i = 0; i < 2; i++) + { + bCaps[i] = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await nodeB.TryAdmitAsync(account, "b" + i, V4B, 10, 0, TenMbpsBps)); + ratesB.Register(account, "b" + i, bCaps[i], TenMbpsBps); + } + + var three = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 3); + var five = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 5); + Assert.All(aCaps, cap => Assert.Equal(three, cap.MaxSendBytesPerSecond)); + Assert.True( + aCaps.Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)) + + bCaps.Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)) + <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + Assert.All(bCaps, cap => Assert.True(cap.MaxSendBytesPerSecond < five)); + + await nodeA.RenewLeasesAsync(); + Assert.All(aCaps, cap => Assert.Equal(five, cap.MaxSendBytesPerSecond)); + Assert.True( + aCaps.Concat(bCaps).Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)) + <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + ratesB.Unregister(account, "b1"); + await nodeB.ReleaseAsync(account, "b1"); + var four = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 4); + Assert.All(aCaps, cap => Assert.Equal(five, cap.MaxSendBytesPerSecond)); + Assert.True( + aCaps.Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)) + + AccountRateFormula.AllocatedBytesPerSecond(bCaps[0].MaxSendBytesPerSecond) + <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + await nodeA.RenewLeasesAsync(); + Assert.All(aCaps, cap => Assert.True(cap.MaxSendBytesPerSecond <= five)); + Assert.True( + aCaps.Append(bCaps[0]).Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)) + <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + Assert.True(four * 4 <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task StaleGenerationRelease_DoesNotChangeLiveCount() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, "nntpd01:a", 10, 0, 2, 0, Now, Lease)).Accepted); + Assert.Equal(1, await _redis.SessionCountAsync(account)); + Assert.Equal(1, await _redis.ReleaseAsync(account, IpA, "nntpd01:a", 1, 0)); + Assert.Equal(1, await _redis.SessionCountAsync(account)); + Assert.Equal(0, await _redis.ReleaseAsync(account, IpA, "nntpd01:a", 2, 0)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ExpiredSession_IsNotCountedAfterPruneOnAdmit() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedSessionAsync(account, "nntpd01:stale", expiryUnixMs: 1_000, generation: 1, count: 4); + var admitted = await _redis.TryAdmitAsync(account, IpA, "nntpd02:live", 10, 0, 1, 0, 2_000, Lease); + Assert.True(admitted.Accepted); + Assert.Equal(1, admitted.SessionTotal); + Assert.Equal(1, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task RapidJoinLeave_ThroughLiveLua() + { + var account = await _redis.CreateAccountAsync(); + for (var i = 0; i < 12; i++) + { + var owner = "nntpd01:" + i; + var admitted = await _redis.TryAdmitAsync(account, IpA, owner, 10, 0, 1, 0, Now, Lease); + Assert.True(admitted.Accepted); + if (i % 3 == 2) + { + _ = await _redis.ReleaseAsync(account, IpA, owner, 1, 0); + } + } + + var remaining = await _redis.SessionCountAsync(account); + Assert.InRange(remaining, 1, 10); + await _redis.DeleteKeysAsync(account); + } + + private sealed class FakeCap : IOutboundRateCap + { + public long MaxSendBytesPerSecond { get; private set; } + + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => MaxSendBytesPerSecond = bytesPerSecond; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingTests.cs new file mode 100644 index 00000000..a25bde3b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/RateLimiting/SessionStateRateLimitingTests.cs @@ -0,0 +1,731 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.RateLimiting; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState.RateLimiting; + +public sealed class SessionStateRateLimitingTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + /// Database account_rate_limit for 10 Mbps: 10,000,000 bits/sec. + private const int TenMbpsBps = 10_000_000; + + [Fact] + public void Policy_RateZero_IsUnlimitedAndDoesNotRequireTracking() + { + var policy = new NntpAccountPolicy("alice", 0, 0, 10, 0, "c"); + Assert.False(policy.RequiresRateTracking); + Assert.True(policy.RequiresAdmission); + Assert.Equal(0, policy.RateLimitBps); + Assert.Equal(0, policy.ByteLimit); + } + + [Fact] + public void Policy_PositiveRate_RequiresTrackingRegardlessOfByteLimit() + { + var policy = new NntpAccountPolicy("alice", 10, 100, 10, 0, "c"); + Assert.True(policy.RequiresRateTracking); + Assert.Equal(10, policy.RateLimitBps); + Assert.Equal(100, policy.ByteLimit); + } + + [Fact] + public void Policy_PositiveRate_RequiresAdmissionWhenSessionLimitsAreZero() + { + var policy = new NntpAccountPolicy("alice", 240, 999, 0, 0, "c"); + Assert.True(policy.RequiresRateTracking); + Assert.True(policy.RequiresAdmission); + } + + [Fact] + public void DatabaseAccount_240Bps_OneSession_LimiterCapIs30() + { + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount("alice", 1); + rates.Register("alice", "s1", cap, 240); + Assert.Equal(30, cap.MaxSendBytesPerSecond); + Assert.NotEqual(30_000_000, cap.MaxSendBytesPerSecond); + } + + [Fact] + public void Allocator_PositiveSubEightBps_IsBlocked() + { + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount("alice", 1); + rates.Register("alice", "s1", cap, 7); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, cap.MaxSendBytesPerSecond); + } + + [Fact] + public void Allocator_TenMillionBps_TenSessions_Is125000Each() + { + var rates = new AccountRateAllocator(); + var caps = Register(rates, "alice", 10_000_000, 10); + Assert.Equal(1_250_000, AccountRateFormula.AccountBytesPerSecond(10_000_000)); + AssertCaps(caps, 125_000); + } + + [Fact] + public void Allocator_TenMillionBps_SevenSessions_Is178571Each() + { + var rates = new AccountRateAllocator(); + var caps = Register(rates, "alice", 10_000_000, 7); + AssertCaps(caps, 178_571); + } + + [Fact] + public void Allocator_TenThenSevenThenEight_UpdatesExistingCaps() + { + var rates = new AccountRateAllocator(); + var caps = Register(rates, "alice", TenMbpsBps, 10); + AssertCaps(caps, AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 10)); + + Unregister(rates, "alice", caps, 3); + rates.ObserveClusterSessionCount("alice", 7); + AssertCaps(caps.Take(7), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 7)); + + var extra = new FakeCap(); + rates.Register("alice", "s10", extra, TenMbpsBps); + rates.ObserveClusterSessionCount("alice", 8); + Assert.Equal(AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8), extra.MaxSendBytesPerSecond); + AssertCaps(caps.Take(7).Append(extra), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8)); + Assert.True(AccountRateFormula.AggregateDoesNotExceed(TenMbpsBps, 8)); + } + + [Fact] + public void Allocator_OneSession_GetsFullAccountRate() + { + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.ObserveClusterSessionCount("alice", 1); + rates.Register("alice", "s1", cap, TenMbpsBps); + Assert.Equal(1_250_000, cap.MaxSendBytesPerSecond); + } + + [Fact] + public void Allocator_ZeroSessions_DoesNotCreatePhantomCap() + { + var rates = new AccountRateAllocator(); + rates.ObserveClusterSessionCount("alice", 0); + Assert.Equal(0, rates.LocalSessionCount("alice")); + var cap = new FakeCap(); + rates.Register("alice", "s1", cap, TenMbpsBps); + rates.ObserveClusterSessionCount("alice", 1); + Assert.Equal(1_250_000, cap.MaxSendBytesPerSecond); + } + + [Fact] + public void Allocator_RateZero_LeavesUnlimitedCap() + { + var rates = new AccountRateAllocator(); + var cap = new FakeCap(); + rates.Register("alice", "s1", cap, 0); + Assert.Equal(0, cap.MaxSendBytesPerSecond); + Assert.Equal(0, rates.LocalSessionCount("alice")); + } + + [Fact] + public async Task Distributed_ExistingLocalSessionsUpdateOnJoinAndLeave() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var caps = new FakeCap[10]; + for (var i = 0; i < 10; i++) + { + caps[i] = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", "s" + i, V4A, sessionLimit: 10, srcIpLimit: 0, TenMbpsBps)); + rates.Register("alice", "s" + i, caps[i], TenMbpsBps); + } + + AssertCaps(caps, 125_000); + + for (var i = 7; i < 10; i++) + { + rates.Unregister("alice", "s" + i); + await node.ReleaseAsync("alice", "s" + i); + } + + AssertCaps(caps.Take(7), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 7)); + + var extra = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", "s10", V4A, 10, 0, TenMbpsBps)); + rates.Register("alice", "s10", extra, TenMbpsBps); + Assert.Equal(AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8), extra.MaxSendBytesPerSecond); + AssertCaps(caps.Take(7).Append(extra), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8)); + } + + [Fact] + public async Task Distributed_UsesActiveCount_NotSessionLimit() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var first = new FakeCap(); + var second = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s1", V4A, 10, 0, TenMbpsBps)); + rates.Register("alice", "s1", first, TenMbpsBps); + Assert.Equal(1_250_000, first.MaxSendBytesPerSecond); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s2", V4B, 10, 0, TenMbpsBps)); + rates.Register("alice", "s2", second, TenMbpsBps); + Assert.Equal(625_000, first.MaxSendBytesPerSecond); + Assert.Equal(625_000, second.MaxSendBytesPerSecond); + } + + [Fact] + public async Task Distributed_SessionLimitExceeded_DoesNotChangeExistingCaps() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var caps = await RegisterAdmitted(rates, node, 10); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await node.TryAdmitAsync("alice", "overflow", V4A, 10, 0, TenMbpsBps)); + AssertCaps(caps, 125_000); + Assert.Equal(10, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task Distributed_RateOnly_TracksSessionsWhenSessionLimitIsZero() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var first = new FakeCap(); + var second = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s1", V4A, 0, 0, TenMbpsBps)); + rates.Register("alice", "s1", first, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s2", V4A, 0, 0, TenMbpsBps)); + rates.Register("alice", "s2", second, TenMbpsBps); + Assert.Equal(625_000, first.MaxSendBytesPerSecond); + Assert.Equal(625_000, second.MaxSendBytesPerSecond); + Assert.Equal(0, node.LocalHotPathAdmits); + Assert.Equal(2, node.DistributedAdmits); + } + + [Fact] + public async Task MultiNode_OtherNodeConvergesOnRenew() + { + var membership = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(); + var ratesB = new AccountRateAllocator(); + var nodeA = CreateNode(membership, "nntpd01", ratesA); + var nodeB = CreateNode(membership, "nntpd02", ratesB); + var a1 = new FakeCap(); + var a2 = new FakeCap(); + var a3 = new FakeCap(); + var b1 = new FakeCap(); + var b2 = new FakeCap(); + + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a1", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a1", a1, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a2", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a2", a2, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a3", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a3", a3, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b1", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b1", b1, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b2", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b2", b2, TenMbpsBps); + + var three = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 3); + var five = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 5); + AssertCaps([a1, a2, a3], three); + Assert.True(Allocated(b1, b2) + Allocated(a1, a2, a3) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + Assert.True(Allocated(b1, b2) < five); + + await nodeA.RenewLeasesAsync(); + AssertCaps([a1, a2, a3], five); + Assert.True(Allocated(a1, a2, a3, b1, b2) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + ratesB.Unregister("alice", "b2"); + await nodeB.ReleaseAsync("alice", "b2"); + AssertCaps([a1, a2, a3], five); + Assert.True(Allocated(a1, a2, a3, b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + await nodeA.RenewLeasesAsync(); + AssertCaps([a1, a2, a3], five); + Assert.True(Allocated(a1, a2, a3, b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task MultiNode_JoinOnOtherNode_UpdatesLocalCapsOnRenew() + { + var membership = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(); + var ratesB = new AccountRateAllocator(); + var nodeA = CreateNode(membership, "nntpd01", ratesA); + var nodeB = CreateNode(membership, "nntpd02", ratesB); + var a1 = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a1", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a1", a1, TenMbpsBps); + Assert.Equal(1_250_000, a1.MaxSendBytesPerSecond); + + var b1 = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b1", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b1", b1, TenMbpsBps); + Assert.Equal(1_250_000, a1.MaxSendBytesPerSecond); + Assert.True(Allocated(b1) <= 3); + Assert.True(Allocated(a1, b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + await nodeA.RenewLeasesAsync(); + Assert.Equal(625_000, a1.MaxSendBytesPerSecond); + Assert.True(Allocated(a1, b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task CrossNodeJoin_TenMillionBps_DoesNotExceedAggregateBeforeRemoteReconcile() + { + var clock = new ManualClock(); + var membership = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(clock); + var ratesB = new AccountRateAllocator(clock); + var nodeA = CreateNode(membership, "nntpd01", ratesA); + var nodeB = CreateNode(membership, "nntpd02", ratesB); + + var aCaps = new FakeCap[7]; + for (var i = 0; i < 7; i++) + { + aCaps[i] = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await nodeA.TryAdmitAsync("alice", "a" + i, V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a" + i, aCaps[i], TenMbpsBps); + } + + var seven = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 7); + AssertCaps(aCaps, seven); + + var b1 = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b1", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b1", b1, TenMbpsBps); + AssertCaps(aCaps, seven); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, b1.MaxSendBytesPerSecond); + Assert.True(Allocated(aCaps) + Allocated(b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + await nodeA.RenewLeasesAsync(); + var eight = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8); + AssertCaps(aCaps, eight); + Assert.True(Allocated(aCaps) + Allocated(b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + clock.Advance(SessionStateDefaults.LeaseTtl); + ratesB.ObserveClusterSessionCount("alice", 8); + Assert.True(b1.MaxSendBytesPerSecond <= 0); + AssertCaps(aCaps, eight); + Assert.True(Allocated(aCaps) + Allocated(b1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task ExactScenario_TenThenSevenThenEight_LocalAndTwoNodes() + // codeql[cs/complex-block] Exact multi-node rate scenario keeps steps inline for readability. + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var caps = await RegisterAdmitted(rates, node, 10); + AssertCaps(caps, 125_000); + Assert.True(10 * 125_000 <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + for (var i = 7; i < 10; i++) + { + rates.Unregister("alice", "s" + i); + await node.ReleaseAsync("alice", "s" + i); + } + + var seven = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 7); + AssertCaps(caps.Take(7), seven); + + var extra = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s10", V4A, 10, 0, TenMbpsBps)); + rates.Register("alice", "s10", extra, TenMbpsBps); + var eight = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 8); + Assert.Equal(eight, extra.MaxSendBytesPerSecond); + AssertCaps(caps.Take(7).Append(extra), eight); + + var clock = new ManualClock(); + var split = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(clock); + var ratesB = new AccountRateAllocator(clock); + var nodeA = CreateNode(split, "nntpd01", ratesA); + var nodeB = CreateNode(split, "nntpd02", ratesB); + var aCaps = new FakeCap[7]; + for (var i = 0; i < 7; i++) + { + aCaps[i] = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("bob", "a" + i, V4A, 10, 0, TenMbpsBps)); + ratesA.Register("bob", "a" + i, aCaps[i], TenMbpsBps); + } + + var bCaps = new FakeCap[3]; + for (var i = 0; i < 3; i++) + { + bCaps[i] = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("bob", "b" + i, V4B, 10, 0, TenMbpsBps)); + ratesB.Register("bob", "b" + i, bCaps[i], TenMbpsBps); + Assert.True(Allocated(aCaps) + Allocated(bCaps.Take(i + 1)) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + await nodeA.RenewLeasesAsync(); + AssertCaps(aCaps, 125_000); + Assert.True(Allocated(aCaps) + Allocated(bCaps) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + clock.Advance(SessionStateDefaults.LeaseTtl); + ratesB.ObserveClusterSessionCount("bob", 10); + AssertCaps(aCaps, 125_000); + Assert.True(Allocated(aCaps) + Allocated(bCaps) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + + for (var i = 0; i < 3; i++) + { + ratesB.Unregister("bob", "b" + i); + await nodeB.ReleaseAsync("bob", "b" + i); + } + + AssertCaps(aCaps, 125_000); + await nodeA.RenewLeasesAsync(); + AssertCaps(aCaps, seven); + + var rejoin = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("bob", "b0", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("bob", "b0", rejoin, TenMbpsBps); + AssertCaps(aCaps, seven); + Assert.True(Allocated(rejoin) <= 3); + Assert.True(Allocated(aCaps) + Allocated(rejoin) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + await nodeA.RenewLeasesAsync(); + AssertCaps(aCaps, eight); + clock.Advance(SessionStateDefaults.LeaseTtl); + ratesB.ObserveClusterSessionCount("bob", 8); + Assert.True(rejoin.MaxSendBytesPerSecond <= 0); + AssertCaps(aCaps, eight); + Assert.True(Allocated(aCaps) + Allocated(rejoin) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task RapidJoinLeave_NeverExceedsAggregate() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var caps = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < 20; i++) + { + var id = "s" + (i % 7); + if (caps.ContainsKey(id)) + { + rates.Unregister("alice", id); + await node.ReleaseAsync("alice", id); + caps.Remove(id); + } + else + { + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", id, V4A, 10, 0, TenMbpsBps)); + var cap = new FakeCap(); + rates.Register("alice", id, cap, TenMbpsBps); + caps[id] = cap; + } + + if (caps.Count > 0) + { + var expected = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, caps.Count); + AssertCaps(caps.Values, expected); + Assert.True(expected * caps.Count <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + } + } + + [Fact] + public async Task SimultaneousJoins_ShareTheSameFloor() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var admits = Enumerable.Range(0, 10) + .Select(i => node.TryAdmitAsync("alice", "s" + i, V4A, 10, 0, TenMbpsBps).AsTask()); + var results = await Task.WhenAll(admits); + Assert.All(results, static result => Assert.Equal(SessionAdmissionResult.Success, result)); + + var caps = new FakeCap[10]; + for (var i = 0; i < 10; i++) + { + caps[i] = new FakeCap(); + rates.Register("alice", "s" + i, caps[i], TenMbpsBps); + } + + rates.ObserveClusterSessionCount("alice", 10); + AssertCaps(caps, 125_000); + } + + [Fact] + public async Task SuccessfulReleaseToZero_DoesNotNeedObserveZero_NextAdmitGetsFullRate() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var first = new FakeCap(); + var second = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s1", V4A, 0, 0, TenMbpsBps)); + rates.Register("alice", "s1", first, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s2", V4A, 0, 0, TenMbpsBps)); + rates.Register("alice", "s2", second, TenMbpsBps); + Assert.Equal(625_000, first.MaxSendBytesPerSecond); + + rates.Unregister("alice", "s1"); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(1_250_000, second.MaxSendBytesPerSecond); + Assert.Equal(1, membership.ActiveSessionCount("alice", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + + rates.Unregister("alice", "s2"); + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(0, rates.LocalSessionCount("alice")); + Assert.Equal(0, membership.ActiveSessionCount("alice", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + var released = node.ReleaseCalls; + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(released, node.ReleaseCalls); + + var again = new FakeCap(); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s3", V4A, 0, 0, TenMbpsBps)); + rates.Register("alice", "s3", again, TenMbpsBps); + Assert.Equal(1_250_000, again.MaxSendBytesPerSecond); + } + + [Fact] + public async Task ReleaseWhenMembershipUnavailable_DoesNotRaiseJoinerToFullRate() + { + var fixture = await AdmitEstablishedThenJoinersAsync(); + fixture.Membership.Unavailable = true; + fixture.RatesB.Unregister("alice", "b2"); + await fixture.NodeB.ReleaseAsync("alice", "b2"); + + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, fixture.B1.MaxSendBytesPerSecond); + AssertCaps([fixture.A1, fixture.A2], fixture.Quarter); + Assert.Equal(1, fixture.NodeB.GetLocalSessionCount("alice")); + Assert.Equal(1, fixture.RatesB.LocalSessionCount("alice")); + Assert.Equal(4, fixture.Membership.ActiveSessionCount("alice", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + Assert.True(Allocated(fixture.A1, fixture.A2, fixture.B1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task UnavailableRelease_ThenSuccessfulRenew_ObservesUnreleasedClusterTotal() + { + var fixture = await AdmitEstablishedThenJoinersAsync(); + fixture.Membership.Unavailable = true; + fixture.RatesB.Unregister("alice", "b2"); + await fixture.NodeB.ReleaseAsync("alice", "b2"); + fixture.Membership.Unavailable = false; + + await fixture.NodeA.RenewLeasesAsync(); + await fixture.NodeB.RenewLeasesAsync(); + + AssertCaps([fixture.A1, fixture.A2], fixture.Quarter); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, fixture.B1.MaxSendBytesPerSecond); + Assert.Equal(4, fixture.Membership.ActiveSessionCount("alice", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + Assert.True(Allocated(fixture.A1, fixture.A2, fixture.B1) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task UnavailableRelease_ThenSuccessfulLaterRelease_DoesNotOvershoot() + { + var fixture = await AdmitEstablishedThenJoinersAsync(); + fixture.Membership.Unavailable = true; + fixture.RatesB.Unregister("alice", "b2"); + await fixture.NodeB.ReleaseAsync("alice", "b2"); + var releasesAfterFail = fixture.NodeB.ReleaseCalls; + await fixture.NodeB.ReleaseAsync("alice", "b2"); + Assert.Equal(releasesAfterFail, fixture.NodeB.ReleaseCalls); + + fixture.Membership.Unavailable = false; + fixture.RatesB.Unregister("alice", "b1"); + await fixture.NodeB.ReleaseAsync("alice", "b1"); + Assert.Equal(0, fixture.NodeB.GetLocalSessionCount("alice")); + Assert.Equal(0, fixture.RatesB.LocalSessionCount("alice")); + Assert.Equal(3, fixture.Membership.ActiveSessionCount("alice", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + + await fixture.NodeA.RenewLeasesAsync(); + AssertCaps([fixture.A1, fixture.A2], fixture.Quarter); + Assert.True(Allocated(fixture.A1, fixture.A2) <= AccountRateFormula.AccountBytesPerSecond(TenMbpsBps)); + } + + [Fact] + public async Task ReleaseDuringRenew_ExistingCapsFollowReleaseTotal() + { + var membership = new InMemorySessionStateStore(); + var rates = new AccountRateAllocator(); + var node = CreateNode(membership, "nntpd01", rates); + var caps = await RegisterAdmitted(rates, node, 10); + membership.NotifyRenewStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + membership.BlockRenew = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var renew = node.RenewLeasesAsync().AsTask(); + await membership.NotifyRenewStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + rates.Unregister("alice", "s9"); + await node.ReleaseAsync("alice", "s9"); + AssertCaps(caps.Take(9), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 9)); + membership.BlockRenew.TrySetResult(); + await renew.WaitAsync(TimeSpan.FromSeconds(2)); + AssertCaps(caps.Take(9), AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 9)); + } + + [Fact] + public async Task FakeRedisWritePath_DoesNotEvaluateOnLimiterWrite() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var rates = new AccountRateAllocator(); + var node = new DistributedSessionStateTracker( + store, + NullLogger.Instance, + "nntpd01", + TimeProvider.System, + rates: rates); + Assert.Equal(SessionAdmissionResult.Success, await node.TryAdmitAsync("alice", "s1", V4A, 10, 0, TenMbpsBps)); + var evals = redis.Database.ScriptEvaluateCount; + await using var inner = new MemoryStream(); + await using var limiter = new OutboundRateLimiter(inner, 0, leaveInnerOpen: true); + rates.Register("alice", "s1", limiter, TenMbpsBps); + limiter.UpdateMaxSendBytesPerSecond(125_000); + await limiter.WriteAsync("281 Authentication accepted\r\n"u8.ToArray()); + rates.ObserveClusterSessionCount("alice", 1); + Assert.Equal(evals, redis.Database.ScriptEvaluateCount); + + await node.RenewLeasesAsync(); + Assert.Equal(evals + 1, redis.Database.ScriptEvaluateCount); + } + + private static async Task AdmitEstablishedThenJoinersAsync() + { + var membership = new InMemorySessionStateStore(); + var ratesA = new AccountRateAllocator(); + var ratesB = new AccountRateAllocator(); + var nodeA = CreateNode(membership, "nntpd01", ratesA); + var nodeB = CreateNode(membership, "nntpd02", ratesB); + var a1 = new FakeCap(); + var a2 = new FakeCap(); + var b1 = new FakeCap(); + var b2 = new FakeCap(); + + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a1", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a1", a1, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeA.TryAdmitAsync("alice", "a2", V4A, 10, 0, TenMbpsBps)); + ratesA.Register("alice", "a2", a2, TenMbpsBps); + AssertCaps([a1, a2], 625_000); + + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b1", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b1", b1, TenMbpsBps); + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "b2", V4B, 10, 0, TenMbpsBps)); + ratesB.Register("alice", "b2", b2, TenMbpsBps); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, b1.MaxSendBytesPerSecond); + Assert.Equal(AccountRateFormula.BlockedBytesPerSecond, b2.MaxSendBytesPerSecond); + + await nodeA.RenewLeasesAsync(); + var quarter = AccountRateFormula.PerSessionBytesPerSecond(TenMbpsBps, 4); + AssertCaps([a1, a2], quarter); + return new TwoNodeRateFixture(membership, ratesA, ratesB, nodeA, nodeB, a1, a2, b1, b2, quarter); + } + + private static async Task RegisterAdmitted( + AccountRateAllocator rates, + DistributedSessionStateTracker node, + int count) + { + var caps = new FakeCap[count]; + for (var i = 0; i < count; i++) + { + caps[i] = new FakeCap(); + Assert.Equal( + SessionAdmissionResult.Success, + await node.TryAdmitAsync("alice", "s" + i, V4A, count, 0, TenMbpsBps)); + rates.Register("alice", "s" + i, caps[i], TenMbpsBps); + } + + return caps; + } + + private static FakeCap[] Register(AccountRateAllocator rates, string account, int rateBps, int count) + { + var caps = new FakeCap[count]; + rates.ObserveClusterSessionCount(account, count); + for (var i = 0; i < count; i++) + { + caps[i] = new FakeCap(); + rates.Register(account, "s" + i, caps[i], rateBps); + } + + return caps; + } + + private static void Unregister(AccountRateAllocator rates, string account, FakeCap[] caps, int remove) + { + for (var i = caps.Length - remove; i < caps.Length; i++) + { + rates.Unregister(account, "s" + i); + } + } + + private static void AssertCaps(IEnumerable caps, long expected) + { + foreach (var cap in caps) + { + Assert.Equal(expected, cap.MaxSendBytesPerSecond); + } + } + + private static long Allocated(params FakeCap[] caps) => + Allocated((IEnumerable)caps); + + private static long Allocated(IEnumerable caps) => + caps.Sum(static cap => AccountRateFormula.AllocatedBytesPerSecond(cap.MaxSendBytesPerSecond)); + + private sealed class ManualClock : TimeProvider + { + private DateTimeOffset _utcNow = DateTimeOffset.UnixEpoch; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfNegative(delta.Ticks); + _utcNow += delta; + } + } + + private static DistributedSessionStateTracker CreateNode( + ISessionStateStore membership, + string nodeId, + IAccountRateAllocator rates) => + new( + membership, + NullLogger.Instance, + nodeId, + TimeProvider.System, + rates: rates); + + private sealed record TwoNodeRateFixture( + InMemorySessionStateStore Membership, + AccountRateAllocator RatesA, + AccountRateAllocator RatesB, + DistributedSessionStateTracker NodeA, + DistributedSessionStateTracker NodeB, + FakeCap A1, + FakeCap A2, + FakeCap B1, + FakeCap B2, + long Quarter); + + private sealed class FakeCap : IOutboundRateCap + { + public long MaxSendBytesPerSecond { get; private set; } + + public void UpdateMaxSendBytesPerSecond(long bytesPerSecond) => MaxSendBytesPerSecond = bytesPerSecond; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateAdmissionCompensationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateAdmissionCompensationTests.cs new file mode 100644 index 00000000..a196f6df --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateAdmissionCompensationTests.cs @@ -0,0 +1,566 @@ +using System.Globalization; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Compensating RELEASE after TRY_ADMIT + OCE: generation, owner, source, and +/// account-gate serialization. +/// +public sealed class SessionStateAdmissionCompensationTests +{ + private const string Src = "src:alice"; + private const string Sess = "sess:alice"; + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const string IpA = "192.0.2.10"; + private const string IpB = "198.51.100.20"; + private static readonly IPAddress V4A = IPAddress.Parse(IpA); + private static readonly IPAddress V4B = IPAddress.Parse(IpB); + + [Fact] + public async Task SameAccountAdmission_WaitsUntilCompensationReleasesGate() + { + var inner = new InMemorySessionStateStore(); + var store = new CompensationStore(inner); + var node = CreateNode(store); + using var cts = new CancellationTokenSource(); + var admitA = node.TryAdmitAsync("alice", "A", V4A, sessionLimit: 5, srcIpLimit: 2, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + + var admitB = node.TryAdmitAsync("alice", "B", V4B, sessionLimit: 5, srcIpLimit: 2).AsTask(); + Assert.False(admitB.IsCompleted); + + Assert.True(inner.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var canceledSessionGeneration, + out var canceledSessionCount)); + Assert.Equal(1, canceledSessionCount); + Assert.True(inner.Engine.TryGetOwnership( + InMemorySessionStateStore.SourceKey("alice"), + SessionStateKeys.SourceField(IpA, node.OwnerId), + out _, + out var canceledSourceGeneration, + out var canceledSourceCount)); + Assert.Equal(1, canceledSourceCount); + + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admitA); + Assert.Equal(SessionAdmissionResult.Success, await admitB); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + Assert.False(inner.HasSourceOwner("alice", IpA, node.OwnerId, NowMs())); + Assert.True(inner.HasSourceOwner("alice", IpB, node.OwnerId, NowMs())); + Assert.Equal(node.OwnerId, store.LastReleaseOwnerId); + Assert.Equal(IpA, store.LastReleaseIp); + Assert.Equal(canceledSessionGeneration, store.LastReleaseSessionGeneration); + Assert.Equal(canceledSourceGeneration, store.LastReleaseSourceGeneration); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.True(inner.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var liveSessionGeneration, + out _)); + Assert.NotEqual(canceledSessionGeneration, liveSessionGeneration); + } + + [Fact] + public async Task CanceledCallerToken_IsNotUsedForCompensatingRelease() + { + var inner = new InMemorySessionStateStore(); + var store = new CompensationStore(inner); + var node = CreateNode(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync("alice", "A", V4A, sessionLimit: 2, srcIpLimit: 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + Assert.True(cts.Token.IsCancellationRequested); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.False(store.LastReleaseToken.CanBeCanceled); + Assert.Equal(1, store.ReleaseCalls); + Assert.Equal(0, inner.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task CompensationWhenRedisUnavailable_LeavesTtlOwnershipAndNoLocalCommit() + { + var inner = new InMemorySessionStateStore(); + var store = new CompensationStore(inner) { UnavailableOnRelease = true }; + var node = CreateNode(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync("alice", "A", V4A, sessionLimit: 2, srcIpLimit: 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(0, node.DistributedAdmits); + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + Assert.True(inner.HasSourceOwner("alice", IpA, node.OwnerId, NowMs())); + Assert.Equal(1, store.ReleaseCalls); + } + + [Fact] + public async Task NonCanceledStoreException_DoesNotCompensate() + { + var inner = new InMemorySessionStateStore(); + var store = new CompensationStore(inner) { ThrowAfterAdmit = new InvalidOperationException("store fault") }; + var node = CreateNode(store); + await Assert.ThrowsAsync( + () => node.TryAdmitAsync("alice", "A", V4A, sessionLimit: 2, srcIpLimit: 2).AsTask()); + Assert.Equal(0, store.ReleaseCalls); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task ObjectDisposedException_DoesNotCompensate() + { + var inner = new InMemorySessionStateStore(); + var store = new CompensationStore(inner) { ThrowAfterAdmit = new ObjectDisposedException("membership") }; + var node = CreateNode(store); + await Assert.ThrowsAsync( + () => node.TryAdmitAsync("alice", "A", V4A, sessionLimit: 2, srcIpLimit: 2).AsTask()); + Assert.Equal(0, store.ReleaseCalls); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public void Engine_SameGenerationCompensation_RemovesCountOneAndDecrementsCountTwo() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.Equal(0, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.False(engine.HasSourceOwner(Src, IpA, O1, 0)); + + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal(2, engine.OwnerSessionCount(Sess, O1, 0)); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.Equal(1, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.True(engine.HasSourceOwner(Src, IpA, O1, 0)); + } + + [Fact] + public void Engine_StaleGenerationCompensation_LeavesReplacedEpoch() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, 30_000, 1, 1); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(IpA, O1), 30_000, 1, 1); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 2, 2)); + Assert.True(engine.TryGetOwnership(Sess, O1, out _, out var sessionGeneration, out var sessionCount)); + Assert.Equal(2, sessionGeneration); + Assert.Equal(1, sessionCount); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.True(engine.TryGetOwnership(Sess, O1, out _, out sessionGeneration, out sessionCount)); + Assert.Equal(2, sessionGeneration); + Assert.Equal(1, sessionCount); + Assert.True(engine.HasSourceOwner(Src, IpA, O1, 0)); + } + + [Fact] + public void Engine_SourceCompensation_LeavesOtherOwnerAndOtherIp() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, IpA, O2, 10, 4, 0, 30_000, 9, 9)); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpB, O1, 10, 4, 0, 30_000, 1, 3)); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.False(engine.HasSourceOwner(Src, IpA, O1, 0)); + Assert.True(engine.HasSourceOwner(Src, IpA, O2, 0)); + Assert.True(engine.HasSourceOwner(Src, IpB, O1, 0)); + Assert.Equal(1, engine.OwnerSessionCount(Sess, O2, 0)); + Assert.Equal(1, engine.OwnerSessionCount(Sess, O1, 0)); + } + + [Fact] + public void Engine_DualOwnershipCompensation_RemovesBothWhenCountIsOne() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.Equal(0, engine.ActiveSessionCount(Sess, 0)); + Assert.False(engine.HasSourceOwner(Src, IpA, O1, 0)); + } + + [Fact] + public void Engine_DualOwnershipCompensation_KeepsSourceWhenAnotherSessionRemains() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal(2, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.True(engine.TryGetOwnership(Src, SessionStateKeys.SourceField(IpA, O1), out _, out _, out var sourceCount)); + Assert.Equal(2, sourceCount); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.Equal(1, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.True(engine.HasSourceOwner(Src, IpA, O1, 0)); + Assert.True(engine.TryGetOwnership(Src, SessionStateKeys.SourceField(IpA, O1), out _, out var sourceGeneration, out sourceCount)); + Assert.Equal(1, sourceGeneration); + Assert.Equal(1, sourceCount); + } + + [Fact] + public void Engine_DualOwnershipCompensation_SessionCountTwoSourceCountOne_RemovesOnlyCanceledIp() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, O1, 10, 4, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpB, O1, 10, 4, 0, 30_000, 1, 3)); + Assert.Equal(2, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.True(engine.TryGetOwnership(Src, SessionStateKeys.SourceField(IpA, O1), out _, out _, out var ipACount)); + Assert.Equal(1, ipACount); + _ = engine.Release(Src, Sess, IpA, O1, 1, 1); + Assert.Equal(1, engine.OwnerSessionCount(Sess, O1, 0)); + Assert.False(engine.HasSourceOwner(Src, IpA, O1, 0)); + Assert.True(engine.HasSourceOwner(Src, IpB, O1, 0)); + } + + [Fact] + public void Engine_DelayedCompensation_AfterReleaseOwner_DoesNotRecreateOrTouchNewIncarnation() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, "nntpd01:old", 10, 4, 0, 30_000, 1, 1)); + _ = engine.ReleaseOwner(Src, Sess, "nntpd01:old"); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, IpA, "nntpd01:new", 10, 4, 0, 30_000, 2, 2)); + _ = engine.Release(Src, Sess, IpA, "nntpd01:old", 1, 1); + Assert.Equal(1, engine.OwnerSessionCount(Sess, "nntpd01:new", 0)); + Assert.True(engine.HasSourceOwner(Src, IpA, "nntpd01:new", 0)); + Assert.False(engine.HasSourceOwner(Src, IpA, "nntpd01:old", 0)); + Assert.Equal(0, engine.OwnerSessionCount(Sess, "nntpd01:old", 0)); + } + + [Fact] + public async Task Tracker_DifferentNodeAdmit_IsIndependentOfCompensation() + { + var membership = new InMemorySessionStateStore(); + var storeA = new CompensationStore(membership); + var nodeA = CreateNode(storeA, "nntpd01", "incA"); + var nodeB = CreateNode(membership, "nntpd02", "incB"); + using var cts = new CancellationTokenSource(); + var admitA = nodeA.TryAdmitAsync("alice", "A", V4A, sessionLimit: 5, srcIpLimit: 2, cts.Token).AsTask(); + await storeA.Admitted.Task; + Assert.Equal(SessionAdmissionResult.Success, await nodeB.TryAdmitAsync("alice", "B", V4A, 5, 2)); + cts.Cancel(); + storeA.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admitA); + Assert.Equal(0, nodeA.GetLocalSessionCount("alice")); + Assert.Equal(1, nodeB.GetLocalSessionCount("alice")); + Assert.False(membership.HasSourceOwner("alice", IpA, nodeA.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", IpA, nodeB.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeB.OwnerId, NowMs())); + } + + [Fact] + public void ReleaseScript_RequiresOwnerAndGeneration() + { + var script = SessionStateScripts.Release; + Assert.Contains("local function decrement(key, field, gen)", script, StringComparison.Ordinal); + Assert.Contains("if storedGen ~= gen then", script, StringComparison.Ordinal); + Assert.Contains("decrement(sessKey, owner, sessionGen)", script, StringComparison.Ordinal); + Assert.Contains("decrement(srcKey, srcField, sourceGen)", script, StringComparison.Ordinal); + Assert.Contains("ip .. '\\31' .. owner", script, StringComparison.Ordinal); + Assert.DoesNotContain("HDEL', sessKey, owner", script, StringComparison.Ordinal); + } + + [Fact] + public async Task RedisStore_StaleGenerationRelease_LeavesReplacedEpoch() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + Assert.True((await store.TryAdmitAsync("alice", IpA, O1, 10, 4, 1, 1, now, ttl)).Accepted); + Assert.True((await store.TryAdmitAsync("alice", IpA, O1, 10, 4, 2, 2, now, ttl)).Accepted); + await store.ReleaseAsync("alice", IpA, O1, 1, 1); + var engine = redis.Database.SessionStateEngine; + var sessKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSession("alice")); + var srcKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSource("alice")); + Assert.True(engine.TryGetOwnership(sessKey, O1, out _, out var sessionGeneration, out var sessionCount)); + Assert.Equal(2, sessionGeneration); + Assert.Equal(1, sessionCount); + Assert.True(engine.HasSourceOwner(srcKey, IpA, O1, 0)); + Assert.True(engine.TryGetOwnership(srcKey, SessionStateKeys.SourceField(IpA, O1), out _, out var sourceGeneration, out var sourceCount)); + Assert.Equal(2, sourceGeneration); + Assert.Equal(1, sourceCount); + } + + [Fact] + public async Task RedisStore_SameGenerationRelease_DecrementsWithoutDeletingOtherOwner() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + Assert.True((await store.TryAdmitAsync("alice", IpA, O1, 10, 4, 1, 1, now, ttl)).Accepted); + Assert.True((await store.TryAdmitAsync("alice", IpA, O1, 10, 4, 1, 1, now, ttl)).Accepted); + Assert.True((await store.TryAdmitAsync("alice", IpA, O2, 10, 4, 9, 9, now, ttl)).Accepted); + await store.ReleaseAsync("alice", IpA, O1, 1, 1); + var engine = redis.Database.SessionStateEngine; + var sessKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSession("alice")); + var srcKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSource("alice")); + Assert.Equal(1, engine.OwnerSessionCount(sessKey, O1, 0)); + Assert.Equal(1, engine.OwnerSessionCount(sessKey, O2, 0)); + Assert.True(engine.HasSourceOwner(srcKey, IpA, O1, 0)); + Assert.True(engine.HasSourceOwner(srcKey, IpA, O2, 0)); + await store.ReleaseAsync("alice", IpA, O1, 1, 1); + Assert.Equal(0, engine.OwnerSessionCount(sessKey, O1, 0)); + Assert.False(engine.HasSourceOwner(srcKey, IpA, O1, 0)); + Assert.Equal(1, engine.OwnerSessionCount(sessKey, O2, 0)); + Assert.True(engine.HasSourceOwner(srcKey, IpA, O2, 0)); + } + + [Fact] + public void LuaDecrementRule_StaleGenerationIsANoOp() + { + var session = new Dictionary(StringComparer.Ordinal) + { + [O1] = SessionStateKeys.Value(30_000, 2, 1), + }; + var source = new Dictionary(StringComparer.Ordinal) + { + [SessionStateKeys.SourceField(IpA, O1)] = SessionStateKeys.Value(30_000, 2, 1), + [SessionStateKeys.SourceField(IpA, O2)] = SessionStateKeys.Value(30_000, 9, 3), + }; + ApplyLuaDecrement(session, O1, 1); + ApplyLuaDecrement(source, SessionStateKeys.SourceField(IpA, O1), 1); + Assert.True(session.TryGetValue(O1, out var sessionValue)); + Assert.True(SessionStateKeys.TrySplitValue(sessionValue, out _, out var sessionGeneration, out var sessionCount)); + Assert.Equal(2, sessionGeneration); + Assert.Equal(1, sessionCount); + Assert.True(source.ContainsKey(SessionStateKeys.SourceField(IpA, O1))); + Assert.True(source.ContainsKey(SessionStateKeys.SourceField(IpA, O2))); + } + + [Fact] + public async Task DelayedCompensation_AfterReleaseOwner_LeavesNewIncarnation() + { + var membership = new InMemorySessionStateStore(); + var crashed = CreateNode(membership, "nntpd01", "old"); + Assert.Equal(SessionAdmissionResult.Success, await crashed.TryAdmitAsync("alice", "A", V4A, 5, 2)); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + crashed.OwnerId, + out _, + out var staleSessionGeneration, + out _)); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SourceKey("alice"), + SessionStateKeys.SourceField(IpA, crashed.OwnerId), + out _, + out var staleSourceGeneration, + out _)); + await crashed.ReleaseAllOwnershipAsync(); + var restarted = CreateNode(membership, "nntpd01", "new"); + Assert.Equal(SessionAdmissionResult.Success, await restarted.TryAdmitAsync("alice", "B", V4A, 5, 2)); + await membership.ReleaseAsync("alice", IpA, crashed.OwnerId, staleSessionGeneration, staleSourceGeneration); + Assert.Equal(1, membership.OwnerSessionCount("alice", restarted.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", IpA, restarted.OwnerId, NowMs())); + Assert.Equal(0, membership.OwnerSessionCount("alice", crashed.OwnerId, NowMs())); + Assert.False(membership.HasSourceOwner("alice", IpA, crashed.OwnerId, NowMs())); + } + + private static void ApplyLuaDecrement(Dictionary hash, string field, long generation) + { + var gen = generation.ToString(CultureInfo.InvariantCulture); + if (!hash.TryGetValue(field, out var current) + || !SessionStateKeys.TrySplitValue(current, out var expiry, out var storedGeneration, out var count) + || storedGeneration.ToString(CultureInfo.InvariantCulture) != gen) + { + return; + } + + if (count <= 1) + { + hash.Remove(field); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count - 1); + } + + private static DistributedSessionStateTracker CreateNode( + ISessionStateStore store, + string nodeId = "nntpd01", + string? incarnation = null) => + new( + store, + NullLogger.Instance, + nodeId, + TimeProvider.System, + incarnation: incarnation); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + private sealed class CompensationStore : ISessionStateStore + { + private readonly InMemorySessionStateStore _inner; + + public CompensationStore(InMemorySessionStateStore inner) + { + _inner = inner; + } + + public TaskCompletionSource Admitted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Continue { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool UnavailableOnRelease { get; set; } + + public Exception? ThrowAfterAdmit { get; set; } + + public int ReleaseCalls { get; private set; } + + public CancellationToken LastReleaseToken { get; private set; } = new(canceled: true); + + public string? LastReleaseOwnerId { get; private set; } + + public string? LastReleaseIp { get; private set; } + + public long LastReleaseSessionGeneration { get; private set; } + + public long LastReleaseSourceGeneration { get; private set; } + + public async ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) + { + var result = await _inner.TryAdmitAsync( + accountName, + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + leaseTtl, + CancellationToken.None, + trackSessions).ConfigureAwait(false); + Admitted.TrySetResult(); + if (ThrowAfterAdmit is { } fault) + { + throw fault; + } + + await Continue.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + return result; + } + + public ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) + { + ReleaseCalls++; + LastReleaseToken = cancellationToken; + LastReleaseOwnerId = ownerId; + LastReleaseIp = normalizedSourceIp; + LastReleaseSessionGeneration = sessionGeneration; + LastReleaseSourceGeneration = sourceGeneration; + if (UnavailableOnRelease) + { + _inner.Unavailable = true; + } + + return _inner.ReleaseAsync( + accountName, + normalizedSourceIp, + ownerId, + sessionGeneration, + sourceGeneration, + cancellationToken); + } + + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + _inner.RenewAsync(accountName, ownerId, sessionGeneration, sources, now, leaseTtl, cancellationToken); + + public ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) => + _inner.RenewAndApplyAsync( + accountName, + ownerId, + sessionGeneration, + sources, + now, + leaseTtl, + batchId, + consumed, + mysqlRemainingAfter, + cancellationToken); + + public ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) => + _inner.ReleaseOwnerAsync(accountName, ownerId, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateBytesAccountingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateBytesAccountingTests.cs new file mode 100644 index 00000000..5329e3d1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateBytesAccountingTests.cs @@ -0,0 +1,428 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Core; +using VectorNNTPD.ReaderService.Hosting; +using VectorNNTPD.ReaderService.Logging; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +[Collection(SerilogCollection.Name)] +public sealed class SessionStateBytesAccountingTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public void Host_RegistersExactlyOneSessionStateService_AndNoAccountByteService() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpReaderHosting(includePlaceholderService: false); + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + Assert.Equal(1, services.Count(static s => s is SessionStateService)); + Assert.DoesNotContain(services, static s => s.GetType().Name == "AccountByteService"); + Assert.DoesNotContain(services, static s => s.GetType().Name is "RateLimitService" or "AccountRateService" or "RateScheduler"); + Assert.NotNull(host.Services.GetService()); + Assert.NotNull(host.Services.GetService()); + Assert.Null(typeof(SessionStateService).Assembly.GetType("VectorNNTPD.ReaderService.AccountBytes.AccountByteService")); + Assert.Null(typeof(SessionStateService).Assembly.GetType("VectorNNTPD.ReaderService.SessionState.RateLimiting.RateLimitService")); + } + + [Fact] + public async Task OneAccountThreeSessions_OneCombinedEvalPerCycle() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 1_000_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V6A)); + bytes.CreateSink("alice").ObserveCopied(100); + bytes.CreateSink("alice").ObserveCopied(200); + bytes.CreateSink("alice").ObserveCopied(300); + var afterAdmit = redis.Database.ScriptEvaluateCount; + + await bytes.CommitDurableAsync(); + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(afterAdmit + 1, redis.Database.ScriptEvaluateCount); + Assert.Equal(1, durable.ConsumeCalls); + Assert.Equal(999_400, durable.Remaining("alice")); + Assert.False(bytes.TryGetCommittedBatch("alice", out _)); + Assert.Equal(999_400, await bytes.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task TwoAccounts_RemainIsolated_OneCombinedEvalEach() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 10_000); + durable.SeedByteAccount("bob", 20_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a1", V4A, "alice")); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "b1", V6A, "bob")); + bytes.CreateSink("alice").ObserveCopied(100); + bytes.CreateSink("bob").ObserveCopied(250); + var afterAdmit = redis.Database.ScriptEvaluateCount; + + await bytes.CommitDurableAsync(); + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(afterAdmit + 2, redis.Database.ScriptEvaluateCount); + Assert.Equal(9_900, durable.Remaining("alice")); + Assert.Equal(19_750, durable.Remaining("bob")); + Assert.Equal(9_900, await bytes.ObserveRemainingAsync("alice")); + Assert.Equal(19_750, await bytes.ObserveRemainingAsync("bob")); + } + + [Fact] + public async Task CombinedEval_AppliesBytesWhenRenewIsLost() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 5_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + bytes.CreateSink("alice").ObserveCopied(40); + await bytes.CommitDurableAsync(); + await node.ReleaseAllOwnershipAsync(); + + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(4_960, durable.Remaining("alice")); + Assert.False(bytes.TryGetCommittedBatch("alice", out _)); + Assert.Equal(4_960, await bytes.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task CombinedEval_StaleGeneration_StillFloorsHighRedis_ThroughTracker() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var cluster = new RedisAccountByteStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 5_000); + var bytes = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(50_000, await cluster.ApplyAsync("alice", "seed-high", 0, 50_000)); + bytes.CreateSink("alice").ObserveCopied(40); + await bytes.CommitDurableAsync(); + Assert.True(bytes.TryGetCommittedBatch("alice", out _)); + Assert.Equal(4_960, durable.Remaining("alice")); + Assert.Equal(50_000, await cluster.ObserveAsync("alice")); + + var sourceKey = System.Text.Encoding.UTF8.GetString(SessionStateKeys.CreateSource("alice")); + Assert.True( + redis.Database.SessionStateEngine.TryGetOwnership( + sourceKey, + SessionStateKeys.SourceField("192.0.2.10", node.OwnerId), + out var expiry, + out var generation, + out var count)); + redis.Database.SessionStateEngine.WriteOwnership( + sourceKey, + SessionStateKeys.SourceField("192.0.2.10", node.OwnerId), + expiry, + generation + 1, + count); + + var afterSetup = redis.Database.ScriptEvaluateCount; + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(afterSetup + 1, redis.Database.ScriptEvaluateCount); + Assert.False(bytes.TryGetCommittedBatch("alice", out _)); + Assert.Equal(4_960, await cluster.ObserveAsync("alice")); + Assert.Equal(4_960, await bytes.ObserveRemainingAsync("alice")); + Assert.Equal(1, durable.ConsumeCalls); + } + + [Fact] + public async Task SessionStateService_LostRenew_StillCompletesApply() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var cluster = new RedisAccountByteStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 8_000); + var bytes = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(99_000, await cluster.ApplyAsync("alice", "seed-high", 0, 99_000)); + bytes.CreateSink("alice").ObserveCopied(80); + await node.ReleaseAllOwnershipAsync(); + var afterRelease = redis.Database.ScriptEvaluateCount; + + var clock = new ControllableTimeProvider(); + var service = new SessionStateService( + node, + bytes, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + await service.StartAsync(CancellationToken.None); + try + { + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!clock.HasScheduledTimers) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + clock.Advance(TimeSpan.FromSeconds(10)); + while (durable.ConsumeCalls < 1 || bytes.TryGetCommittedBatch("alice", out _)) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Equal(afterRelease + 1, redis.Database.ScriptEvaluateCount); + Assert.Equal(7_920, durable.Remaining("alice")); + Assert.Equal(7_920, await cluster.ObserveAsync("alice")); + } + finally + { + await service.StopAsync(CancellationToken.None); + } + + Assert.False(bytes.TryGetCommittedBatch("alice", out _)); + } + + [Fact] + public async Task EmptyCycle_NoPendingNoOwnership_ZeroRedis() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 1_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(store, bytes); + var before = redis.Database.ScriptEvaluateCount; + + await bytes.CommitDurableAsync(); + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(before, redis.Database.ScriptEvaluateCount); + Assert.Equal(0, durable.ConsumeCalls); + Assert.Equal(1_000, durable.Remaining("alice")); + } + + [Fact] + public void PackedReturn_DistinguishesSuccessZeroFromFailureZero() + { + Assert.Equal(0, SessionStateBytePack.Encode(renewed: true, remaining: 0)); + Assert.Equal(-1, SessionStateBytePack.Encode(renewed: false, remaining: 0)); + + var successZero = SessionStateBytePack.Decode(0); + Assert.Equal(SessionStateRenewStatus.Renewed, successZero.Status); + Assert.Equal(0, successZero.Remaining); + + var lostZero = SessionStateBytePack.Decode(-1); + Assert.Equal(SessionStateRenewStatus.Lost, lostZero.Status); + Assert.Equal(0, lostZero.Remaining); + } + + [Theory] + [InlineData(1L)] + [InlineData(100L)] + [InlineData(10_000_000_000_000L)] + [InlineData(long.MaxValue)] + public void PackedReturn_RoundTripsRemaining_ForSuccessAndLoss(long remaining) + { + var success = SessionStateBytePack.Decode(SessionStateBytePack.Encode(renewed: true, remaining)); + Assert.Equal(SessionStateRenewStatus.Renewed, success.Status); + Assert.Equal(remaining, success.Remaining); + + var lost = SessionStateBytePack.Decode(SessionStateBytePack.Encode(renewed: false, remaining)); + Assert.Equal(SessionStateRenewStatus.Lost, lost.Status); + Assert.Equal(remaining, lost.Remaining); + } + + [Fact] + public void PackedReturn_ClampsNegativeRemainingAndLostMaxValueIsMinValue() + { + Assert.Equal(0, SessionStateBytePack.Encode(renewed: true, remaining: -4)); + Assert.Equal(-1, SessionStateBytePack.Encode(renewed: false, remaining: -4)); + Assert.Equal(long.MinValue, SessionStateBytePack.Encode(renewed: false, remaining: long.MaxValue)); + var decoded = SessionStateBytePack.Decode(long.MinValue); + Assert.Equal(SessionStateRenewStatus.Lost, decoded.Status); + Assert.Equal(long.MaxValue, decoded.Remaining); + } + + [Fact] + public void CombinedScript_DoesNotSubtractConsumed() + { + Assert.DoesNotContain("consumed", SessionStateScripts.RenewAndApply, StringComparison.Ordinal); + Assert.DoesNotContain("ARGV[7 + (ipCount * 2)]", SessionStateScripts.RenewAndApply, StringComparison.Ordinal); + Assert.Contains("tonumber(ARGV[8 + (ipCount * 2)])", SessionStateScripts.RenewAndApply, StringComparison.Ordinal); + } + + [Fact] + public async Task ByteOnlyAccount_WithoutSessionOwnership_UsesApplyOnly() + { + var redis = new FakeRedisService(); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 8_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(new RedisSessionStateStore(redis), bytes); + bytes.CreateSink("alice").ObserveCopied(80); + var before = redis.Database.ScriptEvaluateCount; + + await bytes.CommitDurableAsync(); + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + + Assert.Equal(before + 1, redis.Database.ScriptEvaluateCount); + Assert.Equal(7_920, durable.Remaining("alice")); + Assert.Equal(7_920, await bytes.ObserveRemainingAsync("alice")); + } + + [Fact] + public async Task CombinedEval_DuplicateBatchIsIdempotent() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 1_000); + var cluster = new RedisAccountByteStore(redis); + var bytes = new AccountByteTracker(durable, cluster, NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + bytes.CreateSink("alice").ObserveCopied(25); + await bytes.CommitDurableAsync(); + Assert.True(bytes.TryGetCommittedBatch("alice", out var batch)); + + var now = DateTimeOffset.UtcNow; + var first = await store.RenewAndApplyAsync( + "alice", + node.OwnerId, + sessionGeneration: 0, + [], + now, + SessionStateDefaults.LeaseTtl, + batch.BatchId, + batch.Consumed, + batch.MysqlRemaining); + var replay = await store.RenewAndApplyAsync( + "alice", + node.OwnerId, + sessionGeneration: 0, + [], + now, + SessionStateDefaults.LeaseTtl, + batch.BatchId, + batch.Consumed, + batch.MysqlRemaining); + + Assert.Equal(SessionStateRenewStatus.Renewed, first.Renew); + Assert.Equal(first.Remaining, replay.Remaining); + Assert.Equal(975, first.Remaining); + Assert.Equal(1, durable.ConsumeCalls); + } + + [Fact] + public async Task SessionStateCycle_DoesNotProduceTwoRedisOpsPerOwnedAccount() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var durable = new InMemoryAccountByteDurableStore(); + durable.SeedByteAccount("alice", 50_000); + var bytes = new AccountByteTracker( + durable, + new RedisAccountByteStore(redis), + NullLogger.Instance); + var node = CreateNode(store, bytes); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4B)); + bytes.CreateSink("alice").ObserveCopied(10); + var service = new SessionStateService( + node, + bytes, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + var afterAdmit = redis.Database.ScriptEvaluateCount; + + await service.StartAsync(CancellationToken.None); + await bytes.CommitDurableAsync(); + await node.RenewLeasesAsync(); + await bytes.ApplyCommittedAsync(); + await service.StopAsync(CancellationToken.None); + + var cycleOps = redis.Database.ScriptEvaluateCount - afterAdmit; + Assert.True(cycleOps < 4, $"expected fewer than 2N+final ops, observed {cycleOps}"); + Assert.True(cycleOps >= 1); + Assert.Equal(49_990, durable.Remaining("alice")); + } + + [Fact] + public void CombinedScript_DoesNotEarlyReturnBeforeApply() + { + var script = SessionStateScripts.RenewAndApply; + var applyFn = script.IndexOf("local function apply_bytes()", StringComparison.Ordinal); + var firstReturn0 = script.IndexOf("return 0", StringComparison.Ordinal); + var remainingReturn = script.IndexOf("return remaining", StringComparison.Ordinal); + Assert.True(applyFn > 0); + Assert.True(firstReturn0 > applyFn); + Assert.True(remainingReturn > applyFn); + Assert.Contains("return -remaining - 1", script, StringComparison.Ordinal); + Assert.Contains("KEYS[3]", script, StringComparison.Ordinal); + Assert.Contains("HEXISTS", script, StringComparison.Ordinal); + Assert.Contains("if next > mysql then", script, StringComparison.Ordinal); + } + + private static DistributedSessionStateTracker CreateNode( + ISessionStateStore membership, + IAccountByteAccountant bytes) => + new( + membership, + NullLogger.Instance, + "nntpd01", + TimeProvider.System, + bytes: bytes); + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + string account = "alice") => + node.TryAdmitAsync(account, sessionId, ip, sessionLimit: 5, srcIpLimit: 3); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateConcurrencyBoundaryTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateConcurrencyBoundaryTests.cs new file mode 100644 index 00000000..0449bccc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateConcurrencyBoundaryTests.cs @@ -0,0 +1,464 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Deterministic races at the local-tracker / Redis-membership boundary. +/// +public sealed class SessionStateConcurrencyBoundaryTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V4C = IPAddress.Parse("203.0.113.30"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public async Task RedisAdmitSuccess_ThenCancelBeforeLocalCommit_ReleasesDistributedOwnership() + { + var inner = new InMemorySessionStateStore(); + var store = new HoldAfterAdmitStore(inner); + var node = CreateNode(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync("alice", "s1", V4A, sessionLimit: 2, srcIpLimit: 2, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.Equal(1, inner.ActiveSessionCount("alice", NowMs())); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(0, inner.ActiveSessionCount("alice", NowMs())); + Assert.False(inner.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.Equal(1, store.ReleaseCalls); + } + + [Fact] + public async Task RedisReject_DoesNotCreateLocalSession() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "held", V4A, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + await Admit(node, "extra", V4B, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", "198.51.100.20", node.OwnerId, NowMs())); + } + + [Fact] + public async Task ReleaseOneOfTwoSameSource_LeavesOwnership_FiniteAndUnlimited() + { + await AssertReleaseOneOfTwoSameSource(sessionLimit: 5, srcIpLimit: 1); + await AssertReleaseOneOfTwoSameSource(sessionLimit: 0, srcIpLimit: 1); + } + + [Fact] + public async Task ReleaseOneOfTwoDifferentSources_LeavesTheOther_FiniteAndUnlimited() + { + await AssertReleaseOneOfTwoDifferentSources(sessionLimit: 5, srcIpLimit: 2); + await AssertReleaseOneOfTwoDifferentSources(sessionLimit: 0, srcIpLimit: 2); + } + + [Fact] + public async Task ReleaseBlockedAtRedis_ConcurrentSameAccountAdmit_WaitsThenSeesReleasedSlot() + { + var membership = new InMemorySessionStateStore + { + BlockRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 1, srcIpLimit: 2)); + var release = node.ReleaseAsync("alice", "s1").AsTask(); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (node.GetLocalSessionCount("alice") != 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + var admit = Admit(node, "s2", V4B, sessionLimit: 1, srcIpLimit: 2).AsTask(); + Assert.False(admit.IsCompleted); + membership.BlockRelease.TrySetResult(); + await release; + Assert.Equal(SessionAdmissionResult.Success, await admit); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "198.51.100.20", node.OwnerId, NowMs())); + } + + [Fact] + public async Task ReleaseBlockedAtRedis_ConcurrentSameSourceAdmit_KeepsSourceMembership() + { + var membership = new InMemorySessionStateStore + { + BlockRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 5, srcIpLimit: 1)); + var release = node.ReleaseAsync("alice", "s1").AsTask(); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (node.GetLocalSessionCount("alice") != 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + var admit = Admit(node, "s3", V4A, sessionLimit: 5, srcIpLimit: 1).AsTask(); + membership.BlockRelease.TrySetResult(); + await release; + Assert.Equal(SessionAdmissionResult.Success, await admit); + Assert.Equal(2, node.GetLocalSessionCount("alice")); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task RenewSnapshotThenRelease_DoesNotResurrect() + { + var membership = new InMemorySessionStateStore + { + BlockRenew = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + NotifyRenewStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 2, srcIpLimit: 1)); + var renew = node.RenewLeasesAsync().AsTask(); + await membership.NotifyRenewStarted.Task; + + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + membership.BlockRenew.TrySetResult(); + await renew; + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.Null(membership.SessionExpiry("alice", node.OwnerId)); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + } + + [Fact] + public async Task RenewSnapshotThenReleaseOneOfTwo_KeepsRemainingOwnership() + { + var membership = new InMemorySessionStateStore + { + BlockRenew = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + NotifyRenewStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var clock = new ControllableTimeProvider(); + var node = CreateNode(membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4B, sessionLimit: 5, srcIpLimit: 2)); + var renew = node.RenewLeasesAsync().AsTask(); + await membership.NotifyRenewStarted.Task; + + await node.ReleaseAsync("alice", "s1"); + membership.BlockRenew.TrySetResult(); + await renew; + Assert.Equal(1, membership.ActiveSessionCount("alice", clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "198.51.100.20", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task LaterFiniteSessionLimit_DoesNotUseExistingSourceHotPath() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal(1, node.DistributedAdmits); + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal(1, node.LocalHotPathAdmits); + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s3", V4A, sessionLimit: 5, srcIpLimit: 1)); + Assert.Equal(1, node.LocalHotPathAdmits); + Assert.Equal(2, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task SameNode_SessionLimit_ExactlyNOfNPlusOne() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + var results = new SessionAdmissionResult[4]; + var ips = new[] { V4A, V4B, V4C, V6A }; + Parallel.Invoke( + () => results[0] = Admit(node, "s0", ips[0], sessionLimit: 3, srcIpLimit: 8).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(node, "s1", ips[1], sessionLimit: 3, srcIpLimit: 8).AsTask().GetAwaiter().GetResult(), + () => results[2] = Admit(node, "s2", ips[2], sessionLimit: 3, srcIpLimit: 8).AsTask().GetAwaiter().GetResult(), + () => results[3] = Admit(node, "s3", ips[3], sessionLimit: 3, srcIpLimit: 8).AsTask().GetAwaiter().GetResult()); + Assert.Equal(3, results.Count(static r => r == SessionAdmissionResult.Success)); + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.SessionLimitExceeded)); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(3, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task SameNode_SourceLimit_ExactlyNDistinctOfNPlusOne() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + var results = new SessionAdmissionResult[4]; + var ips = new[] { V4A, V4B, V4C, V6A }; + Parallel.Invoke( + () => results[0] = Admit(node, "s0", ips[0], sessionLimit: 10, srcIpLimit: 3).AsTask().GetAwaiter().GetResult(), + () => results[1] = Admit(node, "s1", ips[1], sessionLimit: 10, srcIpLimit: 3).AsTask().GetAwaiter().GetResult(), + () => results[2] = Admit(node, "s2", ips[2], sessionLimit: 10, srcIpLimit: 3).AsTask().GetAwaiter().GetResult(), + () => results[3] = Admit(node, "s3", ips[3], sessionLimit: 10, srcIpLimit: 3).AsTask().GetAwaiter().GetResult()); + Assert.Equal(3, results.Count(static r => r == SessionAdmissionResult.Success)); + Assert.Equal(1, results.Count(static r => r == SessionAdmissionResult.SourceAddressLimitExceeded)); + Assert.Equal(3, membership.ActiveSourceIps("alice", NowMs()).Count); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task SameSourceThreeSessions_ReleaseStepsKeepSourceUntilLast() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 3, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit: 3, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V4A, sessionLimit: 3, srcIpLimit: 1)); + Assert.Equal(SessionAdmissionResult.SessionLimitExceeded, await Admit(node, "s4", V4A, sessionLimit: 3, srcIpLimit: 1)); + Assert.Equal(3, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + await node.ReleaseAsync("alice", "s3"); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.Empty(membership.ActiveSourceIps("alice", NowMs())); + } + + [Fact] + public async Task TwoSources_SourceLimitIndependentOfSessionRelease() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a1", V4A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a2", V4A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "b1", V6A, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.SourceAddressLimitExceeded, await Admit(node, "c1", V4B, sessionLimit: 10, srcIpLimit: 2)); + await node.ReleaseAsync("alice", "a1"); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + await node.ReleaseAsync("alice", "a2"); + Assert.Equal(["2001:db8::10"], membership.ActiveSourceIps("alice", NowMs())); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "c1", V4B, sessionLimit: 10, srcIpLimit: 2)); + Assert.Equal(2, membership.ActiveSourceIps("alice", NowMs()).Count); + } + + [Fact] + public async Task CrossNode_CannotReleaseOrRenewPeerOwnership() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode(membership, clock, "nntpd01", "incA"); + var nodeB = CreateNode(membership, clock, "nntpd02", "incB"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1", V4A, sessionLimit: 5, srcIpLimit: 2)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1", V6A, sessionLimit: 5, srcIpLimit: 2)); + var bExpiry = membership.SessionExpiry("alice", nodeB.OwnerId); + await nodeA.ReleaseAsync("alice", "missing-on-a"); + await nodeA.ReleaseAsync("alice", "b1"); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeB.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", nodeB.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + + clock.Advance(TimeSpan.FromSeconds(10)); + await nodeA.RenewLeasesAsync(); + Assert.Equal(bExpiry, membership.SessionExpiry("alice", nodeB.OwnerId)); + Assert.True(membership.SessionExpiry("alice", nodeA.OwnerId) > bExpiry); + } + + [Fact] + public async Task FailureMatrix_MatchesImplementedPolicy() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit: 2, srcIpLimit: 1)); + var expiry = membership.SessionExpiry("alice", node.OwnerId); + await node.RenewLeasesAsync(); + Assert.True(membership.SessionExpiry("alice", node.OwnerId) >= expiry); + + membership.Unavailable = true; + Assert.Equal(SessionAdmissionResult.Unavailable, await Admit(node, "s2", V4B, sessionLimit: 2, srcIpLimit: 2)); + var frozen = membership.SessionExpiry("alice", node.OwnerId); + await node.RenewLeasesAsync(); + Assert.Equal(frozen, membership.SessionExpiry("alice", node.OwnerId)); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Equal(frozen, membership.SessionExpiry("alice", node.OwnerId)); + await node.ReleaseAllOwnershipAsync(); + Assert.Equal(frozen, membership.SessionExpiry("alice", node.OwnerId)); + } + + private static async Task AssertReleaseOneOfTwoSameSource(int sessionLimit, int srcIpLimit) + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit, srcIpLimit)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A, sessionLimit, srcIpLimit)); + await node.ReleaseAsync("alice", "s1"); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + Assert.True(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + if (sessionLimit > 0) + { + Assert.Equal(1, membership.ActiveSessionCount("alice", NowMs())); + } + + Assert.Equal(["192.0.2.10"], membership.ActiveSourceIps("alice", NowMs())); + await node.ReleaseAsync("alice", "s2"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + Assert.Empty(membership.ActiveSourceIps("alice", NowMs())); + } + + private static async Task AssertReleaseOneOfTwoDifferentSources(int sessionLimit, int srcIpLimit) + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A, sessionLimit, srcIpLimit)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V6A, sessionLimit, srcIpLimit)); + await node.ReleaseAsync("alice", "s1"); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "2001:db8::10", node.OwnerId, NowMs())); + Assert.Equal(1, node.GetLocalSessionCount("alice")); + await node.ReleaseAsync("alice", "s2"); + Assert.Empty(membership.ActiveSourceIps("alice", NowMs())); + } + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + int sessionLimit = 0, + int srcIpLimit = 0) => + node.TryAdmitAsync("alice", sessionId, ip, sessionLimit, srcIpLimit); + + private static DistributedSessionStateTracker CreateNode(ISessionStateStore membership) => + CreateNode(membership, TimeProvider.System); + + private static DistributedSessionStateTracker CreateNode( + ISessionStateStore membership, + TimeProvider time, + string nodeId = "nntpd01", + string? incarnation = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time, + incarnation: incarnation); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + private sealed class HoldAfterAdmitStore : ISessionStateStore + { + private readonly InMemorySessionStateStore _inner; + + public HoldAfterAdmitStore(InMemorySessionStateStore inner) + { + _inner = inner; + } + + public TaskCompletionSource Admitted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Continue { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int ReleaseCalls { get; private set; } + + public async ValueTask TryAdmitAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default, + bool trackSessions = false) + { + var result = await _inner.TryAdmitAsync( + accountName, + normalizedSourceIp, + ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + now, + leaseTtl, + cancellationToken, + trackSessions).ConfigureAwait(false); + Admitted.TrySetResult(); + await Continue.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + return result; + } + + public ValueTask ReleaseAsync( + string accountName, + string normalizedSourceIp, + string ownerId, + long sessionGeneration, + long sourceGeneration, + CancellationToken cancellationToken = default) + { + ReleaseCalls++; + return _inner.ReleaseAsync( + accountName, + normalizedSourceIp, + ownerId, + sessionGeneration, + sourceGeneration, + cancellationToken); + } + + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + _inner.RenewAsync(accountName, ownerId, sessionGeneration, sources, now, leaseTtl, cancellationToken); + + public ValueTask RenewAndApplyAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + DateTimeOffset now, + TimeSpan leaseTtl, + string batchId, + long consumed, + long mysqlRemainingAfter, + CancellationToken cancellationToken = default) => + _inner.RenewAndApplyAsync( + accountName, + ownerId, + sessionGeneration, + sources, + now, + leaseTtl, + batchId, + consumed, + mysqlRemainingAfter, + cancellationToken); + + public ValueTask ReleaseOwnerAsync( + string accountName, + string ownerId, + CancellationToken cancellationToken = default) => + _inner.ReleaseOwnerAsync(accountName, ownerId, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateGenerationEpochTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateGenerationEpochTests.cs new file mode 100644 index 00000000..ffc8f35d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateGenerationEpochTests.cs @@ -0,0 +1,461 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Generation-epoch increment/replace: same generation increments; a new +/// generation for the same owner replaces stale count with 1. Different owners +/// are separate HASH fields and are not replaced. +/// +public sealed class SessionStateGenerationEpochTests +{ + private const string Src = "src:alice"; + private const string Sess = "sess:alice"; + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const string Ip = "192.0.2.10"; + private const long LeaseMs = 30_000; + private static readonly IPAddress V4A = IPAddress.Parse(Ip); + + [Fact] + public void TryAdmitScript_IncrementIsAtomicAndGenerationAware() + { + var script = SessionStateScripts.TryAdmit; + Assert.Contains("local function increment(key, field, gen)", script, StringComparison.Ordinal); + Assert.Contains("if storedGen and storedGen == gen then", script, StringComparison.Ordinal); + Assert.Contains("tostring((count or 0) + 1)", script, StringComparison.Ordinal); + Assert.Contains("tostring(expiry) .. '|' .. gen .. '|1'", script, StringComparison.Ordinal); + var incrementAt = script.IndexOf("local function increment", StringComparison.Ordinal); + var pruneAt = script.IndexOf("prune(srcKey)", StringComparison.Ordinal); + Assert.True(incrementAt > 0 && incrementAt < pruneAt); + Assert.True(script.IndexOf("increment(sessKey, owner, sessionGen)", StringComparison.Ordinal) > pruneAt); + Assert.True(script.IndexOf("increment(srcKey, srcField, sourceGen)", StringComparison.Ordinal) > pruneAt); + } + + [Fact] + public void SameOwnerSameGeneration_IncrementsCountOnEngineAndLuaRule() + { + AssertIncrementParity( + owner: O1, + storedGeneration: 1, + storedCount: 1, + requestedGeneration: 1, + expectedGeneration: 1, + expectedCount: 2); + } + + [Fact] + public void SameOwnerNewGeneration_ReplacesStaleCountOnEngineAndLuaRule() + { + AssertIncrementParity( + owner: O1, + storedGeneration: 1, + storedCount: 1, + requestedGeneration: 2, + expectedGeneration: 2, + expectedCount: 1); + } + + [Fact] + public void SameOwnerNewGeneration_AfterMultipleStaleSessions_ReplacesWithCountOne() + { + AssertIncrementParity( + owner: O1, + storedGeneration: 1, + storedCount: 7, + requestedGeneration: 2, + expectedGeneration: 2, + expectedCount: 1); + } + + [Fact] + public void DifferentOwner_CreatesSeparateField_LeavesExistingOwnerUnchanged() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, expiryUnixMs: 30_000, generation: 1, count: 7); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O1), 30_000, 1, 7); + + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, Ip, O2, sessionLimit: 20, srcIpLimit: 4, 0, LeaseMs, 9, 9)); + + AssertOwnership(engine, Sess, O1, generation: 1, count: 7); + AssertOwnership(engine, Sess, O2, generation: 9, count: 1); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 1, 7); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O2), 9, 1); + Assert.Equal(8, engine.ActiveSessionCount(Sess, 0)); + Assert.Equal([Ip], engine.ActiveSourceIps(Src, 0)); + } + + [Fact] + public void ReleaseAfterReplacement_DeletesNewEpoch_OldCountDoesNotSurvive() + { + var engine = SeedThenReplace(storedCount: 5); + _ = engine.Release(Src, Sess, Ip, O1, 2, 2); + Assert.False(engine.TryGetOwnership(Sess, O1, out _, out _, out _)); + Assert.False(engine.HasSourceOwner(Src, Ip, O1, 0)); + Assert.Equal(0, engine.ActiveSessionCount(Sess, 0)); + } + + [Fact] + public void StaleReleaseAfterReplacement_LeavesNewEpochUnchanged() + { + var engine = SeedThenReplace(storedCount: 5); + AssertOwnership(engine, Sess, O1, 2, 1); + _ = engine.Release(Src, Sess, Ip, O1, 1, 1); + AssertOwnership(engine, Sess, O1, 2, 1); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 2, 1); + _ = engine.Release(Src, Sess, Ip, O1, 2, 2); + Assert.False(engine.TryGetOwnership(Sess, O1, out _, out _, out _)); + Assert.False(engine.HasSourceOwner(Src, Ip, O1, 0)); + } + + [Fact] + public void StaleRenewAfterReplacement_DoesNotExtendOrRecreateOldGeneration() + { + var engine = SeedThenReplace(storedCount: 5); + Assert.True(engine.TryGetOwnership(Sess, O1, out var expiryBefore, out _, out _)); + Assert.Equal(0, engine.Renew(Src, Sess, O1, sessionGeneration: 1, nowUnixMs: 1_000, LeaseMs, [(Ip, 1)])); + Assert.True(engine.TryGetOwnership(Sess, O1, out var expiryAfterStale, out var generation, out var count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + Assert.Equal(expiryBefore, expiryAfterStale); + + Assert.Equal(1, engine.Renew(Src, Sess, O1, sessionGeneration: 2, nowUnixMs: 1_000, LeaseMs, [(Ip, 2)])); + Assert.True(engine.TryGetOwnership(Sess, O1, out var expiryRenewed, out generation, out count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + Assert.True(expiryRenewed > expiryAfterStale); + } + + [Fact] + public void ExpiredOldGeneration_IsPrunedThenNewEpochStartsAtCountOne() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, expiryUnixMs: 1_000, generation: 1, count: 7); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O1), 1_000, 1, 7); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, Ip, O1, 20, 4, nowUnixMs: 2_000, LeaseMs, 2, 2)); + AssertOwnership(engine, Sess, O1, 2, 1, nowUnixMs: 2_000); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 2, 1, nowUnixMs: 2_000); + } + + [Fact] + public void UnexpiredOldGeneration_IsReplacedWithoutWaitingForTtl() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, expiryUnixMs: 80_000, generation: 1, count: 7); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O1), 80_000, 1, 7); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, Ip, O1, 20, 4, nowUnixMs: 1_000, LeaseMs, 2, 2)); + AssertOwnership(engine, Sess, O1, 2, 1, nowUnixMs: 1_000); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 2, 1, nowUnixMs: 1_000); + } + + [Fact] + public void SessionOwnershipReplacement_DoesNotAlterAnotherOwner() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, 30_000, 1, 7); + engine.WriteOwnership(Sess, O2, 30_000, 9, 3); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus(Src, Sess, Ip, O1, sessionLimit: 20, srcIpLimit: 0, 0, LeaseMs, 2, 0)); + AssertOwnership(engine, Sess, O1, 2, 1); + AssertOwnership(engine, Sess, O2, 9, 3); + Assert.False(engine.TryGetOwnership(Src, SessionStateKeys.SourceField(Ip, O1), out _, out _, out _)); + } + + [Fact] + public void SourceOwnershipReplacement_DoesNotAlterAnotherOwnerOfSameIp() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O1), 30_000, 1, 7); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O2), 30_000, 9, 3); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, Ip, O1, sessionLimit: 0, srcIpLimit: 4, 0, LeaseMs, 0, 2)); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 2, 1); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O2), 9, 3); + Assert.False(engine.TryGetOwnership(Sess, O1, out _, out _, out _)); + Assert.Equal([Ip], engine.ActiveSourceIps(Src, 0)); + } + + [Fact] + public async Task RedisStoreDispatch_SameOwnerNewGeneration_ReplacesThroughTryAdmitScript() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + Assert.True((await store.TryAdmitAsync("alice", Ip, O1, 20, 4, 1, 1, now, ttl)).Accepted); + Assert.Equal(1, redis.Database.ScriptEvaluateCount); + SeedStoreEngine(redis, generation: 1, count: 7); + Assert.True((await store.TryAdmitAsync("alice", Ip, O1, 20, 4, 2, 2, now, ttl)).Accepted); + var engine = redis.Database.SessionStateEngine; + var sessKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSession("alice")); + var srcKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSource("alice")); + AssertOwnership(engine, sessKey, O1, 2, 1); + AssertOwnership(engine, srcKey, SessionStateKeys.SourceField(Ip, O1), 2, 1); + Assert.True(redis.Database.ScriptEvaluateCount >= 2); + } + + [Fact] + public async Task Tracker_RedisDownRelease_ReadmitUsesNewEpochThenClears() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership, "inc1"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "A")); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var sessionGeneration1, + out var sessionCount1)); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SourceKey("alice"), + SessionStateKeys.SourceField(Ip, node.OwnerId), + out _, + out var sourceGeneration1, + out var sourceCount1)); + Assert.Equal(1, sessionCount1); + Assert.Equal(1, sourceCount1); + Assert.True(sessionGeneration1 > 0); + Assert.True(sourceGeneration1 > 0); + + membership.Unavailable = true; + await node.ReleaseAsync("alice", "A"); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + AssertOwnership( + membership.Engine, + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + sessionGeneration1, + 1); + AssertOwnership( + membership.Engine, + InMemorySessionStateStore.SourceKey("alice"), + SessionStateKeys.SourceField(Ip, node.OwnerId), + sourceGeneration1, + 1); + + membership.Unavailable = false; + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "B")); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var sessionGeneration2, + out var sessionCount2)); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SourceKey("alice"), + SessionStateKeys.SourceField(Ip, node.OwnerId), + out _, + out var sourceGeneration2, + out var sourceCount2)); + Assert.True(sessionGeneration2 > sessionGeneration1); + Assert.True(sourceGeneration2 > sourceGeneration1); + Assert.Equal(1, sessionCount2); + Assert.Equal(1, sourceCount2); + + await node.ReleaseAsync("alice", "B"); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", Ip, node.OwnerId, NowMs())); + + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "C")); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var generation3, + out var count3)); + Assert.True(generation3 > sessionGeneration2); + Assert.Equal(1, count3); + await node.ReleaseAsync("alice", "C"); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + } + + [Fact] + public async Task Tracker_RepeatedStaleEpochs_NeverAccumulateCounts() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership, "inc1"); + long previousGeneration = 0; + for (var epoch = 1; epoch <= 3; epoch++) + { + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, $"s{epoch}")); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var generation, + out var count)); + Assert.Equal(1, count); + Assert.True(generation > previousGeneration); + previousGeneration = generation; + membership.Unavailable = true; + await node.ReleaseAsync("alice", $"s{epoch}"); + membership.Unavailable = false; + } + + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s4")); + Assert.True(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out var finalGeneration, + out var finalCount)); + Assert.Equal(1, finalCount); + Assert.True(finalGeneration > previousGeneration); + await node.ReleaseAsync("alice", "s4"); + Assert.Equal(0, membership.ActiveSessionCount("alice", NowMs())); + Assert.False(membership.HasSourceOwner("alice", Ip, node.OwnerId, NowMs())); + Assert.False(membership.Engine.TryGetOwnership( + InMemorySessionStateStore.SessionKey("alice"), + node.OwnerId, + out _, + out _, + out _)); + } + + [Fact] + public async Task Tracker_SourceReplacement_DoesNotDropPeerOwnerOfSameIp() + { + var membership = new InMemorySessionStateStore(); + var nodeA = CreateNode(membership, "incA", "nntpd01"); + var nodeB = CreateNode(membership, "incB", "nntpd02"); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a1")); + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeB, "b1")); + Assert.Equal([Ip], membership.ActiveSourceIps("alice", NowMs())); + + membership.Unavailable = true; + await nodeA.ReleaseAsync("alice", "a1"); + membership.Unavailable = false; + Assert.Equal(SessionAdmissionResult.Success, await Admit(nodeA, "a2")); + + Assert.True(membership.HasSourceOwner("alice", Ip, nodeB.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeB.OwnerId, NowMs())); + Assert.Equal(1, membership.OwnerSessionCount("alice", nodeA.OwnerId, NowMs())); + Assert.Equal([Ip], membership.ActiveSourceIps("alice", NowMs())); + Assert.Equal(2, membership.ActiveSessionCount("alice", NowMs())); + } + + private static void AssertIncrementParity( + string owner, + long storedGeneration, + int storedCount, + long requestedGeneration, + long expectedGeneration, + int expectedCount) + { + var sessionField = owner; + var sourceField = SessionStateKeys.SourceField(Ip, owner); + var luaSession = new Dictionary(StringComparer.Ordinal) + { + [sessionField] = SessionStateKeys.Value(30_000, storedGeneration, storedCount), + }; + var luaSource = new Dictionary(StringComparer.Ordinal) + { + [sourceField] = SessionStateKeys.Value(30_000, storedGeneration, storedCount), + }; + ApplyLuaIncrement(luaSession, sessionField, requestedGeneration, expiry: 30_000); + ApplyLuaIncrement(luaSource, sourceField, requestedGeneration, expiry: 30_000); + AssertLuaOwnership(luaSession, sessionField, expectedGeneration, expectedCount); + AssertLuaOwnership(luaSource, sourceField, expectedGeneration, expectedCount); + + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, sessionField, 30_000, storedGeneration, storedCount); + engine.WriteOwnership(Src, sourceField, 30_000, storedGeneration, storedCount); + _ = engine.TryAdmitStatus(Src, Sess, Ip, owner, 20, 4, 0, LeaseMs, requestedGeneration, requestedGeneration); + AssertOwnership(engine, Sess, sessionField, expectedGeneration, expectedCount); + AssertOwnership(engine, Src, sourceField, expectedGeneration, expectedCount); + } + + private static SessionStateEngine SeedThenReplace(int storedCount) + { + var engine = new SessionStateEngine(); + engine.WriteOwnership(Sess, O1, 30_000, 1, storedCount); + engine.WriteOwnership(Src, SessionStateKeys.SourceField(Ip, O1), 30_000, 1, storedCount); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus(Src, Sess, Ip, O1, 20, 4, 0, LeaseMs, 2, 2)); + AssertOwnership(engine, Sess, O1, 2, 1); + AssertOwnership(engine, Src, SessionStateKeys.SourceField(Ip, O1), 2, 1); + return engine; + } + + private static void ApplyLuaIncrement( + Dictionary hash, + string field, + long generation, + long expiry) + { + var gen = generation.ToString(System.Globalization.CultureInfo.InvariantCulture); + if (hash.TryGetValue(field, out var current) + && SessionStateKeys.TrySplitValue(current, out _, out var storedGeneration, out var count) + && storedGeneration.ToString(System.Globalization.CultureInfo.InvariantCulture) == gen) + { + hash[field] = SessionStateKeys.Value(expiry, storedGeneration, count + 1); + return; + } + + hash[field] = SessionStateKeys.Value(expiry, generation, 1); + } + + private static void AssertLuaOwnership( + Dictionary hash, + string field, + long generation, + int count) + { + Assert.True(hash.TryGetValue(field, out var value)); + Assert.True(SessionStateKeys.TrySplitValue(value, out _, out var storedGeneration, out var storedCount)); + Assert.Equal(generation, storedGeneration); + Assert.Equal(count, storedCount); + } + + private static void AssertOwnership( + SessionStateEngine engine, + string key, + string field, + long generation, + int count, + long nowUnixMs = 0) + { + Assert.True(engine.TryGetOwnership(key, field, out var expiry, out var storedGeneration, out var storedCount)); + Assert.Equal(generation, storedGeneration); + Assert.Equal(count, storedCount); + Assert.True(expiry > nowUnixMs); + } + + private static void SeedStoreEngine(FakeRedisService redis, long generation, int count) + { + var engine = redis.Database.SessionStateEngine; + var sessKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSession("alice")); + var srcKey = Encoding.UTF8.GetString(SessionStateKeys.CreateSource("alice")); + engine.WriteOwnership(sessKey, O1, 30_000, generation, count); + engine.WriteOwnership(srcKey, SessionStateKeys.SourceField(Ip, O1), 30_000, generation, count); + } + + private static ValueTask Admit(DistributedSessionStateTracker node, string sessionId) => + node.TryAdmitAsync("alice", sessionId, V4A, sessionLimit: 20, srcIpLimit: 4); + + private static DistributedSessionStateTracker CreateNode( + InMemorySessionStateStore membership, + string incarnation, + string nodeId = "nntpd01") => + new( + membership, + NullLogger.Instance, + nodeId, + TimeProvider.System, + incarnation: incarnation); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateLuaStaticAuditTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateLuaStaticAuditTests.cs new file mode 100644 index 00000000..f774d3fb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateLuaStaticAuditTests.cs @@ -0,0 +1,432 @@ +using System.Net; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Static Lua/C# SessionState parity. These tests do not execute Redis. +/// FakeRedis dispatches EVAL to . +/// +public sealed class SessionStateLuaStaticAuditTests +{ + [Fact] + public async Task ProductionStore_DispatchesExactSessionStateScripts() + { + var redis = new RecordingRedis(); + var store = new RedisSessionStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + + _ = await store.TryAdmitAsync("alice", "192.0.2.10", "nntpd01:a", 10, 4, 1, 1, now, ttl); + Assert.Same(SessionStateScripts.TryAdmit, redis.LastScript); + + await store.ReleaseAsync("alice", "192.0.2.10", "nntpd01:a", 1, 1); + Assert.Same(SessionStateScripts.Release, redis.LastScript); + + _ = await store.RenewAsync("alice", "nntpd01:a", 1, [("192.0.2.10", 1)], now, ttl); + Assert.Same(SessionStateScripts.Renew, redis.LastScript); + + _ = await store.RenewAndApplyAsync( + "alice", + "nntpd01:a", + 1, + [("192.0.2.10", 1)], + now, + ttl, + "batch-a", + 10, + 90); + Assert.Same(SessionStateScripts.RenewAndApply, redis.LastScript); + + await store.ReleaseOwnerAsync("alice", "nntpd01:a"); + Assert.Same(SessionStateScripts.ReleaseOwner, redis.LastScript); + } + + [Fact] + public void Scripts_MutateOnlyInsideSingleEval_AndNeverSetKeyTtl() + { + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("EXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PEXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PERSIST", script, StringComparison.Ordinal); + Assert.DoesNotContain("SET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("GET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("EVAL", script, StringComparison.Ordinal); + Assert.DoesNotContain("redis.call('KEYS'", script, StringComparison.Ordinal); + } + } + + [Fact] + public void Scripts_UseOnlyDocumentedRedisCommands() + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "HGETALL", "HGET", "HSET", "HDEL", "HLEN", "DEL", + "EXISTS", "HEXISTS", "HKEYS", + }; + foreach (var script in AllScripts()) + { + foreach (var command in EnumerateRedisCalls(script)) + { + Assert.Contains(command, allowed); + } + } + } + + [Fact] + public void TryAdmit_ChecksLimitsBeforeAnyIncrement_AndSessionLimitFirst() + { + var script = SessionStateScripts.TryAdmit; + var sessionReject = script.IndexOf("return 1", StringComparison.Ordinal); + var sourceReject = script.IndexOf("return 0", StringComparison.Ordinal); + var incrementSess = script.IndexOf("increment(sessKey, owner, sessionGen)", StringComparison.Ordinal); + var incrementSrc = script.IndexOf("increment(srcKey, srcField, sourceGen)", StringComparison.Ordinal); + var prune = script.IndexOf("prune(srcKey)", StringComparison.Ordinal); + Assert.True(prune > 0 && prune < sessionReject); + Assert.True(sessionReject > 0 && sessionReject < sourceReject); + Assert.True(sourceReject > 0 && sourceReject < incrementSess); + Assert.True(incrementSess > 0 && incrementSess < incrementSrc); + Assert.Contains("if sessionLimit ~= nil and sessionLimit > 0 and sessionTotal >= sessionLimit then", script, StringComparison.Ordinal); + Assert.Contains("if srcLimit ~= nil and srcLimit > 0 and not active[ip] then", script, StringComparison.Ordinal); + } + + [Fact] + public void TryAdmit_IncrementReplacesOnGenerationMismatch() + { + var script = SessionStateScripts.TryAdmit; + Assert.Contains("if storedGen and storedGen == gen then", script, StringComparison.Ordinal); + Assert.Contains("tostring((count or 0) + 1)", script, StringComparison.Ordinal); + Assert.Contains("tostring(expiry) .. '|' .. gen .. '|1'", script, StringComparison.Ordinal); + + var engine = new SessionStateEngine(); + engine.WriteOwnership("sess", "O1", 30_000, 1, 7); + engine.WriteOwnership("src", SessionStateKeys.SourceField("192.0.2.10", "O1"), 30_000, 1, 7); + Assert.Equal( + SessionStateEngine.AcceptedExisting, + engine.TryAdmitStatus("src", "sess", "192.0.2.10", "O1", 20, 4, 0, 30_000, 2, 2)); + Assert.True(engine.TryGetOwnership("sess", "O1", out _, out var generation, out var count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + } + + [Fact] + public void Release_IsGenerationAndOwnerScopedDecrement_NotReleaseOwner() + { + var script = SessionStateScripts.Release; + Assert.Contains("if storedGen ~= gen then", script, StringComparison.Ordinal); + Assert.Contains("decrement(sessKey, owner, sessionGen)", script, StringComparison.Ordinal); + Assert.Contains("decrement(srcKey, srcField, sourceGen)", script, StringComparison.Ordinal); + Assert.Contains("ip .. '\\31' .. owner", script, StringComparison.Ordinal); + Assert.DoesNotContain("HDEL', sessKey, owner", script, StringComparison.Ordinal); + Assert.Contains("return sessionTotal", script, StringComparison.Ordinal); + } + + [Fact] + public void Renew_IsAllOrNothing_AndDoesNotRecreate() + { + var script = SessionStateScripts.Renew; + var firstLost = script.IndexOf("return 0", StringComparison.Ordinal); + var firstWrite = script.IndexOf("renew_field(sessKey", StringComparison.Ordinal); + Assert.True(firstLost > 0 && firstWrite > firstLost); + Assert.Contains("if storedGen ~= gen or exp == nil or exp <= now then", script, StringComparison.Ordinal); + Assert.Contains("if sessionGen ~= '0' then", script, StringComparison.Ordinal); + Assert.DoesNotContain("increment(", script, StringComparison.Ordinal); + } + + [Fact] + public void ReleaseOwner_DeletesOnlyMatchingOwnerFields() + { + var script = SessionStateScripts.ReleaseOwner; + Assert.Contains("redis.call('HDEL', sessKey, owner)", script, StringComparison.Ordinal); + Assert.Contains("if fieldOwner == owner then", script, StringComparison.Ordinal); + Assert.Contains("redis.call('HDEL', srcKey, unpack(owned))", script, StringComparison.Ordinal); + Assert.DoesNotContain("generation", script, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void SourceField_IsNormalizedIpUnitSeparatorOwner_WithNoLuaNormalization() + { + Assert.Equal('\u001f', SessionStateKeys.FieldSeparator); + Assert.Equal("192.0.2.10\u001fnntpd01:a", SessionStateKeys.SourceField("192.0.2.10", "nntpd01:a")); + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("ffff", script, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("MapToIPv4", script, StringComparison.Ordinal); + Assert.DoesNotContain("IPv4", script, StringComparison.Ordinal); + } + + var mapped = IPAddress.Parse("::ffff:192.0.2.10"); + Assert.Equal("192.0.2.10", SourceAddressIdentity.Format(mapped)); + Assert.Equal("192.0.2.10", SourceAddressIdentity.Format(IPAddress.Parse("192.0.2.10"))); + Assert.Equal("2001:db8::10", SourceAddressIdentity.Format(IPAddress.Parse("2001:db8::10"))); + Assert.Equal( + IPAddress.Parse("198.18.0.70").GetAddressBytes(), + SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("198.18.0.70"))); + Assert.Equal(4, SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("198.18.0.70")).Length); + Assert.Equal( + IPAddress.Parse("2001:db8::1234").GetAddressBytes(), + SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("2001:db8::1234"))); + Assert.Equal(16, SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("2001:db8::1234")).Length); + Assert.Equal( + IPAddress.Parse("198.18.0.70").GetAddressBytes(), + SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("::ffff:198.18.0.70"))); + Assert.Equal(4, SourceAddressIdentity.ToNetworkBytes(IPAddress.Parse("::ffff:198.18.0.70")).Length); + } + + [Fact] + public void Engine_AndLuaRules_ShareReturnCodesAndValueFormat() + { + Assert.Equal(3, SessionStateEngine.AcceptedExisting); + Assert.Equal(2, SessionStateEngine.AcceptedNew); + Assert.Equal(1, SessionStateEngine.RejectedSessionLimit); + Assert.Equal(0, SessionStateEngine.RejectedSourceLimit); + Assert.Contains("return 3 + (sessionTotal * 4)", SessionStateScripts.TryAdmit, StringComparison.Ordinal); + Assert.Contains("return 2 + (sessionTotal * 4)", SessionStateScripts.TryAdmit, StringComparison.Ordinal); + Assert.Equal("1000|7|4", SessionStateKeys.Value(1000, 7, 4)); + Assert.True(SessionStateKeys.TrySplitValue("1000|7|4", out var expiry, out var generation, out var count)); + Assert.Equal(1000, expiry); + Assert.Equal(7, generation); + Assert.Equal(4, count); + } + + [Fact] + public void Engine_RejectedAdmit_DoesNotCreateOwnership_AfterExpiredPrune() + { + var engine = new SessionStateEngine(); + engine.WriteOwnership("sess", "O1", expiryUnixMs: 1_000, generation: 1, count: 7); + engine.WriteOwnership("src", SessionStateKeys.SourceField("192.0.2.10", "O1"), 1_000, 1, 7); + engine.WriteOwnership("sess", "O2", expiryUnixMs: 80_000, generation: 9, count: 1); + Assert.Equal( + SessionStateEngine.RejectedSessionLimit, + engine.TryAdmitStatus("src", "sess", "198.51.100.20", "O3", sessionLimit: 1, srcIpLimit: 4, nowUnixMs: 2_000, 30_000, 1, 1)); + Assert.False(engine.TryGetOwnership("sess", "O1", out _, out _, out _)); + Assert.False(engine.HasSourceOwner("src", "192.0.2.10", "O1", 2_000)); + Assert.Equal(1, engine.OwnerSessionCount("sess", "O2", 2_000)); + Assert.False(engine.TryGetOwnership("sess", "O3", out _, out _, out _)); + Assert.False(engine.HasSourceOwner("src", "198.51.100.20", "O3", 2_000)); + } + + [Fact] + public void Engine_RejectedSourceLimit_LeavesExistingSessionAndSourceUnchanged() + { + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus("src", "sess", "192.0.2.10", "O1", 10, 1, 0, 30_000, 1, 1)); + Assert.Equal( + SessionStateEngine.RejectedSourceLimit, + engine.TryAdmitStatus("src", "sess", "198.51.100.20", "O2", 10, 1, 0, 30_000, 2, 2)); + Assert.Equal(1, engine.OwnerSessionCount("sess", "O1", 0)); + Assert.Equal(0, engine.OwnerSessionCount("sess", "O2", 0)); + Assert.True(engine.HasSourceOwner("src", "192.0.2.10", "O1", 0)); + Assert.False(engine.HasSourceOwner("src", "198.51.100.20", "O2", 0)); + Assert.Equal(["192.0.2.10"], engine.ActiveSourceIps("src", 0)); + } + + [Fact] + public void Engine_RenewLost_DoesNotRecreateMissingOrExpiredOrWrongGeneration() + { + var engine = new SessionStateEngine(); + Assert.Equal(0, engine.Renew("src", "sess", "O1", 1, 0, 30_000, [("192.0.2.10", 1)])); + Assert.False(engine.TryGetOwnership("sess", "O1", out _, out _, out _)); + + engine.WriteOwnership("sess", "O1", 30_000, 2, 1); + engine.WriteOwnership("src", SessionStateKeys.SourceField("192.0.2.10", "O1"), 30_000, 2, 1); + Assert.Equal(0, engine.Renew("src", "sess", "O1", 1, 0, 30_000, [("192.0.2.10", 1)])); + Assert.True(engine.TryGetOwnership("sess", "O1", out _, out var generation, out var count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + + engine.WriteOwnership("sess", "O2", expiryUnixMs: 1_000, generation: 1, count: 1); + engine.WriteOwnership("src", SessionStateKeys.SourceField("192.0.2.10", "O2"), 1_000, 1, 1); + Assert.Equal(0, engine.Renew("src", "sess", "O2", 1, nowUnixMs: 2_000, 30_000, [("192.0.2.10", 1)])); + Assert.True(engine.TryGetOwnership("sess", "O2", out var expiry, out generation, out count)); + Assert.Equal(1_000, expiry); + Assert.Equal(1, generation); + Assert.Equal(1, count); + } + + [Fact] + public async Task StoreFailureMapping_IsNotBroadened() + { + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + + var unavailable = new RecordingRedis { BeginSucceeds = false }; + var store = new RedisSessionStateStore(unavailable); + Assert.Equal( + SessionStateAdmitStatus.Unavailable, + (await store.TryAdmitAsync("alice", "192.0.2.10", "O1", 1, 1, 1, 1, now, ttl)).Status); + Assert.Equal( + SessionStateRenewStatus.Unavailable, + (await store.RenewAsync("alice", "O1", 1, [("192.0.2.10", 1)], now, ttl)).Status); + + var redisDown = new RecordingRedis { EvaluateException = new RedisUnavailableException("timeout") }; + store = new RedisSessionStateStore(redisDown); + Assert.Equal( + SessionStateAdmitStatus.Unavailable, + (await store.TryAdmitAsync("alice", "192.0.2.10", "O1", 1, 1, 1, 1, now, ttl)).Status); + + var disposed = new RecordingRedis { EvaluateException = new ObjectDisposedException("mux") }; + store = new RedisSessionStateStore(disposed); + Assert.Equal( + SessionStateAdmitStatus.Unavailable, + (await store.TryAdmitAsync("alice", "192.0.2.10", "O1", 1, 1, 1, 1, now, ttl)).Status); + + var canceled = new RecordingRedis { EvaluateException = new OperationCanceledException() }; + store = new RedisSessionStateStore(canceled); + await Assert.ThrowsAsync( + () => store.TryAdmitAsync("alice", "192.0.2.10", "O1", 1, 1, 1, 1, now, ttl).AsTask()); + } + + [Fact] + public void FakeRedis_DispatchesScriptsToSessionStateEngine() + { + var source = File.ReadAllText( + Path.Join(FindRepositoryRoot(), "tests", "VectorNNTPD.ReaderService.Tests", "TestDoubles", "FakeRedis.cs")); + Assert.Contains("if (script == SessionStateScripts.TryAdmit)", source, StringComparison.Ordinal); + Assert.Contains("return SessionStateEngine.TryAdmit(", source, StringComparison.Ordinal); + Assert.Contains("return SessionStateEngine.Release(", source, StringComparison.Ordinal); + Assert.Contains("return SessionStateEngine.Renew(", source, StringComparison.Ordinal); + Assert.Contains("SessionStateScripts.RenewAndApply", source, StringComparison.Ordinal); + Assert.Contains("AccountByteEngine.Apply(", source, StringComparison.Ordinal); + Assert.Contains("return SessionStateEngine.ReleaseOwner(", source, StringComparison.Ordinal); + Assert.DoesNotContain("ConnectionMultiplexer", source, StringComparison.Ordinal); + } + + [Fact] + public void LeaseConstants_RemainCrashRecoveryNotIdleTimeout() + { + Assert.Equal(TimeSpan.FromSeconds(30), SessionStateDefaults.LeaseTtl); + Assert.Equal(TimeSpan.FromSeconds(10), SessionStateDefaults.RenewalPeriod); + Assert.Equal(TimeSpan.FromSeconds(2), SessionStateDefaults.HotPathSkew); + } + + private static IEnumerable AllScripts() + { + yield return SessionStateScripts.TryAdmit; + yield return SessionStateScripts.Release; + yield return SessionStateScripts.Renew; + yield return SessionStateScripts.RenewAndApply; + yield return SessionStateScripts.ReleaseOwner; + } + + private static IEnumerable EnumerateRedisCalls(string script) + { + const string prefix = "redis.call('"; + var start = 0; + while (true) + { + var at = script.IndexOf(prefix, start, StringComparison.Ordinal); + if (at < 0) + { + yield break; + } + + var commandStart = at + prefix.Length; + var commandEnd = script.IndexOf('\'', commandStart); + Assert.True(commandEnd > commandStart); + yield return script[commandStart..commandEnd]; + start = commandEnd + 1; + } + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Join(directory.FullName, "VectorNNTPD.slnx"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Repository root was not found."); + } + + private sealed class RecordingRedis : IRedisService + { + public string? LastScript { get; private set; } + + public bool BeginSucceeds { get; set; } = true; + + public Exception? EvaluateException { get; set; } + + public IRedisDatabase Database => new RecordingDatabase(this); + + public bool IsUnavailable => !BeginSucceeds; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return BeginSucceeds; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + private sealed class RecordingDatabase : IRedisDatabase + { + private readonly RecordingRedis _owner; + + public RecordingDatabase(RecordingRedis owner) + { + _owner = owner; + } + + public Task PingAsync(CancellationToken cancellationToken = default) => + Task.FromResult(TimeSpan.FromMilliseconds(1)); + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + _owner.LastScript = script; + if (_owner.EvaluateException is { } fault) + { + return ValueTask.FromException(fault); + } + + return ValueTask.FromResult(2L); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateRedisLuaIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateRedisLuaIntegrationTests.cs new file mode 100644 index 00000000..f40716df --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateRedisLuaIntegrationTests.cs @@ -0,0 +1,429 @@ +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Fixtures; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +/// +/// Live EVAL of through +/// against the dedicated test Redis. +/// +public sealed class SessionStateRedisLuaIntegrationTests : IClassFixture +{ + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const string IpA = "192.0.2.10"; + private const string IpB = "198.51.100.20"; + private const string Ip6 = "2001:db8::10"; + private const long Now = 10_000; + private const long Lease = 30_000; + private readonly SessionStateRedisIntegrationFixture _redis; + + public SessionStateRedisLuaIntegrationTests(SessionStateRedisIntegrationFixture redis) + { + _redis = redis; + } + + [SessionStateRedisIntegrationFact] + public void ConnectsToAuthorizedTestRedis() + { + Assert.True(_redis.IsConfigured); + Assert.Equal(SessionStateRedisIntegration.AuthorizedEndpoint, _redis.Endpoint); + Assert.False(string.IsNullOrWhiteSpace(_redis.ServerVersion)); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_NewSessionAndSource_WritesBothHashes() + { + var account = await _redis.CreateAccountAsync(); + var engine = new SessionStateEngine(); + Assert.Equal( + SessionStateEngine.AcceptedNew, + engine.TryAdmitStatus("src", "sess", IpA, O1, 10, 4, Now, Lease, 1, 1)); + var result = await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease); + Assert.Equal(SessionStateAdmitStatus.AcceptedNew, result.Status); + var session = await _redis.ReadSessionAsync(account, O1); + var source = await _redis.ReadSourceAsync(account, IpA, O1); + AssertOwnership(session, Now + Lease, 1, 1); + AssertOwnership(source, Now + Lease, 1, 1); + Assert.Null(await _redis.KeyTimeToLiveAsync(account, session: true)); + Assert.Null(await _redis.KeyTimeToLiveAsync(account, session: false)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_SecondSessionSameSource_IncrementsSessionKeepsOneIp() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Now + Lease, 1, 2); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), Now + Lease, 1, 2); + Assert.Equal(2, await _redis.SessionCountAsync(account)); + Assert.Equal(1, await _redis.DistinctSourceCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_NewSource_AddsSecondSourceField() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpB, O1, 10, 4, 1, 3, Now, Lease)).Accepted); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Now + Lease, 1, 2); + Assert.NotNull(await _redis.ReadSourceAsync(account, IpA, O1)); + Assert.NotNull(await _redis.ReadSourceAsync(account, IpB, O1)); + Assert.Equal(2, await _redis.DistinctSourceCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_SessionLimit_AcceptsExactlyN() + { + var account = await _redis.CreateAccountAsync(); + for (var i = 0; i < 3; i++) + { + Assert.True((await _redis.TryAdmitAsync(account, IpA, "owner:" + i, 3, 4, 1, 1, Now, Lease)).Accepted); + } + + Assert.Equal( + SessionStateAdmitStatus.RejectedSessionLimit, + (await _redis.TryAdmitAsync(account, IpA, "owner:3", 3, 4, 1, 1, Now, Lease)).Status); + Assert.Equal(3, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_SourceLimit_AcceptsExactlyNDistinctIps() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, "203.0.113.1", O1, 20, 2, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, "203.0.113.2", O1, 20, 2, 1, 2, Now, Lease)).Accepted); + Assert.Equal( + SessionStateAdmitStatus.RejectedSourceLimit, + (await _redis.TryAdmitAsync(account, "203.0.113.3", O1, 20, 2, 1, 3, Now, Lease)).Status); + Assert.Equal(2, await _redis.SessionCountAsync(account)); + Assert.Equal(2, await _redis.DistinctSourceCountAsync(account)); + Assert.Null(await _redis.ReadSourceAsync(account, "203.0.113.3", O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_BothLimitsViolated_IsSessionLimitFirst() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 1, 1, 1, 1, Now, Lease)).Accepted); + var result = await _redis.TryAdmitAsync(account, IpB, O2, 1, 1, 1, 1, Now, Lease); + Assert.Equal(SessionStateAdmitStatus.RejectedSessionLimit, result.Status); + Assert.Equal(1, await _redis.SessionCountAsync(account)); + Assert.Null(await _redis.ReadSessionAsync(account, O2)); + Assert.Null(await _redis.ReadSourceAsync(account, IpB, O2)); + Assert.NotNull(await _redis.ReadSessionAsync(account, O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_GenerationReplacement_ReplacesCountAndIgnoresStaleRelease() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedSessionAsync(account, O1, 30_000, 1, 7); + await _redis.SeedSourceAsync(account, IpA, O1, 30_000, 1, 7); + var engine = new SessionStateEngine(); + engine.WriteOwnership("sess", O1, 30_000, 1, 7); + engine.WriteOwnership("src", SessionStateKeys.SourceField(IpA, O1), 30_000, 1, 7); + Assert.Equal(SessionStateEngine.AcceptedExisting, engine.TryAdmitStatus("src", "sess", IpA, O1, 20, 4, 0, Lease, 2, 2)); + + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 20, 4, 2, 2, 0, Lease)).Accepted); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Lease, 2, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(account, IpA, O1, 1, 1); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Lease, 2, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(account, IpA, O1, 2, 2); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Release_SameGeneration_DecrementsThenRemoves() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + await _redis.ReleaseAsync(account, IpA, O1, 1, 1); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Now + Lease, 1, 1); + await _redis.ReleaseAsync(account, IpA, O1, 1, 1); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Release_DifferentOwner_LeavesPeer() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O2, 10, 4, 9, 9, Now, Lease)).Accepted); + await _redis.ReleaseAsync(account, IpA, O1, 1, 1); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + AssertOwnership(await _redis.ReadSessionAsync(account, O2), Now + Lease, 9, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O2), Now + Lease, 9, 1); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Release_SourceField_LeavesOtherOwnerAndOtherIp() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O2, 10, 4, 9, 9, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpB, O1, 10, 4, 1, 3, Now, Lease)).Accepted); + await _redis.ReleaseAsync(account, IpA, O1, 1, 1); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + Assert.NotNull(await _redis.ReadSourceAsync(account, IpA, O2)); + Assert.NotNull(await _redis.ReadSourceAsync(account, IpB, O1)); + Assert.Equal("192.0.2.10\u001fnntpd02:b", SessionStateKeys.SourceField(IpA, O2)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_Valid_ExtendsExpiryUsingSuppliedNow() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.Equal(SessionStateRenewStatus.Renewed, (await _redis.RenewAsync(account, O1, 1, [(IpA, 1)], 20_000, Lease)).Status); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), 50_000, 1, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), 50_000, 1, 1); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_WrongGeneration_IsLostAndLeavesG2() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedSessionAsync(account, O1, 30_000, 2, 1); + await _redis.SeedSourceAsync(account, IpA, O1, 30_000, 2, 1); + Assert.Equal(SessionStateRenewStatus.Lost, (await _redis.RenewAsync(account, O1, 1, [(IpA, 1)], Now, Lease)).Status); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), 30_000, 2, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), 30_000, 2, 1); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_Missing_IsLostAndDoesNotCreate() + { + var account = await _redis.CreateAccountAsync(); + Assert.Equal(SessionStateRenewStatus.Lost, (await _redis.RenewAsync(account, O1, 1, [(IpA, 1)], Now, Lease)).Status); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_Expired_IsLostAndDoesNotRecreate() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedSessionAsync(account, O1, 1_000, 1, 1); + await _redis.SeedSourceAsync(account, IpA, O1, 1_000, 1, 1); + Assert.Equal(SessionStateRenewStatus.Lost, (await _redis.RenewAsync(account, O1, 1, [(IpA, 1)], 2_000, Lease)).Status); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), 1_000, 1, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), 1_000, 1, 1); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Renew_OneInvalidField_IsAllOrNothing() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, IpB, O1, 10, 4, 1, 3, Now, Lease)).Accepted); + var before = await _redis.ReadSessionAsync(account, O1); + Assert.Equal( + SessionStateRenewStatus.Lost, + (await _redis.RenewAsync(account, O1, 1, [(IpA, 1), (IpB, 99)], 20_000, Lease)).Status); + Assert.Equal(before, await _redis.ReadSessionAsync(account, O1)); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), Now + Lease, 1, 1); + AssertOwnership(await _redis.ReadSourceAsync(account, IpB, O1), Now + Lease, 3, 1); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ReleaseOwner_RemovesOnlyThatOwnerAcrossAccountsAndIps() + { + var alice = await _redis.CreateAccountAsync(); + var bob = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(alice, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(alice, IpB, O1, 10, 4, 1, 3, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(alice, IpA, O2, 10, 4, 9, 9, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(bob, IpA, O1, 10, 4, 2, 2, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(bob, IpA, O2, 10, 4, 8, 8, Now, Lease)).Accepted); + await _redis.ReleaseOwnerAsync(alice, O1); + await _redis.ReleaseOwnerAsync(bob, O1); + Assert.Null(await _redis.ReadSessionAsync(alice, O1)); + Assert.Null(await _redis.ReadSourceAsync(alice, IpA, O1)); + Assert.Null(await _redis.ReadSourceAsync(alice, IpB, O1)); + Assert.NotNull(await _redis.ReadSessionAsync(alice, O2)); + Assert.NotNull(await _redis.ReadSourceAsync(alice, IpA, O2)); + Assert.Null(await _redis.ReadSessionAsync(bob, O1)); + Assert.NotNull(await _redis.ReadSessionAsync(bob, O2)); + await _redis.DeleteKeysAsync(alice); + await _redis.DeleteKeysAsync(bob); + } + + [SessionStateRedisIntegrationFact] + public async Task ConcurrentTryAdmit_SessionLimit_AcceptsExactlyTen() + { + var account = await _redis.CreateAccountAsync(); + var tasks = Enumerable.Range(0, 20) + .Select(i => _redis.TryAdmitAsync(account, IpA, "node:" + i, 10, 0, 1, 0, Now, Lease).AsTask()); + var results = await Task.WhenAll(tasks); + Assert.Equal(10, results.Count(static result => result.Accepted)); + Assert.Equal(10, results.Count(static result => result.Status == SessionStateAdmitStatus.RejectedSessionLimit)); + Assert.Equal(10, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ConcurrentTryAdmit_SourceLimit_AcceptsExactlyTenDistinctIps() + { + var account = await _redis.CreateAccountAsync(); + var tasks = Enumerable.Range(1, 20) + .Select(i => _redis.TryAdmitAsync(account, "203.0.113." + i, "node:" + i, 0, 10, 0, 1, Now, Lease).AsTask()); + var results = await Task.WhenAll(tasks); + Assert.Equal(10, results.Count(static result => result.Accepted)); + Assert.Equal(10, results.Count(static result => result.Status == SessionStateAdmitStatus.RejectedSourceLimit)); + Assert.Equal(10, await _redis.DistinctSourceCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task ConcurrentTryAdmit_BothLimits_SessionLimitWinsForUniqueIps() + { + var account = await _redis.CreateAccountAsync(); + var tasks = Enumerable.Range(1, 20) + .Select(i => _redis.TryAdmitAsync(account, "203.0.113." + i, "node:" + i, 6, 6, 1, 1, Now, Lease).AsTask()); + var results = await Task.WhenAll(tasks); + Assert.Equal(6, results.Count(static result => result.Accepted)); + Assert.Equal(6, await _redis.SessionCountAsync(account)); + Assert.Equal(6, await _redis.DistinctSourceCountAsync(account)); + Assert.DoesNotContain(results, static result => result.Status == SessionStateAdmitStatus.RejectedSourceLimit); + Assert.Equal(14, results.Count(static result => result.Status == SessionStateAdmitStatus.RejectedSessionLimit)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task TryAdmit_PrunesExpiredWithoutConsumingCapacity() + { + var account = await _redis.CreateAccountAsync(); + await _redis.SeedSessionAsync(account, O1, 1_000, 1, 7); + await _redis.SeedSourceAsync(account, IpA, O1, 1_000, 1, 7); + await _redis.SeedSessionAsync(account, O2, 80_000, 9, 1); + await _redis.SeedSourceAsync(account, IpB, O2, 80_000, 9, 1); + var engine = new SessionStateEngine(); + engine.WriteOwnership("sess", O1, 1_000, 1, 7); + engine.WriteOwnership("src", SessionStateKeys.SourceField(IpA, O1), 1_000, 1, 7); + engine.WriteOwnership("sess", O2, 80_000, 9, 1); + engine.WriteOwnership("src", SessionStateKeys.SourceField(IpB, O2), 80_000, 9, 1); + Assert.Equal(SessionStateEngine.AcceptedNew, engine.TryAdmitStatus("src", "sess", IpA, "O3", 2, 4, 2_000, Lease, 1, 1)); + + Assert.True((await _redis.TryAdmitAsync(account, IpA, "nntpd03:c", 2, 4, 1, 1, 2_000, Lease)).Accepted); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + AssertOwnership(await _redis.ReadSessionAsync(account, O2), 80_000, 9, 1); + Assert.NotNull(await _redis.ReadSessionAsync(account, "nntpd03:c")); + Assert.Equal(2, await _redis.SessionCountAsync(account)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task SourceIdentity_MappedIpv6SharesIpv4_NativeIpv6IsIndependent() + { + var account = await _redis.CreateAccountAsync(); + var mapped = SessionStateRedisIntegrationFixture.Format("::ffff:192.0.2.10"); + var v4 = SessionStateRedisIntegrationFixture.Format("192.0.2.10"); + var v6 = SessionStateRedisIntegrationFixture.Format("2001:db8::10"); + Assert.Equal(IpA, mapped); + Assert.Equal(IpA, v4); + Assert.Equal(Ip6, v6); + Assert.True((await _redis.TryAdmitAsync(account, mapped, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, v4, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(account, v6, O1, 10, 4, 1, 3, Now, Lease)).Accepted); + Assert.Equal(3, await _redis.SessionCountAsync(account)); + Assert.Equal(2, await _redis.DistinctSourceCountAsync(account)); + AssertOwnership(await _redis.ReadSourceAsync(account, IpA, O1), Now + Lease, 1, 2); + Assert.NotNull(await _redis.ReadSourceAsync(account, Ip6, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, "::ffff:192.0.2.10", O1)); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task RenewAndApply_FloorsBytesEvenWhenRenewIsLost_AndReplayIsIdempotent() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + var now = DateTimeOffset.FromUnixTimeMilliseconds(Now); + var ttl = TimeSpan.FromMilliseconds(Lease); + var applied = await _redis.Store.RenewAndApplyAsync( + account, + O1, + 1, + [(IpA, 1)], + now, + ttl, + "live-batch-1", + consumed: 25, + mysqlRemainingAfter: 50_000); + Assert.Equal(SessionStateRenewStatus.Renewed, applied.Renew); + Assert.Equal(50_000, applied.Remaining); + AssertOwnership(await _redis.ReadSessionAsync(account, O1), Now + Lease, 1, 1); + + await _redis.Store.ReleaseOwnerAsync(account, O1); + var lost = await _redis.Store.RenewAndApplyAsync( + account, + O1, + 1, + [(IpA, 1)], + now, + ttl, + "live-batch-2", + consumed: 25, + mysqlRemainingAfter: 800); + Assert.Equal(SessionStateRenewStatus.Lost, lost.Renew); + Assert.Equal(800, lost.Remaining); + + var replay = await _redis.Store.RenewAndApplyAsync( + account, + O1, + 1, + [(IpA, 1)], + now, + ttl, + "live-batch-2", + consumed: 25, + mysqlRemainingAfter: 800); + Assert.Equal(SessionStateRenewStatus.Lost, replay.Renew); + Assert.Equal(800, replay.Remaining); + await _redis.DeleteKeysAsync(account); + } + + [SessionStateRedisIntegrationFact] + public async Task Cleanup_RemovesOnlyThisSuiteAccounts() + { + var account = await _redis.CreateAccountAsync(); + Assert.True((await _redis.TryAdmitAsync(account, IpA, O1, 10, 4, 1, 1, Now, Lease)).Accepted); + await _redis.DeleteKeysAsync(account); + Assert.Null(await _redis.ReadSessionAsync(account, O1)); + Assert.Null(await _redis.ReadSourceAsync(account, IpA, O1)); + } + + private static void AssertOwnership(Ownership? actual, long expiryUnixMs, long generation, int count) + { + Assert.True(actual.HasValue); + var ownership = actual.GetValueOrDefault(); + Assert.Equal(expiryUnixMs, ownership.ExpiryUnixMs); + Assert.Equal(generation, ownership.Generation); + Assert.Equal(count, ownership.Count); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateServiceTests.cs new file mode 100644 index 00000000..f800fcb3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateServiceTests.cs @@ -0,0 +1,250 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +public sealed class SessionStateServiceTests +{ + private static readonly IPAddress V4A = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress V4B = IPAddress.Parse("198.51.100.20"); + private static readonly IPAddress V6A = IPAddress.Parse("2001:db8::10"); + + [Fact] + public void Types_UseSessionStateNamespace() + { + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(SessionStateService).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(ISessionStateLeaseManager).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(ISessionStateTracker).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(DistributedSessionStateTracker).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(InMemorySessionStateTracker).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(SessionAdmissionResult).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.SessionState", typeof(SourceAddressIdentity).Namespace); + } + + [Fact] + public async Task Start_RenewsThenStopReleasesOwnership() + { + var leases = new FakeLeaseManager(); + var clock = new ControllableTimeProvider(); + var service = new SessionStateService( + leases, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + + await service.StartAsync(CancellationToken.None); + Assert.Equal("SessionState", service.Name); + Assert.NotNull(service.Execution); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (Volatile.Read(ref leases.Renewals) < 3) + { + wait.Token.ThrowIfCancellationRequested(); + if (!clock.HasScheduledTimers) + { + await Task.Yield(); + continue; + } + + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + } + + await service.StopAsync(CancellationToken.None); + Assert.True(leases.Renewals >= 3); + Assert.Equal(1, leases.ReleaseAll); + } + + [Fact] + public async Task Start_IsIdempotent() + { + var leases = new FakeLeaseManager(); + var service = new SessionStateService( + leases, + NullAccountByteAccountant.Instance, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(1, leases.ReleaseAll); + } + + [Fact] + public async Task OneAccountMultipleSessionsAndIps_OneRedisEvalPerCycle() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var node = CreateNode(store); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s3", V6A)); + var afterAdmit = redis.Database.ScriptEvaluateCount; + + await node.RenewLeasesAsync(); + Assert.Equal(afterAdmit + 1, redis.Database.ScriptEvaluateCount); + + await node.RenewLeasesAsync(); + Assert.Equal(afterAdmit + 2, redis.Database.ScriptEvaluateCount); + } + + [Fact] + public async Task MultipleAccounts_EachGetsOneRedisEval() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var node = CreateNode(store); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "a1", V4A, "alice")); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "b1", V6A, "bob")); + var afterAdmit = redis.Database.ScriptEvaluateCount; + + await node.RenewLeasesAsync(); + Assert.Equal(afterAdmit + 2, redis.Database.ScriptEvaluateCount); + } + + [Fact] + public async Task AfterSessionRelease_RemainingOwnershipStillRenews() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4B)); + await node.ReleaseAsync("alice", "s1"); + var before = membership.RenewCalls; + + await node.RenewLeasesAsync(); + Assert.Equal(before + 1, membership.RenewCalls); + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, NowMs())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, NowMs())); + Assert.True(membership.HasSourceOwner("alice", "198.51.100.20", node.OwnerId, NowMs())); + } + + [Fact] + public async Task LastSessionRelease_StopsAccountRenewal() + { + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + await node.ReleaseAsync("alice", "s1"); + var before = membership.RenewCalls; + + await node.RenewLeasesAsync(); + Assert.Equal(before, membership.RenewCalls); + Assert.Equal(0, node.GetLocalSessionCount("alice")); + } + + [Fact] + public async Task ServiceRenewal_DoesNotResurrectReleasedSession() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemorySessionStateStore(); + var node = CreateNode(membership, clock); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s2", V4B)); + await node.ReleaseAsync("alice", "s1"); + + var leases = new CountingLeaseManager(node); + var service = new SessionStateService( + leases, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + await service.StartAsync(CancellationToken.None); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (Volatile.Read(ref leases.Renewals) < 1) + { + wait.Token.ThrowIfCancellationRequested(); + if (!clock.HasScheduledTimers) + { + await Task.Yield(); + continue; + } + + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + } + + Assert.Equal(1, membership.OwnerSessionCount("alice", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(membership.HasSourceOwner("alice", "192.0.2.10", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.True(membership.HasSourceOwner("alice", "198.51.100.20", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task RedisUnavailableDuringRenew_DoesNotRecreateOwnership() + { + var redis = new FakeRedisService(); + var store = new RedisSessionStateStore(redis); + var node = CreateNode(store); + Assert.Equal(SessionAdmissionResult.Success, await Admit(node, "s1", V4A)); + await node.ReleaseAllOwnershipAsync(); + redis.IsUnavailable = true; + + await node.RenewLeasesAsync(); + redis.Recover(); + Assert.Equal( + SessionAdmissionResult.Success, + await Admit(CreateNode(store, incarnation: "other"), "take", V6A)); + } + + private static DistributedSessionStateTracker CreateNode( + ISessionStateStore membership, + TimeProvider? time = null, + string? incarnation = null) => + new( + membership, + NullLogger.Instance, + "nntpd01", + time ?? TimeProvider.System, + incarnation: incarnation); + + private static ValueTask Admit( + DistributedSessionStateTracker node, + string sessionId, + IPAddress ip, + string account = "alice") => + node.TryAdmitAsync(account, sessionId, ip, sessionLimit: 5, srcIpLimit: 3); + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + private sealed class FakeLeaseManager : ISessionStateLeaseManager + { + public int Renewals; + public int ReleaseAll; + + public ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Renewals); + return ValueTask.CompletedTask; + } + + public ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref ReleaseAll); + return ValueTask.CompletedTask; + } + } + + private sealed class CountingLeaseManager : ISessionStateLeaseManager + { + private readonly DistributedSessionStateTracker _inner; + + public CountingLeaseManager(DistributedSessionStateTracker inner) + { + _inner = inner; + } + + public int Renewals; + + public ValueTask RenewLeasesAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Renewals); + return _inner.RenewLeasesAsync(cancellationToken); + } + + public ValueTask ReleaseAllOwnershipAsync(CancellationToken cancellationToken = default) => + _inner.ReleaseAllOwnershipAsync(cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateTrackerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateTrackerTests.cs new file mode 100644 index 00000000..49705563 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/SessionState/SessionStateTrackerTests.cs @@ -0,0 +1,126 @@ +using System.Net; +using VectorNNTPD.ReaderService.SessionState; + +namespace VectorNNTPD.ReaderService.Tests.SessionState; + +public sealed class SessionStateTrackerTests +{ + [Fact] + public void ZeroLimits_AreUnlimited() + { + var tracker = new InMemorySessionStateTracker(); + for (var i = 0; i < 8; i++) + { + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", $"s{i}", IPAddress.Parse("203.0.113.10"), sessionLimit: 0, srcIpLimit: 0)); + } + } + + [Fact] + public void SessionLimit_IsAtomicAndExcludesReleasedSlots() + { + var tracker = new InMemorySessionStateTracker(); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", IPAddress.Loopback, sessionLimit: 1, srcIpLimit: 0)); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + tracker.TryAdmit("alice", "b", IPAddress.Loopback, sessionLimit: 1, srcIpLimit: 0)); + tracker.Release("alice", "a"); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "b", IPAddress.Loopback, sessionLimit: 1, srcIpLimit: 0)); + } + + [Fact] + public void SourceIpLimit_CountsDistinctAddresses_NotSessionsOnSameIp() + { + var tracker = new InMemorySessionStateTracker(); + var ip = IPAddress.Parse("2001:db8::1"); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", ip, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "b", ip, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + tracker.TryAdmit("alice", "c", IPAddress.Parse("2001:db8::2"), sessionLimit: 0, srcIpLimit: 1)); + } + + [Fact] + public void Ipv4MappedAddress_CountsAsIpv4() + { + var tracker = new InMemorySessionStateTracker(); + var v4 = IPAddress.Parse("192.0.2.10"); + var mapped = IPAddress.Parse("::ffff:192.0.2.10"); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", v4, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "b", mapped, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + tracker.TryAdmit("alice", "c", IPAddress.Parse("192.0.2.11"), sessionLimit: 0, srcIpLimit: 1)); + } + + [Fact] + public void ConcurrentAdmit_RespectsSessionLimit() + { + var tracker = new InMemorySessionStateTracker(); + var success = 0; + Parallel.For(0, 32, i => + { + if (tracker.TryAdmit("alice", $"s{i}", IPAddress.Loopback, sessionLimit: 2, srcIpLimit: 0) + == SessionAdmissionResult.Success) + { + Interlocked.Increment(ref success); + } + }); + + Assert.Equal(2, success); + } + + [Fact] + public void CompressedAndExpandedIpv6_AreOneSource() + { + var tracker = new InMemorySessionStateTracker(); + var compressed = IPAddress.Parse("2001:db8::10"); + var expanded = IPAddress.Parse("2001:0db8:0000:0000:0000:0000:0000:0010"); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", compressed, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "b", expanded, sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + tracker.TryAdmit("alice", "c", IPAddress.Parse("2001:db8::11"), sessionLimit: 0, srcIpLimit: 1)); + } + + [Fact] + public void Ipv4AndIpv6_AreDistinctSources() + { + var tracker = new InMemorySessionStateTracker(); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", IPAddress.Parse("192.0.2.10"), sessionLimit: 0, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.SourceAddressLimitExceeded, + tracker.TryAdmit("alice", "b", IPAddress.Parse("2001:db8::10"), sessionLimit: 0, srcIpLimit: 1)); + } + + [Fact] + public void AccountsAreIsolated() + { + var tracker = new InMemorySessionStateTracker(); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("alice", "a", IPAddress.Loopback, sessionLimit: 1, srcIpLimit: 1)); + Assert.Equal( + SessionAdmissionResult.Success, + tracker.TryAdmit("bob", "b", IPAddress.Loopback, sessionLimit: 1, srcIpLimit: 1)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementConnectionTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementConnectionTests.cs new file mode 100644 index 00000000..d33bc8f3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementConnectionTests.cs @@ -0,0 +1,764 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Transport.Vatp; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Tests.Storage; + +public sealed class ArticlePlacementConnectionTests +{ + [Fact] + public async Task Several_articles_reuse_one_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + for (var i = 0; i < 3; i++) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal(ArticlePlacementKind.Accepted, result.Kind); + } + + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + Assert.Equal(3, server.Results); + } + + [Fact] + public async Task Two_targets_do_not_share_a_connection() + { + await using var left = StoreServer.Start(); + await using var right = StoreServer.Start(); + await using var client = Client(maxOutstanding: 1); + var a = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var b = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(a.Record, Entry(left.Port, "cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(b.Record, Entry(right.Port, "cache02.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(2, client.ConnectCount); + Assert.Equal(1, left.AcceptedConnections); + Assert.Equal(1, right.AcceptedConnections); + } + + [Fact] + public async Task Fqdn_case_uses_one_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 1); + var lower = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var upper = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(lower.Record, Entry(server.Port, "cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(upper.Record, Entry(server.Port, "Cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + } + + [Fact] + public async Task Limit_one_does_not_overlap_stores() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await Task.Delay(50); + Assert.Equal(1, server.PeakOutstanding); + server.ReleaseResults(); + Assert.Equal(ArticlePlacementKind.Accepted, (await firstTask).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await secondTask).Kind); + Assert.Equal(1, server.PeakOutstanding); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Two_stores_complete_with_the_result_for_their_stream() + { + await using var server = StoreServer.Start(holdResult: true, duplicateSecond: true); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(2); + Assert.Equal(2, server.PeakOutstanding); + server.ReleaseResults(); + var results = await Task.WhenAll(firstTask, secondTask); + Assert.Contains(results, static result => result.Kind == ArticlePlacementKind.Accepted); + Assert.Contains(results, static result => result.Kind == ArticlePlacementKind.Duplicate); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(0, server.ParseFailures); + } + + [Fact] + public async Task Dead_connection_is_replaced_once() + { + await using var server = StoreServer.Start(closeAfterAccepted: 1); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(first.Record, target, CancellationToken.None)).Kind); + var replaced = ArticlePlacementKind.TransportFailure; + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline && replaced != ArticlePlacementKind.Accepted) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + replaced = (await client.PlaceAsync(article.Record, target, CancellationToken.None)).Kind; + } + + Assert.Equal(ArticlePlacementKind.Accepted, replaced); + Assert.Equal(2, client.ConnectCount); + Assert.Equal(2, server.AcceptedConnections); + } + + [Fact] + public async Task Closed_port_does_not_dial_again_during_backoff() + { + await using var client = new ArticlePlacementClient( + NullLogger.Instance, + new VatpClientOptions(), + ArticleTransferLimits.Default, + maxOutstandingStores: 1) + { + TestTcpConnectHost = "127.0.0.1", + }; + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var target = Entry(port); + var first = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal(ArticlePlacementKind.TransportFailure, first.Kind); + Assert.StartsWith("SocketException:", first.Failure, StringComparison.Ordinal); + var second = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal("backoff", second.Failure); + Assert.Equal(1, client.DialAttempts); + } + + [Fact] + public async Task Drop_after_end_is_not_success() + { + await using var server = StoreServer.Start(dropAfterEnd: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Remote_rejection_is_not_success() + { + await using var server = StoreServer.Start(reject: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.RejectedInvalid, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Cancel_while_waiting_for_a_slot_sends_nothing() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var held = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var heldTask = client.PlaceAsync(held.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + var queued = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(queued.Record, target, cts.Token); + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(1, server.Stores); + server.ReleaseResults(); + Assert.Equal(ArticlePlacementKind.Accepted, (await heldTask).Kind); + } + + [Fact] + public async Task Shutdown_cancels_an_outstanding_store_and_rejects_new_calls() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + await client.DisposeAsync(); + var result = await pending; + Assert.NotEqual(ArticlePlacementKind.Accepted, result.Kind); + var later = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.TransportFailure, later.Kind); + Assert.Equal("shutdown", later.Failure); + } + + [Fact] + public async Task Stream_ids_are_not_reused_on_a_live_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + for (var i = 0; i < 3; i++) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(article.Record, target, CancellationToken.None)).Kind); + } + + var ids = server.StreamIds; + Assert.Equal(new uint[] { 1, 2, 3 }, ids); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Cancel_during_dial_does_not_open_a_second_connection() + { + await using var server = StoreServer.Start(handshakeDelay: TimeSpan.FromMilliseconds(400)); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + using var cts = new CancellationTokenSource(); + var firstArticle = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var secondArticle = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var first = client.PlaceAsync(firstArticle.Record, target, cts.Token).AsTask(); + var started = DateTime.UtcNow; + while (client.DialAttempts < 1) + { + if (DateTime.UtcNow - started > TimeSpan.FromSeconds(2)) + { + throw new TimeoutException("Dial did not start."); + } + + await Task.Delay(5); + } + + var second = client.PlaceAsync(secondArticle.Record, target, CancellationToken.None).AsTask(); + await cts.CancelAsync(); + var firstResult = await first; + var secondResult = await second; + Assert.Equal(ArticlePlacementKind.Cancelled, firstResult.Kind); + Assert.Equal(ArticlePlacementKind.Accepted, secondResult.Kind); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + } + + [Fact] + public async Task Two_streams_keep_independent_window_credit() + { + var limits = new ArticleTransferLimits { InitialStreamWindowBytes = 32 }; + await using var server = StoreServer.Start(initialCredit: 32); + await using var client = Client(maxOutstanding: 2, limits: limits); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('a', 200) + "\r\n"); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('b', 200) + "\r\n"); + var results = await Task.WhenAll( + client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(), + client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask()); + Assert.All(results, static result => Assert.Equal(ArticlePlacementKind.Accepted, result.Kind)); + Assert.Equal(0, server.ParseFailures); + Assert.Equal(2, server.PeakOutstanding); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Connection_loss_after_end_is_ambiguous_for_every_outstanding_store() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await server.WaitForEndsAsync(2); + server.DropConnections(); + var results = await Task.WhenAll(firstTask, secondTask); + Assert.All(results, static result => Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind)); + Assert.All(results, static result => Assert.False(result.Succeeded)); + } + + [Fact] + public async Task Cancel_while_blocked_on_credit_does_not_report_success() + { + var limits = new ArticleTransferLimits { InitialStreamWindowBytes = 16 }; + await using var server = StoreServer.Start(grantWindows: false, initialCredit: 16); + await using var client = Client(maxOutstanding: 1, limits: limits, ioTimeout: TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('c', 400) + "\r\n"); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), cts.Token).AsTask(); + await server.WaitForDataAsync(1); + await cts.CancelAsync(); + var result = await pending; + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(0, server.Ends); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Client_does_not_send_the_body_before_window_credit() + { + await using var server = StoreServer.Start(initialCredit: 0); + await using var client = Client(maxOutstanding: 1, ioTimeout: TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(); + var article = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + body: new string('d', 400) + "\r\n"); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), cts.Token).AsTask(); + await server.WaitForStoresAsync(1); + Assert.Equal(0, server.DataFrames); + await cts.CancelAsync(); + var result = await pending; + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(0, server.DataFrames); + } + + [Fact] + public async Task Timeout_after_end_is_ambiguous() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1, ioTimeout: TimeSpan.FromMilliseconds(300)); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Timeout_before_handshake_is_not_success() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var accept = listener.AcceptTcpClientAsync(); + await using var client = new ArticlePlacementClient( + NullLogger.Instance, + new VatpClientOptions + { + ConnectTimeout = TimeSpan.FromMilliseconds(200), + TlsHandshakeTimeout = TimeSpan.FromMilliseconds(200), + IoTimeout = TimeSpan.FromMilliseconds(200), + }, + ArticleTransferLimits.Default, + maxOutstandingStores: 1) + { + TestTcpConnectHost = "127.0.0.1", + ServerCertificateValidationCallback = static (_, _, _, _) => true, + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var place = client.PlaceAsync(article.Record, Entry(port), CancellationToken.None).AsTask(); + using var accepted = await accept; + var result = await place; + listener.Stop(); + Assert.Equal(ArticlePlacementKind.TransportFailure, result.Kind); + Assert.Equal("timeout", result.Failure); + Assert.False(result.Succeeded); + } + + [Fact] + public void Production_placement_stays_disabled() + { + var property = typeof(VectorNNTPD.ReaderService.ArticleIngestion.IncomingSpoolWriterService) + .GetProperty("ArticlePlacementEnabled", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + Assert.NotNull(property); + Assert.False((bool)property.GetValue(null)!); + } + + [Fact] + public void Outstanding_limit_rejects_values_other_than_one_or_two() + { + Assert.Throws(() => + new ArticlePlacementClient(NullLogger.Instance, 64)); + } + + private static ArticlePlacementClient Client( + int maxOutstanding, + ArticleTransferLimits? limits = null, + TimeSpan? ioTimeout = null) => + new( + NullLogger.Instance, + new VatpClientOptions + { + ConnectTimeout = TimeSpan.FromSeconds(2), + TlsHandshakeTimeout = TimeSpan.FromSeconds(2), + IoTimeout = ioTimeout ?? TimeSpan.FromSeconds(2), + }, + limits ?? ArticleTransferLimits.Default, + maxOutstanding) + { + TestTcpConnectHost = "127.0.0.1", + ServerCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static StorageServerFleetEntry Entry(int port, string fqdn = "cache01.usenet.ninja") => + new(1, fqdn, 10_000, 0, 10_000, DateTimeOffset.UtcNow, port); + + private sealed class StoreServer : IAsyncDisposable + { + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _accept; + private readonly ConcurrentDictionary _streams = new(); + private readonly ConcurrentQueue _streamIds = new(); + private int _outstanding; + private int _peak; + private int _stores; + private int _ends; + private int _dataFrames; + private int _results; + private int _acceptedConnections; + private int _parseFailures; + + private StoreServer( + TcpListener listener, + bool holdResult, + bool duplicateSecond, + bool dropAfterEnd, + bool reject, + int closeAfterAccepted, + bool grantWindows, + int initialCredit, + TimeSpan handshakeDelay) + { + _listener = listener; + HoldResult = holdResult; + DuplicateSecond = duplicateSecond; + DropAfterEnd = dropAfterEnd; + Reject = reject; + CloseAfterAccepted = closeAfterAccepted; + GrantWindows = grantWindows; + InitialCredit = initialCredit; + HandshakeDelay = handshakeDelay; + Port = ((IPEndPoint)listener.LocalEndpoint).Port; + _accept = Task.Run(AcceptLoop); + } + + public int Port { get; } + + public bool HoldResult { get; } + + public bool DuplicateSecond { get; } + + public bool DropAfterEnd { get; } + + public bool Reject { get; } + + public int CloseAfterAccepted { get; } + + public bool GrantWindows { get; } + + public int InitialCredit { get; } + + public TimeSpan HandshakeDelay { get; } + + public int PeakOutstanding => Volatile.Read(ref _peak); + + public int Stores => Volatile.Read(ref _stores); + + public int Ends => Volatile.Read(ref _ends); + + public int DataFrames => Volatile.Read(ref _dataFrames); + + public uint[] StreamIds => _streamIds.ToArray(); + + public int Results => Volatile.Read(ref _results); + + public int AcceptedConnections => Volatile.Read(ref _acceptedConnections); + + public int ParseFailures => Volatile.Read(ref _parseFailures); + + private TaskCompletionSource ResultsReleased { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public static StoreServer Start( + bool holdResult = false, + bool duplicateSecond = false, + bool dropAfterEnd = false, + bool reject = false, + int closeAfterAccepted = 0, + bool grantWindows = true, + int initialCredit = 1024 * 1024, + TimeSpan handshakeDelay = default) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return new StoreServer( + listener, + holdResult, + duplicateSecond, + dropAfterEnd, + reject, + closeAfterAccepted, + grantWindows, + initialCredit, + handshakeDelay); + } + + public void DropConnections() + { + foreach (var stream in _streams.Keys) + { + stream.Close(); + } + } + + public Task WaitForStoresAsync(int count) => WaitForAsync(() => Stores, count, "STORE frames did not arrive."); + + public Task WaitForEndsAsync(int count) => WaitForAsync(() => Ends, count, "END frames did not arrive."); + + public Task WaitForDataAsync(int count) => WaitForAsync(() => DataFrames, count, "DATA frames did not arrive."); + + private static async Task WaitForAsync(Func read, int count, string message) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (read() < count) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException(message); + } + + await Task.Delay(10); + } + } + + public void ReleaseResults() => ResultsReleased.TrySetResult(); + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _listener.Stop(); + try + { + await _accept.WaitAsync(TimeSpan.FromSeconds(2)); + } + catch (Exception ex) when (ex is OperationCanceledException or TimeoutException or SocketException or ObjectDisposedException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private async Task AcceptLoop() + { + while (!_stop.IsCancellationRequested) + { + TcpClient tcp; + try + { + tcp = await _listener.AcceptTcpClientAsync(_stop.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + return; + } + + Interlocked.Increment(ref _acceptedConnections); + _ = HandleAsync(tcp); + } + } + + private async Task HandleAsync(TcpClient tcp) + { + var certificate = VatpTestCertificates.CreateServerCertificate(); + using var tcpDisposable = tcp; + // codeql[cs/local-not-disposed] SslStream ownership transfers to the VATP/TLS session fixture for connection lifetime. + var ssl = new SslStream(tcp.GetStream(), leaveInnerStreamOpen: false); + _streams.TryAdd(ssl, 0); + try + { + if (HandshakeDelay > TimeSpan.Zero) + { + await Task.Delay(HandshakeDelay, _stop.Token); + } + + await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions + { + ServerCertificate = certificate, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }); + var buffer = new byte[64 * 1024]; + var held = new List(); + var credit = new Dictionary(); + var order = new List(); + var pendingResults = new List(); + Task? readTask = null; + while (!_stop.IsCancellationRequested) + { + if (HoldResult && ResultsReleased.Task.IsCompleted) + { + await WritePendingResultsAsync(ssl, pendingResults, order); + } + + var parsed = VatpFrameParser.ParseOneFrame(held.ToArray(), VatpProtocol.DefaultMaxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + Interlocked.Increment(ref _parseFailures); + return; + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } frame) + { + readTask ??= ssl.ReadAsync(buffer, _stop.Token).AsTask(); + if (HoldResult && !ResultsReleased.Task.IsCompleted) + { + await Task.WhenAny(readTask, ResultsReleased.Task); + if (!readTask.IsCompleted) + { + continue; + } + } + + var read = await readTask; + readTask = null; + if (read == 0) + { + return; + } + + for (var i = 0; i < read; i++) + { + held.Add(buffer[i]); + } + + continue; + } + + held.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + if (frame.Header.Type == VatpFrameType.Hello) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeHello(VatpProtocol.DefaultMaxFramePayload)); + continue; + } + + if (frame.Header.Type == VatpFrameType.Store) + { + var now = Interlocked.Increment(ref _outstanding); + int previous; + do + { + previous = Volatile.Read(ref _peak); + if (now <= previous) + { + break; + } + } + while (Interlocked.CompareExchange(ref _peak, now, previous) != previous); + + Interlocked.Increment(ref _stores); + credit[frame.Header.StreamId] = InitialCredit; + _streamIds.Enqueue(frame.Header.StreamId); + order.Add(frame.Header.StreamId); + if (InitialCredit > 0) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)InitialCredit)); + } + + continue; + } + + if (frame.Header.Type == VatpFrameType.Data) + { + var length = frame.Payload.Length; + if (!credit.TryGetValue(frame.Header.StreamId, out var available) || length > available) + { + Interlocked.Increment(ref _parseFailures); + return; + } + + credit[frame.Header.StreamId] = available - length; + Interlocked.Increment(ref _dataFrames); + if (length > 0 && GrantWindows) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)length)); + credit[frame.Header.StreamId] += length; + } + + continue; + } + + if (frame.Header.Type != VatpFrameType.End) + { + continue; + } + + Interlocked.Increment(ref _ends); + if (DropAfterEnd) + { + return; + } + + if (HoldResult && !ResultsReleased.Task.IsCompleted) + { + pendingResults.Add(frame.Header.StreamId); + continue; + } + + if (await WriteResultAsync(ssl, frame.Header.StreamId, order)) + { + return; + } + } + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or ObjectDisposedException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + _streams.TryRemove(ssl, out _); + await ssl.DisposeAsync(); + } + } + + private async Task WritePendingResultsAsync(SslStream ssl, List pendingResults, List order) + { + while (pendingResults.Count > 0) + { + var streamId = pendingResults[0]; + pendingResults.RemoveAt(0); + if (await WriteResultAsync(ssl, streamId, order)) + { + return; + } + } + } + + private async Task WriteResultAsync(SslStream ssl, uint streamId, List order) + { + var index = order.IndexOf(streamId); + var outcome = Reject + ? (byte)ArticlePlacementKind.RejectedInvalid + : DuplicateSecond && index == 1 + ? (byte)ArticlePlacementKind.Duplicate + : (byte)ArticlePlacementKind.Accepted; + await WriteAsync(ssl, VatpFrameEncoder.EncodeResult(streamId, outcome)); + Interlocked.Decrement(ref _outstanding); + var sent = Interlocked.Increment(ref _results); + return CloseAfterAccepted > 0 && sent >= CloseAfterAccepted; + } + + private static async Task WriteAsync(Stream stream, VatpFrameEncoder.EncodedFrame frame) + { + if (!frame.Header.IsEmpty) + { + await stream.WriteAsync(frame.Header); + } + + if (!frame.Payload.IsEmpty) + { + await stream.WriteAsync(frame.Payload); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementTests.cs new file mode 100644 index 00000000..761a6f2a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Storage/ArticlePlacementTests.cs @@ -0,0 +1,1183 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Reflection; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Tests.Storage; + +public sealed class ArticlePlacementTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.Parse("2026-09-30T00:00:00Z"); + + [Fact] + public void Selector_PrefersHighestAvailableBytes_ThenServerId_ThenFqdn() + { + var entries = new StorageServerFleetEntry[] + { + Entry("cache02.example", 2, available: 10, port: 563), + Entry("cache09.example", 9, available: 50, port: 563), + Entry("cache03.example", 3, available: 50, port: 563), + }; + Assert.True(StorageServerPlacementSelector.TrySelect(entries, out var selected)); + Assert.Equal(3, selected.ServerId); + + var tiedId = new StorageServerFleetEntry[] + { + Entry("m.example", 4, available: 20, port: 563), + Entry("a.example", 4, available: 20, port: 563), + }; + Assert.True(StorageServerPlacementSelector.TrySelect(tiedId, out var fqdn)); + Assert.Equal("a.example", fqdn.Fqdn); + } + + [Fact] + public void Selector_ExcludesMissingPort_AndEmptySetDoesNotSelect() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 100, port: null), Now); + registry.ApplyAdvertisement(Advertisement("stale.example", 2, 500, port: 563), Now.AddSeconds(-10)); + var active = registry.GetActive(Now); + Assert.Contains(active, static entry => entry.Fqdn == "cache01.example"); + Assert.DoesNotContain(active, static entry => entry.Fqdn == "stale.example"); + Assert.False(StorageServerPlacementSelector.TrySelect(active, out _)); + Assert.False(StorageServerPlacementSelector.TrySelect([], out _)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + public async Task LocalProducer_PlacesOnceAfterPersist(InboundArticleProducer producer) + { + var client = new RecordingPlacement(); + var registry = RegistryWithTarget(); + var article = CanonicalArticleText.CreateQueued("", producer); + await RunAsync(article, client, registry); + Assert.Equal(0, client.Calls); + Assert.False(client.SawPersist); + } + + [Fact] + public async Task BackFiller_StoresOnceOnTheLowestServerId() + { + var client = new RecordingPlacement(); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Fact] + public async Task NoEligibleServer_DoesNotDial() + { + var client = new RecordingPlacement { FailIfCalled = true }; + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 100, port: null), Now); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, registry, time: Now); + Assert.Equal(0, client.Calls); + } + + [Fact] + public async Task PlacementFailure_DoesNotRequeue() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Conflict), + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunAsync(article, client, RegistryWithTarget()); + Assert.Equal(0, client.Calls); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + public async Task LocalProducer_PlacesOnceAndDoesNotSelectASecondTarget(InboundArticleProducer producer) + { + var client = new RecordingPlacement(); + var article = CanonicalArticleText.CreateQueued("", producer); + var logs = new ListLogger(); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Fact] + public async Task BackFiller_Duplicate_IsOneStoreAndDoesNotReplicate() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Duplicate), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2621, logs.EventIds); + } + + [Fact] + public async Task BackFiller_Conflict_LogsAndDoesNotStoreAgain() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Conflict), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2622, logs.EventIds); + } + + [Theory] + [InlineData(ArticlePlacementKind.RejectedCapacity, 2623)] + [InlineData(ArticlePlacementKind.RejectedInvalid, 2625)] + public async Task BackFiller_Rejection_DoesNotTryAnotherServer(ArticlePlacementKind kind, int eventId) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(kind), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(eventId, logs.EventIds); + } + + [Fact] + public async Task BackFiller_TransportFailure_DoesNotTryAnotherServer() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "IOException"), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2627, logs.EventIds); + } + + [Fact] + public async Task PeerProducer_PlacesOnceOnTheHighestFreeSpace() + { + var client = new RecordingPlacement(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunObservedAsync(article, client, DivergentTargets()); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + } + + [Fact] + public async Task BackFiller_ConcurrentArticles_DoNotShareStoreState() + { + var client = new ConcurrentPlacement(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunManyAsync([first, second], client, DivergentTargets(), workers: 2); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + Assert.NotEqual(first.Record.ArtId, second.Record.ArtId); + } + + [Fact] + public async Task BackFiller_SameArticleConcurrently_StoresTwice() + { + var client = new ConcurrentPlacement(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + Assert.Equal(first.Record.ArtId, second.Record.ArtId); + await RunManyAsync([first, second], client, DivergentTargets(), workers: 2); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Records); + Assert.Empty(client.Targets); + } + + [Fact] + public async Task NoSecondEligible_LeavesSingleCopy() + { + var client = new RecordingPlacement(); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, RegistryWithTarget(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Theory] + [InlineData(ArticlePlacementKind.Conflict, 2622)] + [InlineData(ArticlePlacementKind.RejectedCapacity, 2623)] + [InlineData(ArticlePlacementKind.RejectedInvalid, 2625)] + [InlineData(ArticlePlacementKind.TransportFailure, 2627)] + [InlineData(ArticlePlacementKind.AcknowledgementNotObserved, 2629)] + public async Task FirstFailure_DoesNotAttemptReplica(ArticlePlacementKind kind, int eventId) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(kind, kind == ArticlePlacementKind.TransportFailure ? "IOException" : null), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(eventId, logs.EventIds); + } + + [Fact] + public async Task CancellationBeforeFirstAcceptance_DoesNotAttemptReplica() + { + using var cts = new CancellationTokenSource(); + var client = new RecordingPlacement + { + CancelAt = CancelPoint.FirstCallReturnsCancelled, + Cancel = cts, + }; + var logs = new ListLogger(); + await PlaceDirectAsync( + CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis), + client, + TwoTargets(), + cts.Token, + logs); + Assert.Equal(1, client.Calls); + Assert.Equal(ArticlePlacementKind.Cancelled, client.Outcomes[0].Kind); + Assert.Equal("cache01.example", client.Targets[0].Fqdn); + Assert.Contains(2628, logs.EventIds); + Assert.Single(client.Targets); + Assert.DoesNotContain(logs.EventIds, static id => id >= 2630); + } + + [Fact] + public async Task DuplicatePrimary_CompletesAfterOnePlacement() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Duplicate), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2621, logs.EventIds); + } + + [Fact] + public async Task CancellationAfterFirstAcceptance_DoesNotStartReplica() + { + using var cts = new CancellationTokenSource(); + var client = new RecordingPlacement + { + CancelAt = CancelPoint.AfterFirstAccepted, + Cancel = cts, + }; + var logs = new ListLogger(); + await PlaceDirectAsync( + CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis), + client, + TwoTargets(), + cts.Token, + logs); + Assert.Equal(1, client.Calls); + Assert.Equal(ArticlePlacementKind.Accepted, client.Outcomes[0].Kind); + Assert.Equal("cache01.example", client.Targets[0].Fqdn); + Assert.Contains(2620, logs.EventIds); + Assert.Single(client.Targets); + Assert.DoesNotContain(logs.EventIds, static id => id >= 2630); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_RetriesOnceOnTheSameTargetAndPayload(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(TimeSpan.FromSeconds(5), delay.Requested); + Assert.Equal(1, delay.Calls); + AssertSamePlacement(article, client, producer); + Assert.Equal(ArticlePlacementKind.Accepted, client.Outcomes[1].Kind); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=Accepted"); + Assert.Equal(2, logs.EventIds.Count(static id => id is 2624 or 2620)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_SecondDuplicate_SettlesOnTheSameTarget(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.Duplicate)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(1, delay.Calls); + Assert.Equal(TimeSpan.FromSeconds(5), delay.Requested); + AssertSamePlacement(article, client, producer); + Assert.Equal(ArticlePlacementKind.Duplicate, client.Outcomes[1].Kind); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=Duplicate"); + Assert.Contains(2621, logs.EventIds); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_SecondPressure_IsTerminal(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(2, client.Calls); + Assert.Equal(1, delay.Calls); + AssertSamePlacement(article, client, producer); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=RejectedPressure"); + Assert.Equal(2, logs.Messages.Count(static message => message.Contains("outcome=RejectedPressure", StringComparison.Ordinal))); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=3", StringComparison.Ordinal)); + } + + [Theory] + [MemberData(nameof(TerminalPlacementOutcomes))] + public async Task TerminalOutcomes_AreNotRetried(InboundArticleProducer producer, ArticlePlacementKind kind) + { + var delay = new ImmediatePressureDelay(); + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult( + kind, + kind == ArticlePlacementKind.TransportFailure ? "IOException" : null), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(1, client.Calls); + Assert.Equal(0, delay.Calls); + AssertOriginalTarget(producer, client.Targets[0]); + AssertSameArtData(article.Record, client.Records[0]); + Assert.Contains(logs.Messages, message => message.Contains("attempt=1", StringComparison.Ordinal)); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=2", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task CancellationDuringPressureDelay_DoesNotStartAttemptTwo(InboundArticleProducer producer) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + }; + var logs = new ListLogger(); + using var cts = new CancellationTokenSource(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var article = CanonicalArticleText.CreateQueued($"", producer); + var pending = InvokePlacementAsync( + article, + client, + async (delay, token) => + { + Assert.Equal(TimeSpan.FromSeconds(5), delay); + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token).ConfigureAwait(false); + }, + cts.Token, + logs); + + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await pending.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, client.Calls); + AssertOriginalTarget(producer, client.Targets[0]); + AssertSameArtData(article.Record, client.Records[0]); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + Assert.Contains( + logs.Messages, + static message => message.Contains("placement cancelled", StringComparison.Ordinal) + && message.Contains("attempt=1", StringComparison.Ordinal)); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=2", StringComparison.Ordinal)); + } + + public static IEnumerable TerminalPlacementOutcomes() + { + var producers = new[] { InboundArticleProducer.TakeThis, InboundArticleProducer.BackFiller }; + var kinds = new[] + { + ArticlePlacementKind.RejectedCapacity, + ArticlePlacementKind.Conflict, + ArticlePlacementKind.RejectedInvalid, + ArticlePlacementKind.TransportFailure, + ArticlePlacementKind.AcknowledgementNotObserved, + ArticlePlacementKind.Cancelled, + }; + foreach (var producer in producers) + { + foreach (var kind in kinds) + { + yield return [producer, kind]; + } + } + } + + [Fact] + public void Placement_DoesNotReferenceLookupOrReadPool() + { + var placementFields = typeof(ArticlePlacementClient).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(placementFields, static field => field.FieldType.Name.Contains("Lookup", StringComparison.Ordinal)); + var writerFields = typeof(IncomingSpoolWriterService).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(writerFields, static field => field.FieldType.Name.Contains("Lookup", StringComparison.Ordinal)); + Assert.DoesNotContain( + typeof(StorageArticleLookupService).GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic), + static method => method.Name.Contains("Replica", StringComparison.Ordinal) + || method.Name.Contains("Primary", StringComparison.Ordinal)); + } + + [Fact] + public async Task PlacementClient_SocketException_ReportsErrorCodeHResultAndMessage() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + + var client = new ArticlePlacementClient(NullLogger.Instance) + { + TestTcpConnectHost = "127.0.0.1", + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry("cache01.usenet.ninja", 1, 100, port), CancellationToken.None); + + Assert.Equal(ArticlePlacementKind.TransportFailure, result.Kind); + Assert.NotNull(result.Failure); + Assert.StartsWith("SocketException: SocketErrorCode=", result.Failure, StringComparison.Ordinal); + Assert.Contains("SocketErrorCode=ConnectionRefused", result.Failure, StringComparison.Ordinal); + Assert.Contains(" HResult=", result.Failure, StringComparison.Ordinal); + Assert.Contains(" Message=", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void PlacementClient_DoesNotUseReadConnectionPool() + { + var fields = typeof(ArticlePlacementClient).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(fields, static field => field.FieldType == typeof(VatpConnectionPool)); + Assert.False(typeof(VatpArticleClient).IsAssignableTo(typeof(IArticlePlacementClient))); + } + + [Fact] + public async Task Transfer_ResultAccepted_AndWindowPacing() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + var limits = new ArticleTransferLimits + { + InitialStreamWindowBytes = 32, + MaxStreamCreditBytes = 1024 * 1024, + DefaultMaxFramePayload = 64, + }; + Assert.True(article.ArtSize > 32); + await using var link = new Link(); + var server = Task.Run(() => ScriptedServer.RunAsync(link.Server, result: (byte)ArticlePlacementKind.Accepted)); + var result = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + limits, + TimeSpan.FromSeconds(5), + CancellationToken.None); + Assert.Equal(ArticlePlacementKind.Accepted, result.Kind); + Assert.True(await server.WaitAsync(TimeSpan.FromSeconds(5))); + } + + [Fact] + public async Task Transfer_DropAfterEnd_IsAcknowledgementNotObserved() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + await using var link = new Link(); + var server = Task.Run(() => ScriptedServer.RunAsync(link.Server, result: null)); + var placed = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + ArticleTransferLimits.Default, + TimeSpan.FromSeconds(5), + CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, placed.Kind); + Assert.False(placed.Succeeded); + await server.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Transfer_CancelBeforeSend_DoesNotReportSuccess() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + await using var link = new Link(); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + var placed = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + ArticleTransferLimits.Default, + TimeSpan.FromSeconds(5), + cts.Token); + Assert.Equal(ArticlePlacementKind.Cancelled, placed.Kind); + Assert.False(placed.Succeeded); + } + + private static Task RunAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs = null, + DateTimeOffset? time = null) => + RunCoreAsync(article, client, registry, logs, time); + + private static async Task PlaceDirectAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + CancellationToken cancellationToken, + ListLogger? logs = null) + { + var writer = CreateWriter(client, registry, logs, time: null); + var place = typeof(IncomingSpoolWriterService).GetMethod( + "PlaceAfterPersistAsync", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(place); + try + { + var pending = (Task)place.Invoke(writer, [article, cancellationToken])!; + await pending.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static IncomingSpoolWriterService CreateWriter( + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs, + DateTimeOffset? time) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + return new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(time ?? Now), + placement: client, + placementRegistry: registry); + } + + private static async Task RunObservedAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs = null) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task RunManyAsync( + InboundArticle[] articles, + ConcurrentPlacement client, + StorageServerRegistry registry, + int workers) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions(workers)), + NullLogger.Instance, + timeProvider: new FixedTime(Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + foreach (var article in articles) + { + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task RunCoreAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs, + DateTimeOffset? time) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(time ?? Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task InvokePlacementAsync( + InboundArticle article, + RecordingPlacement client, + Func delay, + CancellationToken cancellationToken, + ListLogger? logs = null) + { + var writer = CreateWriter( + client, + TargetsFor(article.Producer), + logs, + time: null); + writer.PressureRetryDelay = delay; + var methodName = article.Producer == InboundArticleProducer.BackFiller + ? "StoreBackFillerAsync" + : "PlaceAfterPersistAsync"; + var place = typeof(IncomingSpoolWriterService).GetMethod( + methodName, + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(place); + try + { + var pending = (Task)place.Invoke(writer, [article, cancellationToken])!; + await pending; + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static RecordingPlacement SequenceClient(params ArticlePlacementResult[] sequence) => + new() + { + Sequence = sequence, + ExpectedCalls = sequence.Length, + }; + + private static StorageServerRegistry TargetsFor(InboundArticleProducer producer) => + producer == InboundArticleProducer.BackFiller ? DivergentTargets() : TwoTargets(); + + private static void AssertOriginalTarget(InboundArticleProducer producer, StorageServerFleetEntry target) + { + if (producer == InboundArticleProducer.BackFiller) + { + Assert.Equal(2, target.ServerId); + Assert.Equal("lowid.example", target.Fqdn); + return; + } + + Assert.Equal(1, target.ServerId); + Assert.Equal("cache01.example", target.Fqdn); + } + + private static void AssertSamePlacement( + InboundArticle article, + RecordingPlacement client, + InboundArticleProducer producer) + { + Assert.Equal(2, client.Calls); + Assert.Equal(2, client.Targets.Count); + Assert.Equal(2, client.Records.Count); + AssertOriginalTarget(producer, client.Targets[0]); + Assert.Equal(client.Targets[0].ServerId, client.Targets[1].ServerId); + Assert.Equal(client.Targets[0].Fqdn, client.Targets[1].Fqdn); + Assert.Equal(client.Targets[0].VatpPort, client.Targets[1].VatpPort); + Assert.Equal(article.Record.ArtId, client.Records[0].ArtId); + Assert.Equal(article.Record.ArtId, client.Records[1].ArtId); + AssertSameArtData(article.Record, client.Records[0]); + AssertSameArtData(article.Record, client.Records[1]); + AssertSameArtData(client.Records[0], client.Records[1]); + } + + private static void AssertAttempt(IReadOnlyList messages, int attempt, string outcome) => + Assert.Contains( + messages, + message => message.Contains($"attempt={attempt}", StringComparison.Ordinal) + && message.Contains(outcome, StringComparison.Ordinal)); + + private static void AssertSameArtData(ArticleRecord expected, ArticleRecord actual) + { + Assert.True(MemoryMarshal.TryGetArray(expected.ArtData, out var expectedArray)); + Assert.True(MemoryMarshal.TryGetArray(actual.ArtData, out var actualArray)); + Assert.Same(expectedArray.Array, actualArray.Array); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + Assert.Equal(expectedArray.Count, actualArray.Count); + } + + private static NntpdOptions PlacementOptions(int workers = 1) => + new() + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = workers, + MaxWorkers = workers, + ScaleIntervalSeconds = 3600, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + }, + }; + + private static StorageServerRegistry TwoTargets() + { + var registry = RegistryWithTarget(); + registry.ApplyAdvertisement(Advertisement("cache02.example", 2, 100, 564), Now); + return registry; + } + + private static StorageServerRegistry DivergentTargets() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("lowid.example", 2, 10, 564), Now); + registry.ApplyAdvertisement(Advertisement("highspace.example", 9, 5000, 563), Now); + return registry; + } + + private static StorageServerRegistry RegistryWithTarget() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 1000, 563), Now); + return registry; + } + + private static StorageServerAdvertisement Advertisement(string fqdn, int serverId, long available, int? port) => + new(1, serverId, fqdn, 10_000, 10_000 - available, available, Now, port); + + private static StorageServerFleetEntry Entry(string fqdn, int serverId, long available, int? port) => + new(serverId, fqdn, 10_000, 10_000 - available, available, Now, port); + + private sealed class FixedTime(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } + + private enum CancelPoint + { + None, + FirstCallReturnsCancelled, + AfterFirstAccepted, + } + + private sealed class ConcurrentPlacement : IArticlePlacementClient + { + private readonly Barrier _barrier = new(2); + private readonly List _targets = []; + private readonly List _records = []; + private readonly object _gate = new(); + + public int Calls { get; private set; } + + public IReadOnlyList Targets + { + get + { + lock (_gate) + { + return [.. _targets]; + } + } + } + + public IReadOnlyList Records + { + get + { + lock (_gate) + { + return [.. _records]; + } + } + } + + public TaskCompletionSource BothStored { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + lock (_gate) + { + Calls++; + _targets.Add(target); + _records.Add(record); + } + + _barrier.SignalAndWait(cancellationToken); + BothStored.TrySetResult(); + return ValueTask.FromResult(new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + } + } + + private sealed class RecordingPlacement : IArticlePlacementClient + { + private readonly List _targets = []; + private readonly List _records = []; + private readonly List _outcomes = []; + + public int Calls { get; private set; } + + public bool SawPersist { get; private set; } + + public bool FailIfCalled { get; init; } + + public ArticleRecord Last { get; private set; } + + public ArticlePlacementResult Result { get; init; } = new(ArticlePlacementKind.Accepted); + + public ArticlePlacementResult[]? Sequence { get; init; } + + public int ExpectedCalls { get; init; } = 1; + + public CancelPoint CancelAt { get; init; } + + public CancellationTokenSource? Cancel { get; init; } + + public Action? BeforeStore { get; init; } + + public IReadOnlyList Targets => _targets; + + public IReadOnlyList Records => _records; + + public IReadOnlyList Outcomes => _outcomes; + + public TaskCompletionSource Finished { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + if (FailIfCalled) + { + throw new InvalidOperationException("Placement dialed without an eligible server."); + } + + var call = ++Calls; + SawPersist = true; + Last = record; + _targets.Add(target); + _records.Add(record); + BeforeStore?.Invoke(call, target); + + ArticlePlacementResult outcome; + if (CancelAt == CancelPoint.FirstCallReturnsCancelled && call == 1) + { + Cancel!.Cancel(); + outcome = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + else if (CancelAt == CancelPoint.AfterFirstAccepted && call == 1) + { + outcome = new ArticlePlacementResult(ArticlePlacementKind.Accepted); + Cancel!.Cancel(); + } + else if (cancellationToken.IsCancellationRequested) + { + outcome = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + else + { + outcome = Sequence is { } sequence && call - 1 < sequence.Length + ? sequence[call - 1] + : Result; + } + + _outcomes.Add(outcome); + if (call >= ExpectedCalls) + { + Finished.TrySetResult(); + } + + return ValueTask.FromResult(outcome); + } + } + + private sealed class ImmediatePressureDelay + { + public int Calls { get; private set; } + + public TimeSpan Requested { get; private set; } + + public Task Delay(TimeSpan delay, CancellationToken cancellationToken) + { + Calls++; + Requested = delay; + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + private sealed class ListLogger : ILogger + { + public List EventIds { get; } = []; + + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + EventIds.Add(eventId.Id); + Messages.Add(formatter(state, exception)); + } + } + + private sealed class Link : IAsyncDisposable + { + private readonly Pipe _toServer = new(); + private readonly Pipe _toClient = new(); + + public Link() + { + Client = new PipeStream(_toClient.Reader, _toServer.Writer); + Server = new PipeStream(_toServer.Reader, _toClient.Writer); + } + + public Stream Client { get; } + + public Stream Server { get; } + + public async ValueTask DisposeAsync() + { + await Client.DisposeAsync(); + await Server.DisposeAsync(); + } + } + + private sealed class PipeStream(PipeReader reader, PipeWriter writer) : Stream + { + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + if (result.Buffer.IsEmpty && result.IsCompleted) + { + return 0; + } + + var length = (int)Math.Min(buffer.Length, result.Buffer.Length); + var written = 0; + foreach (var segment in result.Buffer.Slice(0, length)) + { + segment.Span.CopyTo(buffer.Span[written..]); + written += segment.Length; + } + + reader.AdvanceTo(result.Buffer.GetPosition(length)); + return length; + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + var copied = buffer.ToArray(); + await writer.WriteAsync(copied, cancellationToken).ConfigureAwait(false); + var flushed = await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + if (flushed.IsCompleted) + { + throw new IOException("peer closed"); + } + } + + public override async ValueTask DisposeAsync() + { + try + { + await writer.CompleteAsync().ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + try + { + await reader.CompleteAsync().ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + } + + private static class ScriptedServer + { + public static async Task RunAsync(Stream stream, byte? result) + { + var pending = new List(); + var buffer = new byte[8192]; + var windows = 0; + while (true) + { + var read = await stream.ReadAsync(buffer); + if (read == 0) + { + return windows > 0 || result is null; + } + + pending.AddRange(buffer.AsSpan(0, read).ToArray()); + while (pending.Count > 0) + { + var parsed = VatpFrameParser.ParseOneFrame(pending.ToArray(), VatpProtocol.DefaultMaxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } frame) + { + return false; + } + + pending.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + if (frame.Header.Type == VatpFrameType.Hello) + { + var hello = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeHello(VatpProtocol.DefaultMaxFramePayload)); + await stream.WriteAsync(hello); + } + else if (frame.Header.Type == VatpFrameType.Meta + && TryDecodeMeta(frame.Payload, out var meta) + && meta.ArtSize > 0) + { + var grant = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)meta.ArtSize)); + await stream.WriteAsync(grant); + } + else if (frame.Header.Type == VatpFrameType.Data && frame.Payload.Length > 0) + { + windows++; + var window = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, checked((uint)frame.Payload.Length))); + await stream.WriteAsync(window); + } + else if (frame.Header.Type == VatpFrameType.End) + { + if (result is byte outcome) + { + var encoded = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeResult(frame.Header.StreamId, outcome)); + await stream.WriteAsync(encoded); + } + else + { + await stream.DisposeAsync(); + } + + return true; + } + } + } + } + + private static bool TryDecodeMeta(ReadOnlySequence payload, out ArticleCanonicalTransferMeta meta) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return VatpMetaCodec.TryDecode(bytes, out meta, out _); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageArticleLookupServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageArticleLookupServiceTests.cs new file mode 100644 index 00000000..082b7f73 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageArticleLookupServiceTests.cs @@ -0,0 +1,498 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Storage; + +public sealed class StorageArticleLookupServiceTests +{ + private static readonly ArticleId ArticleX = ArticleId.FromMessageId(""u8); + + [Fact] + public async Task MultipleOwners_FirstPositiveCompletes_AndDuplicateIsIgnored() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, publication.Exchange); + Assert.Equal(string.Empty, publication.RoutingKey); + Assert.Equal(CacheFleetTopology.LookupExpirationMilliseconds, publication.Expiration); + Assert.NotNull(service.CurrentReplyTo); + Assert.Equal(service.CurrentReplyTo, publication.ReplyTo); + + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + Assert.NotNull(request); + + var first = CreateResponse(request!, 2, "cache02.usenet.ninja"); + var second = CreateResponse(request!, 3, "cache03.usenet.ninja"); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(second)); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.Equal("cache02.usenet.ninja", result.Fqdn); + Assert.Equal(2, result.ServerId); + Assert.Equal(1, service.OutstandingCorrelations); + Assert.NotNull(result.Alternates); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 4, "cache04.usenet.ninja"))); + + var alternate = await result.Alternates!.WaitForAlternateAsync(CancellationToken.None); + Assert.True(alternate.HasValue); + var alternateCopy = alternate.GetValueOrDefault(); + Assert.Equal(3, alternateCopy.ServerId); + Assert.Equal("cache03.usenet.ninja", alternateCopy.Fqdn); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 4, "cache04.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task TwoPresentCopies_AreRetainedByExistingFanout() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + Assert.NotNull(request); + + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 2, "cache02.usenet.ninja"))); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 3, "cache03.usenet.ninja"))); + + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.Equal(2, result.ServerId); + Assert.Equal("cache02.usenet.ninja", result.Fqdn); + Assert.NotNull(result.Alternates); + var alternate = await result.Alternates!.WaitForAlternateAsync(CancellationToken.None); + Assert.True(alternate.HasValue); + var alternateCopy = alternate.GetValueOrDefault(); + Assert.Equal(3, alternateCopy.ServerId); + Assert.Equal("cache03.usenet.ninja", alternateCopy.Fqdn); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task NobodyOwns_TimesOutCleanly() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + _ = await WaitForPublicationAsync(factory); + for (var attempt = 0; attempt < 3 && !lookupTask.IsCompleted; attempt++) + { + time.Advance(CacheFleetTopology.LookupTimeout); + await Task.Yield(); + } + + var result = await lookupTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task LateResponse_AfterDeadline_IsIgnored() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 1, "cache01.usenet.ninja"))); + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.NotNull(result.Alternates); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates!.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 2, "cache02.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task AlternateWait_CompletesWhenOriginalWindowClosesWithoutSecondCandidate() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 1, "cache01.usenet.ninja"))); + var result = await lookupTask; + Assert.NotNull(result.Alternates); + + var alternateTask = result.Alternates!.WaitForAlternateAsync(CancellationToken.None).AsTask(); + Assert.False(alternateTask.IsCompleted); + time.Advance(CacheFleetTopology.LookupTimeout); + var alternate = await alternateTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Null(alternate); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 2, "cache02.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task ConcurrentLookups_CorrelateIndependently() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance); + var articleA = ArticleId.FromMessageId(""u8); + var articleB = ArticleId.FromMessageId(""u8); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupA = service.LookupAsync(articleA, CancellationToken.None); + var lookupB = service.LookupAsync(articleB, CancellationToken.None); + var publications = await WaitForPublicationCountAsync(factory, 2); + var requestA = publications.Single(p => + StorageArticleLookupWireProtocol.TryParseRequestV1(p.Body, out var r, out _) + && r!.ArticleId == articleA); + var requestB = publications.Single(p => + StorageArticleLookupWireProtocol.TryParseRequestV1(p.Body, out var r, out _) + && r!.ArticleId == articleB); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(requestA.Body, out var parsedA, out _)); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(requestB.Body, out var parsedB, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + + await consume.DeliverAsync( + requestB.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + CreateResponse(parsedB!, 3, "cache03.usenet.ninja"))); + await consume.DeliverAsync( + requestA.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + CreateResponse(parsedA!, 1, "cache01.usenet.ninja"))); + + var resultA = await lookupA; + var resultB = await lookupB; + Assert.Equal("cache01.usenet.ninja", resultA.Fqdn); + Assert.Equal("cache03.usenet.ninja", resultB.Fqdn); + Assert.NotEqual(resultA.RequestId, resultB.RequestId); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DuplicateServerId_AndSameEndpoint_AreNotSecondCandidates() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var first = CreateResponse(request!, 2, "cache02.usenet.ninja"); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + var result = await lookupTask; + + var duplicateServer = new StorageArticleLookupResponse( + 1, + request!.RequestId, + 2, + "cache02.usenet.ninja", + request.ArticleId, + 1192); + var sameEndpoint = new StorageArticleLookupResponse( + 1, + request.RequestId, + 9, + first.Fqdn, + request.ArticleId, + first.VatpPort); + Assert.False(service.TryDispatchResponse(publication.CorrelationId, duplicateServer)); + Assert.False(service.TryDispatchResponse(publication.CorrelationId, sameEndpoint)); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates!.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + var alternate = await result.Alternates.WaitForAlternateAsync(CancellationToken.None); + Assert.Null(alternate); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task InvalidCandidateResponses_AreIgnoredUntilAValidOneArrives() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + new FakeTimeProvider()); + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var otherArticle = ArticleId.FromMessageId(""u8); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 0, "cache00.usenet.ninja"))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 256, "cache256.usenet.ninja"))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + Guid.NewGuid(), + 4, + "cache04.usenet.ninja", + request!.ArticleId, + 1191))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + request.RequestId, + 5, + "cache05.usenet.ninja", + otherArticle, + 1191))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + request.RequestId, + 6, + "cache06.usenet.ninja", + request.ArticleId, + 0))); + Assert.False(lookupTask.IsCompleted); + + Assert.True(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request, 7, "cache07.usenet.ninja"))); + var result = await lookupTask; + Assert.Equal(7, result.ServerId); + Assert.Equal(request.RequestId, result.RequestId); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task EmptyRegistry_DoesNotPublish_AndReturnsExistingMiss() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + storageServers: new StorageServerRegistry()); + + var result = await service.LookupAsync(ArticleX, CancellationToken.None); + + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Equal("Storage article lookup timed out with no positive response.", result.Error); + Assert.Null(factory.LastConnection); + Assert.Null(service.CurrentReplyTo); + } + + [Fact] + public async Task RegisteredStorageServer_StillPublishesFleetLookup() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var registry = new StorageServerRegistry(); + var now = DateTimeOffset.Parse("2026-10-04T12:00:00Z"); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 1000, 100, 900, now, 1191), + now); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time, + registry); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, publication.Exchange); + Assert.Equal(string.Empty, publication.RoutingKey); + + for (var attempt = 0; attempt < 3 && !lookupTask.IsCompleted; attempt++) + { + time.Advance(CacheFleetTopology.LookupTimeout); + await Task.Yield(); + } + + var result = await lookupTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Equal("Storage article lookup timed out with no positive response.", result.Error); + await service.StopAsync(CancellationToken.None); + } + + private static StorageArticleLookupResponse CreateResponse( + StorageArticleLookupRequest request, + int serverId, + string fqdn) => + new( + 1, + request.RequestId, + serverId, + fqdn, + request.ArticleId, + 1191); + + private static RabbitMqService CreateRabbitMq(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static NntpdOptions CreateOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + return options; + } + + private static async Task WaitForPublicationAsync(FakeRabbitMqConnectionFactory factory) + { + var publications = await WaitForPublicationCountAsync(factory, 1); + return publications[0]; + } + + private static async Task> WaitForPublicationCountAsync( + FakeRabbitMqConnectionFactory factory, + int count) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2); + while (DateTime.UtcNow < deadline) + { + var connection = factory.LastConnection; + if (connection is not null) + { + var publications = connection.RpcChannels + .SelectMany(static channel => channel.Publications) + .Where(static p => p.Exchange == CacheFleetTopology.RequestsExchangeName) + .ToList(); + if (publications.Count >= count) + { + return publications; + } + } + + await Task.Delay(10); + } + + throw new TimeoutException($"Expected {count} cache.requests publications."); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs new file mode 100644 index 00000000..51d5090d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs @@ -0,0 +1,200 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.Storage; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.RabbitMq; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Storage; + +public sealed class StorageServerFleetConsumerServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresEphemeralQueue_BoundToBroadcast() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var options = Options.Create(CreateNntpdOptions(1)); + var consumer = new StorageServerFleetConsumerService( + rabbit, + options, + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + var channel = Assert.Single(connection.ManualAckChannels); + Assert.Equal("cache.nntpd01.usenet.ninja", consumer.CurrentQueueName); + Assert.Equal("cache.nntpd01.usenet.ninja", channel.ConsumedQueue); + Assert.Contains( + channel.Exchanges, + static e => e.Name == CacheBroadcastTopology.ExchangeName && e.Type == "fanout" && e.Durable); + var queue = Assert.Single(channel.Queues); + Assert.Equal("cache.nntpd01.usenet.ninja", queue.Name); + Assert.False(queue.Durable); + Assert.True(queue.Exclusive); + Assert.True(queue.AutoDelete); + var binding = Assert.Single(channel.Bindings); + Assert.Equal(queue.Name, binding.Queue); + Assert.Equal(CacheBroadcastTopology.ExchangeName, binding.Exchange); + + await consumer.StopAsync(CancellationToken.None); + Assert.Equal(1, channel.DisposeCount); + Assert.Null(consumer.CurrentQueueName); + } + + [Fact] + public async Task IndependentNntpdInstances_UseDistinctQueues() + { + var factoryA = new FakeRabbitMqConnectionFactory(); + var factoryB = new FakeRabbitMqConnectionFactory(); + await using var rabbitA = CreateRabbitMq(factoryA); + await using var rabbitB = CreateRabbitMq(factoryB); + var consumerA = new StorageServerFleetConsumerService( + rabbitA, + Options.Create(CreateNntpdOptions(1)), + new StorageServerRegistry(), + NullLogger.Instance); + var consumerB = new StorageServerFleetConsumerService( + rabbitB, + Options.Create(CreateNntpdOptions(2)), + new StorageServerRegistry(), + NullLogger.Instance); + + await rabbitA.StartAsync(CancellationToken.None); + await rabbitB.StartAsync(CancellationToken.None); + await consumerA.StartAsync(CancellationToken.None); + await consumerB.StartAsync(CancellationToken.None); + + Assert.Equal("cache.nntpd01.usenet.ninja", consumerA.CurrentQueueName); + Assert.Equal("cache.nntpd02.usenet.ninja", consumerB.CurrentQueueName); + Assert.NotEqual(consumerA.CurrentQueueName, consumerB.CurrentQueueName); + + await consumerA.StopAsync(CancellationToken.None); + await consumerB.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Delivery_UpdatesRegistry_AndAcks() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var consumer = new StorageServerFleetConsumerService( + rabbit, + Options.Create(CreateNntpdOptions(1)), + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + + var channel = Assert.Single(factory.LastConnection!.ManualAckChannels); + var body = StorageServerAdvertisementWireProtocol.SerializeV1( + new StorageServerAdvertisement( + 1, + 7, + "cache07.usenet.ninja", + 1_000_000, + 250_000, + 750_000, + DateTimeOffset.Parse("2026-09-29T12:00:00Z"))); + await channel.DeliverAsync(42, body); + + Assert.Contains(42UL, channel.Acks); + Assert.True(registry.TryGet("cache07.usenet.ninja", out var entry)); + Assert.Equal(7, entry.ServerId); + Assert.Equal(1_000_000, entry.TotalBytes); + Assert.Equal(750_000, entry.AvailableBytes); + + await consumer.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task LifecycleDelivery_UpdatesSelection() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var consumer = new StorageServerFleetConsumerService( + rabbit, + Options.Create(CreateNntpdOptions(1)), + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + var channel = Assert.Single(factory.LastConnection!.ManualAckChannels); + var advertisement = StorageServerAdvertisementWireProtocol.SerializeV1( + new StorageServerAdvertisement( + 1, + 7, + "cache07.usenet.ninja", + 1_000, + 100, + 900, + DateTimeOffset.Parse("2026-09-29T12:00:00Z"), + 1191)); + await channel.DeliverAsync(1, advertisement); + var selected = Assert.Single(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:01Z"))); + Assert.Equal(900, selected.AvailableBytes); + Assert.Equal(1191, selected.VatpPort); + Assert.False(selected.IsDraining); + + var draining = StorageServerLifecycleWireProtocol.SerializeV1( + new StorageServerLifecycleAnnouncement( + 1, + 7, + "cache07.usenet.ninja", + StorageServerLifecycleState.Draining, + DateTimeOffset.Parse("2026-09-29T12:00:02Z"), + 1191)); + await channel.DeliverAsync(2, draining); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:02Z"))); + Assert.True(registry.TryGet("cache07.usenet.ninja", out var drained)); + Assert.True(drained.IsDraining); + Assert.Equal(900, drained.AvailableBytes); + Assert.Equal(1191, drained.VatpPort); + + var ready = """{"version":1,"serverId":7,"fqdn":"cache07.usenet.ninja","state":"Ready","timestamp":"2026-09-29T12:00:04.0000000Z","vatpPort":1191}"""u8.ToArray(); + await channel.DeliverAsync(3, ready); + Assert.Contains(3UL, channel.Acks); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:04Z"))); + Assert.True(registry.TryGet("cache07.usenet.ninja", out drained)); + Assert.True(drained.IsDraining); + Assert.Equal(900, drained.AvailableBytes); + + var invalid = """{"version":1,"serverId":7,"fqdn":"cache07.usenet.ninja","state":"Offline","timestamp":"2026-09-29T12:00:03.0000000Z","vatpPort":1191}"""u8.ToArray(); + await channel.DeliverAsync(4, invalid); + Assert.Contains(4UL, channel.Acks); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:03Z"))); + + await consumer.StopAsync(CancellationToken.None); + } + + private static RabbitMqService CreateRabbitMq(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.ReaderService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static NntpdOptions CreateNntpdOptions(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + return options; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerRegistryTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerRegistryTests.cs new file mode 100644 index 00000000..0e08f9db --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Storage/StorageServerRegistryTests.cs @@ -0,0 +1,279 @@ +using System.Reflection; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.Storage; + +namespace VectorNNTPD.ReaderService.Tests.Storage; + +public sealed class StorageServerRegistryTests +{ + [Fact] + public void ApplyAdvertisement_CreatesAndUpdatesEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + var first = new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 1000, 100, 900, t0); + registry.ApplyAdvertisement(first, t0); + + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.Equal(1, entry.ServerId); + Assert.Equal(1000, entry.TotalBytes); + Assert.Equal(100, entry.UsedBytes); + Assert.Equal(900, entry.AvailableBytes); + Assert.Equal(t0, entry.LastSeen); + Assert.True(entry.IsActive(t0)); + + var t1 = t0.AddSeconds(1); + var second = first with { UsedBytes = 200, AvailableBytes = 800, Timestamp = t1 }; + registry.ApplyAdvertisement(second, t1); + + Assert.True(registry.TryGet("cache01.usenet.ninja", out entry)); + Assert.Equal(200, entry.UsedBytes); + Assert.Equal(800, entry.AvailableBytes); + Assert.Equal(t1, entry.LastSeen); + Assert.Equal(StorageServerFleetState.Active, entry.GetState(t1)); + } + + [Fact] + public void GetActive_ExcludesStaleServers_AndKeepsMultipleActive() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 100, 10, 90, t0), + t0); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 2, "cache02.usenet.ninja", 200, 20, 180, t0), + t0); + + var activeAtStart = registry.GetActive(t0); + Assert.Equal(2, activeAtStart.Count); + + var staleNow = t0 + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(staleNow)); + Assert.Equal(2, registry.Snapshot().Count); + Assert.Equal(StorageServerFleetState.Stale, registry.Snapshot()[0].GetState(staleNow)); + + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 2, "cache02.usenet.ninja", 200, 50, 150, staleNow), + staleNow); + var active = Assert.Single(registry.GetActive(staleNow)); + Assert.Equal("cache02.usenet.ninja", active.Fqdn); + Assert.Equal(50, active.UsedBytes); + } + + [Fact] + public void FirstAdvertisement_MakesTheServerSelectable_WithoutASecondRegistryEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + + var selected = Assert.Single(registry.GetActive(t0)); + Assert.Equal(1, selected.ServerId); + Assert.Equal(1191, selected.VatpPort); + Assert.False(selected.IsDraining); + Assert.True(StorageServerPlacementSelector.TrySelect(registry.GetActive(t0), out var target)); + Assert.Equal("cache01.usenet.ninja", target.Fqdn); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 900 }, + t0.AddSeconds(1)); + var updated = Assert.Single(registry.Snapshot()); + Assert.Equal(900, updated.AvailableBytes); + Assert.False(updated.IsDraining); + Assert.Equal("cache01.usenet.ninja", Assert.Single(registry.GetActive(t0.AddSeconds(1))).Fqdn); + } + + [Fact] + public void Draining_RemovesTheServerFromNewSelection_Immediately() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyAdvertisement(Advertisement("cache02.usenet.ninja", 2, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddTicks(1)), t0); + + var selected = Assert.Single(registry.GetActive(t0)); + Assert.Equal("cache02.usenet.ninja", selected.Fqdn); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var drained)); + Assert.True(drained.IsDraining); + Assert.Equal(500, drained.AvailableBytes); + Assert.True(drained.IsActive(t0)); + } + + [Fact] + public void DelayedAdvertisement_DoesNotClearDraining() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + var drainAt = t0.AddSeconds(2); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 1 }, + drainAt); + + Assert.Empty(registry.GetActive(drainAt)); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(drainAt, entry.LastSeen); + } + + [Fact] + public void CrashWithoutDraining_ExpiresThroughTheLivenessWindow() + { + var registry = new StorageServerRegistry(); + var seen = DateTimeOffset.Parse("2026-09-29T12:00:01Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, seen), seen); + + Assert.Single(registry.GetActive(seen)); + var expired = seen + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(expired)); + Assert.Single(registry.Snapshot()); + Assert.False(registry.Snapshot()[0].IsDraining); + } + + [Fact] + public void Restart_BecomesSelectableFromANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), t0.AddSeconds(1)); + Assert.Empty(registry.GetActive(t0.AddSeconds(1))); + + var restarted = t0.AddSeconds(3); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, restarted) with { AvailableBytes = 42 }, + restarted); + var selected = Assert.Single(registry.GetActive(restarted)); + Assert.False(selected.IsDraining); + Assert.Equal(42, selected.AvailableBytes); + + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), restarted); + Assert.Single(registry.GetActive(restarted)); + Assert.Equal(42, Assert.Single(registry.Snapshot()).AvailableBytes); + + var drainedAgain = restarted.AddSeconds(1); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainedAgain), drainedAgain); + Assert.Empty(registry.GetActive(drainedAgain)); + Assert.Equal(42, Assert.Single(registry.Snapshot()).AvailableBytes); + } + + [Fact] + public void DelayedAdvertisement_AfterRestart_DoesNotReplaceANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var drainAt = t0.AddSeconds(10); + var restartedAt = drainAt.AddSeconds(1); + var received = restartedAt.AddSeconds(1); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, restartedAt) with { AvailableBytes = 42 }, + received); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 1 }, + received.AddSeconds(1)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), received.AddSeconds(1)); + + var entry = Assert.Single(registry.GetActive(received.AddSeconds(1))); + Assert.False(entry.IsDraining); + Assert.Equal(received, entry.LastSeen); + Assert.Equal(42, entry.AvailableBytes); + Assert.Single(registry.Snapshot()); + } + + [Fact] + public void AdvertisementNotNewerThanDraining_StaysUnselectable() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var drainAt = t0.AddSeconds(10); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, drainAt) with { AvailableBytes = 7 }, + drainAt.AddSeconds(1)); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(9)) with { AvailableBytes = 8 }, + drainAt.AddSeconds(2)); + + Assert.Empty(registry.GetActive(drainAt.AddSeconds(2))); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(drainAt, entry.LastSeen); + } + + [Fact] + public void OlderAdvertisement_DoesNotReplaceANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var newer = t0.AddSeconds(2); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, newer) with { AvailableBytes = 80 }, newer); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0) with { AvailableBytes = 1 }, + newer.AddSeconds(1)); + + var entry = Assert.Single(registry.GetActive(newer.AddSeconds(1))); + Assert.Equal(80, entry.AvailableBytes); + Assert.Equal(newer, entry.LastSeen); + Assert.False(entry.IsDraining); + } + + [Fact] + public void RepeatedDraining_StaysOneEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(2)), t0.AddSeconds(2)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(3)), t0.AddSeconds(3)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(2)), t0.AddSeconds(4)); + + Assert.Empty(registry.GetActive(t0.AddSeconds(4))); + var entry = Assert.Single(registry.Snapshot()); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(t0.AddSeconds(3), entry.NewestMessageTimestamp); + } + + [Fact] + public void LivenessExpiry_DoesNotConvertDrainingToSelectable() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), t0.AddSeconds(1)); + + Assert.Empty(registry.GetActive(t0.AddSeconds(1))); + var expired = t0.AddSeconds(1) + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(expired)); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + } + + [Fact] + public void Registry_HasNoReadyEpochState() + { + const BindingFlags publicStatic = BindingFlags.Public | BindingFlags.Static; + Assert.Null(typeof(StorageServerFleetEntry).GetProperty("LifecycleOriginTimestamp")); + Assert.Null(typeof(StorageServerFleetEntry).GetProperty("SupersededDrainingTimestamp")); + Assert.Null(typeof(StorageServerLifecycleState).GetField("Ready", publicStatic)); + Assert.NotNull(typeof(StorageServerLifecycleState).GetField(nameof(StorageServerLifecycleState.Draining), publicStatic)); + Assert.Equal(1, StorageServerLifecycleWireProtocol.CurrentVersion); + } + + private static StorageServerAdvertisement Advertisement(string fqdn, int serverId, DateTimeOffset timestamp) => + new(1, serverId, fqdn, 1000, 500, 500, timestamp, 1191); + + private static StorageServerLifecycleAnnouncement Draining(string fqdn, int serverId, DateTimeOffset timestamp) => + new(1, serverId, fqdn, StorageServerLifecycleState.Draining, timestamp, 1191); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/CountingPostFilter.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/CountingPostFilter.cs new file mode 100644 index 00000000..f9cc1940 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/CountingPostFilter.cs @@ -0,0 +1,19 @@ +using VectorNNTPD.ReaderService.PostFilter; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Counts calls. Accepts without reserving. +internal sealed class CountingPostFilter : IPostFilter +{ + public int EvaluateCalls { get; private set; } + + public ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + _ = request; + cancellationToken.ThrowIfCancellationRequested(); + EvaluateCalls++; + return ValueTask.FromResult(PostFilterResult.AcceptedDisabled()); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeApplicationService.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeApplicationService.cs new file mode 100644 index 00000000..d47dc700 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeApplicationService.cs @@ -0,0 +1,118 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +internal sealed class FakeApplicationService : IApplicationService +{ + private readonly Func? _onStart; + private readonly Func? _onStop; + private readonly TaskCompletionSource? _executionTcs; + private int _startCount; + private int _stopCount; + private int _stopAttemptCount; + + public FakeApplicationService( + string name, + Func? onStart = null, + Func? onStop = null, + bool withExecution = false) + { + Name = name; + _onStart = onStart; + _onStop = onStop; + if (withExecution) + { + _executionTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + public string Name { get; } + + public Task? Execution => _executionTcs?.Task; + + public int StartCount => Volatile.Read(ref _startCount); + + public int StopCount => Volatile.Read(ref _stopCount); + + /// Number of times was entered, including canceled/aborted attempts. + public int StopAttemptCount => Volatile.Read(ref _stopAttemptCount); + + public List SharedStartOrder { get; set; } = []; + + public List SharedStopOrder { get; set; } = []; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StopGate { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StopEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool HoldStopUntilReleased { get; set; } + + /// + /// When set with , stop waits on without + /// observing the cancellation token (non-cooperative stop). + /// + public bool IgnoreStopCancellation { get; set; } + + public void CompleteExecution(Exception? exception = null) + { + if (_executionTcs is null) + { + throw new InvalidOperationException("Execution was not enabled for this fake."); + } + + if (exception is null) + { + _executionTcs.TrySetResult(); + } + else + { + _executionTcs.TrySetException(exception); + } + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (_onStart is not null) + { + await _onStart(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _startCount); + SharedStartOrder.Add(Name); + Started.TrySetResult(); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _stopAttemptCount); + StopEntered.TrySetResult(); + + if (HoldStopUntilReleased) + { + if (IgnoreStopCancellation) + { + await StopGate.Task.ConfigureAwait(false); + } + else + { + await StopGate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (_onStop is not null) + { + await _onStop(cancellationToken).ConfigureAwait(false); + } + + if (!IgnoreStopCancellation) + { + cancellationToken.ThrowIfCancellationRequested(); + } + + Interlocked.Increment(ref _stopCount); + SharedStopOrder.Add(Name); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeCloudflareDnsClient.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeCloudflareDnsClient.cs new file mode 100644 index 00000000..25b16bff --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeCloudflareDnsClient.cs @@ -0,0 +1,343 @@ +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// +/// In-memory Cloudflare DNS client for offline tests. Never performs network I/O. +/// +internal sealed class FakeCloudflareDnsClient : ICloudflareDnsClient +{ + private readonly object _sync = new(); + private readonly List _records = []; + private int _nextId = 1; + + public int ListCallCount { get; private set; } + public int ListAllCallCount { get; private set; } + public int CreateCallCount { get; private set; } + public int UpdateCallCount { get; private set; } + public int DeleteCallCount { get; private set; } + + public Func? OnMutate { get; set; } + public Exception? ListException { get; set; } + public Exception? ListAllException { get; set; } + public Exception? CreateException { get; set; } + public Exception? UpdateException { get; set; } + public Exception? DeleteException { get; set; } + public Exception? FailListAfterMutationsException { get; set; } + public int MutationsBeforeListFailure { get; set; } = int.MaxValue; + + /// + /// How many delete calls should fail when is set. + /// Use -1 (default) for unlimited failures; a positive count fails that many times then succeeds. + /// + public int DeleteFailRemaining { get; set; } = -1; + + /// + /// When true, a failing delete still removes the record then throws (timeout after apply). + /// + public bool ApplyDeleteThenFailUncertain { get; set; } + + /// When set, create fails only for this DNS record type (e.g. AAAA). + public string? CreateFailType { get; set; } + + /// When set with , throws this instead of . + public Exception? CreateFailTypeException { get; set; } + + /// + /// How many matching creates should fail before succeeding. + /// Use for permanent failure. + /// + public int CreateFailTypeRemaining { get; set; } = int.MaxValue; + + /// + /// When true, a failing create for still mutates the store, then throws + /// (simulates timeout after Cloudflare applied the change). + /// + public bool ApplyCreateThenFailUncertain { get; set; } + + public int SuccessfulCreateCount { get; private set; } + public int SuccessfulDeleteCount { get; private set; } + + /// Ordered mutation labels such as Create:A or Delete for successful mutations. + public List MutationOrder { get; } = []; + + /// + /// Optional transform applied to typed list results (e.g. inject an external record during verification). + /// Arguments: record type, listed records, current list call count. + /// + public Func, int, IReadOnlyList>? + TransformListedRecords + { get; set; } + + /// + /// Optional transform applied to results. + /// Arguments: listed records, current list-all call count. + /// + public Func, int, IReadOnlyList>? + TransformListedAllRecords + { get; set; } + + public IReadOnlyList Snapshot() + { + lock (_sync) + { + return _records.Select(Clone).ToArray(); + } + } + + public void Seed(params CloudflareDnsRecord[] records) + { + lock (_sync) + { + _records.Clear(); + foreach (var record in records) + { + _records.Add(Clone(record)); + } + } + } + + /// Adds a record without clearing existing ones (simulates an external DNS actor). + public void Inject(CloudflareDnsRecord record) + { + lock (_sync) + { + _records.Add(Clone(record)); + } + } + + public Task> ListRecordsAsync( + string zoneId, + string fqdn, + string type, + CancellationToken cancellationToken) + { + ListCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + + if (ListException is not null) + { + throw ListException; + } + + if (CreateCallCount + UpdateCallCount + DeleteCallCount >= MutationsBeforeListFailure + && FailListAfterMutationsException is not null) + { + throw FailListAfterMutationsException; + } + + IReadOnlyList matches; + lock (_sync) + { + var normalized = fqdn.Trim().TrimEnd('.').ToLowerInvariant(); + matches = _records + .Where(r => + string.Equals(r.Type, type, StringComparison.OrdinalIgnoreCase) + && string.Equals(r.Name.Trim().TrimEnd('.'), normalized, StringComparison.OrdinalIgnoreCase)) + .Select(Clone) + .ToArray(); + } + + if (TransformListedRecords is not null) + { + matches = TransformListedRecords(type, matches, ListCallCount); + } + + return Task.FromResult(matches); + } + + public Task> ListAllRecordsForNameAsync( + string zoneId, + string fqdn, + CancellationToken cancellationToken) + { + ListAllCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + + if (ListAllException is not null) + { + throw ListAllException; + } + + if (ListException is not null) + { + throw ListException; + } + + if (CreateCallCount + UpdateCallCount + DeleteCallCount >= MutationsBeforeListFailure + && FailListAfterMutationsException is not null) + { + throw FailListAfterMutationsException; + } + + IReadOnlyList matches; + lock (_sync) + { + var normalized = fqdn.Trim().TrimEnd('.').ToLowerInvariant(); + matches = _records + .Where(r => + string.Equals(r.Name.Trim().TrimEnd('.'), normalized, StringComparison.OrdinalIgnoreCase)) + .Select(Clone) + .ToArray(); + } + + if (TransformListedAllRecords is not null) + { + matches = TransformListedAllRecords(matches, ListAllCallCount); + } + + return Task.FromResult(matches); + } + + public async Task CreateRecordAsync( + string zoneId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + CreateCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + var failForThisType = CreateFailType is not null + && CreateFailTypeRemaining > 0 + && string.Equals(request.Type, CreateFailType, StringComparison.OrdinalIgnoreCase); + var failEx = failForThisType + ? CreateFailTypeException ?? new CloudflareDnsException($"create {request.Type} failed") + : CreateFailType is null + ? CreateException + : null; + + if (failEx is not null) + { + if (failForThisType) + { + CreateFailTypeRemaining--; + } + + if (ApplyCreateThenFailUncertain && failForThisType) + { + lock (_sync) + { + _records.Add(new CloudflareDnsRecord + { + Id = $"rec-{_nextId++}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + Ttl = request.Ttl, + Proxied = request.Proxied, + }); + SuccessfulCreateCount++; + MutationOrder.Add($"Create:{request.Type}"); + } + } + + throw failEx; + } + + lock (_sync) + { + var record = new CloudflareDnsRecord + { + Id = $"rec-{_nextId++}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + Ttl = request.Ttl, + Proxied = request.Proxied, + }; + _records.Add(record); + SuccessfulCreateCount++; + MutationOrder.Add($"Create:{request.Type}"); + return Clone(record); + } + } + + public async Task UpdateRecordAsync( + string zoneId, + string recordId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + UpdateCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + if (UpdateException is not null) + { + throw UpdateException; + } + + lock (_sync) + { + var existing = _records.FirstOrDefault(r => r.Id == recordId) + ?? throw new CloudflareDnsException($"Record '{recordId}' was not found."); + existing.Type = request.Type; + existing.Name = request.Name; + existing.Content = request.Content; + existing.Ttl = request.Ttl; + existing.Proxied = request.Proxied; + MutationOrder.Add($"Update:{request.Type}"); + return Clone(existing); + } + } + + public async Task DeleteRecordAsync( + string zoneId, + string recordId, + CancellationToken cancellationToken) + { + DeleteCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + if (DeleteException is not null && DeleteFailRemaining != 0) + { + if (DeleteFailRemaining > 0) + { + DeleteFailRemaining--; + } + + if (ApplyDeleteThenFailUncertain) + { + lock (_sync) + { + _records.RemoveAll(r => r.Id == recordId); + SuccessfulDeleteCount++; + MutationOrder.Add("Delete"); + } + } + + throw DeleteException; + } + + lock (_sync) + { + _records.RemoveAll(r => r.Id == recordId); + SuccessfulDeleteCount++; + MutationOrder.Add("Delete"); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private static CloudflareDnsRecord Clone(CloudflareDnsRecord record) => + new() + { + Id = record.Id, + Type = record.Type, + Name = record.Name, + Content = record.Content, + Ttl = record.Ttl, + Proxied = record.Proxied, + }; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs new file mode 100644 index 00000000..440d04ee --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs @@ -0,0 +1,40 @@ +using System.Net; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// +/// Deterministic local-address probe for configuration tests. +/// +internal sealed class FakeLocalIpAddressAssignee : ILocalIpAddressAssignee +{ + private readonly HashSet _assigned; + private readonly bool _assignAll; + private readonly IReadOnlyList _unicast; + + public FakeLocalIpAddressAssignee(bool assignAll) + { + _assignAll = assignAll; + _assigned = new HashSet(); + _unicast = assignAll + ? [TestHostFactory.TestIpv4, TestHostFactory.TestIpv6] + : Array.Empty(); + } + + public FakeLocalIpAddressAssignee(params IPAddress[] assigned) + { + _assignAll = false; + _assigned = new HashSet(assigned); + _unicast = assigned; + } + + public bool IsLocallyAssigned(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return _assignAll || _assigned.Contains(address); + } + + public IReadOnlyList GetAssignedUnicastAddresses() => _unicast; +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeNntpDb.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeNntpDb.cs new file mode 100644 index 00000000..cfeac8fc --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeNntpDb.cs @@ -0,0 +1,447 @@ +using MySqlConnector; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Moderation; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.PostFilter; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// In-memory MySQL connection factory for offline NntpDB tests. +internal sealed class FakeNntpDbConnectionFactory : INntpDbConnectionFactory +{ + private readonly object _sync = new(); + private readonly List _connections = []; + + public int OpenCount { get; private set; } + + public int OpenAttemptCount { get; private set; } + + public Exception? OpenException { get; set; } + + public int RemainingOpenFailures { get; set; } + + public TaskCompletionSource? BlockOpen { get; set; } + + public TaskCompletionSource? OpenStarted { get; set; } + + public TaskCompletionSource? BlockSelectOne { get; set; } + + public TaskCompletionSource? SelectOneStarted { get; set; } + + public int NextHealthCheckResult { get; set; } = 1; + + public Exception? NextHealthCheckException { get; set; } + + public IReadOnlyList Newsgroups { get; set; } = []; + + public Exception? QueryNewsgroupsException { get; set; } + + public Dictionary Users { get; } = new(StringComparer.Ordinal); + + public Exception? QueryUserException { get; set; } + + public Exception? ConsumeAccountBytesException { get; set; } + + public IReadOnlyList Moderators { get; set; } = []; + + public Exception? QueryModeratorsException { get; set; } + + public PostFilterPolicyRecord? PostFilterPolicy { get; set; } = + new(1, DateTimeOffset.UtcNow, new PostFilterOptions()); + + public Dictionary PostFilterRevisions { get; } = []; + + public long? PostFilterCurrentRevision { get; set; } + + public Exception? QueryPostFilterPolicyException { get; set; } + + public Exception? InsertPostFilterRejectionException { get; set; } + + public TaskCompletionSource? BlockQueryNewsgroups { get; set; } + + public TaskCompletionSource? QueryNewsgroupsStarted { get; set; } + + public IReadOnlyList? SharedConfigurationRows { get; set; } + + public TransitCatalogueRecord? TransitCatalogue { get; set; } = TransitCatalogueRecord.Empty; + + public Exception? SharedConfigurationException { get; set; } + + public IReadOnlyList Connections + { + get + { + lock (_sync) + { + return _connections.ToArray(); + } + } + } + + public int QueryNewsgroupsCount + { + get + { + lock (_sync) + { + var total = 0; + foreach (var connection in _connections) + { + total += connection.QueryNewsgroupsCount; + } + + return total; + } + } + } + + public async Task OpenAsync(string connectionString, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + OpenAttemptCount++; + OpenStarted?.TrySetResult(); + if (BlockOpen is not null) + { + await BlockOpen.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (RemainingOpenFailures > 0) + { + RemainingOpenFailures--; + throw new NntpDbUnavailableException("down"); + } + + if (OpenException is not null) + { + throw OpenException; + } + + var connection = new FakeNntpDbConnection + { + HealthCheckResult = NextHealthCheckResult, + HealthCheckException = NextHealthCheckException, + BlockSelectOne = BlockSelectOne, + SelectOneStarted = SelectOneStarted, + Newsgroups = Newsgroups, + QueryNewsgroupsException = QueryNewsgroupsException, + Users = Users, + QueryUserException = QueryUserException, + ConsumeAccountBytesException = ConsumeAccountBytesException, + Moderators = Moderators, + QueryModeratorsException = QueryModeratorsException, + PostFilterPolicy = PostFilterPolicy, + PostFilterRevisions = PostFilterRevisions, + PostFilterCurrentRevision = PostFilterCurrentRevision, + QueryPostFilterPolicyException = QueryPostFilterPolicyException, + InsertPostFilterRejectionException = InsertPostFilterRejectionException, + BlockQueryNewsgroups = BlockQueryNewsgroups, + QueryNewsgroupsStarted = QueryNewsgroupsStarted, + SharedConfigurationRows = SharedConfigurationRows, + SharedConfigurationException = SharedConfigurationException, + TransitCatalogue = TransitCatalogue, + }; + lock (_sync) + { + OpenCount++; + _connections.Add(connection); + } + + return connection; + } +} + +/// In-memory logical MySQL connection with failure injection. +internal sealed class FakeNntpDbConnection : INntpDbConnection, INntpSharedConfigurationRowSource +{ + public int SelectOneCount { get; private set; } + + public int DisposeCount { get; private set; } + + public int HealthCheckResult { get; set; } = 1; + + public Exception? HealthCheckException { get; set; } + + public TaskCompletionSource? BlockSelectOne { get; set; } + + public TaskCompletionSource? SelectOneStarted { get; set; } + + public IReadOnlyList Newsgroups { get; set; } = []; + + public Exception? QueryNewsgroupsException { get; set; } + + public Dictionary Users { get; set; } = new(StringComparer.Ordinal); + + public Exception? QueryUserException { get; set; } + + public Exception? ConsumeAccountBytesException { get; set; } + + public int ConsumeAccountBytesCount { get; private set; } + + public int QueryAccountByteRemainingCount { get; private set; } + + public IReadOnlyList Moderators { get; set; } = []; + + public Exception? QueryModeratorsException { get; set; } + + public int QueryModeratorsCount { get; private set; } + + public PostFilterPolicyRecord? PostFilterPolicy { get; set; } + + public Dictionary PostFilterRevisions { get; set; } = []; + + public long? PostFilterCurrentRevision { get; set; } + + public Exception? QueryPostFilterPolicyException { get; set; } + + public int QueryPostFilterPolicyCount { get; private set; } + + public TaskCompletionSource? BlockQueryNewsgroups { get; set; } + + public TaskCompletionSource? QueryNewsgroupsStarted { get; set; } + + public int QueryNewsgroupsCount { get; private set; } + + public int QueryUserCount { get; private set; } + + public async ValueTask SelectOneAsync(CancellationToken cancellationToken) + { + SelectOneStarted?.TrySetResult(); + if (BlockSelectOne is not null) + { + await BlockSelectOne.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + SelectOneCount++; + if (HealthCheckException is not null) + { + throw HealthCheckException; + } + + return HealthCheckResult; + } + + public async ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken) + { + QueryNewsgroupsStarted?.TrySetResult(); + if (BlockQueryNewsgroups is not null) + { + await BlockQueryNewsgroups.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + QueryNewsgroupsCount++; + if (QueryNewsgroupsException is not null) + { + throw QueryNewsgroupsException; + } + + return Newsgroups; + } + + public ValueTask QueryUserAccountAsync(string accountName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + QueryUserCount++; + if (QueryUserException is not null) + { + throw QueryUserException; + } + + return Users.TryGetValue(accountName, out var record) + ? ValueTask.FromResult(record) + : ValueTask.FromResult(null); + } + + public ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + cancellationToken.ThrowIfCancellationRequested(); + ConsumeAccountBytesCount++; + if (ConsumeAccountBytesException is not null) + { + throw ConsumeAccountBytesException; + } + + lock (Users) + { + if (!Users.TryGetValue(accountName, out var record)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var current = record.ByteLimit < 0 ? 0 : record.ByteLimit; + var consumed = bytes > current ? current : bytes; + var remaining = current - consumed; + Users[accountName] = WithByteLimit(record, remaining); + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, consumed)); + } + } + + public ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + QueryAccountByteRemainingCount++; + if (QueryUserException is not null) + { + throw QueryUserException; + } + + if (!Users.TryGetValue(accountName, out var record)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var remaining = record.ByteLimit < 0 ? 0 : record.ByteLimit; + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, 0)); + } + + public ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryModeratorsCount++; + if (QueryModeratorsException is not null) + { + throw QueryModeratorsException; + } + + return ValueTask.FromResult(Moderators); + } + + public ValueTask QueryPostFilterPolicyAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryPostFilterPolicyCount++; + if (QueryPostFilterPolicyException is not null) + { + throw QueryPostFilterPolicyException; + } + + if (PostFilterRevisions.Count > 0) + { + if (PostFilterCurrentRevision is not { } revision) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult( + PostFilterRevisions.TryGetValue(revision, out var versioned) ? versioned : null); + } + + return ValueTask.FromResult(PostFilterPolicy); + } + + public TransitCatalogueRecord? TransitCatalogue { get; set; } + + public Exception? QueryTransitCatalogueException { get; set; } + + public ValueTask QueryTransitCatalogueAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueryTransitCatalogueException is not null) + { + throw QueryTransitCatalogueException; + } + + return ValueTask.FromResult(TransitCatalogue); + } + + public List Rejections { get; } = []; + + public Exception? InsertPostFilterRejectionException { get; set; } + + public int InsertPostFilterRejectionCount { get; private set; } + + public ValueTask InsertPostFilterRejectionAsync( + PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(evidence); + cancellationToken.ThrowIfCancellationRequested(); + InsertPostFilterRejectionCount++; + if (InsertPostFilterRejectionException is not null) + { + throw InsertPostFilterRejectionException; + } + + Rejections.Add(evidence); + return ValueTask.CompletedTask; + } + + public IReadOnlyList? SharedConfigurationRows { get; set; } + + public Exception? SharedConfigurationException { get; set; } + + public MySqlCommand CreateCommand() => + throw new NotSupportedException("Fake NntpDB connections do not create provider commands."); + + public ValueTask BeginTransactionAsync(CancellationToken cancellationToken) => + throw new NotSupportedException("Fake NntpDB connections do not begin transactions."); + + public ValueTask BeginTransactionAsync( + System.Data.IsolationLevel isolationLevel, + CancellationToken cancellationToken) => + throw new NotSupportedException("Fake NntpDB connections do not begin transactions."); + + public ValueTask> ReadCandidatesAsync( + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (SharedConfigurationException is not null) + { + throw SharedConfigurationException; + } + + IReadOnlyList rows = SharedConfigurationRows + ?? [new NntpSharedConfigurationCandidate( + 5 * 1024 * 1024, + "news.usenet.ninja", + null, + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "5811a29d39a0732afb5f160c9b137c3d", + "usenet.ninja")]; + return ValueTask.FromResult(rows); + } + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + + private static NntpUserRecord WithByteLimit(NntpUserRecord record, long byteLimit) => + new( + record.AccountName, + record.AccountPassword, + record.AllowAuthPlain, + record.AllowAuthScram256, + record.ScramSalt, + record.ScramIterations, + record.ScramStoredKey, + record.ScramServerKey, + record.RateLimitBps, + byteLimit, + record.SessionLimit, + record.SrcIpLimit, + record.IsEnabled, + record.CustomerId, + record.AllowedArtTypes); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRabbitMq.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRabbitMq.cs new file mode 100644 index 00000000..d606dc5b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRabbitMq.cs @@ -0,0 +1,1216 @@ +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.RabbitMq; +using VectorNNTPD.ReaderService.RabbitMq.ArticleWork; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// In-memory RabbitMQ connection factory for offline tests. +internal sealed class FakeRabbitMqConnectionFactory : IRabbitMqConnectionFactory +{ + private int _connectCount; + private int _attemptCount; + private readonly SemaphoreSlim _attemptPulse = new(0, int.MaxValue); + + /// Gets the number of successful connects. + public int ConnectCount => Volatile.Read(ref _connectCount); + + /// Gets the number of connect attempts, including failures. + public int AttemptCount => Volatile.Read(ref _attemptCount); + + /// Gets the most recently created connection. + public FakeRabbitMqConnection? LastConnection { get; private set; } + + /// Gets every connection created by this factory. + public List Connections { get; } = []; + + /// When set, every connect attempt throws this exception. + public Exception? ConnectException { get; set; } + + /// + /// Remaining connect attempts that throw or a default broker failure. + /// After the budget is exhausted, connects succeed unless is set. + /// + public int RemainingConnectFailures { get; set; } + + /// Signaled when a connect attempt begins, before . + public TaskCompletionSource? ConnectStarted { get; set; } + + /// When set, connect waits on this source before completing. + public TaskCompletionSource? BlockConnect { get; set; } + + /// Signaled after each successful connect with the new . + public TaskCompletionSource? Connected { get; set; } + + /// When , created connections report IsOpen == false. + public bool ReturnUnusableConnection { get; set; } + + /// Copied onto each created connection as . + public Exception? QueueDeclareException { get; set; } + + /// + /// Copied onto each created connection as . + /// + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// Waits until at least connect attempts have started. + public async Task WaitForAttemptsAsync(int count, CancellationToken cancellationToken) + { + while (AttemptCount < count) + { + await _attemptPulse.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + public async Task ConnectAsync( + RabbitMqOptions options, + string connectionName, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(connectionName); + + Interlocked.Increment(ref _attemptCount); + _attemptPulse.Release(); + ConnectStarted?.TrySetResult(); + if (BlockConnect is not null) + { + await BlockConnect.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (RemainingConnectFailures > 0) + { + RemainingConnectFailures--; + throw ConnectException ?? new InvalidOperationException("broker down"); + } + + if (ConnectException is not null) + { + throw ConnectException; + } + + var count = Interlocked.Increment(ref _connectCount); + var hosts = options.Hosts ?? []; + var connection = new FakeRabbitMqConnection( + hosts.Length > 0 ? hosts[0] : "127.0.0.1", + options.Port ?? 5672, + string.IsNullOrWhiteSpace(options.VirtualHost) ? "/" : options.VirtualHost.Trim(), + connectionName) + { + IsOpen = !ReturnUnusableConnection, + QueueDeclareException = QueueDeclareException, + }; + foreach (var pair in PassiveConsumerCounts) + { + connection.PassiveConsumerCounts[pair.Key] = pair.Value; + } + + LastConnection = connection; + Connections.Add(connection); + Connected?.TrySetResult(count); + return connection; + } +} + +/// In-memory RabbitMQ connection for lifecycle tests. +internal sealed class FakeRabbitMqConnection : IRabbitMqConnection +{ + /// Initializes a new fake connection. + public FakeRabbitMqConnection(string host, int port, string virtualHost, string clientProvidedName) + { + Host = host; + Port = port; + VirtualHost = virtualHost; + ClientProvidedName = clientProvidedName; + } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// + public string Host { get; } + + /// + public int Port { get; } + + /// + public string VirtualHost { get; } + + /// + public string ClientProvidedName { get; } + + /// + public event EventHandler? ConnectionBlocked; + + /// + public event EventHandler? ConnectionUnblocked; + + /// Gets how many times the connection was disposed. + public int DisposeCount { get; private set; } + + /// Signaled when begins, before . + public TaskCompletionSource? DisposeStarted { get; set; } + + /// When set, waits on this source before completing. + public TaskCompletionSource? BlockDispose { get; set; } + + /// + public event EventHandler? ConnectionLost; + + /// Gets every topology channel created on this connection. + public List TopologyChannels { get; } = []; + + /// Gets every RPC channel created on this connection. + public List RpcChannels { get; } = []; + + /// Gets every confirm-enabled publish channel created on this connection. + public List PublishChannels { get; } = []; + + /// Gets every async-confirm publish channel created on this connection. + public List AsyncConfirmPublishChannels { get; } = []; + + /// Gets every manual-ack channel created on this connection. + public List ManualAckChannels { get; } = []; + + /// When set, throws this exception. + public Exception? CreateRpcChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreateManualAckChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreatePublishChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreateAsyncConfirmPublishChannelException { get; set; } + + /// Optional configurator invoked for every newly created publish channel. + public Action? ConfigurePublishChannel { get; set; } + + /// Optional configurator invoked for every newly created async-confirm channel. + public Action? ConfigureAsyncConfirmPublishChannel { get; set; } + + /// When set, throws this exception. + public Exception? CreateTopologyChannelException { get; set; } + + /// When set, the next created topology channel throws this exception from queue declare. + public Exception? QueueDeclareException { get; set; } + + /// When set, the next created topology channel throws this exception from exchange declare. + public Exception? ExchangeDeclareException { get; set; } + + /// When set, the next created topology channel throws this exception from queue bind. + public Exception? QueueBindException { get; set; } + + /// + /// Passive-declare consumer counts copied onto each new topology channel (ArticleWork tests). + /// + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// When set, throws on new topology channels. + public Exception? QueueDeclarePassiveException { get; set; } + + /// + /// Queue names for which passive declare throws NOT_FOUND on each new topology channel + /// (simulates missing provider queues under BackFiller-owned topology). + /// + public HashSet PassiveDeclareNotFoundQueues { get; } = new(StringComparer.Ordinal); + + /// + public Task CreateTopologyChannelAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateTopologyChannelException is not null) + { + throw CreateTopologyChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for topology declaration."); + } + + var channel = new FakeRabbitMqTopologyChannel + { + ExchangeDeclareException = ExchangeDeclareException, + QueueDeclareException = QueueDeclareException, + QueueBindException = QueueBindException, + QueueDeclarePassiveException = QueueDeclarePassiveException, + }; + foreach (var pair in PassiveConsumerCounts) + { + channel.PassiveConsumerCounts[pair.Key] = pair.Value; + } + + foreach (var queue in PassiveDeclareNotFoundQueues) + { + channel.PassiveDeclareNotFoundQueues.Add(queue); + } + + TopologyChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateRpcChannelAsync(long generation, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateRpcChannelException is not null) + { + throw CreateRpcChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for article-work RPC."); + } + + var channel = new FakeRabbitMqRpcChannel(generation); + RpcChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateManualAckChannelAsync( + long generation, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateManualAckChannelException is not null) + { + throw CreateManualAckChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for channel creation."); + } + + var channel = new FakeRabbitMqManualAckChannel(generation); + ManualAckChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreatePublishChannelAsync(long generation, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreatePublishChannelException is not null) + { + throw CreatePublishChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for publication."); + } + + var channel = new FakeRabbitMqPublishChannel(generation); + ConfigurePublishChannel?.Invoke(channel); + PublishChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateAsyncConfirmPublishChannelAsync( + long generation, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateAsyncConfirmPublishChannelException is not null) + { + throw CreateAsyncConfirmPublishChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for publication."); + } + + var channel = new FakeRabbitMqAsyncConfirmPublishChannel(generation); + ConfigureAsyncConfirmPublishChannel?.Invoke(channel); + AsyncConfirmPublishChannels.Add(channel); + return Task.FromResult(channel); + } + + /// Raises as a peer-initiated disconnect. + public void SimulateLost(ushort replyCode = 320, string replyText = "CONNECTION FORCED") + { + IsOpen = false; + ConnectionLost?.Invoke(this, new RabbitMqConnectionLostEventArgs(replyCode, replyText, "Peer")); + } + + /// + public async ValueTask DisposeAsync() + { + IsOpen = false; + DisposeStarted?.TrySetResult(); + if (BlockDispose is not null) + { + await BlockDispose.Task.ConfigureAwait(false); + } + + DisposeCount++; + } + + /// Gets every exchange declared on any topology channel of this connection. + public IReadOnlyList ExchangeDeclarations => + TopologyChannels.SelectMany(static channel => channel.Exchanges).ToList(); + + /// Gets every queue declared on any topology channel of this connection. + public IReadOnlyList QueueDeclarations => + TopologyChannels.SelectMany(static channel => channel.Queues).ToList(); + + /// Gets every binding declared on any topology channel of this connection. + public IReadOnlyList BindingDeclarations => + TopologyChannels.SelectMany(static channel => channel.Bindings).ToList(); +} + +/// In-memory declare-only RabbitMQ channel for topology tests. +internal sealed class FakeRabbitMqTopologyChannel : IRabbitMqTopologyChannel +{ + /// Gets recorded exchange declarations in call order. + public List Exchanges { get; } = []; + + /// Gets recorded queue declarations in call order. + public List Queues { get; } = []; + + /// Gets recorded queue bindings in call order. + public List Bindings { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? ExchangeDeclareException { get; set; } + + /// When set, throws this exception. + public Exception? QueueDeclareException { get; set; } + + /// When set, throws this exception. + public Exception? QueueBindException { get; set; } + + /// Passive-declare consumer counts keyed by queue name (tests). + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// When set, throws this exception. + public Exception? QueueDeclarePassiveException { get; set; } + + /// Queues that throw a broker-style NOT_FOUND on passive declare. + public HashSet PassiveDeclareNotFoundQueues { get; } = new(StringComparer.Ordinal); + + /// Number of passive queue declares performed on this channel. + public int PassiveDeclareCount { get; private set; } + + /// Passive-declare call targets in order. + public List PassiveDeclareQueues { get; } = []; + + /// + public Task ExchangeDeclareAsync( + string exchange, + string type, + bool durable, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (ExchangeDeclareException is not null) + { + throw ExchangeDeclareException; + } + + Exchanges.Add(new FakeRabbitMqExchangeDeclaration( + exchange, + type, + durable, + autoDelete, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueueDeclareException is not null) + { + throw QueueDeclareException; + } + + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclarePassiveAsync(string queue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + PassiveDeclareCount++; + PassiveDeclareQueues.Add(queue); + if (QueueDeclarePassiveException is not null) + { + throw QueueDeclarePassiveException; + } + + if (PassiveDeclareNotFoundQueues.Contains(queue)) + { + throw new InvalidOperationException($"NOT_FOUND - no queue '{queue}' in vhost '/'"); + } + + PassiveConsumerCounts.TryGetValue(queue, out var consumers); + return Task.FromResult(new RabbitMqQueueStats(MessageCount: 0, ConsumerCount: consumers)); + } + + /// + public Task QueueBindAsync( + string queue, + string exchange, + string routingKey, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueueBindException is not null) + { + throw QueueBindException; + } + + Bindings.Add(new FakeRabbitMqBindingDeclaration( + queue, + exchange, + routingKey, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + + private static IReadOnlyDictionary? CopyArguments( + IReadOnlyDictionary? arguments) + { + if (arguments is null) + { + return null; + } + + return new Dictionary(arguments, StringComparer.Ordinal); + } +} + +/// In-memory RabbitMQ channel for article-work RPC tests. +internal sealed class FakeRabbitMqRpcChannel : IRabbitMqRpcChannel +{ + /// Initializes a new fake RPC channel. + public FakeRabbitMqRpcChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// Gets recorded queue declarations. + public List Queues { get; } = []; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// Gets the consume handler when a consumer has started. + public Func? DeliveryHandler { get; private set; } + + /// Gets the most recent delivery passed to the consumer, when any. + public RabbitMqRpcDelivery? LastDelivery { get; private set; } + + /// Gets the consumed queue name. + public string? ConsumedQueue { get; private set; } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task PublishAsync( + string exchange, + string routingKey, + string correlationId, + string requestId, + string replyTo, + string contentType, + string expiration, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqRpcChannel)); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqRpcPublication( + exchange, + routingKey, + correlationId, + requestId, + replyTo, + contentType, + expiration, + body.ToArray())); + return Task.CompletedTask; + } + + /// + public Task ConsumeAsync( + string queue, + Func onDelivery, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConsumedQueue = queue; + DeliveryHandler = onDelivery; + return Task.FromResult("fake-consumer"); + } + + /// Delivers one message to the registered consumer. + public Task DeliverAsync( + string? correlationId, + ReadOnlyMemory body, + string? contentType = "application/json", + string? requestId = null, + string? expiration = ArticleWorkRpcAmqp.ExpirationMilliseconds) + { + if (DeliveryHandler is null) + { + throw new InvalidOperationException("No RPC consumer is registered."); + } + + var delivery = new RabbitMqRpcDelivery( + correlationId, + requestId, + contentType, + expiration, + body.ToArray(), + Generation); + LastDelivery = delivery; + return DeliveryHandler(delivery); + } + + /// + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// In-memory confirm-enabled publish channel for OverviewDB handoff tests. +internal sealed class FakeRabbitMqPublishChannel : IRabbitMqPublishChannel +{ + /// Initializes a new fake publish channel. + public FakeRabbitMqPublishChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// + /// Remaining publications that throw or a default nack. + /// After the budget is exhausted, publications succeed unless stays set + /// and is zero with a sticky exception. + /// + public int RemainingPublishFailures { get; set; } + + /// + /// When , waits until cancellationToken is cancelled + /// instead of completing a confirm. + /// + public bool HoldUntilCancelled { get; set; } + + /// + /// When , marks the channel closed at the start of the next + /// publish so the not-open failure path is exercised after the channel was obtained. + /// + public bool CloseBeforePublish { get; set; } + + /// + /// Optional await injected after open checks and before recording a successful confirm. + /// Used by concurrency tests to observe in-flight publishes. + /// + public Func? BeforeConfirmAsync { get; set; } + + /// + public Task PublishConfirmedAsync( + string exchange, + string routingKey, + string messageId, + string appId, + string expiration, + ReadOnlyMemory body, + CancellationToken cancellationToken) => + PublishConfirmedAsync( + new RabbitMqConfirmedPublication( + exchange, + routingKey, + messageId, + appId, + CorrelationId: null, + ContentType: null, + RequestIdHeader: null, + expiration, + Persistent: true, + Mandatory: true, + body), + cancellationToken); + + /// + public async Task PublishConfirmedAsync( + RabbitMqConfirmedPublication publication, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(publication); + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqPublishChannel)); + } + + if (CloseBeforePublish) + { + IsOpen = false; + } + + if (HoldUntilCancelled) + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ publish channel is not open."); + } + + if (BeforeConfirmAsync is not null) + { + await BeforeConfirmAsync().ConfigureAwait(false); + } + + if (RemainingPublishFailures > 0) + { + RemainingPublishFailures--; + throw PublishException ?? new InvalidOperationException("RabbitMQ negatively acknowledged the publication."); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqConfirmedPublication( + publication.Exchange, + publication.RoutingKey, + publication.MessageId, + publication.AppId, + publication.CorrelationId, + publication.ContentType, + publication.RequestIdHeader, + publication.ExpirationMilliseconds, + publication.Persistent, + publication.Mandatory, + publication.Body.ToArray())); + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// +/// In-memory async-confirm publish channel. returns after the write; +/// auto-acks by default so outstanding windows drain without a real broker. +/// +internal sealed class FakeRabbitMqAsyncConfirmPublishChannel : IRabbitMqAsyncConfirmPublishChannel +{ + private long _nextSequence = 1; + private AsyncEventHandler? _acks; + private AsyncEventHandler? _nacks; + private AsyncEventHandler? _returns; + + public FakeRabbitMqAsyncConfirmPublishChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When true (default), raises BasicAck after each successful publish. + public bool AutoAck { get; set; } = true; + + /// Delay before auto-ack (simulates quorum confirm latency). + public TimeSpan AutoAckDelay { get; set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// Remaining publications that throw before succeeding. + public int RemainingPublishFailures { get; set; } + + /// Optional await before completing a publish write. + public Func? BeforePublishAsync { get; set; } + + /// + public event AsyncEventHandler BasicAcksAsync + { + add => _acks += value; + remove => _acks -= value; + } + + /// + public event AsyncEventHandler BasicNacksAsync + { + add => _nacks += value; + remove => _nacks -= value; + } + + /// + public event AsyncEventHandler BasicReturnAsync + { + add => _returns += value; + remove => _returns -= value; + } + + /// + public ValueTask GetNextPublishSequenceNumberAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var seq = (ulong)Interlocked.Increment(ref _nextSequence) - 1; + return ValueTask.FromResult(seq); + } + + /// + public async Task PublishAsync( + string exchange, + string routingKey, + string messageId, + string appId, + string expiration, + ulong publishSequenceNumber, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqAsyncConfirmPublishChannel)); + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ publish channel is not open."); + } + + if (BeforePublishAsync is not null) + { + await BeforePublishAsync().ConfigureAwait(false); + } + + if (RemainingPublishFailures > 0) + { + RemainingPublishFailures--; + throw PublishException + ?? new InvalidOperationException("RabbitMQ publish write failed."); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqConfirmedPublication( + exchange, + routingKey, + messageId, + appId, + CorrelationId: null, + ContentType: null, + RequestIdHeader: null, + expiration, + Persistent: true, + Mandatory: OverviewDbTopology.Mandatory, + body.ToArray())); + + if (AutoAck) + { + var delay = AutoAckDelay; + var seq = publishSequenceNumber; + _ = Task.Run(async () => + { + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay).ConfigureAwait(false); + } + + await RaiseAckAsync(seq, multiple: false).ConfigureAwait(false); + }); + } + } + + /// Raises a Basic.Ack for tests. + public Task RaiseAckAsync(ulong deliveryTag, bool multiple) => + _acks?.Invoke(this, new BasicAckEventArgs(deliveryTag, multiple, CancellationToken.None)) + ?? Task.CompletedTask; + + /// Raises a Basic.Nack for tests. + public Task RaiseNackAsync(ulong deliveryTag, bool multiple) => + _nacks?.Invoke(this, new BasicNackEventArgs(deliveryTag, multiple, requeue: false, CancellationToken.None)) + ?? Task.CompletedTask; + + /// Raises a Basic.Return for tests. + public Task RaiseReturnAsync(ulong publishSequenceNumber, ReadOnlyMemory body) + { + var properties = new BasicProperties + { + Headers = new Dictionary + { + // Mirror production AMQP table representation (invariant decimal string). + [Constants.PublishSequenceNumberHeader] = + RabbitMqClientAsyncConfirmPublishChannel.FormatPublishSequenceNumberHeader( + publishSequenceNumber), + }, + }; + return _returns?.Invoke( + this, + new BasicReturnEventArgs( + 312, + "NO_ROUTE", + OverviewDbTopology.DefaultExchange, + OverviewDbTopology.RoutingKey, + properties, + body, + CancellationToken.None)) + ?? Task.CompletedTask; + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// Recorded article-work RPC publication. +internal sealed record FakeRabbitMqRpcPublication( + string Exchange, + string RoutingKey, + string CorrelationId, + string RequestId, + string ReplyTo, + string ContentType, + string Expiration, + byte[] Body); + +/// In-memory manual-ack RabbitMQ channel for consume/settle tests. +internal sealed class FakeRabbitMqManualAckChannel : IRabbitMqManualAckChannel +{ + /// Initializes a new fake manual-ack channel. + public FakeRabbitMqManualAckChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded exchange declarations. + public List Exchanges { get; } = []; + + /// Gets recorded queue declarations. + public List Queues { get; } = []; + + /// Gets recorded bindings. + public List Bindings { get; } = []; + + /// Gets recorded acknowledgements. + public List Acks { get; } = []; + + /// Gets recorded negative acknowledgements. + public List<(ulong DeliveryTag, bool Requeue)> Nacks { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// Gets the consume handler when a consumer has started. + public Func? DeliveryHandler { get; private set; } + + /// Gets the consumed queue name. + public string? ConsumedQueue { get; private set; } + + /// Gets the configured prefetch. + public ushort PrefetchCount { get; private set; } + + /// + public Task ExchangeDeclareAsync( + string exchange, + string type, + bool durable, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Exchanges.Add(new FakeRabbitMqExchangeDeclaration( + exchange, + type, + durable, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task QueueBindAsync( + string queue, + string exchange, + string routingKey, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Bindings.Add(new FakeRabbitMqBindingDeclaration( + queue, + exchange, + routingKey, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task BasicConsumeAsync( + string queue, + ushort prefetchCount, + Func onDelivery, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConsumedQueue = queue; + PrefetchCount = prefetchCount; + DeliveryHandler = onDelivery; + return Task.FromResult("fake-manual-ack-consumer"); + } + + /// + public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + + /// + public Task BasicAckAsync(ulong deliveryTag, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Acks.Add(deliveryTag); + return Task.CompletedTask; + } + + /// + public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Nacks.Add((deliveryTag, requeue)); + return Task.CompletedTask; + } + + /// + public Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Nacks.Add((deliveryTag, requeue)); + return Task.CompletedTask; + } + + /// Delivers one message to the registered manual-ack consumer. + public Task DeliverAsync( + ulong deliveryTag, + ReadOnlyMemory body, + string? contentType = StorageServerAdvertisementWireProtocol.JsonContentType, + string? exchange = CacheFleetTopology.BroadcastExchangeName, + string? routingKey = "", + string? correlationId = null, + string? replyTo = null) + { + if (DeliveryHandler is null) + { + throw new InvalidOperationException("No manual-ack consumer is registered."); + } + + var delivery = new RabbitMqManualAckDelivery( + deliveryTag, + body.ToArray(), + correlationId, + replyTo, + contentType, + RequestIdHeader: null, + Redelivered: false, + routingKey, + exchange, + ConsumerTag: "fake-manual-ack-consumer", + Generation); + return DeliveryHandler(delivery); + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// Recorded confirm-enabled publication. +internal sealed record FakeRabbitMqConfirmedPublication( + string Exchange, + string RoutingKey, + string MessageId, + string? AppId, + string? CorrelationId, + string? ContentType, + string? RequestIdHeader, + string Expiration, + bool Persistent, + bool Mandatory, + byte[] Body); + +/// Recorded exchange declaration. +internal sealed record FakeRabbitMqExchangeDeclaration( + string Name, + string Type, + bool Durable, + bool AutoDelete, + IReadOnlyDictionary? Arguments); + +/// Recorded queue declaration. +internal sealed record FakeRabbitMqQueueDeclaration( + string Name, + bool Durable, + bool Exclusive, + bool AutoDelete, + IReadOnlyDictionary? Arguments); + +/// Recorded queue binding. +internal sealed record FakeRabbitMqBindingDeclaration( + string Queue, + string Exchange, + string RoutingKey, + IReadOnlyDictionary? Arguments); diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRedis.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRedis.cs new file mode 100644 index 00000000..f60a1441 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeRedis.cs @@ -0,0 +1,536 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Text; +using VectorNNTPD.ReaderService.SessionState.BytesAccounting; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.PostFilter.Quota; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Redis; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// In-memory Redis connection factory for offline tests. +internal sealed class FakeRedisConnectionFactory : IRedisConnectionFactory +{ + public int ConnectCount { get; private set; } + + public Exception? ConnectException { get; set; } + + public FakeRedisConnection? LastConnection { get; private set; } + + public Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + if (ConnectException is not null) + { + throw ConnectException; + } + + ConnectCount++; + LastConnection = new FakeRedisConnection(); + return Task.FromResult(LastConnection); + } +} + +/// In-memory Redis connection wrapping one . +internal sealed class FakeRedisConnection : IRedisConnection +{ + public FakeRedisDatabase Database { get; } = new(); + + public int DisposeCount { get; private set; } + + public IRedisDatabase GetDatabase() => Database; + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// In-memory Redis database with optional failure injection. +internal sealed class FakeRedisDatabase : IRedisDatabase +{ + private readonly ConcurrentDictionary _keys = + new(StringComparer.Ordinal); + + public int PingCount { get; set; } + + public int KeyExistsCount { get; set; } + + private int _existsStartedCount; + + public int ExistsStartedCount => Volatile.Read(ref _existsStartedCount); + + public ConcurrentDictionary BlockExistsByKey { get; } = new(StringComparer.Ordinal); + + public int SetCount { get; set; } + + public int GetCount { get; set; } + + public Exception? ExistsException { get; set; } + + public Exception? GetException { get; set; } + + public Exception? SetException { get; set; } + + public TimeSpan LastSetExpiry { get; private set; } + + public TaskCompletionSource? BlockSet { get; set; } + + public TaskCompletionSource? BlockExists { get; set; } + + public TaskCompletionSource? ExistsStarted { get; set; } + + public int NotifyExistsStartedAt { get; set; } + + public TaskCompletionSource? ExistsReached { get; set; } + + public int ScriptEvaluateCount { get; set; } + + public Exception? ScriptException { get; set; } + + public TaskCompletionSource? BlockScript { get; set; } + + public TaskCompletionSource? ScriptStarted { get; set; } + + internal SessionStateEngine SessionStateEngine { get; } = new(); + + internal TransitPeerStateEngine TransitPeerStateEngine { get; } = new(); + + internal AccountByteEngine AccountByteEngine { get; } = new(); + + internal PostFilterQuotaEngine PostFilterQuotaEngine { get; } = new(); + + public Task PingAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + PingCount++; + return Task.FromResult(TimeSpan.FromMilliseconds(1)); + } + + public async ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + var started = Interlocked.Increment(ref _existsStartedCount); + ExistsStarted?.TrySetResult(); + if (NotifyExistsStartedAt > 0 && started >= NotifyExistsStartedAt) + { + ExistsReached?.TrySetResult(); + } + if (BlockExistsByKey.TryGetValue(ToKey(key), out var keyedBlock)) + { + await keyedBlock.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + else if (BlockExists is not null) + { + await BlockExists.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + KeyExistsCount++; + if (ExistsException is not null) + { + throw ExistsException; + } + + if (!_keys.TryGetValue(ToKey(key), out var entry)) + { + return false; + } + + if (entry.Expiry <= DateTimeOffset.UtcNow) + { + _keys.TryRemove(ToKey(key), out _); + return false; + } + + return true; + } + + public ValueTask GetAsync( + ReadOnlyMemory key, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + GetCount++; + if (GetException is not null) + { + return ValueTask.FromException(GetException); + } + + var name = ToKey(key); + if (!_keys.TryGetValue(name, out var entry)) + { + return ValueTask.FromResult(null); + } + + if (entry.Expiry <= DateTimeOffset.UtcNow) + { + _keys.TryRemove(name, out _); + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(entry.Value); + } + + public async ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) + { + if (BlockSet is not null) + { + await BlockSet.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (SetException is not null) + { + SetCount++; + throw SetException; + } + + LastSetExpiry = expiry; + _keys[ToKey(key)] = (value.ToArray(), DateTimeOffset.UtcNow.Add(expiry)); + SetCount++; + } + + public async ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(script); + ArgumentNullException.ThrowIfNull(keys); + ArgumentNullException.ThrowIfNull(values); + ScriptStarted?.TrySetResult(); + if (BlockScript is not null) + { + await BlockScript.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + ScriptEvaluateCount++; + if (ScriptException is not null) + { + throw ScriptException; + } + + if (keys.Length == 1) + { + var key = Encoding.UTF8.GetString(keys[0].Span); + if (script == AccountByteScripts.Apply) + { + return AccountByteEngine.Apply(key, Utf8(values[0]), ParseLong(values[1]), ParseLong(values[2])); + } + + if (script == AccountByteScripts.Observe) + { + return AccountByteEngine.Observe(key); + } + + if (script == AccountByteScripts.Delete) + { + return AccountByteEngine.Delete(key); + } + + return EvaluateTransit(script, key, values); + } + + if (keys.Length >= 3 && script == SessionStateScripts.RenewAndApply) + { + return EvaluateRenewAndApply(keys, values); + } + + if (keys.Length == 2 && TryEvaluateQuota(script, keys, values, out var quotaResult)) + { + return quotaResult; + } + + if (keys.Length < 2) + { + throw new InvalidOperationException("Admission EVAL requires KEYS[1] source and KEYS[2] session."); + } + + var sourceKey = Encoding.UTF8.GetString(keys[0].Span); + var sessionKey = Encoding.UTF8.GetString(keys[1].Span); + if (script == SessionStateScripts.TryAdmit) + { + return SessionStateEngine.TryAdmit( + sourceKey, + sessionKey, + Utf8(values[0]), + Utf8(values[1]), + ParseInt(values[2]), + ParseInt(values[3]), + ParseLong(values[4]), + ParseLong(values[5]), + ParseLong(values[6]), + ParseLong(values[7]), + trackSessions: values.Length > 8 && ParseInt(values[8]) == 1); + } + + if (script == SessionStateScripts.Release) + { + return SessionStateEngine.Release( + sourceKey, + sessionKey, + Utf8(values[0]), + Utf8(values[1]), + ParseLong(values[2]), + ParseLong(values[3])); + } + + if (script == SessionStateScripts.Renew) + { + var ipCount = ParseInt(values[4]); + var sources = new (string Ip, long Generation)[ipCount]; + for (var i = 0; i < ipCount; i++) + { + sources[i] = (Utf8(values[5 + (i * 2)]), ParseLong(values[6 + (i * 2)])); + } + + return SessionStateEngine.Renew( + sourceKey, + sessionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + sources); + } + + if (script == SessionStateScripts.ReleaseOwner) + { + return SessionStateEngine.ReleaseOwner(sourceKey, sessionKey, Utf8(values[0])); + } + + throw new NotSupportedException("FakeRedis only evaluates cluster admission scripts."); + } + + private bool TryEvaluateQuota( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + out long result) + { + result = 0; + var quotaKey = Encoding.UTF8.GetString(keys[0].Span); + var multipostKey = Encoding.UTF8.GetString(keys[1].Span); + if (script == PostFilterQuotaScripts.Reserve) + { + var reservation = ParseReservation(Utf8(values[2]), ParseLong(values[3])); + result = PostFilterQuotaEngine.Reserve( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0]), + ParseLong(values[1]), + new PostFilterQuotaWindows(ParseLong(values[8]), ParseLong(values[9])), + new PostFilterQuotaCeilings( + ParseLong(values[10]), + ParseLong(values[11]), + ParseLong(values[12]), + ParseLong(values[13]), + ParseLong(values[14]), + ParseLong(values[15])), + ParseLong(values[4]), + ParseLong(values[5]), + ParseInt(values[6]), + Utf8(values[7])); + return true; + } + + if (script == PostFilterQuotaScripts.Commit) + { + var reservation = ParseReservation(Utf8(values[1]), ParseLong(values[2])); + result = PostFilterQuotaEngine.Commit( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0]), + new PostFilterQuotaWindows(ParseLong(values[3]), ParseLong(values[4])), + new PostFilterQuotaCeilings( + ParseLong(values[5]), + ParseLong(values[6]), + ParseLong(values[7]), + ParseLong(values[8]), + ParseLong(values[9]), + ParseLong(values[10]))); + return true; + } + + if (script == PostFilterQuotaScripts.Release) + { + var reservation = ParseReservation(Utf8(values[1]), ParseLong(values[2])); + result = PostFilterQuotaEngine.Release( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0])); + return true; + } + + return false; + } + + private static PostFilterReservationId ParseReservation(string token, long generation) + { + var last = token.LastIndexOf(':'); + var mid = token.LastIndexOf(':', last - 1); + return new PostFilterReservationId(token[..mid], token[(mid + 1)..last], generation); + } + + private long EvaluateRenewAndApply(ReadOnlyMemory[] keys, ReadOnlyMemory[] values) + { + var sourceKey = Encoding.UTF8.GetString(keys[0].Span); + var sessionKey = Encoding.UTF8.GetString(keys[1].Span); + var bytesKey = Encoding.UTF8.GetString(keys[2].Span); + var ipCount = ParseInt(values[4]); + var sources = new (string Ip, long Generation)[ipCount]; + for (var i = 0; i < ipCount; i++) + { + sources[i] = (Utf8(values[5 + (i * 2)]), ParseLong(values[6 + (i * 2)])); + } + + var renewed = SessionStateEngine.Renew( + sourceKey, + sessionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + sources); + var remaining = AccountByteEngine.Apply( + bytesKey, + Utf8(values[5 + (ipCount * 2)]), + ParseLong(values[6 + (ipCount * 2)]), + ParseLong(values[7 + (ipCount * 2)])); + return SessionStateBytePack.Encode(renewed == 1, remaining); + } + + private long EvaluateTransit(string script, string connectionKey, ReadOnlyMemory[] values) + { + if (script == TransitPeerStateScripts.TryAdmit) + { + return TransitPeerStateEngine.TryAdmit( + connectionKey, + Utf8(values[0]), + ParseInt(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + ParseLong(values[4])); + } + + if (script == TransitPeerStateScripts.Release) + { + return TransitPeerStateEngine.Release(connectionKey, Utf8(values[0]), ParseLong(values[1])); + } + + if (script == TransitPeerStateScripts.Renew) + { + return TransitPeerStateEngine.Renew( + connectionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3])); + } + + if (script == TransitPeerStateScripts.ReleaseOwner) + { + return TransitPeerStateEngine.ReleaseOwner(connectionKey, Utf8(values[0])); + } + + throw new NotSupportedException("FakeRedis only evaluates cluster admission scripts."); + } + + public void Seed(ReadOnlyMemory key, TimeSpan? expiry = null) => + Seed(key, new byte[] { 1 }, expiry); + + public void Seed(ReadOnlyMemory key, ReadOnlyMemory value, TimeSpan? expiry = null) => + _keys[ToKey(key)] = (value.ToArray(), DateTimeOffset.UtcNow.Add(expiry ?? TimeSpan.FromHours(2))); + + public bool Contains(ReadOnlyMemory key) => _keys.ContainsKey(ToKey(key)); + + public bool TryGetStored(ReadOnlyMemory key, out byte[] value, out DateTimeOffset expiry) + { + if (_keys.TryGetValue(ToKey(key), out var entry)) + { + value = entry.Value; + expiry = entry.Expiry; + return true; + } + + value = []; + expiry = default; + return false; + } + + public void Expire(ReadOnlyMemory key) => _keys.TryRemove(ToKey(key), out _); + + private static string ToKey(ReadOnlyMemory key) => Convert.ToHexString(key.Span); + + private static string Utf8(ReadOnlyMemory value) => Encoding.UTF8.GetString(value.Span); + + private static int ParseInt(ReadOnlyMemory value) => + int.Parse(Utf8(value), CultureInfo.InvariantCulture); + + private static long ParseLong(ReadOnlyMemory value) => + long.Parse(Utf8(value), CultureInfo.InvariantCulture); +} + +/// Direct double for HistoryDB tests. +internal sealed class FakeRedisService : IRedisService +{ + public FakeRedisDatabase Database { get; } = new(); + + IRedisDatabase IRedisService.Database => Database; + + public bool IsUnavailable { get; set; } + + public int BeginRejectCount { get; private set; } + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + if (IsUnavailable) + { + BeginRejectCount++; + return false; + } + + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + if (!succeeded) + { + IsUnavailable = true; + } + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public void Recover() => IsUnavailable = false; + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + Database.SetAsync(key, value, expiry, cancellationToken); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeSmtpServer.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeSmtpServer.cs new file mode 100644 index 00000000..ecc1b9cb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/FakeSmtpServer.cs @@ -0,0 +1,594 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +public sealed class FakeSmtpServerOptions +{ + public bool ImplicitTls { get; set; } + + public bool AdvertiseStartTls { get; set; } + + public bool AdvertiseAuthPlain { get; set; } + + public bool AdvertiseAuthLogin { get; set; } + + public bool RequireTlsForAuthentication { get; set; } = true; + + public string? ExpectedUsername { get; set; } + + public string? ExpectedPassword { get; set; } + + public long? Size { get; set; } + + public int MailFromCode { get; set; } = 250; + + public int DefaultRcptCode { get; set; } = 250; + + public Dictionary RcptCodes { get; } = new(StringComparer.OrdinalIgnoreCase); + + public int DataIntroCode { get; set; } = 354; + + public int DataDoneCode { get; set; } = 250; + + public int TransientDataFailures { get; set; } + + public bool CloseAfterGreeting { get; set; } + + public bool CloseAfterEhlo { get; set; } + + public string Greeting { get; set; } = "220 test.example ESMTP"; + + public X509Certificate2? Certificate { get; set; } +} + +public sealed class FakeSmtpServer : IAsyncDisposable +{ + private readonly FakeSmtpServerOptions _options; + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly List _commands = []; + private readonly List _rawLines = []; + private readonly Task? _run; + private int _dataAttempts; + private bool _sawPlaintextBeforeTls; + + public FakeSmtpServer(FakeSmtpServerOptions? options = null) + { + _options = options ?? new FakeSmtpServerOptions(); + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _run = Task.Run(() => AcceptLoopAsync(_cts.Token)); + } + + public int Port { get; } + + public IReadOnlyList Commands + { + get + { + lock (_commands) + { + return [.. _commands]; + } + } + } + + public IReadOnlyList RawLines + { + get + { + lock (_rawLines) + { + return [.. _rawLines]; + } + } + } + + public byte[]? LastMessage { get; private set; } + + public bool Authenticated { get; private set; } + + public bool StartedTls { get; private set; } + + public bool SawPlaintextSmtpBeforeTls => _sawPlaintextBeforeTls; + + public X509Certificate2? Certificate => _options.Certificate; + + public SmtpOptions ClientOptions( + SmtpSecurityMode security, + string? username = null, + string? password = null) => + new() + { + Host = "127.0.0.1", + Port = Port, + Security = security, + Username = username ?? string.Empty, + Password = password ?? string.Empty, + RequireTlsForAuthentication = security != SmtpSecurityMode.None, + ConnectTimeout = TimeSpan.FromSeconds(5), + CommandTimeout = TimeSpan.FromSeconds(5), + MaxAttempts = 1, + }; + + public static X509Certificate2[] TrustAnchors(X509Certificate2 certificate) + { + ArgumentNullException.ThrowIfNull(certificate); + return [X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert))]; + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync().ConfigureAwait(false); + _listener.Stop(); + if (_run is not null) + { + try + { + await _run.WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + } + catch (TimeoutException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + _cts.Dispose(); + _options.Certificate?.Dispose(); + } + + public static X509Certificate2 CreateSelfSignedCertificate( + string dnsName = "localhost", + bool includeLoopbackIp = true) + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + dnsName, + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + var san = new SubjectAlternativeNameBuilder(); + san.AddDnsName(dnsName); + if (includeLoopbackIp) + { + san.AddIpAddress(IPAddress.Loopback); + } + + request.CertificateExtensions.Add(san.Build()); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, + critical: false)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, critical: false)); + using var ephemeral = request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow.AddDays(30)); + return X509CertificateLoader.LoadPkcs12( + ephemeral.Export(X509ContentType.Pfx, "vectornntp-test"), + "vectornntp-test", + X509KeyStorageFlags.Exportable | X509KeyStorageFlags.UserKeySet | X509KeyStorageFlags.PersistKeySet); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var tcp = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + _ = Task.Run(() => HandleAsync(tcp, cancellationToken), CancellationToken.None); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (SocketException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private async Task HandleAsync(TcpClient tcp, CancellationToken cancellationToken) + { + using (tcp) + try + { + Stream stream = tcp.GetStream(); + var tls = false; + if (_options.ImplicitTls) + { + stream = await ServerTlsAsync(stream, cancellationToken).ConfigureAwait(false); + tls = true; + StartedTls = true; + } + + await WriteLineAsync(stream, _options.Greeting, cancellationToken).ConfigureAwait(false); + if (_options.CloseAfterGreeting) + { + return; + } + + while (!cancellationToken.IsCancellationRequested) + { + var line = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false); + if (line is null) + { + return; + } + + if (!tls) + { + _sawPlaintextBeforeTls = true; + } + + Record(line, redact: line.StartsWith("AUTH ", StringComparison.OrdinalIgnoreCase)); + var verb = Verb(line); + if (verb.Equals("EHLO", StringComparison.OrdinalIgnoreCase) + || verb.Equals("HELO", StringComparison.OrdinalIgnoreCase)) + { + await WriteEhloAsync(stream, cancellationToken).ConfigureAwait(false); + if (_options.CloseAfterEhlo) + { + return; + } + + continue; + } + + if (verb.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + if (!_options.AdvertiseStartTls) + { + await WriteLineAsync(stream, "502 STARTTLS not available", cancellationToken) + .ConfigureAwait(false); + continue; + } + + await WriteLineAsync(stream, "220 Ready to start TLS", cancellationToken).ConfigureAwait(false); + stream = await ServerTlsAsync(stream, cancellationToken).ConfigureAwait(false); + tls = true; + StartedTls = true; + continue; + } + + if (verb.Equals("AUTH", StringComparison.OrdinalIgnoreCase)) + { + if (_options.RequireTlsForAuthentication && !tls) + { + await WriteLineAsync(stream, "538 5.7.11 Encryption required for requested authentication mechanism", cancellationToken) + .ConfigureAwait(false); + continue; + } + + await HandleAuthAsync(stream, line, cancellationToken).ConfigureAwait(false); + continue; + } + + if (verb.Equals("MAIL", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, _options.MailFromCode + " OK", cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("RCPT", StringComparison.OrdinalIgnoreCase)) + { + var mailbox = ExtractMailbox(line); + var code = _options.RcptCodes.TryGetValue(mailbox, out var mapped) + ? mapped + : _options.DefaultRcptCode; + await WriteLineAsync(stream, code + " " + (code < 400 ? "OK" : "rejected"), cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("DATA", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, _options.DataIntroCode + " Start mail input", cancellationToken) + .ConfigureAwait(false); + if (_options.DataIntroCode >= 400) + { + continue; + } + + LastMessage = await ReadDataAsync(stream, cancellationToken).ConfigureAwait(false); + _dataAttempts++; + var done = _dataAttempts <= _options.TransientDataFailures + ? 451 + : _options.DataDoneCode; + await WriteLineAsync(stream, done + " " + (done < 400 ? "OK" : "failed"), cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, "221 Bye", cancellationToken).ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "500 unrecognized", cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (AuthenticationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + private async Task HandleAuthAsync(Stream stream, string line, CancellationToken cancellationToken) + { + var parts = line.Split(' ', 3, StringSplitOptions.RemoveEmptyEntries); + var mechanism = parts.Length >= 2 ? parts[1] : string.Empty; + if (mechanism.Equals("PLAIN", StringComparison.OrdinalIgnoreCase)) + { + string payload; + if (parts.Length >= 3) + { + payload = parts[2]; + } + else + { + await WriteLineAsync(stream, "334 ", cancellationToken).ConfigureAwait(false); + payload = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(payload, redact: true); + } + + if (TryDecodePlain(payload, out var user, out var password) + && user == _options.ExpectedUsername + && password == _options.ExpectedPassword) + { + Authenticated = true; + await WriteLineAsync(stream, "235 2.7.0 Authentication successful", cancellationToken) + .ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "535 5.7.8 Authentication failed", cancellationToken) + .ConfigureAwait(false); + return; + } + + if (mechanism.Equals("LOGIN", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, "334 VXNlcm5hbWU6", cancellationToken).ConfigureAwait(false); + var userB64 = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(userB64, redact: true); + await WriteLineAsync(stream, "334 UGFzc3dvcmQ6", cancellationToken).ConfigureAwait(false); + var passB64 = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(passB64, redact: true); + var user = DecodeUtf8(userB64); + var password = DecodeUtf8(passB64); + if (user == _options.ExpectedUsername && password == _options.ExpectedPassword) + { + Authenticated = true; + await WriteLineAsync(stream, "235 Authentication successful", cancellationToken) + .ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "535 Authentication failed", cancellationToken).ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "504 Unrecognized authentication type", cancellationToken) + .ConfigureAwait(false); + } + + private async Task WriteEhloAsync(Stream stream, CancellationToken cancellationToken) + { + var lines = new List { "250-test.example" }; + if (_options.AdvertiseStartTls) + { + lines.Add("250-STARTTLS"); + } + + if (_options.Size is { } size) + { + lines.Add("250-SIZE " + size); + } + + lines.Add("250-8BITMIME"); + if (_options.AdvertiseAuthPlain || _options.AdvertiseAuthLogin) + { + var mechanisms = new List(); + if (_options.AdvertiseAuthPlain) + { + mechanisms.Add("PLAIN"); + } + + if (_options.AdvertiseAuthLogin) + { + mechanisms.Add("LOGIN"); + } + + lines.Add("250-AUTH " + string.Join(' ', mechanisms)); + } + + lines.Add("250 PIPELINING"); + foreach (var line in lines) + { + await WriteLineAsync(stream, line, cancellationToken).ConfigureAwait(false); + } + } + + private async Task ServerTlsAsync(Stream inner, CancellationToken cancellationToken) + { + var cert = _options.Certificate ?? CreateSelfSignedCertificate(); + _options.Certificate ??= cert; + var ssl = new SslStream(inner, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsServerAsync( + new SslServerAuthenticationOptions + { + ServerCertificate = cert, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }, + cancellationToken) + .ConfigureAwait(false); + return ssl; + } + + private void Record(string line, bool redact) + { + lock (_rawLines) + { + _rawLines.Add(redact ? "REDACTED" : line); + } + + lock (_commands) + { + if (redact) + { + _commands.Add("AUTH"); + return; + } + + _commands.Add(line); + } + } + + private static async Task WriteLineAsync(Stream stream, string line, CancellationToken cancellationToken) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadLineAsync(Stream stream, CancellationToken cancellationToken) + { + var buffer = new List(128); + var one = new byte[1]; + while (true) + { + var read = await stream.ReadAsync(one, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return buffer.Count == 0 ? null : Encoding.ASCII.GetString(buffer.ToArray()); + } + + if (one[0] == (byte)'\n') + { + if (buffer.Count > 0 && buffer[^1] == (byte)'\r') + { + buffer.RemoveAt(buffer.Count - 1); + } + + return Encoding.ASCII.GetString(buffer.ToArray()); + } + + buffer.Add(one[0]); + } + } + + private static async Task ReadDataAsync(Stream stream, CancellationToken cancellationToken) + { + var output = new List(); + var line = new List(); + var one = new byte[1]; + while (true) + { + var read = await stream.ReadAsync(one, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + break; + } + + line.Add(one[0]); + if (line.Count >= 2 && line[^2] == (byte)'\r' && line[^1] == (byte)'\n') + { + if (line.Count == 3 && line[0] == (byte)'.') + { + break; + } + + var payload = line.ToArray().AsSpan()[..^2]; + if (payload.Length > 0 && payload[0] == (byte)'.') + { + payload = payload[1..]; + } + + output.AddRange(payload.ToArray()); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + line.Clear(); + } + } + + return [.. output]; + } + + private static string Verb(string line) + { + var space = line.IndexOf(' '); + return space < 0 ? line : line[..space]; + } + + private static string ExtractMailbox(string line) + { + var start = line.IndexOf('<'); + var end = line.IndexOf('>'); + return start >= 0 && end > start ? line[(start + 1)..end] : string.Empty; + } + + private static bool TryDecodePlain(string payload, out string user, out string password) + { + user = string.Empty; + password = string.Empty; + try + { + var bytes = Convert.FromBase64String(payload); + var text = Encoding.UTF8.GetString(bytes); + var parts = text.Split('\0'); + if (parts.Length < 3) + { + return false; + } + + user = parts[1]; + password = parts[2]; + return true; + } + catch (FormatException) + { + return false; + } + } + + private static string DecodeUtf8(string base64) + { + try + { + return Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + } + catch (FormatException) + { + return string.Empty; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs new file mode 100644 index 00000000..f9385f0a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs @@ -0,0 +1,76 @@ +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// +/// AUTHINFO USER/PASS map used by moderation and ordinary POST tests. +/// Passwords are never logged or asserted. +/// +public sealed class MapNntpAuthenticationProvider : INntpAuthenticationProvider +{ + /// Ordinary authenticated poster who is not a moderator. + public const string NormalUser = "normal-user"; + + /// Moderator principal for group A mappings. + public const string ModeratorA = "moderator-a"; + + /// Moderator principal for group B mappings. + public const string ModeratorB = "moderator-b"; + + /// Password for . + public const string NormalUserPassword = "normal-user-secret"; + + /// Password for . + public const string ModeratorAPassword = "moderator-a-secret"; + + /// Password for . + public const string ModeratorBPassword = "moderator-b-secret"; + + /// Authenticated reader + poster privileges (no transit, no control-cancel). + public static NntpAuthorization PosterPrivileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private readonly Dictionary _accounts = + new(StringComparer.Ordinal); + + /// Creates the standard normal-user / moderator-a / moderator-b map. + public static MapNntpAuthenticationProvider CreateStandard() + { + var provider = new MapNntpAuthenticationProvider(); + provider.Add(NormalUser, NormalUserPassword, PosterPrivileges); + provider.Add(ModeratorA, ModeratorAPassword, PosterPrivileges); + provider.Add(ModeratorB, ModeratorBPassword, PosterPrivileges); + return provider; + } + + /// Registers one username/password and the privileges granted on success. + public void Add(string username, string password, NntpAuthorization authorization) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(authorization); + _accounts[username] = (password, authorization); + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + if (_accounts.TryGetValue(username, out var account) && account.Password == password) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, account.Authorization)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs new file mode 100644 index 00000000..686074aa --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs @@ -0,0 +1,75 @@ +using VectorNNTPD.ReaderService.Moderation; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// In-memory for catalogue tests. +internal sealed class MemoryNntpModeratorRepository : INntpModeratorRepository +{ + private readonly List _rows = []; + + public int QueryCount { get; private set; } + + public Exception? Exception { get; set; } + + public void Add(MemoryModeratorRecord record) + { + ArgumentNullException.ThrowIfNull(record); + _rows.Add(record); + } + + public ValueTask> GetEnabledAsync( + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryCount++; + if (Exception is not null) + { + throw Exception; + } + + var enabled = _rows + .Where(static row => row.Enabled) + .OrderBy(static row => row.ModeratorId) + .Select(static row => new NntpModeratorRow( + row.ModeratorId, + row.GroupPattern, + row.ModeratorAddress, + row.AccountName)) + .ToArray(); + return ValueTask.FromResult>(enabled); + } +} + +/// One in-memory moderator row, including disabled rows for filter tests. +internal sealed class MemoryModeratorRecord +{ + public long ModeratorId { get; init; } + + public string GroupPattern { get; init; } = string.Empty; + + public string ModeratorAddress { get; init; } = string.Empty; + + public string AccountName { get; init; } = string.Empty; + + public bool Enabled { get; init; } = true; +} + +/// Builds a from test mappings. +internal static class ModeratorTestSnapshot +{ + public static ModeratorSnapshot Create(params (string Pattern, string Address, string Username)[] mappings) + { + ArgumentNullException.ThrowIfNull(mappings); + var rows = new NntpModeratorRow[mappings.Length]; + for (var i = 0; i < mappings.Length; i++) + { + rows[i] = new NntpModeratorRow( + i + 1, + mappings[i].Pattern, + mappings[i].Address, + mappings[i].Username); + } + + return ModeratorSnapshot.Create(rows); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs new file mode 100644 index 00000000..3e9c3da1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs @@ -0,0 +1,68 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.ReaderService.Authentication; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// In-memory for authentication tests. +internal sealed class MemoryNntpUserRecordStore : INntpUserRecordStore +{ + private readonly Dictionary _users = new(StringComparer.Ordinal); + + public int LookupCount { get; private set; } + + public Exception? Exception { get; set; } + + public void Add(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + _users[record.AccountName] = record; + } + + public ValueTask TryGetUserAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + LookupCount++; + if (Exception is not null) + { + throw Exception; + } + + return _users.TryGetValue(accountName, out var record) + ? ValueTask.FromResult(record) + : ValueTask.FromResult(null); + } + + public static NntpUserRecord Create( + string name, + string password, + bool enabled = true, + bool allowPlain = true, + bool allowScram = true, + int sessionLimit = 0, + int srcIpLimit = 0, + int rateLimitBps = 0, + long byteLimit = 1_000_000_000, + ReadOnlyMemory scramSalt = default, + int scramIterations = 0, + ReadOnlyMemory scramStoredKey = default, + ReadOnlyMemory scramServerKey = default, + string customerId = "11111111-1111-1111-1111-111111111111", + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) => + new( + name, + password, + allowPlain, + allowScram, + scramSalt, + scramIterations, + scramStoredKey, + scramServerKey, + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + enabled, + customerId, + allowedArtTypes); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingEmailService.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingEmailService.cs new file mode 100644 index 00000000..3d3fb6a1 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingEmailService.cs @@ -0,0 +1,58 @@ +using VectorNNTPD.ReaderService.Email; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Captures without SMTP or spool I/O. +internal sealed class RecordingEmailService : IEmailService +{ + private readonly List _sent = []; + + public EmailEnqueueStatus Status { get; set; } = EmailEnqueueStatus.Accepted; + + public Exception? Throw { get; set; } + + public TaskCompletionSource? Hold { get; set; } + + public TaskCompletionSource SentOnce { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int SendThreadId { get; private set; } + + public int SendCount { get; private set; } + + public IReadOnlyList Sent + { + get + { + lock (_sent) + { + return [.. _sent]; + } + } + } + + public async ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + if (Hold is not null) + { + await Hold.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Throw is not null) + { + throw Throw; + } + + SendThreadId = Environment.CurrentManagedThreadId; + lock (_sent) + { + _sent.Add(message); + SendCount++; + } + + SentOnce.TrySetResult(); + return new EmailEnqueueResult(Status, "test"); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingModerationSubmissionService.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingModerationSubmissionService.cs new file mode 100644 index 00000000..c42b272f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingModerationSubmissionService.cs @@ -0,0 +1,36 @@ +using VectorNNTPD.ReaderService.Moderation; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Records moderation submissions and returns a configured result. +public sealed class RecordingModerationSubmissionService : IModerationSubmissionService +{ + /// Initializes a new instance of the class. + public RecordingModerationSubmissionService( + ModerationSubmissionStatus status = ModerationSubmissionStatus.Accepted, + string detail = "recorded") + { + Status = status; + Detail = detail; + } + + /// Gets the status returned to callers. + public ModerationSubmissionStatus Status { get; set; } + + /// Gets the detail returned to callers. + public string Detail { get; set; } + + /// Gets recorded submissions in call order. + public List Submissions { get; } = []; + + /// + public ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + cancellationToken.ThrowIfCancellationRequested(); + Submissions.Add(submission); + return ValueTask.FromResult(new ModerationSubmissionResult(Status, Detail)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNewsLogWriter.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNewsLogWriter.cs new file mode 100644 index 00000000..f80f9c68 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNewsLogWriter.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Captures news events for post-queue ingress tests. +internal sealed class RecordingNewsLogWriter : INewsLogWriter +{ + private readonly List _events = []; + private readonly TaskCompletionSource? _gate; + private int _writeCalls; + private int _flushCalls; + + public RecordingNewsLogWriter(Task? gate = null) + { + _gate = gate is null + ? null + : TaskCompletionSourceFrom(gate); + } + + public IReadOnlyList Events + { + get + { + lock (_events) + { + return _events.ToArray(); + } + } + } + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public int FlushCalls => Volatile.Read(ref _flushCalls); + + public void Write(in NewsLogEvent evt) + { + if (_gate is not null) + { + _gate.Task.GetAwaiter().GetResult(); + } + + Interlocked.Increment(ref _writeCalls); + lock (_events) + { + _events.Add( + new NewsLogEvent( + evt.Disposition, + evt.MessageId.ToArray(), + evt.Feed.ToArray(), + evt.Sites.ToArray(), + evt.Timestamp, + evt.ResponseCode, + evt.Reason.ToArray(), + evt.Size)); + } + } + + public void Flush() => Interlocked.Increment(ref _flushCalls); + + private static TaskCompletionSource TaskCompletionSourceFrom(Task gate) + { + var source = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + gate.ContinueWith( + static (completed, state) => + { + var tcs = (TaskCompletionSource)state!; + if (completed.IsFaulted) + { + tcs.TrySetException(completed.Exception!.InnerExceptions); + } + else if (completed.IsCanceled) + { + tcs.TrySetCanceled(); + } + else + { + tcs.TrySetResult(); + } + }, + source, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + return source; + } +} + +/// News writer that throws to prove already-accepted articles are not reprocessed. +internal sealed class ThrowingNewsLogWriter : INewsLogWriter +{ + public int WriteCalls { get; private set; } + + public void Write(in NewsLogEvent evt) + { + WriteCalls++; + throw new IOException("news writer test failure"); + } + + public void Flush() + { + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs new file mode 100644 index 00000000..f3ba3413 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs @@ -0,0 +1,37 @@ +using System.Net; +using VectorNNTPD.ReaderService.Session.Authentication; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Counts reader-provider AUTHINFO invocations. Never logs passwords. +internal sealed class RecordingNntpAuthenticationProvider : INntpAuthenticationProvider +{ + private readonly INntpAuthenticationProvider _inner; + + public RecordingNntpAuthenticationProvider(INntpAuthenticationProvider inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + public int AuthenticateCount { get; private set; } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + AuthenticateCount++; + return _inner.AuthenticateAsync(username, password, cancellationToken); + } + + public ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + AuthenticateCount++; + return _inner.AuthenticateAsync(username, password, clientIp, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..205d68da --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs @@ -0,0 +1,192 @@ +using VectorNNTPD.ReaderService.ArticleIngestion.OverviewDb; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Captures OverviewDB handoff payloads and can nack a budgeted number of publishes. +internal sealed class RecordingOverviewDbHandoffPublisher : IOverviewDbHandoffPublisher +{ + private int _attempts; + private int _outstanding; + private readonly object _gate = new(); + private readonly Queue _failures = new(); + private readonly TaskCompletionSource _firstAttempt = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _confirmed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Gets captured successfully confirmed payloads in publish order. + public List Payloads { get; } = []; + + /// Gets the number of publish attempts, including nacks. + public int AttemptCount => Volatile.Read(ref _attempts); + + /// Completes when the first publish attempt begins. + public Task FirstAttempt => _firstAttempt.Task; + + /// Completes when the first payload is recorded as confirmed. + public Task Confirmed => _confirmed.Task; + + private int _confirmWaitCount; + private TaskCompletionSource? _confirmWait; + + /// + /// Completes when at least payloads have been confirmed. + /// + public Task WaitUntilConfirmedAsync(int count, CancellationToken cancellationToken) + { + lock (_gate) + { + if (Payloads.Count >= count) + { + return Task.CompletedTask; + } + + _confirmWaitCount = count; + _confirmWait = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + return _confirmWait.Task.WaitAsync(cancellationToken); + } + } + + /// Remaining attempts that enqueue a failure instead of confirming. + public int RemainingFailures { get; set; } + + /// + /// Exception thrown for each budgeted attempt. + /// + public Exception? TransientPublishException { get; set; } + + /// When set, every attempt throws this exception after the remaining-failure budget. + public Exception? PublishException { get; set; } + + /// + /// When set, a failing attempt waits on this source after recording the attempt + /// and before enqueueing the failure. + /// + public TaskCompletionSource? BlockOnFailure { get; set; } + + /// + /// Artificial confirmation delay applied after returns. + /// Simulates quorum confirm latency without blocking the publish call. + /// + public TimeSpan ConfirmDelay { get; set; } + + /// Maximum outstanding unconfirmed publishes (window). Zero means unbounded. + public int OutstandingWindow { get; set; } + + /// + public int OutstandingCount => Volatile.Read(ref _outstanding); + + /// + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + cancellationToken.ThrowIfCancellationRequested(); + + if (OutstandingWindow > 0) + { + while (Volatile.Read(ref _outstanding) >= OutstandingWindow) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Delay(1, cancellationToken).ConfigureAwait(false); + } + } + + var attempt = Interlocked.Increment(ref _attempts); + _firstAttempt.TrySetResult(attempt); + Interlocked.Increment(ref _outstanding); + + if (RemainingFailures > 0) + { + RemainingFailures--; + if (BlockOnFailure is not null) + { + await BlockOnFailure.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + Interlocked.Decrement(ref _outstanding); + if (TransientPublishException is not null || PublishException is not null) + { + throw TransientPublishException + ?? PublishException + ?? new InvalidOperationException("RabbitMQ negatively acknowledged the OverviewDB handoff."); + } + + EnqueueFailure(item); + return; + } + + if (PublishException is not null) + { + if (BlockOnFailure is not null) + { + await BlockOnFailure.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + Interlocked.Decrement(ref _outstanding); + throw PublishException; + } + + // Fire-and-forget confirm so PublishAsync is not serialized behind ConfirmDelay. + _ = ConfirmLaterAsync(item, cancellationToken); + } + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + lock (_gate) + { + if (_failures.Count == 0) + { + item = null!; + return false; + } + + item = _failures.Dequeue(); + return true; + } + } + + /// + public void AbandonOutstanding() + { + Volatile.Write(ref _outstanding, 0); + lock (_gate) + { + _failures.Clear(); + } + } + + private async Task ConfirmLaterAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + try + { + if (ConfirmDelay > TimeSpan.Zero) + { + await Task.Delay(ConfirmDelay, CancellationToken.None).ConfigureAwait(false); + } + + lock (_gate) + { + Payloads.Add(item.Payload.ToArray()); + _confirmed.TrySetResult(); + if (_confirmWait is not null && Payloads.Count >= _confirmWaitCount) + { + _confirmWait.TrySetResult(); + _confirmWait = null; + } + } + } + finally + { + Interlocked.Decrement(ref _outstanding); + } + } + + private void EnqueueFailure(OverviewDbWorkItem item) + { + lock (_gate) + { + _failures.Enqueue(item); + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingPathSurveyWriter.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingPathSurveyWriter.cs new file mode 100644 index 00000000..20c30823 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingPathSurveyWriter.cs @@ -0,0 +1,126 @@ +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Logging; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Captures Path-survey observations without pretending to persist them. +internal sealed class RecordingPathSurveyWriter : IPathSurveyWriter +{ + private readonly List _paths = []; + private int _writeCalls; + private int _flushCalls; + + public IReadOnlyList Paths + { + get + { + lock (_paths) + { + return _paths.ToArray(); + } + } + } + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public int FlushCalls => Volatile.Read(ref _flushCalls); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + lock (_paths) + { + _paths.Add(canonicalPath.ToArray()); + } + } + + public void Flush() => Interlocked.Increment(ref _flushCalls); +} + +/// Counts Path-survey writes without retaining the observation stream. +internal sealed class CountingPathSurveyWriter : IPathSurveyWriter +{ + private int _writeCalls; + private long _bytes; + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public long Bytes => Interlocked.Read(ref _bytes); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + Interlocked.Add(ref _bytes, canonicalPath.Length); + } + + public void Flush() + { + } +} + +/// Path-survey writer that throws to prove already-accepted articles are not reprocessed. +internal sealed class ThrowingPathSurveyWriter : IPathSurveyWriter +{ + public int WriteCalls { get; private set; } + + public void Write(ReadOnlySpan canonicalPath) + { + WriteCalls++; + throw new IOException("path survey writer test failure"); + } + + public void Flush() + { + } +} + +/// Records completed Path-survey files handed off before gzip. +internal sealed class RecordingCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler +{ + private readonly List _completed = []; + private readonly List _gzipExisted = []; + + public IReadOnlyList Completed + { + get + { + lock (_completed) + { + return _completed.ToArray(); + } + } + } + + public IReadOnlyList GzipExistedAtHandoff + { + get + { + lock (_gzipExisted) + { + return _gzipExisted.ToArray(); + } + } + } + + public void OnCompletedFile(string completedFilePath) + { + var gzip = completedFilePath + ".gz"; + lock (_completed) + { + _gzipExisted.Add(File.Exists(gzip)); + _completed.Add(completedFilePath); + } + } +} + +/// Completed-file handler that throws after recording the path. +internal sealed class ThrowingCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler +{ + public string? Path { get; private set; } + + public void OnCompletedFile(string completedFilePath) + { + Path = completedFilePath; + throw new InvalidOperationException("completed-file handler test failure"); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs new file mode 100644 index 00000000..7f2123a2 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs @@ -0,0 +1,21 @@ +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Counts Transit AUTHINFO invocations. Never logs credentials. +internal sealed class RecordingTransitPeerAuthenticator : ITransitPeerAuthenticator +{ + public int AuthenticateCount { get; private set; } + + public NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization) + { + AuthenticateCount++; + return TransitPeerAuthenticator.Instance.Authenticate(policy, username, password, currentAuthorization); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs new file mode 100644 index 00000000..8506d009 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.ReaderService.Newsgroups; + +namespace VectorNNTPD.ReaderService.Tests.TestDoubles; + +/// Test catalogue that publishes snapshots without MySQL. +internal sealed class StaticNewsgroupCatalogue : INewsgroupCatalogue +{ + private NewsgroupSnapshot _current; + + public StaticNewsgroupCatalogue(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + _current = snapshot; + } + + public int CurrentReadCount { get; private set; } + + public NewsgroupSnapshot Current + { + get + { + CurrentReadCount++; + return Volatile.Read(ref _current); + } + } + + public void Publish(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Interlocked.Exchange(ref _current, snapshot); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/NewsfeedsPatternTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/NewsfeedsPatternTests.cs new file mode 100644 index 00000000..f0d68e0c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/NewsfeedsPatternTests.cs @@ -0,0 +1,89 @@ +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// newsfeeds(5) / INN uwildmat_poison matching matrix. +/// Source: https://www.eyrie.org/~eagle/software/inn/docs/libinn-uwildmat.html +/// and https://www.eyrie.org/~eagle/software/inn/docs/newsfeeds.html +/// +public sealed class NewsfeedsPatternTests +{ + [Theory] + [InlineData("*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "alt.comp", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.sources.*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources.*", "comp.sources", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.sources*", "comp.sources", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.sourcesfoo", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.lang.c", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.*,!comp.sources.*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*,!comp.sources.*", "comp.sources.unix", NewsfeedsPatternDecision.Exclude)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "alt.fan", NewsfeedsPatternDecision.Match)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "alt.binaries.warez", NewsfeedsPatternDecision.Poison)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "misc.test", NewsfeedsPatternDecision.Match)] + [InlineData("!control", "control", NewsfeedsPatternDecision.Exclude)] + [InlineData("!control", "control.cancel", NewsfeedsPatternDecision.NoMatch)] + [InlineData("!control.*", "control.cancel", NewsfeedsPatternDecision.Exclude)] + [InlineData("!control.*", "control", NewsfeedsPatternDecision.NoMatch)] + [InlineData("news.*,!news.misc", "news.misc", NewsfeedsPatternDecision.Exclude)] + [InlineData("news.*,!news.misc,*.misc", "news.misc", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*,news.*", "news.misc", NewsfeedsPatternDecision.Match)] + [InlineData("exact.group", "exact.group", NewsfeedsPatternDecision.Match)] + [InlineData("exact.group", "exact.group.extra", NewsfeedsPatternDecision.NoMatch)] + [InlineData("a", "aa", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.?", "comp.x", NewsfeedsPatternDecision.Match)] + [InlineData("comp.?", "comp.xy", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.[abc]", "comp.a", NewsfeedsPatternDecision.Match)] + [InlineData("comp.[^a]", "comp.b", NewsfeedsPatternDecision.Match)] + [InlineData("comp.[^a]", "comp.a", NewsfeedsPatternDecision.NoMatch)] + public void Evaluate_MatchesNewsfeedsSemantics(string expression, string group, NewsfeedsPatternDecision expected) + { + Assert.True(NewsfeedsPattern.TryParse(expression, out var pattern, out var error), error); + Assert.NotNull(pattern); + Assert.Equal(expected, pattern.Evaluate(group)); + Assert.Equal(expected == NewsfeedsPatternDecision.Match, pattern.MatchesNewsgroup(group)); + } + + [Fact] + public void EvaluateArticle_PoisonWinsOverOtherMatchedGroups() + { + Assert.True(NewsfeedsPattern.TryParse("alt.*,@alt.binaries.warez,misc.*", out var pattern, out _)); + Assert.Equal( + NewsfeedsPatternDecision.Poison, + pattern!.EvaluateArticle(["misc.test", "alt.binaries.warez"])); + Assert.Equal( + NewsfeedsPatternDecision.Match, + pattern.EvaluateArticle(["misc.test", "alt.fan"])); + Assert.Equal( + NewsfeedsPatternDecision.NoMatch, + pattern.EvaluateArticle(["rec.sport"])); + } + + [Theory] + [InlineData("")] + [InlineData(",")] + [InlineData("comp.*,")] + [InlineData(",comp.*")] + [InlineData("!")] + [InlineData("@")] + [InlineData("comp.[abc")] + [InlineData(@"comp.\")] + public void TryParse_RejectsEmptyOrInvalidExpressions(string expression) + { + Assert.False(NewsfeedsPattern.TryParse(expression, out var pattern, out var error)); + Assert.Null(pattern); + Assert.False(string.IsNullOrWhiteSpace(error)); + } + + [Fact] + public void TryParse_AcceptsEscapedComma() + { + Assert.True(NewsfeedsPattern.TryParse(@"foo\,bar", out var pattern, out var error), error); + Assert.True(pattern!.MatchesNewsgroup("foo,bar")); + Assert.False(pattern.MatchesNewsgroup("foo")); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAllowFromAndDnsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAllowFromAndDnsTests.cs new file mode 100644 index 00000000..6f1217c7 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAllowFromAndDnsTests.cs @@ -0,0 +1,384 @@ +using System.Net; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitAllowFromAndDnsTests +{ + [Fact] + public void LiteralIpv4Address_MatchesExactly() + { + var authz = Create("192.0.2.10"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.11")).TransitPeerName); + } + + [Fact] + public void LiteralIpv4Cidr_MatchesPrefix() + { + var authz = Create("192.0.2.0/24"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.200")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + } + + [Fact] + public void LiteralIpv6AddressAndCidr_Match() + { + var authz = Create("2001:db8::10", "2001:db8:1234::/48"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8:1234::abcd")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("2001:db8:9999::1")).TransitPeerName); + } + + [Fact] + public void PrivateAndDocumentationAddresses_AreNotRejected() + { + var authz = Create("10.0.0.0/8", "192.168.1.5", "fd00::/8", "198.18.0.70"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("10.1.2.3")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.168.1.5")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("fd00::1")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("198.18.0.70")).TransitPeerName); + } + + [Fact] + public async Task DnsHostname_AuthorizesAllResolvedAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success( + [IPAddress.Parse("192.0.2.10"), IPAddress.Parse("2001:db8::10")], + TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + } + + [Fact] + public async Task MixedDnsAndLiterals_AuthorizeTogether() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("203.0.113.5")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot( + "peer", + TransitTestPeers.Peer(allowFrom: ["news.example.net", "192.0.2.10", "2001:db8::10"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + } + + [Fact] + public async Task DnsRefresh_ReplacesAddressesAtomically() + { + var addresses = new List { IPAddress.Parse("192.0.2.10") }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([.. addresses], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Equal(["192.0.2.10"], cache.GetResolved("news.example.net").Select(static a => a.ToString())); + + addresses.Clear(); + addresses.Add(IPAddress.Parse("198.51.100.20")); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var resolved = cache.GetResolved("news.example.net"); + Assert.DoesNotContain(IPAddress.Parse("192.0.2.10"), resolved); + Assert.Contains(IPAddress.Parse("198.51.100.20"), resolved); + } + + [Fact] + public async Task DnsTransientFailure_PreservesLastAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + resolver.Handler = _ => TransitDnsResolveResult.TransientFailure(); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Contains(IPAddress.Parse("192.0.2.10"), cache.GetResolved("news.example.net")); + } + + [Fact] + public async Task DnsEmptyResult_ClearsAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + resolver.Handler = _ => TransitDnsResolveResult.Empty(); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Empty(cache.GetResolved("news.example.net")); + } + + [Fact] + public async Task DnsRefresh_HonorsMinimumSixtySecondsEvenWhenTtlIsShorter() + { + var clock = new MutableTimeProvider { UtcNow = DateTimeOffset.UnixEpoch }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromSeconds(5)), + }; + var cache = new TransitDnsAddressCache( + resolver, + NullLogger.Instance, + TimeSpan.FromSeconds(60), + clock); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + + clock.UtcNow = clock.UtcNow.AddSeconds(30); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + + clock.UtcNow = clock.UtcNow.AddSeconds(31); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(2, resolver.Calls); + Assert.Equal(TimeSpan.FromSeconds(60), cache.GetDelayUntilNextRefresh(snapshot)); + clock.UtcNow = clock.UtcNow.AddSeconds(60); + Assert.Equal(TimeSpan.Zero, cache.GetDelayUntilNextRefresh(snapshot)); + } + + [Fact] + public async Task DnsRefresh_HonorsTtlWhenGreaterThanSixtySeconds() + { + var clock = new MutableTimeProvider { UtcNow = DateTimeOffset.UnixEpoch }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(10)), + }; + var cache = new TransitDnsAddressCache( + resolver, + NullLogger.Instance, + TimeSpan.FromSeconds(60), + clock); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + clock.UtcNow = clock.UtcNow.AddMinutes(9); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + clock.UtcNow = clock.UtcNow.AddMinutes(2); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(2, resolver.Calls); + } + + [Fact] + public async Task ConnectionAuthorization_DoesNotInvokeDnsResolver() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var callsAfterRefresh = resolver.Calls; + resolver.Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"); + + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + Assert.Equal(callsAfterRefresh, resolver.Calls); + } + + [Fact] + public void ConnectionAuthorization_UsesMaterializedIpAcl_ThrowingResolverNeverCalled() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"), + }; + var cache = new SeededDnsCache(); + cache.Seed("news.example.net", IPAddress.Parse("192.0.2.10")); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + Assert.Equal(0, resolver.Calls); + Assert.Equal(0, cache.RefreshCalls); + } + + [Fact] + public void ConnectionAuthorization_UnresolvedHostname_DoesNotInvokeResolver() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.False(authz.Resolve(IPAddress.Parse("192.0.2.10")).AuthorizedTransit); + Assert.Equal(0, resolver.Calls); + } + + [Fact] + public async Task DnsFailure_LogsWarningForPeerOnEveryAttempt() + { + var logger = new CollectingLogger(); + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.TransientFailure(), + }; + var cache = new TransitDnsAddressCache(resolver, logger); + var snapshot = TransitTestPeers.Snapshot( + "giganews", + TransitTestPeers.Peer(allowFrom: ["news.example.net"], peerName: "Giganews, Inc.")); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + + var warnings = logger.Messages.Where(static m => m.Contains("AllowFrom DNS resolution failed", StringComparison.Ordinal)).ToList(); + Assert.Equal(2, warnings.Count); + Assert.All( + warnings, + static w => + { + Assert.Contains("Giganews, Inc.", w, StringComparison.Ordinal); + Assert.Contains("news.example.net", w, StringComparison.Ordinal); + Assert.DoesNotContain("Password", w, StringComparison.Ordinal); + Assert.DoesNotContain("s3cret", w, StringComparison.Ordinal); + }); + } + + [Fact] + public void AmbiguousIpMatch_DeniesTransit_AndWarnsEveryResolve() + { + var logger = new CollectingLogger(); + var snapshot = new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer("one", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer("two", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + }); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, logger: logger); + var source = IPAddress.Parse("192.0.2.10"); + + Assert.Null(authz.Resolve(source).TransitPeerName); + Assert.False(authz.Resolve(source).AuthorizedTransit); + Assert.Equal(2, logger.Messages.Count); + Assert.All( + logger.Messages, + static m => + { + Assert.Contains("ambiguous", m, StringComparison.OrdinalIgnoreCase); + Assert.Contains("192.0.2.10", m, StringComparison.Ordinal); + Assert.Contains("one", m, StringComparison.Ordinal); + Assert.Contains("two", m, StringComparison.Ordinal); + }); + } + + [Fact] + public void EmptyAllowFrom_DoesNotMatchInbound() + { + var authz = TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot("outbound-only", TransitTestPeers.Peer(allowFrom: []))); + Assert.True(authz.IsEnabled); + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + } + + private static TransitPeerAuthorization Create(params string[] allowFrom) => + TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: allowFrom))); + + private sealed class SeededDnsCache : ITransitDnsAddressCache + { + private readonly Dictionary> _addresses = + new(StringComparer.OrdinalIgnoreCase); + + public int RefreshCalls { get; private set; } + + public void Seed(string hostname, params IPAddress[] addresses) => + _addresses[hostname] = addresses.ToHashSet(); + + public IReadOnlySet GetResolved(string hostname) => + _addresses.TryGetValue(hostname, out var set) ? set : new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) + { + RefreshCalls++; + throw new InvalidOperationException("connection-time DNS refresh is forbidden"); + } + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) + { + RefreshCalls++; + throw new InvalidOperationException("connection-time DNS refresh is forbidden"); + } + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + + private sealed class FakeTransitDnsResolver : ITransitDnsResolver + { + public Func Handler { get; set; } = + static _ => TransitDnsResolveResult.TransientFailure(); + + public int Calls { get; private set; } + + public Task ResolveAsync(string hostname, CancellationToken cancellationToken) + { + Calls++; + return Task.FromResult(Handler(hostname)); + } + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add($"{logLevel}: {formatter(state, exception)}"); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } + + private sealed class MutableTimeProvider : TimeProvider + { + public DateTimeOffset UtcNow { get; set; } = DateTimeOffset.UnixEpoch; + + public override DateTimeOffset GetUtcNow() => UtcNow; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAuthinfoAndCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAuthinfoAndCommandTests.cs new file mode 100644 index 00000000..ad530e5b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitAuthinfoAndCommandTests.cs @@ -0,0 +1,594 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitAuthinfoAndCommandTests +{ + private static readonly IPAddress PeerAddress = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress OtherAddress = IPAddress.Parse("198.51.100.1"); + private const string CommandNotAvailable = "502 Command not available on this server"; + + [Fact] + public async Task ReaderRole_ModeStream_IsUnavailable_AndMatchingPeerCredentialsDoNotAuthenticate() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.NotEqual(NntpAuthenticationAuthority.Transit, session.AuthenticationAuthority); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.Authentication.Username); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ReaderRole_ModeStream_IsUnavailable_AndInvalidPeerPasswordDoesNotAuthenticate() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.NotEqual(NntpAuthenticationAuthority.Transit, session.AuthenticationAuthority); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ReaderRole_PeerAuthinfo_DoesNotLogPassword() + { + var recording = new RecordingLoggerFactory(); + var peers = CreatePeer(username: "feed", password: "peer-secret-xyz"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers, loggerFactory: recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS peer-secret-xyz"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("peer-secret-xyz", joined, StringComparison.Ordinal); + } + + [Fact] + public async Task Authinfo_IsPublic_ForReaderStreamAndNonTransit() + { + var user = NntpCommandTestParse.ParseCommand("AUTHINFO USER x"); + var pass = NntpCommandTestParse.ParseCommand("AUTHINFO PASS y"); + Assert.True(user.IsValid); + Assert.True(pass.IsValid); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(user.Verb, user.Qualifier)); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(pass.Verb, pass.Qualifier)); + + await using var readerDuplex = await CommandDuplex.CreateAsync(OtherAddress); + var reader = readerDuplex.CreateSession(CreatePeer()); + var readerRun = reader.RunAsync(); + _ = await readerDuplex.ReadClientLineAsync(); + await readerDuplex.WriteClientLineAsync("MODE READER"); + Assert.StartsWith("201 ", await readerDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await readerDuplex.WriteClientLineAsync("AUTHINFO USER reader"); + Assert.StartsWith("381 ", await readerDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await readerDuplex.WriteClientLineAsync("QUIT"); + _ = await readerDuplex.ReadClientLineAsync(); + await readerRun; + + var transit = CreatePeer(); + await using var streamDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var stream = streamDuplex.CreateSession(transit); + var streamRun = stream.RunAsync(); + _ = await streamDuplex.ReadClientLineAsync(); + await streamDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await streamDuplex.ReadClientLineAsync()); + await streamDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await streamDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await streamDuplex.WriteClientLineAsync("QUIT"); + _ = await streamDuplex.ReadClientLineAsync(); + await streamRun; + + await using var anonDuplex = await CommandDuplex.CreateAsync(OtherAddress); + var anon = anonDuplex.CreateSession(CreatePeer()); + Assert.Null(anon.Authorization.TransitPeerName); + var anonRun = anon.RunAsync(); + _ = await anonDuplex.ReadClientLineAsync(); + await anonDuplex.WriteClientLineAsync("AUTHINFO USER anybody"); + Assert.StartsWith("381 ", await anonDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await anonDuplex.WriteClientLineAsync("QUIT"); + _ = await anonDuplex.ReadClientLineAsync(); + await anonRun; + } + + [Fact] + public async Task ReaderRole_PeerCredentialsWithWrongOrIncompleteConfiguration_NeverAuthenticate() + { + var both = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(both); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER not-feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + await using var userOnlyDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var userOnly = userOnlyDuplex.CreateSession(CreatePeer(username: "feed", password: "")); + var userOnlyRun = userOnly.RunAsync(); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await userOnlyDuplex.ReadClientLineAsync()); + await userOnlyDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyDuplex.WriteClientLineAsync("AUTHINFO PASS anything"); + Assert.Equal("481 Authentication failed", await userOnlyDuplex.ReadClientLineAsync()); + await userOnlyDuplex.WriteClientLineAsync("QUIT"); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyRun; + + await using var passOnlyDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var passOnly = passOnlyDuplex.CreateSession(CreatePeer(username: "", password: "s3cret")); + var passOnlyRun = passOnly.RunAsync(); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await passOnlyDuplex.ReadClientLineAsync()); + await passOnlyDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyDuplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await passOnlyDuplex.ReadClientLineAsync()); + await passOnlyDuplex.WriteClientLineAsync("QUIT"); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyRun; + } + + [Fact] + public async Task NonTransitSource_PeerUsernamePassword_DoesNotCreateTransitIdentity() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(peers); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedTransit); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NonTransitAuthinfoSuccess_DoesNotGrantTransit() + { + var provider = new ReaderOnlyAuthenticationProvider(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(CreatePeer(username: "feed", password: "s3cret"), authenticationProvider: provider); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER reader"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ReaderRole_BlankPeerCredentials_DoNotCreatePeerAuth() + { + var peers = CreatePeer(); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + Assert.False(session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS anything"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.AuthorizedTransit); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NamedTransitPeer_OnReader_StreamCheckTakeThisIhave_AreUnavailable() + { + var peers = CreatePeer(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers, queue); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + Assert.Equal(PeerAddress, session.ClientAddress); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + // CHECK and stream-plane TAKETHIS from a transit-authorized peer are admitted by the session windows ahead of + // the dispatcher; that state cannot occur on a Reader host (TransitPeerAuthorization.Disabled), so it is + // asserted separately below with the production Reader authorization. + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ReaderHost_WithDisabledPeerAuthorization_AllTransitCommandsAreUnavailable() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(TransitPeerAuthorization.Disabled, queue); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedTransit); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + foreach (var command in new[] + { + "MODE STREAM", + "CHECK ", + "IHAVE ", + "TAKETHIS ", + }) + { + await duplex.WriteClientLineAsync(command); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + } + + Assert.Equal(0, queue.Count); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + [Fact] + public async Task NonTransitClient_StillRejectedForTransitCommands() + { + var peers = CreatePeer(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(peers); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AuthenticatedNonTransit_TransitCommandsReturn502() + { + var provider = new ReaderOnlyAuthenticationProvider(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(CreatePeer(), authenticationProvider: provider); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER reader"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void ChangingNamedPeerPolicy_AffectsNewIdentificationOnly() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], patterns: "comp.*"))); + var peers = TransitPeerAuthorization.CreateForStore(store); + + var first = peers.Resolve(PeerAddress); + Assert.Equal(TransitTestPeers.DefaultPeerName, first.TransitPeerName); + Assert.True(first.TransitPeerPolicy!.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(first.TransitPeerPolicy.Patterns.MatchesNewsgroup("alt.test")); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], patterns: "alt.*"))); + + Assert.True(first.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + var second = peers.Resolve(PeerAddress); + Assert.Equal(TransitTestPeers.DefaultPeerName, second.TransitPeerName); + Assert.True(second.TransitPeerPolicy!.Patterns.MatchesNewsgroup("alt.test")); + Assert.False(second.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + } + + private static ITransitPeerAuthorization CreatePeer(string username = "", string password = "") => + TransitTestPeers.ForAllowFrom( + PeerAddress, + username: username, + password: password); + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) => new RecordingLogger(Messages); + + public void Dispose() + { + } + + private sealed class RecordingLogger(ConcurrentBag messages) : ILogger + { + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } + } + + private sealed class ReaderOnlyAuthenticationProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + if (username == "reader" && password == "secret") + { + return ValueTask.FromResult(NntpAuthenticationResult.Success( + "reader", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false))); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class CommandDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly IPAddress _clientAddress; + + private CommandDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress clientAddress) => + Task.FromResult(new CommandDuplex(clientAddress)); + + public NntpSession CreateSession( + ITransitPeerAuthorization peers, + IArticleIngestionQueue? queue = null, + INntpAuthenticationProvider? authenticationProvider = null, + ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientAddress, 40000))); + return new NntpSession( + connection, + loggerFactory?.CreateLogger() ?? NullLogger.Instance, + authenticationProvider: authenticationProvider, + loggerFactory: loggerFactory, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientBytesAsync(ReadOnlyMemory bytes) + { + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitCatalogueTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitCatalogueTests.cs new file mode 100644 index 00000000..e4437d50 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitCatalogueTests.cs @@ -0,0 +1,523 @@ +using System.Data; +using System.Net; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitCatalogueTests +{ + private const long SharedSendMaxArticleBytes = 5_242_880; + + [Fact] + public void MigratedPublication_ReconstructsTheThreePeers() + { + var snapshot = TransitCatalogueCompiler.Compile(MigratedPublication()); + + Assert.Equal(1, snapshot.PublicationId); + Assert.True(snapshot.WantTrash); + Assert.True(snapshot.LogTrash); + Assert.Equal(3, snapshot.Peers.Count); + Assert.Equal( + ["blueworld-hosting", "giganews", "usenet-ninja"], + snapshot.Peers.Keys.OrderBy(static key => key, StringComparer.Ordinal)); + + var blueworld = snapshot.Peers["blueworld-hosting"]; + Assert.Equal("blueworld-hosting", blueworld.Identifier); + Assert.Equal("Blueworld Hosting", blueworld.PeerName); + Assert.Equal(["usenet.blueworldhosting.com"], blueworld.DnsHostnames); + Assert.Empty(blueworld.LiteralPrefixes); + Assert.Equal(10, blueworld.MaxIncomingConnections); + Assert.False(blueworld.HasPeerCredentials); + Assert.True(blueworld.DeferOnDuplicate); + Assert.Equal(1_048_576, blueworld.MaxSize); + Assert.Equal("usenet.blueworldhosting.com", blueworld.ConnectTo[0].Host); + Assert.Equal(119, blueworld.ConnectTo[0].Port); + Assert.Equal(10, blueworld.MaxOutgoingConnections); + Assert.Equal(TransitSslMode.None, blueworld.Ssl); + Assert.False(blueworld.HasSendCredentials); + Assert.Equal("*,!unidata.*,!control.*,!junk", blueworld.Patterns.Expression); + Assert.Equal(TransitMessageTypes.All, blueworld.MessageTypes); + Assert.Equal("usenet.blueworldhosting.com", blueworld.PathToken); + Assert.Empty(blueworld.PathExclusions); + + var giganews = snapshot.Peers["giganews"]; + Assert.Equal("giganews", giganews.Identifier); + Assert.Equal("Giganews, Inc.", giganews.PeerName); + Assert.Equal(["news-out.nntp.giganews.com"], giganews.DnsHostnames); + Assert.Equal(80, giganews.MaxIncomingConnections); + Assert.Equal(5_242_880, giganews.MaxSize); + Assert.Equal("opticnetworks-in.nntp.ord.giganews.com", giganews.ConnectTo[0].Host); + Assert.Equal(119, giganews.ConnectTo[0].Port); + Assert.Equal(80, giganews.MaxOutgoingConnections); + Assert.Equal("*,!unidata.*", giganews.Patterns.Expression); + Assert.Equal("nntp.giganews.com", giganews.PathToken); + + var local = snapshot.Peers["usenet-ninja"]; + Assert.Equal("usenet-ninja", local.Identifier); + Assert.Equal("Usenet Ninja", local.PeerName); + Assert.Empty(local.DnsHostnames); + Assert.True(local.LiteralPrefixes[0].Contains(IPAddress.Parse("198.18.1.1"))); + Assert.False(local.LiteralPrefixes[0].Contains(IPAddress.Parse("203.0.113.1"))); + Assert.Equal(10, local.MaxIncomingConnections); + Assert.Equal(5_242_880, local.MaxSize); + Assert.Empty(local.ConnectTo); + Assert.Equal(10, local.MaxOutgoingConnections); + Assert.Equal("*", local.Patterns.Expression); + Assert.Equal("nntp.usenet.ninja", local.PathToken); + } + + [Fact] + public void MigratedPublication_KeepsIntentionalDefaultsDistinctFromJson() + { + var snapshot = TransitCatalogueCompiler.Compile(MigratedPublication()); + foreach (var peer in snapshot.Peers.Values) + { + Assert.Equal(TransitMessageTypes.All, peer.ReceiveArticleTypes); + Assert.Equal("*", peer.ReceivePatterns.Expression); + Assert.Equal(SharedSendMaxArticleBytes, peer.SendMaxArticleBytes); + } + + Assert.NotEqual( + snapshot.Peers["blueworld-hosting"].MaxSize, + snapshot.Peers["blueworld-hosting"].SendMaxArticleBytes); + } + + [Fact] + public void Planes_KeepArticleMasksSizesAndPatternsIndependent() + { + var peer = Peer( + identifier: "planes", + receiveArticleTypes: (int)TransitMessageTypes.Default, + sendArticleTypes: (int)TransitMessageTypes.All, + receiveMaxArticleBytes: 1_048_576, + sendMaxArticleBytes: SharedSendMaxArticleBytes, + receivePatterns: "*", + sendPatterns: "*,!unidata.*"); + var compiled = TransitCatalogueCompiler.Compile(Publication(peer)).Peers["planes"]; + + Assert.Equal(TransitMessageTypes.Default, compiled.ReceiveArticleTypes); + Assert.Equal(TransitMessageTypes.All, compiled.MessageTypes); + Assert.Equal(1_048_576, compiled.MaxSize); + Assert.Equal(SharedSendMaxArticleBytes, compiled.SendMaxArticleBytes); + Assert.True(compiled.ReceivePatterns.MatchesNewsgroup("unidata.maps")); + Assert.False(compiled.Patterns.MatchesNewsgroup("unidata.maps")); + Assert.True(compiled.Patterns.MatchesNewsgroup("comp.lang.c")); + } + + [Fact] + public void ArticleTypeMasks_MatchTheCapabilityContract() + { + Assert.Equal(65535, (int)TransitMessageTypes.All); + Assert.Equal(1, (int)TransitMessageTypes.Default); + Assert.Equal(0, (int)TransitMessageTypes.None); + Assert.Equal(65535, TransitCatalogueCompiler.UnrestrictedArticleTypes); + + Assert.True(ArticleTypeCapabilities.Allows((ArticleType)65535, ArticleType.Binary | ArticleType.YEncoded)); + Assert.True(ArticleTypeCapabilities.Allows((ArticleType)1, ArticleType.Default)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)1, ArticleType.Control)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleType.Default)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleType.Control)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleTypeCapabilities.All)); + + var star = TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: "*", sendPatterns: "*"))); + Assert.True(star.Peers["peer"].ReceivePatterns.MatchesNewsgroup("comp.lang.c")); + Assert.True(star.Peers["peer"].Patterns.MatchesNewsgroup("alt.test")); + } + + [Fact] + public void ZeroConnectionLimits_CloseThatPlane() + { + var compiled = TransitCatalogueCompiler.Compile(Publication(Peer(maxInbound: 0, maxOutbound: 0))) + .Peers["peer"]; + Assert.Equal(0, compiled.MaxIncomingConnections); + Assert.Equal(0, compiled.MaxOutgoingConnections); + + var engine = new TransitPeerStateEngine(); + Assert.Equal( + TransitPeerStateEngine.Rejected, + engine.TryAdmit("peer", "owner", compiled.MaxIncomingConnections, 1, 30_000, 1)); + Assert.Equal( + TransitPeerStateEngine.Accepted, + engine.TryAdmit("peer", "owner", 1, 1, 30_000, 1)); + } + + [Fact] + public void InvalidArticleTypesPatternsAndDuplicateEndpoints_AreRejected() + { + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receiveArticleTypes: 65536)))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendArticleTypes: 65536)))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: "")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: " ")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendPatterns: "*,")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer( + endpoints: + [ + new TransitCatalogueEndpointRecord("News.Example", 119), + new TransitCatalogueEndpointRecord("news.example", 119), + ])))); + } + + [Theory] + [InlineData(null)] + [InlineData(1_000_000L)] + [InlineData(5_242_880L)] + public void SendMaxArticleBytes_PreservesUnlimitedAndExplicitCeilings(long? sendMaxArticleBytes) + { + var compiled = TransitCatalogueCompiler.Compile( + Publication(Peer(sendMaxArticleBytes: sendMaxArticleBytes))).Peers["peer"]; + Assert.Equal(sendMaxArticleBytes, compiled.SendMaxArticleBytes); + Assert.Equal(SharedSendMaxArticleBytes, compiled.MaxSize); + } + + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + [InlineData(2_147_483_648L)] + public void ExplicitSendMaxArticleBytes_RejectsNonPositiveAndAboveIntMax(long sendMaxArticleBytes) + { + var ex = Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendMaxArticleBytes: sendMaxArticleBytes)))); + Assert.Contains("send max_article_bytes", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void ReceiveMaxArticleBytes_StaysAnExplicitCeiling() + { + var ex = Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receiveMaxArticleBytes: 0)))); + Assert.Contains("receive max_article_bytes", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Service_PublishesUnlimitedSendSizeWithoutANumericDefault() + { + var factory = new FakeNntpDbConnectionFactory { TransitCatalogue = PeerSendPublication() }; + var database = await StartDatabaseAsync(factory); + var store = new TransitConfigurationStore(); + var services = new ServiceCollection(); + services.AddOptions(); + using var provider = services.BuildServiceProvider(); + var service = new TransitCatalogueService( + database, + store, + provider.GetRequiredService>(), + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(2, store.Current.PublicationId); + Assert.Null(store.Current.Peers["giganews"].SendMaxArticleBytes); + Assert.Equal(50, store.Current.Peers["giganews"].MaxOutgoingConnections); + Assert.Equal(1_000_000, store.Current.Peers["blueworld-hosting"].SendMaxArticleBytes); + Assert.Equal(10, store.Current.Peers["blueworld-hosting"].MaxOutgoingConnections); + Assert.Equal(SharedSendMaxArticleBytes, store.Current.Peers["usenet-ninja"].SendMaxArticleBytes); + Assert.Equal(5_242_880, store.Current.Peers["giganews"].MaxSize); + Assert.Equal(1_048_576, store.Current.Peers["blueworld-hosting"].MaxSize); + await service.DisposeAsync(); + await database.DisposeAsync(); + } + + [Fact] + public void SqlSendSize_RoundTripsToSendPolicy() + { + using var table = new DataTable(); + table.Columns.Add("max_article_bytes", typeof(int)); + table.Rows.Add(DBNull.Value); + table.Rows.Add(1_000_000); + table.Rows.Add(5_242_880); + using var reader = table.CreateDataReader(); + var sizes = new List(); + while (reader.Read()) + { + sizes.Add(NntpTransitCatalogueLoader.ReadSendMaxArticleBytes(reader, 0)); + } + + var snapshot = TransitCatalogueCompiler.Compile(new TransitCatalogueRecord( + 2, + 1, + 2, + 1, + true, + true, + [ + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 50, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com", + sendMaxArticleBytes: (long?)sizes[0]), + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com", + sendMaxArticleBytes: (long?)sizes[1]), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja", + sendMaxArticleBytes: (long?)sizes[2]), + ])); + + Assert.Null(snapshot.Peers["giganews"].SendMaxArticleBytes); + Assert.Equal("*,!unidata.*", snapshot.Peers["giganews"].Patterns.Expression); + Assert.Equal(TransitMessageTypes.All, snapshot.Peers["giganews"].MessageTypes); + Assert.Equal(1_000_000, snapshot.Peers["blueworld-hosting"].SendMaxArticleBytes); + Assert.Equal("*,!unidata.*,!control.*,!junk", snapshot.Peers["blueworld-hosting"].Patterns.Expression); + Assert.False(snapshot.Peers["blueworld-hosting"].Patterns.Expression.Contains("!local", StringComparison.Ordinal)); + Assert.Equal(SharedSendMaxArticleBytes, snapshot.Peers["usenet-ninja"].SendMaxArticleBytes); + Assert.Equal(80, snapshot.Peers["giganews"].MaxIncomingConnections); + Assert.Equal(10, snapshot.Peers["blueworld-hosting"].MaxIncomingConnections); + } + + [Fact] + public void LeftoverJson_IsRejected_ProcessLocalDepthRemains() + { + var peers = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Transit:alpha:PeerName"] = "Alpha", + }).Build(); + Assert.True(new TransitLeftoverConfigurationValidator(peers).Validate(null, new NntpdOptions()).Failed); + + var flags = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Nntpd:Transit:LogTrash"] = "false", + }).Build(); + var flagsResult = new TransitLeftoverConfigurationValidator(flags).Validate(null, new NntpdOptions()); + Assert.True(flagsResult.Failed); + Assert.Contains("nntptransitglobalrevision", flagsResult.FailureMessage, StringComparison.Ordinal); + + var local = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Nntpd:Transit:StreamOutstandingArticleDepth"] = "8", + ["Nntpd:TransitQueueMemoryLimit"] = "4294967296", + }).Build(); + Assert.True(new TransitLeftoverConfigurationValidator(local).Validate(null, new NntpdOptions()).Succeeded); + } + + [Fact] + public async Task Service_PublishesOneCatalogue_AndMissingCurrentFailsStartup() + { + var factory = new FakeNntpDbConnectionFactory { TransitCatalogue = MigratedPublication() }; + var database = await StartDatabaseAsync(factory); + var store = new TransitConfigurationStore(); + var services = new ServiceCollection(); + services.AddOptions(); + using var provider = services.BuildServiceProvider(); + var monitor = provider.GetRequiredService>(); + var service = new TransitCatalogueService( + database, + store, + monitor, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(3, store.Current.Peers.Count); + Assert.Equal(1, store.Current.PublicationId); + Assert.True(monitor.CurrentValue.Transit.WantTrash); + Assert.True(monitor.CurrentValue.Transit.LogTrash); + await service.DisposeAsync(); + await database.DisposeAsync(); + + var missingFactory = new FakeNntpDbConnectionFactory { TransitCatalogue = null }; + var missingDatabase = await StartDatabaseAsync(missingFactory); + var missing = new TransitCatalogueService( + missingDatabase, + new TransitConfigurationStore(), + monitor, + NullLogger.Instance); + var ex = await Assert.ThrowsAsync(() => missing.StartAsync(CancellationToken.None)); + Assert.Contains("nntptransitcurrent", ex.Message, StringComparison.Ordinal); + await missing.DisposeAsync(); + await missingDatabase.DisposeAsync(); + } + + [Fact] + public void EmptyJsonReload_DoesNotWipeThePublishedSnapshot() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitCatalogueCompiler.Compile(MigratedPublication())); + using var manager = new ConfigurationManager(); + var services = new ServiceCollection(); + services.AddLogging(static builder => builder.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(store); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + _ = provider.GetRequiredService(); + + manager["Nntpd:Transit:StreamOutstandingArticleDepth"] = "8"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(3, store.Current.Peers.Count); + Assert.Equal("nntp.giganews.com", store.Current.Peers["giganews"].PathToken); + } + + private static async Task StartDatabaseAsync(FakeNntpDbConnectionFactory factory) + { + var database = new NntpDbService( + factory, + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await database.StartAsync(CancellationToken.None); + return database; + } + + private static TransitCatalogueRecord PeerSendPublication() => + new( + 2, + 1, + 2, + 1, + true, + true, + [ + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com", + sendMaxArticleBytes: 1_000_000), + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 50, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com", + sendMaxArticleBytes: null), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja"), + ]); + + private static TransitCatalogueRecord MigratedPublication() => + new( + 1, + 1, + 1, + 1, + true, + true, + [ + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com"), + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 80, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com"), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja"), + ]); + + private static TransitCatalogueRecord Publication(TransitCataloguePeerRecord peer) => + new(1, 1, 1, 1, true, true, [peer]); + + private static TransitCataloguePeerRecord Peer( + string identifier = "peer", + string peerName = "Peer", + int maxInbound = 1, + long receiveMaxArticleBytes = SharedSendMaxArticleBytes, + IReadOnlyList? allowFrom = null, + int maxOutbound = 1, + IReadOnlyList? endpoints = null, + string sendPatterns = "*", + string pathToken = "peer.example", + int receiveArticleTypes = 65535, + int sendArticleTypes = 65535, + long? sendMaxArticleBytes = SharedSendMaxArticleBytes, + string receivePatterns = "*") => + new( + identifier, + peerName, + maxInbound, + string.Empty, + string.Empty, + true, + receiveMaxArticleBytes, + receiveArticleTypes, + receivePatterns, + allowFrom ?? [], + maxOutbound, + "None", + string.Empty, + string.Empty, + sendMaxArticleBytes, + sendArticleTypes, + sendPatterns, + pathToken, + endpoints ?? [], + []); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs new file mode 100644 index 00000000..1863f54b --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs @@ -0,0 +1,331 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// Role-isolation coverage for Receive DeferOnDuplicate: a Reader session answers every Transit feed +/// command with 502 Command not available on this server before history is consulted, so the +/// duplicate/defer replies (431/438/436/435/239) are never produced on the Reader service. +/// +/// The 431/438/436/435/239 behavior is covered by the Transit service tests. +public sealed class TransitDeferOnDuplicateCommandTests +{ + private const string CommandNotAvailable = "502 Command not available on this server"; + private static readonly IPAddress PeerAddress = IPAddress.Parse("198.18.0.81"); + private static readonly byte[] SeenId = ""u8.ToArray(); + + // CHECK is covered with TransitPeerAuthorization.Disabled, the authorization a Reader-only host registers. + // A transit-authorized peer is never produced there; such a peer's CHECK would be admitted by the session + // CHECK window ahead of the dispatcher, which is not a reachable Reader state. + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Check_Seen_Returns502_AndDoesNotReplyDuplicate(bool readerMode) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, TransitPeerAuthorization.Disabled, readerMode); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Check_Miss_Returns502_AndDoesNotConsultHistory() + { + var history = CreateHistory(new FakeRedisService()); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, TransitPeerAuthorization.Disabled, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Check_HistoryUnavailable_Returns502_NotDeferral() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("down"); + var history = CreateHistory(redis); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, TransitPeerAuthorization.Disabled, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task IHave_Seen_Returns502_AndDoesNotRequestTheBody(bool defer) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task IHave_Miss_Returns502_AndNothingIsIngested(bool defer) + { + var history = CreateHistory(new FakeRedisService()); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task TakeThis_Seen_Returns502_AndNothingIsIngested(bool serial) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, TransitPeerAuthorization.Disabled, readerMode: serial); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + // No article body is sent: the Reader never reads one for an unavailable command. + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ExistingConnection_AndFreshConnection_BothReturn502_AfterCatalogueReplace() + { + var store = new TransitConfigurationStore(); + store.Replace(Snapshot(defer: true)); + var peers = TransitPeerAuthorization.CreateForStore(store); + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + + await using var existingDuplex = await CommandDuplex.CreateAsync(); + var existing = existingDuplex.CreateSession(history, peers, readerMode: false); + var existingRun = existing.RunAsync(); + _ = await existingDuplex.ReadClientLineAsync(); + Assert.True(existing.Authorization.TransitPeerPolicy!.DeferOnDuplicate); + + await existingDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await existingDuplex.ReadClientLineAsync()); + + store.Replace(Snapshot(defer: false)); + Assert.True(existing.Authorization.TransitPeerPolicy.DeferOnDuplicate); + await existingDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await existingDuplex.ReadClientLineAsync()); + + await using var freshDuplex = await CommandDuplex.CreateAsync(); + var fresh = freshDuplex.CreateSession(history, peers, readerMode: false); + var freshRun = fresh.RunAsync(); + _ = await freshDuplex.ReadClientLineAsync(); + Assert.False(fresh.Authorization.TransitPeerPolicy!.DeferOnDuplicate); + await freshDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal(CommandNotAvailable, await freshDuplex.ReadClientLineAsync()); + + await QuitAsync(freshDuplex, freshRun); + await QuitAsync(existingDuplex, existingRun); + } + private static HistoryDb CreateHistory(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private static TransitConfigurationSnapshot Snapshot(bool defer) => + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], deferOnDuplicate: defer)); + + private static async Task QuitAsync(CommandDuplex duplex, Task run) + { + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run; + } + + private sealed class CommandDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new CommandDuplex()); + + public NntpSession CreateSession(IHistoryDb history, bool defer, bool readerMode) => + CreateSession( + history, + TransitTestPeers.ForAllowFrom(PeerAddress, deferOnDuplicate: defer), + readerMode); + + public NntpSession CreateSession(IHistoryDb history, ITransitPeerAuthorization peers, bool readerMode) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 1024 * 1024, + }); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(PeerAddress, 40000))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers, + historyDb: history); + if (readerMode) + { + session.SetMode(NntpSessionMode.Reader); + } + + return session; + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitHotReloadTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitHotReloadTests.cs new file mode 100644 index 00000000..9504922f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitHotReloadTests.cs @@ -0,0 +1,398 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Commands; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitHotReloadTests +{ + [Fact] + public void Reload_AddModifyRemovePeerAndAllowFrom_ReplacesSnapshotAtomically() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:alpha:PeerName"] = "Alpha", + ["Transit:alpha:MaxIncomingConnections"] = "4", + ["Transit:alpha:MaxOutgoingConnections"] = "1", + ["Transit:alpha:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + _ = provider.GetRequiredService(); + var authz = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + provider.GetRequiredService(), + NullLogger.Instance); + + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal(4, store.Current.Peers["alpha"].MaxIncomingConnections); + + manager["Transit:alpha:MaxIncomingConnections"] = "2"; + manager["Transit:alpha:AllowFrom:0"] = "198.51.100.10"; + manager["Transit:beta:PeerName"] = "Beta"; + manager["Transit:beta:MaxIncomingConnections"] = "1"; + manager["Transit:beta:MaxOutgoingConnections"] = "0"; + manager["Transit:beta:AllowFrom:0"] = "203.0.113.5"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + Assert.Equal("beta", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + Assert.Equal(2, store.Current.Peers["alpha"].MaxIncomingConnections); + + manager["Transit:alpha:Ssl"] = "not-a-mode"; + ((IConfigurationRoot)manager).Reload(); + Assert.Equal(2, store.Current.Peers["alpha"].MaxIncomingConnections); + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + + store.Replace(TransitTestPeers.Snapshot("beta", TransitTestPeers.Peer(allowFrom: ["203.0.113.5"]))); + Assert.False(store.Current.Peers.ContainsKey("alpha")); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + Assert.Equal("beta", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + } + + [Fact] + public void Reload_AllowFromChange_NewConnectionLosesPeerA_ExistingSessionUnchanged() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:test-peer:PeerName"] = TransitTestPeers.DefaultPeerDisplayName, + ["Transit:test-peer:MaxIncomingConnections"] = "10", + ["Transit:test-peer:MaxOutgoingConnections"] = "0", + ["Transit:test-peer:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + var hotReload = provider.GetRequiredService(); + var peers = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + hotReload, + NullLogger.Instance); + + var source = IPAddress.Parse("192.0.2.10"); + var existing = peers.Resolve(source); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.TransitPeerName); + + manager["Transit:test-peer:AllowFrom:0"] = "198.51.100.10"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.TransitPeerName); + Assert.True(existing.AuthorizedTransit); + var fresh = peers.Resolve(source); + Assert.Null(fresh.TransitPeerName); + Assert.False(fresh.AuthorizedTransit); + Assert.Equal(TransitTestPeers.DefaultPeerName, peers.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + } + + [Fact] + public async Task Reload_AllowFromChange_ExistingSessionKeepsTransit_NewSessionDoesNot() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:test-peer:PeerName"] = TransitTestPeers.DefaultPeerDisplayName, + ["Transit:test-peer:MaxIncomingConnections"] = "10", + ["Transit:test-peer:MaxOutgoingConnections"] = "0", + ["Transit:test-peer:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + var hotReload = provider.GetRequiredService(); + var peers = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + hotReload, + NullLogger.Instance); + + var source = IPAddress.Parse("192.0.2.10"); + await using var existingDuplex = await SessionDuplex.CreateAsync(source); + var existing = existingDuplex.CreateSession(peers); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + var existingRun = existing.RunAsync(); + _ = await existingDuplex.ReadClientLineAsync(); + + await existingDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("502 Command not available on this server", await existingDuplex.ReadClientLineAsync()); + + manager["Transit:test-peer:AllowFrom:0"] = "198.51.100.10"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + Assert.True(existing.Authorization.AuthorizedTransit); + await existingDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("502 Command not available on this server", await existingDuplex.ReadClientLineAsync()); + + await using var freshDuplex = await SessionDuplex.CreateAsync(source); + var fresh = freshDuplex.CreateSession(peers); + Assert.Null(fresh.Authorization.TransitPeerName); + Assert.False(fresh.Authorization.AuthorizedTransit); + var freshRun = fresh.RunAsync(); + _ = await freshDuplex.ReadClientLineAsync(); + await freshDuplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("502 Command not available on this server", await freshDuplex.ReadClientLineAsync()); + + await existingDuplex.WriteClientLineAsync("QUIT"); + _ = await existingDuplex.ReadClientLineAsync(); + await existingRun; + await freshDuplex.WriteClientLineAsync("QUIT"); + _ = await freshDuplex.ReadClientLineAsync(); + await freshRun; + } + + [Fact] + public void Reload_UpdatesPathTokenMaxSizeAndMessageTypesAtomically() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + allowFrom: ["192.0.2.10"], + pathToken: "old.token", + maxSize: 1024, + messageTypes: ["default"]))); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("old.token", existing.TransitPeerPolicy!.PathToken); + Assert.Equal(1024, existing.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Default, existing.TransitPeerPolicy.MessageTypes); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + allowFrom: ["192.0.2.10"], + pathToken: "new.token", + maxSize: 2048, + messageTypes: ["control", "binary"]))); + + Assert.Equal("old.token", existing.TransitPeerPolicy.PathToken); + var fresh = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("new.token", fresh.TransitPeerPolicy!.PathToken); + Assert.Equal(2048, fresh.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Control | TransitMessageTypes.Binary, fresh.TransitPeerPolicy.MessageTypes); + } + + [Fact] + public void Reload_UpdatesRemainingPeerFieldsAtomically() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 4, + maxOutgoing: 2, + allowFrom: ["192.0.2.10"], + username: "old-user", + password: "old-pass", + ssl: "TLS", + patterns: "comp.*", + connectTo: ["news.example.net:119"], + deferOnDuplicate: true))); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("old-user", existing.TransitPeerPolicy!.Username); + Assert.Equal("old-pass", existing.TransitPeerPolicy.Password); + Assert.Equal(TransitSslMode.Tls, existing.TransitPeerPolicy.Ssl); + Assert.True(existing.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.True(existing.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(4, existing.TransitPeerPolicy.MaxIncomingConnections); + Assert.Equal(2, existing.TransitPeerPolicy.MaxOutgoingConnections); + Assert.Equal("news.example.net", existing.TransitPeerPolicy.ConnectTo.Single().Host); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 8, + maxOutgoing: 3, + allowFrom: ["192.0.2.10"], + username: "new-user", + password: "new-pass", + ssl: "STARTTLS", + patterns: "alt.*", + connectTo: ["other.example.net:563"], + deferOnDuplicate: false))); + + Assert.Equal("old-user", existing.TransitPeerPolicy.Username); + Assert.Equal("old-pass", existing.TransitPeerPolicy.Password); + Assert.True(existing.TransitPeerPolicy.DeferOnDuplicate); + var fresh = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("new-user", fresh.TransitPeerPolicy!.Username); + Assert.Equal("new-pass", fresh.TransitPeerPolicy.Password); + Assert.Equal(TransitSslMode.StartTls, fresh.TransitPeerPolicy.Ssl); + Assert.True(fresh.TransitPeerPolicy.Patterns.MatchesNewsgroup("alt.test")); + Assert.False(fresh.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(fresh.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(8, fresh.TransitPeerPolicy.MaxIncomingConnections); + Assert.Equal(3, fresh.TransitPeerPolicy.MaxOutgoingConnections); + Assert.Equal("other.example.net", fresh.TransitPeerPolicy.ConnectTo.Single().Host); + } + + [Fact] + public void Reload_InvalidConfiguration_KeepsLastValidSnapshot() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitTestPeers.Snapshot("alpha", TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]))); + var validator = new TransitPeersOptionsValidator(); + var invalid = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: "not-a-mode")); + var result = validator.Validate(null, invalid); + Assert.True(result.Failed); + Assert.Equal("alpha", store.Current.Peers.Keys.Single()); + } + + private sealed class EmptyDnsCache : ITransitDnsAddressCache + { + public IReadOnlySet GetResolved(string hostname) => new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + + private sealed class SessionDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly IPAddress _clientAddress; + + private SessionDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress clientAddress) => + Task.FromResult(new SessionDuplex(clientAddress)); + + public NntpSession CreateSession(ITransitPeerAuthorization peers) + { + var connection = new ReloadPipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + _clientAddress); + return new NntpSession( + connection, + NullLogger.Instance, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class ReloadPipeConnection(PipeReader input, PipeWriter output, IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitMessageTypesTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitMessageTypesTests.cs new file mode 100644 index 00000000..567c3e2a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitMessageTypesTests.cs @@ -0,0 +1,133 @@ +using VectorNNTPD.ReaderService.Configuration; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitMessageTypesTests +{ + public static TheoryData DiabloTokens => new() + { + { "none", TransitMessageTypes.None }, + { "default", TransitMessageTypes.Default }, + { "control", TransitMessageTypes.Control }, + { "cancel", TransitMessageTypes.Cancel }, + { "mime", TransitMessageTypes.Mime }, + { "binary", TransitMessageTypes.Binary }, + { "binaries", TransitMessageTypes.Binary }, + { "uuencode", TransitMessageTypes.UuEncode }, + { "base64", TransitMessageTypes.Base64 }, + { "yenc", TransitMessageTypes.Yenc }, + { "bommanews", TransitMessageTypes.BommaNews }, + { "unidata", TransitMessageTypes.UniData }, + { "multipart", TransitMessageTypes.Multipart }, + { "html", TransitMessageTypes.Html }, + { "ps", TransitMessageTypes.PostScript }, + { "binhex", TransitMessageTypes.BinHex }, + { "partial", TransitMessageTypes.Partial }, + { "pgp", TransitMessageTypes.PgpMessage }, + { "all", TransitMessageTypes.All }, + }; + + [Theory] + [MemberData(nameof(DiabloTokens))] + public void ParseToken_MapsDiabloNames(string token, TransitMessageTypes expected) + { + Assert.True(TransitMessageTypesParser.TryParseToken(token, out var actual)); + Assert.Equal(expected, actual); + Assert.True(TransitMessageTypesParser.TryParseToken(" " + token.ToUpperInvariant() + " ", out var padded)); + Assert.Equal(expected, padded); + } + + [Fact] + public void BinaryAndBinaries_AreTheSameFlag() + { + Assert.True(TransitMessageTypesParser.TryParseToken("binary", out var binary)); + Assert.True(TransitMessageTypesParser.TryParseToken("binaries", out var binaries)); + Assert.Equal(TransitMessageTypes.Binary, binary); + Assert.Equal(binary, binaries); + } + + [Fact] + public void All_IsUnionOfConcreteFlags() + { + const TransitMessageTypes concrete = + TransitMessageTypes.Default + | TransitMessageTypes.Control + | TransitMessageTypes.Cancel + | TransitMessageTypes.Mime + | TransitMessageTypes.Binary + | TransitMessageTypes.UuEncode + | TransitMessageTypes.Base64 + | TransitMessageTypes.Yenc + | TransitMessageTypes.BommaNews + | TransitMessageTypes.UniData + | TransitMessageTypes.Multipart + | TransitMessageTypes.Html + | TransitMessageTypes.PostScript + | TransitMessageTypes.BinHex + | TransitMessageTypes.Partial + | TransitMessageTypes.PgpMessage; + Assert.Equal(concrete, TransitMessageTypes.All); + Assert.NotEqual(TransitMessageTypes.None, TransitMessageTypes.Default); + Assert.NotEqual(TransitMessageTypes.Default, TransitMessageTypes.All); + } + + [Fact] + public void Parse_DefaultArrayAndDuplicates() + { + Assert.True(TransitMessageTypesParser.TryParse(null, out var omitted, out _)); + Assert.Equal(TransitMessageTypes.Default, omitted); + + Assert.True(TransitMessageTypesParser.TryParse([], out var empty, out _)); + Assert.Equal(TransitMessageTypes.Default, empty); + + Assert.True(TransitMessageTypesParser.TryParse(["default"], out var defaults, out _)); + Assert.Equal(TransitMessageTypes.Default, defaults); + + Assert.True(TransitMessageTypesParser.TryParse(["binary", "BINARY", "binaries"], out var dupes, out _)); + Assert.Equal(TransitMessageTypes.Binary, dupes); + + Assert.True(TransitMessageTypesParser.TryParse(["default", "control", "binary"], out var combo, out _)); + Assert.Equal( + TransitMessageTypes.Default | TransitMessageTypes.Control | TransitMessageTypes.Binary, + combo); + } + + [Fact] + public void Parse_RejectsUnknownAndBlankTokens() + { + Assert.False(TransitMessageTypesParser.TryParse(["default", "audio"], out _, out var error)); + Assert.Contains("[1]", error, StringComparison.Ordinal); + Assert.False(TransitMessageTypesParser.TryParse([" "], out _, out _)); + } + + [Fact] + public void CanonicalNames_DoNotInventTypes() + { + Assert.Equal(["none"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.None)); + Assert.Equal(["all"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.All)); + Assert.Equal(["binary"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.Binary)); + Assert.DoesNotContain("binaries", TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.Binary)); + } + + [Fact] + public void EnumMemberValues_AreStablePowersOfTwo() + { + Assert.Equal(0, (int)TransitMessageTypes.None); + Assert.Equal(1, (int)TransitMessageTypes.Default); + Assert.Equal(2, (int)TransitMessageTypes.Control); + Assert.Equal(4, (int)TransitMessageTypes.Cancel); + Assert.Equal(8, (int)TransitMessageTypes.Mime); + Assert.Equal(16, (int)TransitMessageTypes.Binary); + Assert.Equal(32, (int)TransitMessageTypes.UuEncode); + Assert.Equal(64, (int)TransitMessageTypes.Base64); + Assert.Equal(128, (int)TransitMessageTypes.Yenc); + Assert.Equal(256, (int)TransitMessageTypes.BommaNews); + Assert.Equal(512, (int)TransitMessageTypes.UniData); + Assert.Equal(1024, (int)TransitMessageTypes.Multipart); + Assert.Equal(2048, (int)TransitMessageTypes.Html); + Assert.Equal(4096, (int)TransitMessageTypes.PostScript); + Assert.Equal(8192, (int)TransitMessageTypes.BinHex); + Assert.Equal(16384, (int)TransitMessageTypes.Partial); + Assert.Equal(32768, (int)TransitMessageTypes.PgpMessage); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs new file mode 100644 index 00000000..d7a2e78f --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs @@ -0,0 +1,353 @@ +using System.Net; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerIdentificationAndLimitTests +{ + [Fact] + public void Identify_MatchingSource_ReturnsNamedPeerPolicy() + { + var snapshot = new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + [TransitTestPeers.DefaultPeerName] = TransitConfigurationSnapshot.CreatePeer( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 3, + username: "feed", + password: "secret", + patterns: "comp.*,!comp.sources.*", + ssl: "TLS", + allowFrom: ["192.0.2.10"])), + }); + var authz = TransitPeerAuthorization.CreateStatic(snapshot).Resolve(IPAddress.Parse("192.0.2.10")); + Assert.False(authz.AuthorizedTransit); + Assert.True(authz.StreamingPermitted); + Assert.Equal(TransitTestPeers.DefaultPeerName, authz.TransitPeerName); + Assert.NotNull(authz.TransitPeerPolicy); + Assert.Equal(TransitTestPeers.DefaultPeerName, authz.TransitPeerPolicy.Identifier); + Assert.Equal(TransitTestPeers.DefaultPeerDisplayName, authz.TransitPeerPolicy.PeerName); + Assert.Equal(3, authz.TransitPeerPolicy.MaxIncomingConnections); + Assert.True(authz.TransitPeerPolicy.HasPeerCredentials); + Assert.Equal(TransitSslMode.Tls, authz.TransitPeerPolicy.Ssl); + Assert.True(authz.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(string.Empty, authz.TransitPeerPolicy.PathToken); + Assert.Equal(TransitPeerOptions.DefaultMaxSize, authz.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Default, authz.TransitPeerPolicy.MessageTypes); + Assert.True(authz.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(authz.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.sources.unix")); + } + + [Fact] + public void Identify_NonMatchingSource_IsNotTransit() + { + var authz = TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]))); + var result = authz.Resolve(IPAddress.Parse("198.51.100.1")); + Assert.False(result.AuthorizedTransit); + Assert.Null(result.TransitPeerName); + Assert.Null(result.TransitPeerPolicy); + } + + [Fact] + public void Identify_AmbiguousRuntimeOverlap_DeniesTransit() + { + // Validation would reject literal overlap; construct a snapshot that simulates + // two peers matching the same address without going through the validator. + var snapshot = new TransitConfigurationSnapshot(new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer("one", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer("two", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + }); + var result = TransitPeerAuthorization.CreateStatic(snapshot).Resolve(IPAddress.Parse("192.0.2.10")); + Assert.False(result.AuthorizedTransit); + Assert.Null(result.TransitPeerName); + } + + [Fact] + public async Task Limit_BelowAtAndOver_IdentifyPeerFromSourceIp() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 2); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + + var first = CreateSession(peers, source); + Assert.Equal(TransitTestPeers.DefaultPeerName, first.Authorization.TransitPeerName); + var lease1 = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease1.Admitted); + + var second = CreateSession(peers, source); + var lease2 = await TransitConnectionAdmission.TryAdmitAsync(limiter, second); + Assert.True(lease2.Admitted); + Assert.Equal(2, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + + var over = CreateSession(peers, source); + var lease3 = await TransitConnectionAdmission.TryAdmitAsync(limiter, over); + Assert.False(lease3.Admitted); + Assert.False(lease3.Lease.IsHeld); + + await lease1.Lease.DisposeAsync(); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + var reused = CreateSession(peers, source); + var lease4 = await TransitConnectionAdmission.TryAdmitAsync(limiter, reused); + Assert.True(lease4.Admitted); + await lease2.Lease.DisposeAsync(); + await lease4.Lease.DisposeAsync(); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Limit_AmbiguousMatch_DoesNotConsumeSlot() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = new TransitConfigurationStore(); + store.Replace( + new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer( + "one", + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer( + "two", + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.0/24"])), + })); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var session = CreateSession(peers, source); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedTransit); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + Assert.Equal(0, limiter.GetCount("one")); + Assert.Equal(0, limiter.GetCount("two")); + Assert.Equal(0, membership.ActiveCount("one", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.ActiveCount("two", TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task Limit_NonMatchingSource_AdmittedWithoutSlot() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var other = CreateSession(peers, IPAddress.Parse("198.51.100.1")); + Assert.Null(other.Authorization.TransitPeerName); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, other); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + Assert.Equal(0, membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task Limit_RemovedPeer_NewConnectionIsNotIdentified() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, existing); + Assert.True(lease.Admitted); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + + store.Replace(TransitConfigurationSnapshot.Empty); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + var fresh = CreateSession(peers, source); + Assert.Null(fresh.Authorization.TransitPeerName); + var uncounted = await TransitConnectionAdmission.TryAdmitAsync(limiter, fresh); + Assert.True(uncounted.Admitted); + Assert.False(uncounted.Lease.IsHeld); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + await lease.Lease.DisposeAsync(); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Limit_LoweredOnReload_AffectsNewConnectionsOnly() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 2); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, existing); + Assert.True(lease.Admitted); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: [source.ToString()]))); + + Assert.Equal(2, existing.Authorization.TransitPeerPolicy!.MaxIncomingConnections); + var blocked = CreateSession(peers, source); + Assert.Equal(1, blocked.Authorization.TransitPeerPolicy!.MaxIncomingConnections); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked)).Admitted); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + await lease.Lease.DisposeAsync(); + var next = CreateSession(peers, source); + var reused = await TransitConnectionAdmission.TryAdmitAsync(limiter, next); + Assert.True(reused.Admitted); + await reused.Lease.DisposeAsync(); + } + + [Fact] + public async Task Limit_ConcurrentBoundary_DoesNotOverAdmit() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 8); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var leases = new TransitInboundConnectionLease[32]; + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 32).Select(async i => + { + await Task.Yield(); + var session = CreateSession(peers, source); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + if (outcome.Admitted) + { + Interlocked.Increment(ref admitted); + leases[i] = outcome.Lease; + } + })); + Assert.Equal(8, admitted); + Assert.Equal(8, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + foreach (var lease in leases.Where(static lease => lease is not null)) + { + await lease.DisposeAsync(); + } + + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Admission_Writes400WhenLimitExceeded() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 0); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + var connection = new LimitPipeConnection(input.Reader, output.Writer, source); + var session = new NntpSession( + connection, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + transitPeerAuthorization: peers); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, session)).Admitted); + await TransitConnectionAdmission.WriteUnavailableAsync(connection, CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await VectorNNTPD.ReaderService.Session.CommandProcessor.NntpCommandLineReader.ReadLineAsync(output.Reader, cts.Token); + Assert.Equal("400 Service temporarily unavailable", line); + } + + [Fact] + public async Task Limit_CancellationStyleRelease_ViaDispose() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var first = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease.Admitted); + await using (lease.Lease) + { + var blocked = CreateSession(peers, source); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked)).Admitted); + } + + var after = CreateSession(peers, source); + var reused = await TransitConnectionAdmission.TryAdmitAsync(limiter, after); + Assert.True(reused.Admitted); + await reused.Lease.DisposeAsync(); + } + + private static TransitConfigurationStore CreatePeerStore(IPAddress source, int maxIncoming) + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: maxIncoming, allowFrom: [source.ToString()]))); + return store; + } + + private static NntpSession CreateSession(ITransitPeerAuthorization peers, IPAddress source) + { + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + var connection = new LimitPipeConnection(input.Reader, output.Writer, source); + return new NntpSession( + connection, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + transitPeerAuthorization: peers); + } + + private sealed class LimitPipeConnection(System.IO.Pipelines.PipeReader input, System.IO.Pipelines.PipeWriter output, IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public System.IO.Pipelines.PipeReader Input { get; } = input; + public System.IO.Pipelines.PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs new file mode 100644 index 00000000..5f5e63b5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs @@ -0,0 +1,217 @@ +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateCompensationTests +{ + private const string Peer = "peer-a"; + + [Fact] + public async Task SameIdentifierAdmission_WaitsUntilCompensationReleasesGate() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admitA = node.TryAdmitAsync(Peer, 5, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + + var admitB = node.TryAdmitAsync(Peer, 5).AsTask(); + Assert.False(admitB.IsCompleted); + + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var canceledGeneration, + out var canceledCount)); + Assert.Equal(1, canceledCount); + + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admitA); + var admitted = await admitB; + Assert.True(admitted.Accepted); + Assert.Equal(1, node.GetLocalCount(Peer)); + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(node.OwnerId, store.LastReleaseOwnerId); + Assert.Equal(canceledGeneration, store.LastReleaseGeneration); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var liveGeneration, + out _)); + Assert.NotEqual(canceledGeneration, liveGeneration); + } + + [Fact] + public async Task CanceledCallerToken_IsNotUsedForCompensatingRelease() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync(Peer, 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.False(store.LastReleaseToken.CanBeCanceled); + Assert.Equal(1, store.ReleaseCalls); + Assert.Equal(0, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + } + + [Fact] + public async Task CompensationWhenRedisUnavailable_LeavesTtlOwnershipAndNoLocalCommit() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner) { UnavailableOnRelease = true }; + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync(Peer, 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, store.ReleaseCalls); + } + + [Fact] + public async Task NonCanceledStoreException_DoesNotCompensate() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner) { ThrowAfterAdmit = new InvalidOperationException("store fault") }; + var node = TransitPeerStateTestFactory.CreateTracker(store); + await Assert.ThrowsAsync(() => node.TryAdmitAsync(Peer, 2).AsTask()); + Assert.Equal(0, store.ReleaseCalls); + Assert.Equal(0, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task CompensationRelease_DoesNotAffectNewerGeneration() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admitA = node.TryAdmitAsync(Peer, 5, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var canceledGeneration, + out _)); + + cts.Cancel(); + store.Continue.TrySetResult(); + var admitB = node.TryAdmitAsync(Peer, 5).AsTask(); + await Assert.ThrowsAnyAsync(() => admitA); + Assert.True((await admitB).Accepted); + Assert.Equal(canceledGeneration, store.LastReleaseGeneration); + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var liveGeneration, + out var liveCount)); + Assert.NotEqual(canceledGeneration, liveGeneration); + Assert.Equal(1, liveCount); + } + + private sealed class CompensationStore : ITransitPeerStateStore + { + private readonly InMemoryTransitPeerStateStore _inner; + + public CompensationStore(InMemoryTransitPeerStateStore inner) + { + _inner = inner; + } + + public TaskCompletionSource Admitted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Continue { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool UnavailableOnRelease { get; set; } + + public Exception? ThrowAfterAdmit { get; set; } + + public int ReleaseCalls { get; private set; } + + public string? LastReleaseOwnerId { get; private set; } + + public long LastReleaseGeneration { get; private set; } + + public CancellationToken LastReleaseToken { get; private set; } = new(true); + + public async ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + var result = await _inner.TryAdmitAsync( + identifier, + ownerId, + maxIncoming, + generation, + now, + leaseTtl, + CancellationToken.None).ConfigureAwait(false); + if (ThrowAfterAdmit is { } fault) + { + throw fault; + } + + Admitted.TrySetResult(); + await Continue.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + return result; + } + + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + ReleaseCalls++; + LastReleaseOwnerId = ownerId; + LastReleaseGeneration = generation; + LastReleaseToken = cancellationToken; + if (UnavailableOnRelease) + { + return; + } + + await _inner.ReleaseAsync(identifier, ownerId, generation, cancellationToken).ConfigureAwait(false); + } + + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + _inner.RenewAsync(identifier, ownerId, generation, now, leaseTtl, cancellationToken); + + public ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) => + _inner.ReleaseOwnerAsync(identifier, ownerId, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs new file mode 100644 index 00000000..4868e7e3 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs @@ -0,0 +1,166 @@ +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateEngineTests +{ + private const string Key = "tconn:peer-a"; + private const string Other = "tconn:peer-b"; + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const long Now = 10_000; + private const long Lease = 30_000; + + [Fact] + public void TryAdmit_FirstConnection_WritesCountOne() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 4, Now, Lease, 1)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expiry, out var generation, out var count)); + Assert.Equal(Now + Lease, expiry); + Assert.Equal(1, generation); + Assert.Equal(1, count); + } + + [Fact] + public void TryAdmit_BelowLimit_IncrementsSameGeneration() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 3, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 3, Now, Lease, 1)); + Assert.True(engine.TryGetOwnership(Key, O1, out _, out var generation, out var count)); + Assert.Equal(1, generation); + Assert.Equal(2, count); + } + + [Fact] + public void TryAdmit_ExactlyAtLimit_ThenOverLimit() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 2, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O2, 2, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, 2, Now, Lease, 1)); + Assert.Equal(2, engine.ActiveCount(Key, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void TryAdmit_MaxZero_RejectsWithoutMutation() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, 0, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, -1, Now, Lease, 1)); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + Assert.Equal(0, engine.ActiveCount(Key, Now)); + } + + [Fact] + public void Release_DecrementsThenDeletesFieldAndHash() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.Release(Key, O1, 1); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + _ = engine.Release(Key, O1, 1); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + Assert.Empty(engine.OwnerFields(Key)); + } + + [Fact] + public void Release_StaleGeneration_IsNoOp() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 2); + _ = engine.Release(Key, O1, 1); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void TryAdmit_NewGeneration_ReplacesStaleCount() + { + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership(Key, O1, Now + Lease, 1, 7); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 20, Now, Lease, 2)); + Assert.True(engine.TryGetOwnership(Key, O1, out _, out var generation, out var count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + } + + [Fact] + public void Renew_ExtendsMatchingUnexpiredOwnership() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + Assert.Equal(1, engine.Renew(Key, O1, 1, Now + 5_000, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expiry, out var generation, out var count)); + Assert.Equal(Now + 5_000 + Lease, expiry); + Assert.Equal(1, generation); + Assert.Equal(1, count); + } + + [Fact] + public void Renew_MissingExpiredOrStale_DoesNotRecreate() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + + engine.WriteOwnership(Key, O1, Now, 1, 2); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expired, out _, out var count)); + Assert.Equal(Now, expired); + Assert.Equal(2, count); + + engine.WriteOwnership(Key, O1, Now + Lease, 2, 2); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var still, out var generation, out _)); + Assert.Equal(Now + Lease, still); + Assert.Equal(2, generation); + } + + [Fact] + public void TryAdmit_PrunesExpiredBeforeLimit() + { + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership(Key, O2, Now, 1, 4); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 1, Now, Lease, 1)); + Assert.Equal(0, engine.OwnerCount(Key, O2, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void DifferentPeers_AreIsolated() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Other, O1, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O2, 1, Now, Lease, 1)); + Assert.Equal(1, engine.ActiveCount(Key, Now)); + Assert.Equal(1, engine.ActiveCount(Other, Now)); + } + + [Fact] + public void ReleaseOwner_RemovesOnlyThisOwner() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.TryAdmit(Key, O2, 4, Now, Lease, 1); + _ = engine.ReleaseOwner(Key, O1); + Assert.Equal(0, engine.OwnerCount(Key, O1, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now)); + _ = engine.ReleaseOwner(Other, O2); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now)); + } + + [Fact] + public void Expiry_FreesCapacity() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 1, Now, Lease, 1); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O2, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O2, 1, Now + Lease, Lease, 1)); + Assert.Equal(0, engine.OwnerCount(Key, O1, Now + Lease)); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now + Lease)); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs new file mode 100644 index 00000000..369c05da --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs @@ -0,0 +1,118 @@ +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateFailureTests +{ + private const string Peer = "peer-a"; + private const string Other = "peer-b"; + + [Fact] + public async Task RedisUnavailableDuringAdmission_FailsClosedWithoutLocalSlot() + { + var membership = new InMemoryTransitPeerStateStore { Unavailable = true }; + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var result = await node.TryAdmitAsync(Peer, 4); + Assert.Equal(TransitPeerStateAdmitStatus.Unavailable, result.Status); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(1, node.UnavailableRejects); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task RedisEvalFailureDuringAdmission_FailsClosed() + { + var redis = new FakeRedisService(); + redis.Database.ScriptException = new RedisUnavailableException("eval failed"); + var store = new RedisTransitPeerStateStore(redis); + var node = TransitPeerStateTestFactory.CreateTracker(store); + var result = await node.TryAdmitAsync(Peer, 4); + Assert.Equal(TransitPeerStateAdmitStatus.Unavailable, result.Status); + Assert.Equal(0, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task OceAfterSuccessfulEval_CompensatesAndRethrows() + { + var membership = new InMemoryTransitPeerStateStore + { + ThrowAfterAdmit = new OperationCanceledException(), + }; + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await Assert.ThrowsAnyAsync(() => node.TryAdmitAsync(Peer, 4).AsTask()); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.ReleaseCalls); + } + + [Fact] + public async Task RedisUnavailableDuringRelease_FinalizesLocally() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var admitted = await node.TryAdmitAsync(Peer, 4); + Assert.True(admitted.Accepted); + membership.Unavailable = true; + await node.ReleaseAsync(Peer, admitted.Generation); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task RedisUnavailableDuringRenewal_DoesNotExtend() + { + var clock = new VectorNNTPD.ReaderService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var expiry = membership.Expiry(Peer, node.OwnerId); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(expiry, membership.Expiry(Peer, node.OwnerId)); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task ShutdownCleanup_ReleaseOwnerClearsRemainingOwnership() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + await node.ReleaseAllOwnershipAsync(); + Assert.False(node.IsAccepting); + Assert.Equal(0, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False((await node.TryAdmitAsync(Peer, 4)).Accepted); + } + + [Fact] + public async Task ReleaseOwner_IsIsolatedToThisOwnerAndPeer() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + Assert.True((await a.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await b.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await a.TryAdmitAsync(Other, 4)).Accepted); + await a.ReleaseAllOwnershipAsync(); + Assert.Equal(0, membership.OwnerCount(Peer, a.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.OwnerCount(Other, a.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.OwnerCount(Peer, b.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.True((await b.TryAdmitAsync(Other, 4)).Accepted); + } + + [Fact] + public async Task NeverAdmitted_NeverIssuesRelease() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, node.ReleaseCalls); + Assert.Equal(0, membership.ReleaseCalls); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs new file mode 100644 index 00000000..5764ef4a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs @@ -0,0 +1,300 @@ +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateLifecycleTests +{ + private static readonly IPAddress Source = IPAddress.Parse("192.0.2.10"); + + [Fact] + public async Task Quit_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await live.ReadClientLineAsync(), StringComparison.Ordinal); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(NntpSessionLifetime.Finalized, live.Session.Lifetime); + Assert.Equal(1, live.Tracker.ReleaseCalls); + Assert.Equal(0, live.Membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task RemoteEof_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task ConnectionReset_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionReset)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task SocketException_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionAborted)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task IoException_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new IOException("peer disappeared")); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task IdleTimeout_ReleasesAdmittedConnectionOnce() + { + var clock = new ControllableTimeProvider(); + await using var live = await StartAdmittedAsync(clock); + await live.Session.IdleWatchArmed; + clock.Advance(TimeSpan.FromSeconds(2)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(TcpDisconnectReason.IdleTimeout, live.Session.CloseReasonForTests); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task Cancellation_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CancelRunAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task ListenerShutdown_FinalizesThenReleaseOwner() + { + await using var live = await StartAdmittedAsync(); + await live.CancelRunAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(0, live.Membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + await live.Tracker.ReleaseAllOwnershipAsync(); + Assert.False(live.Tracker.IsAccepting); + Assert.False((await live.Tracker.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 4)).Accepted); + } + + [Fact] + public async Task ApplicationShutdown_ServiceStopClearsOwnership() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 2)).Accepted); + var service = new TransitPeerStateService( + node, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount(TransitTestPeers.DefaultPeerName, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False(node.IsAccepting); + } + + [Fact] + public async Task DoubleFinalization_ReleasesOnce() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var admitted = await node.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 2); + var lease = new TransitInboundConnectionLease(ct => node.ReleaseAsync(TransitTestPeers.DefaultPeerName, admitted.Generation, ct)); + await lease.DisposeAsync(); + await lease.DisposeAsync(); + // codeql[cs/dispose-not-called-on-throw] Intentional multi-dispose contract exercise for TransitInboundConnectionLease. + lease.Dispose(); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(0, node.GetLocalCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task NeverAdmitted_DoesNotRelease() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await TransitInboundConnectionLease.None.DisposeAsync(); + await node.ReleaseAsync(TransitTestPeers.DefaultPeerName, 1); + Assert.Equal(0, node.ReleaseCalls); + Assert.Equal(0, membership.ReleaseCalls); + } + + private static async Task StartAdmittedAsync(ControllableTimeProvider? clock = null) + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: 4, allowFrom: [Source.ToString()]))); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var live = new LiveConnection(limiter, tracker, membership, peers, clock); + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, live.Session); + Assert.True(outcome.Admitted); + live.AttachLease(outcome.Lease); + live.Start(); + var greeting = await live.ReadClientLineAsync(); + Assert.True( + greeting.StartsWith("200 ", StringComparison.Ordinal) + || greeting.StartsWith("201 ", StringComparison.Ordinal), + greeting); + return live; + } + + private sealed class LiveConnection : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly CancellationTokenSource _runCts = new(); + private TransitInboundConnectionLease _lease = TransitInboundConnectionLease.None; + + public LiveConnection( + ITransitInboundConnectionLimiter limiter, + DistributedTransitPeerStateTracker tracker, + InMemoryTransitPeerStateStore membership, + ITransitPeerAuthorization peers, + ControllableTimeProvider? clock) + { + Limiter = limiter; + Tracker = tracker; + Membership = membership; + Connection = new LifecyclePipeConnection(_clientToServer.Reader, _serverToClient.Writer, Source); + Session = new NntpSession( + Connection, + NullLogger.Instance, + transitPeerAuthorization: peers, + commandIdleTimeout: clock is null ? null : TimeSpan.FromSeconds(1), + timeProvider: clock); + } + + public ITransitInboundConnectionLimiter Limiter { get; } + + public DistributedTransitPeerStateTracker Tracker { get; } + + public InMemoryTransitPeerStateStore Membership { get; } + + public NntpSession Session { get; } + + public LifecyclePipeConnection Connection { get; } + + public Task Run { get; private set; } = Task.CompletedTask; + + public void AttachLease(TransitInboundConnectionLease lease) => _lease = lease; + + public void Start() => Run = Session.RunAsync(_runCts.Token); + + public Task CancelRunAsync() => _runCts.CancelAsync(); + + public async Task FinalizeLeaseAsync() => await _lease.DisposeAsync(); + + public async Task WriteClientLineAsync(string line) + { + var payload = System.Text.Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(payload); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + return await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token) + ?? throw new InvalidOperationException("Server closed."); + } + + public async Task CompleteClientInputAsync(Exception? exception = null) + { + await _clientToServer.Writer.CompleteAsync(exception); + } + + public async ValueTask DisposeAsync() + { + await FinalizeLeaseAsync(); + _runCts.Dispose(); + await Connection.DisposeAsync(); + } + } + + internal sealed class LifecyclePipeConnection( + PipeReader input, + PipeWriter output, + IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs new file mode 100644 index 00000000..2c7a3ecb --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs @@ -0,0 +1,270 @@ +using VectorNNTPD.ReaderService.Redis; +using VectorNNTPD.ReaderService.Transit; +using VectorNNTPD.ReaderService.Tests.TestDoubles; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// Static Lua/C# TransitPeerState parity. These tests do not execute live Redis. +/// FakeRedis dispatches EVAL to . +/// +public sealed class TransitPeerStateLuaStaticAuditTests +{ + [Fact] + public async Task ProductionStore_DispatchesExactTransitPeerStateScripts() + { + var redis = new RecordingRedis(); + var store = new RedisTransitPeerStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + + _ = await store.TryAdmitAsync("peer-a", "nntpd01:a", 10, 1, now, ttl); + Assert.Same(TransitPeerStateScripts.TryAdmit, redis.LastScript); + Assert.Equal(1, redis.LastKeyCount); + + await store.ReleaseAsync("peer-a", "nntpd01:a", 1); + Assert.Same(TransitPeerStateScripts.Release, redis.LastScript); + + _ = await store.RenewAsync("peer-a", "nntpd01:a", 1, now, ttl); + Assert.Same(TransitPeerStateScripts.Renew, redis.LastScript); + + await store.ReleaseOwnerAsync("peer-a", "nntpd01:a"); + Assert.Same(TransitPeerStateScripts.ReleaseOwner, redis.LastScript); + } + + [Fact] + public void Scripts_NeverSetNativeKeyTtl() + { + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("EXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PEXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PERSIST", script, StringComparison.Ordinal); + Assert.DoesNotContain("SET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("GET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("EVAL", script, StringComparison.Ordinal); + Assert.DoesNotContain("redis.call('KEYS'", script, StringComparison.Ordinal); + } + } + + [Fact] + public void Scripts_UseOnlyDocumentedRedisCommands() + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "HGETALL", "HGET", "HSET", "HDEL", "HLEN", "DEL", + }; + foreach (var script in AllScripts()) + { + foreach (var command in EnumerateRedisCalls(script)) + { + Assert.Contains(command, allowed); + } + } + } + + [Fact] + public void TryAdmit_PrunesThenRejectsMaxZeroThenLimitBeforeIncrement() + { + var script = TransitPeerStateScripts.TryAdmit; + var prune = script.IndexOf("prune(key)", StringComparison.Ordinal); + var maxZero = script.IndexOf("if maxIncoming == nil or maxIncoming <= 0 then", StringComparison.Ordinal); + var rejectLimit = script.IndexOf("if total >= maxIncoming then", StringComparison.Ordinal); + var increment = script.IndexOf("increment(key, owner, gen)", StringComparison.Ordinal); + Assert.True(prune > 0 && prune < maxZero); + Assert.True(maxZero > 0 && maxZero < rejectLimit); + Assert.True(rejectLimit > 0 && rejectLimit < increment); + Assert.Contains("return 0", script, StringComparison.Ordinal); + Assert.Contains("return 1", script, StringComparison.Ordinal); + } + + [Fact] + public void Renew_DoesNotRecreateMissingOwnership() + { + var script = TransitPeerStateScripts.Renew; + Assert.Contains("if not cur then", script, StringComparison.Ordinal); + Assert.Contains("return 0", script, StringComparison.Ordinal); + var missing = script.IndexOf("if not cur then", StringComparison.Ordinal); + var write = script.IndexOf("redis.call('HSET'", StringComparison.Ordinal); + Assert.True(missing > 0 && missing < write); + Assert.Contains("storedGen ~= gen or exp == nil or exp <= now", script, StringComparison.Ordinal); + } + + [Fact] + public void Release_RequiresGenerationMatch() + { + var script = TransitPeerStateScripts.Release; + Assert.Contains("if storedGen == gen then", script, StringComparison.Ordinal); + Assert.Contains("redis.call('HDEL', key, owner)", script, StringComparison.Ordinal); + Assert.Contains("if redis.call('HLEN', key) == 0 then", script, StringComparison.Ordinal); + } + + [Fact] + public void ReleaseOwner_DeletesOnlyThisOwnerField() + { + var script = TransitPeerStateScripts.ReleaseOwner; + Assert.Contains("redis.call('HDEL', key, owner)", script, StringComparison.Ordinal); + Assert.DoesNotContain("HGETALL", script, StringComparison.Ordinal); + } + + [Fact] + public async Task EngineAndFakeRedisEval_AreSemanticallyIdentical() + { + var engine = new TransitPeerStateEngine(); + var redis = new FakeRedisService(); + var store = new RedisTransitPeerStateStore(redis); + const string peer = "parity-peer"; + const string owner = "nntpd01:a"; + var now = DateTimeOffset.FromUnixTimeMilliseconds(10_000); + var ttl = TimeSpan.FromMilliseconds(30_000); + Assert.Equal( + TransitPeerStateEngine.Accepted, + engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.True((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + AssertOwnership(engine, "k", owner, 40_000, 1, 1); + AssertOwnership(redis.Database.TransitPeerStateEngine, EncodingKey(peer), owner, 40_000, 1, 1); + + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.True((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.False((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit("k", owner, 0, 10_000, 30_000, 1)); + Assert.False((await store.TryAdmitAsync(peer, owner, 0, 1, now, ttl)).Accepted); + + _ = engine.Release("k", owner, 1); + await store.ReleaseAsync(peer, owner, 1); + + engine.WriteOwnership("k", owner, 40_000, 1, 1); + redis.Database.TransitPeerStateEngine.WriteOwnership(EncodingKey(peer), owner, 40_000, 1, 1); + Assert.Equal(1, engine.Renew("k", owner, 1, 15_000, 30_000)); + Assert.Equal(TransitPeerStateRenewStatus.Renewed, await store.RenewAsync(peer, owner, 1, DateTimeOffset.FromUnixTimeMilliseconds(15_000), ttl)); + Assert.Equal(0, engine.Renew("k", owner, 9, 15_000, 30_000)); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await store.RenewAsync(peer, owner, 9, DateTimeOffset.FromUnixTimeMilliseconds(15_000), ttl)); + + _ = engine.ReleaseOwner("k", owner); + await store.ReleaseOwnerAsync(peer, owner); + Assert.False(engine.TryGetOwnership("k", owner, out _, out _, out _)); + Assert.False(redis.Database.TransitPeerStateEngine.TryGetOwnership(EncodingKey(peer), owner, out _, out _, out _)); + } + + private static void AssertOwnership( + TransitPeerStateEngine engine, + string key, + string owner, + long expiry, + long generation, + int count) + { + Assert.True(engine.TryGetOwnership(key, owner, out var storedExpiry, out var storedGeneration, out var storedCount)); + Assert.Equal(expiry, storedExpiry); + Assert.Equal(generation, storedGeneration); + Assert.Equal(count, storedCount); + } + + private static string EncodingKey(string identifier) => + System.Text.Encoding.UTF8.GetString(TransitPeerStateKeys.Create(identifier)); + + private static IEnumerable AllScripts() => + [ + TransitPeerStateScripts.TryAdmit, + TransitPeerStateScripts.Release, + TransitPeerStateScripts.Renew, + TransitPeerStateScripts.ReleaseOwner, + ]; + + private static IEnumerable EnumerateRedisCalls(string script) + { + const string prefix = "redis.call('"; + var index = 0; + while (true) + { + var start = script.IndexOf(prefix, index, StringComparison.Ordinal); + if (start < 0) + { + yield break; + } + + start += prefix.Length; + var end = script.IndexOf('\'', start); + yield return script[start..end]; + index = end + 1; + } + } + + private sealed class RecordingRedis : IRedisService + { + public string? LastScript { get; private set; } + + public int LastKeyCount { get; private set; } + + public IRedisDatabase Database => new RecordingDatabase(this); + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + private sealed class RecordingDatabase : IRedisDatabase + { + private readonly RecordingRedis _owner; + + public RecordingDatabase(RecordingRedis owner) + { + _owner = owner; + } + + public Task PingAsync(CancellationToken cancellationToken = default) => + Task.FromResult(TimeSpan.FromMilliseconds(1)); + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + _owner.LastScript = script; + _owner.LastKeyCount = keys.Length; + return ValueTask.FromResult(1L); + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs new file mode 100644 index 00000000..36ba883c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs @@ -0,0 +1,162 @@ +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// Live EVAL of through +/// against the dedicated test Redis. +/// +public sealed class TransitPeerStateRedisLuaIntegrationTests : IClassFixture +{ + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const long Now = 10_000; + private const long Lease = 30_000; + private readonly TransitPeerStateRedisIntegrationFixture _redis; + + public TransitPeerStateRedisLuaIntegrationTests(TransitPeerStateRedisIntegrationFixture redis) + { + _redis = redis; + } + + [TransitPeerStateRedisIntegrationFact] + public void ConnectsToAuthorizedTestRedis() + { + Assert.True(_redis.IsConfigured); + Assert.Equal(TransitPeerStateRedisIntegration.AuthorizedEndpoint, _redis.Endpoint); + Assert.False(string.IsNullOrWhiteSpace(_redis.ServerVersion)); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task TryAdmit_WritesHashedOwnership_WithoutNativeTtl() + { + var peer = await _redis.CreateIdentifierAsync(); + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", O1, 4, Now, Lease, 1)); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 1, 1); + Assert.Null(await _redis.KeyTimeToLiveAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task ConcurrentAdmissions_EnforceExactClusterLimit() + { + var peer = await _redis.CreateIdentifierAsync(); + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 20).Select(async i => + { + if ((await _redis.TryAdmitAsync(peer, "owner:" + i, 5, 1, Now, Lease)).Accepted) + { + Interlocked.Increment(ref admitted); + } + })); + Assert.Equal(5, admitted); + Assert.Equal(5, await _redis.CountAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task SamePeerAcrossOwners_SharesLimit_DifferentPeersAreIsolated() + { + var peer = await _redis.CreateIdentifierAsync(); + var other = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 1, 1, Now, Lease)).Accepted); + Assert.False((await _redis.TryAdmitAsync(peer, O2, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(other, O2, 1, 1, Now, Lease)).Accepted); + Assert.Equal(1, await _redis.CountAsync(peer)); + Assert.Equal(1, await _redis.CountAsync(other)); + await _redis.DeleteKeyAsync(peer); + await _redis.DeleteKeyAsync(other); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task GenerationReplacement_ReplacesCountAndIgnoresStaleRelease() + { + var peer = await _redis.CreateIdentifierAsync(); + await _redis.SeedAsync(peer, O1, 30_000, 1, 7); + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership("k", O1, 30_000, 1, 7); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", O1, 20, 0, Lease, 2)); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 20, 2, 0, Lease)).Accepted); + AssertOwnership(await _redis.ReadAsync(peer, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(peer, O1, 1); + AssertOwnership(await _redis.ReadAsync(peer, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(peer, O1, 2); + Assert.Null(await _redis.ReadAsync(peer, O1)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task StaleRenewal_DoesNotRecreateOrExtend() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await _redis.RenewAsync(peer, O1, 1, Now, Lease)); + Assert.Null(await _redis.ReadAsync(peer, O1)); + + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 2, Now, Lease)).Accepted); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await _redis.RenewAsync(peer, O1, 1, Now, Lease)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 2, 1); + Assert.Equal(TransitPeerStateRenewStatus.Renewed, await _redis.RenewAsync(peer, O1, 2, Now + 5_000, Lease)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + 5_000 + Lease, 2, 1); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task PruneExpired_FreesCapacity() + { + var peer = await _redis.CreateIdentifierAsync(); + await _redis.SeedAsync(peer, O2, Now, 1, 4); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 1, 1, Now, Lease)).Accepted); + Assert.Null(await _redis.ReadAsync(peer, O2)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 1, 1); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task MaxZero_RejectsWithoutMutation() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.False((await _redis.TryAdmitAsync(peer, O1, 0, 1, Now, Lease)).Accepted); + Assert.Null(await _redis.ReadAsync(peer, O1)); + Assert.Equal(0, await _redis.FieldCountAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task ReleaseOwner_RemovesOnlyThisOwner() + { + var peer = await _redis.CreateIdentifierAsync(); + var other = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(peer, O2, 4, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(other, O1, 4, 1, Now, Lease)).Accepted); + await _redis.ReleaseOwnerAsync(peer, O1); + Assert.Null(await _redis.ReadAsync(peer, O1)); + Assert.NotNull(await _redis.ReadAsync(peer, O2)); + Assert.NotNull(await _redis.ReadAsync(other, O1)); + await _redis.DeleteKeyAsync(peer); + await _redis.DeleteKeyAsync(other); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task Cleanup_DeletesOnlyTrackedKeys() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + await _redis.DeleteKeyAsync(peer); + Assert.Empty(await _redis.RemainingTestKeysAsync()); + } + + private static void AssertOwnership(Ownership? ownership, long expiry, long generation, int count) + { + Assert.True(ownership.HasValue); + var value = ownership.GetValueOrDefault(); + Assert.Equal(expiry, value.ExpiryUnixMs); + Assert.Equal(generation, value.Generation); + Assert.Equal(count, value.Count); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs new file mode 100644 index 00000000..ba11d20d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateServiceTests +{ + [Fact] + public void Namespace_IsTransit() + { + Assert.Equal("VectorNNTPD.ReaderService.Transit", typeof(TransitPeerStateService).Namespace); + Assert.Equal("VectorNNTPD.ReaderService.Transit", typeof(DistributedTransitPeerStateTracker).Namespace); + Assert.NotEqual(typeof(VectorNNTPD.ReaderService.SessionState.SessionStateService), typeof(TransitPeerStateService)); + } + + [Fact] + public async Task RenewalPass_ExtendsIdleOwnershipWithoutTraffic() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync("peer-a", 1)).Accepted); + var first = membership.Expiry("peer-a", node.OwnerId); + var service = new TransitPeerStateService( + node, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + await service.StartAsync(CancellationToken.None); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!clock.HasScheduledTimers) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + clock.Advance(TimeSpan.FromSeconds(10)); + using var renewWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (membership.RenewCalls == 0) + { + renewWait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.True(membership.Expiry("peer-a", node.OwnerId) > first); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount("peer-a", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(node.IsAccepting); + } + + [Fact] + public async Task Stop_StopsRenewalThenReleaseOwner() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync("peer-a", 2)).Accepted); + var service = new TransitPeerStateService( + node, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount("peer-a", node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False(node.IsAccepting); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs new file mode 100644 index 00000000..58d6e236 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +internal static class TransitPeerStateTestFactory +{ + public static (TransitInboundConnectionLimiter Limiter, DistributedTransitPeerStateTracker Tracker, InMemoryTransitPeerStateStore Membership) + CreateLimiter(TransitConfigurationStore store, string nodeId = "nntpd01", string incarnation = "test") + { + var membership = new InMemoryTransitPeerStateStore(); + var tracker = CreateTracker(membership, nodeId, incarnation); + return (new TransitInboundConnectionLimiter(store, tracker), tracker, membership); + } + + public static DistributedTransitPeerStateTracker CreateTracker( + ITransitPeerStateStore membership, + string nodeId = "nntpd01", + string incarnation = "a", + TimeProvider? time = null, + TimeSpan? leaseTtl = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System, + leaseTtl, + incarnation); + + public static long NowMs(TimeProvider? time = null) => + (time ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs new file mode 100644 index 00000000..5f126013 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs @@ -0,0 +1,312 @@ +using System.Net; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; +using Microsoft.Extensions.Logging.Abstractions; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeerStateTrackerTests +{ + private const string Peer = "peer-a"; + private const string Other = "peer-b"; + + [Fact] + public async Task FirstConnection_BelowAtAndOver_ThenReleaseAndReconnect() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, node.GetLocalCount(Peer)); + Assert.Equal(2, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + + await node.ReleaseAsync(Peer, 1); + Assert.Equal(1, node.GetLocalCount(Peer)); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + await node.ReleaseAsync(Peer, 1); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task MaxZero_IsClosed_NotUnlimited() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.False((await node.TryAdmitAsync(Peer, 0)).Accepted); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, node.DistributedRejects); + } + + [Fact] + public async Task SamePeerAcrossTwoNodes_SharesClusterLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + Assert.True((await a.TryAdmitAsync(Peer, 2)).Accepted); + Assert.True((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await a.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + await a.ReleaseAsync(Peer, 1); + Assert.True((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, membership.OwnerCount(Peer, b.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task SamePeerAcrossNNodes_EnforcesExactGlobalLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var nodes = Enumerable.Range(1, 5) + .Select(i => TransitPeerStateTestFactory.CreateTracker(membership, $"nntpd{i:00}", i.ToString())) + .ToArray(); + var admitted = 0; + // codeql[cs/linq/missed-where] Async TryAdmit filter; Where cannot await. + foreach (var node in nodes) + { + if ((await node.TryAdmitAsync(Peer, 3)).Accepted) + { + admitted++; + } + } + + Assert.Equal(3, admitted); + Assert.Equal(3, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.False((await nodes[0].TryAdmitAsync(Peer, 3)).Accepted); + } + + [Fact] + public async Task DifferentPeers_RemainIsolated() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.True((await node.TryAdmitAsync(Other, 1)).Accepted); + Assert.False((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.ActiveCount(Other, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task ConcurrentAdmissionsAcrossNodes_DoNotExceedLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 40).Select(async i => + { + await Task.Yield(); + var node = i % 2 == 0 ? a : b; + if ((await node.TryAdmitAsync(Peer, 7)).Accepted) + { + Interlocked.Increment(ref admitted); + } + })); + Assert.Equal(7, admitted); + Assert.Equal(7, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(7, a.GetLocalCount(Peer) + b.GetLocalCount(Peer)); + } + + [Fact] + public async Task TwoSourceIpsMappedToOneIdentifier_ShareOneLimit() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + Peer, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10", "198.51.100.20"]))); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var first = CreateSession(peers, IPAddress.Parse("192.0.2.10")); + var second = CreateSession(peers, IPAddress.Parse("198.51.100.20")); + Assert.Equal(Peer, first.Authorization.TransitPeerName); + Assert.Equal(Peer, second.Authorization.TransitPeerName); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease.Admitted); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, second)).Admitted); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + await lease.Lease.DisposeAsync(); + } + + [Fact] + public async Task Ipv4AndIpv6_RemainDistinctAclIdentities_ButShareIdentifierLimit() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + Peer, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10", "2001:db8::10"]))); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var v4 = CreateSession(peers, IPAddress.Parse("192.0.2.10")); + var v6 = CreateSession(peers, IPAddress.Parse("2001:db8::10")); + Assert.Equal(Peer, v4.Authorization.TransitPeerName); + Assert.Equal(Peer, v6.Authorization.TransitPeerName); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, v4); + Assert.True(lease.Admitted); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, v6)).Admitted); + await lease.Lease.DisposeAsync(); + } + + [Fact] + public async Task IdleConnection_RemainsCounted_UntilReleased() + { + var clock = new VectorNNTPD.ReaderService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(1, membership.ActiveCount(Peer, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task Renewal_ExtendsLease_IndependentlyOfTraffic() + { + var clock = new VectorNNTPD.ReaderService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var first = membership.Expiry(Peer, node.OwnerId); + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + var renewed = membership.Expiry(Peer, node.OwnerId); + Assert.True(renewed > first); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(1, membership.ActiveCount(Peer, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task RenewalFailure_DoesNotExtendOwnership() + { + var clock = new VectorNNTPD.ReaderService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var first = membership.Expiry(Peer, node.OwnerId); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(first, membership.Expiry(Peer, node.OwnerId)); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task SameOwnerNewGeneration_ReplacesStaleGeneration() + { + var membership = new InMemoryTransitPeerStateStore(); + membership.Engine.WriteOwnership(InMemoryTransitPeerStateStore.ConnectionKey(Peer), "nntpd01:a", 40_000, 9, 7); + var node = TransitPeerStateTestFactory.CreateTracker(membership, incarnation: "a"); + Assert.True((await node.TryAdmitAsync(Peer, 20)).Accepted); + Assert.True(membership.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var generation, + out var count)); + Assert.Equal(1, generation); + Assert.Equal(1, count); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task StaleReleaseAndRenew_DoNotAffectNewerGeneration() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + await membership.ReleaseAsync(Peer, node.OwnerId, 99); + Assert.Equal(1, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal( + TransitPeerStateRenewStatus.Lost, + await membership.RenewAsync(Peer, node.OwnerId, 99, DateTimeOffset.UtcNow, TimeSpan.FromSeconds(30))); + Assert.Equal(1, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task UnknownPeer_DoesNotCreateState() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitTestPeers.Snapshot(Peer, TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10"]))); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + Assert.False((await limiter.TryAcquireAsync("missing")).Admitted); + Assert.Equal(0, membership.ActiveCount("missing", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.AdmitCalls); + } + + private static NntpSession CreateSession(ITransitPeerAuthorization peers, IPAddress source) + { + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + return new NntpSession( + new IdentityPipeConnection(input.Reader, output.Writer, source), + NullLogger.Instance, + transitPeerAuthorization: peers); + } + + private sealed class IdentityPipeConnection( + System.IO.Pipelines.PipeReader input, + System.IO.Pipelines.PipeWriter output, + IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public System.IO.Pipelines.PipeReader Input { get; } = input; + public System.IO.Pipelines.PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeersOptionsTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeersOptionsTests.cs new file mode 100644 index 00000000..0058de89 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitPeersOptionsTests.cs @@ -0,0 +1,490 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +public sealed class TransitPeersOptionsTests +{ + [Fact] + public void Bind_ExactTopLevelDictionaryShape_MultiplePeersAndDefaults() + { + const string json = """ + { + "Transit": { + "news-example": { + "PeerName": "News Example", + "MaxIncomingConnections": 10, + "MaxOutgoingConnections": 2, + "AllowFrom": [ "news.example.net", "192.0.2.0/24", "2001:db8:1234::/48" ], + "ConnectTo": [ "news.example.net:563" ], + "Username": "", + "Password": "", + "Ssl": "TLS", + "Patterns": "*", + "DeferOnDuplicate": true, + "PathToken": "peer.example", + "MaxSize": 10485760, + "MessageTypes": [ "default" ] + }, + "second": { + "PeerName": "Second Peer", + "MaxIncomingConnections": 1, + "MaxOutgoingConnections": 0, + "AllowFrom": [ "198.51.100.10" ] + } + } + } + """; + + var options = Bind(json); + Assert.Equal(2, options.Count); + Assert.True(options.TryGetValue("news-example", out var first)); + Assert.True(options.TryGetValue("second", out var second)); + Assert.Equal("News Example", first.PeerName); + Assert.Equal("Second Peer", second.PeerName); + Assert.Equal(10, first.MaxIncomingConnections); + Assert.Equal(2, first.MaxOutgoingConnections); + Assert.Equal(["news.example.net", "192.0.2.0/24", "2001:db8:1234::/48"], first.AllowFrom); + Assert.Equal(["news.example.net:563"], first.ConnectTo); + Assert.Equal("TLS", first.Ssl); + Assert.Equal("*", first.Patterns); + Assert.True(first.DeferOnDuplicate); + Assert.Equal(string.Empty, first.Username); + Assert.Equal(string.Empty, first.Password); + Assert.Equal("peer.example", first.PathToken); + Assert.Equal(10_485_760, first.MaxSize); + Assert.Equal(["default"], first.MessageTypes); + Assert.Equal("*", second.Patterns); + Assert.True(second.DeferOnDuplicate); + Assert.Equal([], second.ConnectTo); + Assert.Equal(string.Empty, second.PathToken); + Assert.Equal(TransitPeerOptions.DefaultMaxSize, second.MaxSize); + Assert.Empty(second.MessageTypes); + Assert.True(TransitMessageTypesParser.TryParse(second.MessageTypes, out var secondTypes, out _)); + Assert.Equal(TransitMessageTypes.Default, secondTypes); + } + + [Fact] + public void Bind_EmptyTransit_IsValid() + { + var options = Bind("""{ "Transit": { } }"""); + Assert.Empty(options); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData("usenet-ninja")] + [InlineData("giganews")] + [InlineData("blueworld-hosting")] + [InlineData("news-example")] + [InlineData("GIGANEWS")] + [InlineData("peer.example.com")] + [InlineData("192.0.2.1")] + public void Validate_AcceptsProtocolSafeIdentifiers(string identifier) + { + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + Assert.True(options.ContainsKey(identifier)); + } + + [Theory] + [InlineData("Giganews, Inc.")] + [InlineData("Blueworld Hosting")] + [InlineData("Usenet Ninja")] + [InlineData("peer/with slash")] + [InlineData("peer name")] + [InlineData(" ")] + [InlineData("peer\nname")] + [InlineData("Caf\u00e9")] + public void Validate_RejectsInvalidIdentifiers(string identifier) + { + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("identifier", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_RejectsOverlongIdentifier() + { + var identifier = new string('a', TransitPeersOptionsValidator.MaxPeerIdentifierLength + 1); + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("identifier", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_IdentifierMatch_IsExactOrdinal() + { + var options = new TransitPeersOptions + { + ["GIGANEWS"] = TransitTestPeers.Peer(peerName: "Giganews, Inc."), + }; + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + var snapshot = TransitConfigurationSnapshot.Create(options); + Assert.True(snapshot.Peers.TryGetValue("GIGANEWS", out var giganeews)); + Assert.False(snapshot.Peers.ContainsKey("giganews")); + Assert.Equal("GIGANEWS", giganeews.Identifier); + Assert.Equal("Giganews, Inc.", giganeews.PeerName); + } + + [Theory] + [InlineData("Giganews, Inc.")] + [InlineData("Blueworld Hosting")] + [InlineData("Usenet Ninja")] + [InlineData("Caf\u00e9 \u2014 \u65e5\u672c\u8a9e")] + public void Validate_AcceptsPeerNameWithSpacesAndPunctuation(string peerName) + { + var options = new TransitPeersOptions + { + ["usenet-ninja"] = TransitTestPeers.Peer(peerName: peerName), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + var snapshot = TransitConfigurationSnapshot.Create(options); + Assert.Equal(peerName, snapshot.Peers["usenet-ninja"].PeerName); + Assert.Equal("usenet-ninja", snapshot.Peers["usenet-ninja"].Identifier); + } + + [Fact] + public void Validate_RejectsMissingPeerName() + { + var options = TransitTestPeers.Dictionary("alpha", new TransitPeerOptions + { + MaxIncomingConnections = 1, + MaxOutgoingConnections = 0, + AllowFrom = ["192.0.2.1"], + }); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Validate_RejectsEmptyOrWhitespacePeerName(string peerName) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: peerName)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsControlCharacterPeerName() + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: "peer\nname")); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("control character", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsOverlongPeerName() + { + var peerName = new string('a', TransitPeersOptionsValidator.MaxPeerNameLength + 1); + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: peerName)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("maximum length", StringComparison.Ordinal)); + } + + [Fact] + public void Bind_PreservesIdentifierAndExactPeerName() + { + const string json = """ + { + "Transit": { + "giganews": { + "PeerName": "Giganews, Inc.", + "MaxIncomingConnections": 50, + "MaxOutgoingConnections": 50, + "PathToken": "giganews.example" + }, + "blueworld-hosting": { + "PeerName": "Blueworld Hosting", + "MaxIncomingConnections": 10, + "MaxOutgoingConnections": 10, + "PathToken": "blueworld.example" + } + } + } + """; + + var options = Bind(json); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(options.TryGetValue("giganews", out var giganews)); + Assert.True(options.TryGetValue("blueworld-hosting", out var blueworld)); + Assert.False(options.ContainsKey("Giganews, Inc.")); + Assert.False(options.ContainsKey("Blueworld Hosting")); + Assert.Equal("Giganews, Inc.", giganews.PeerName); + Assert.Equal("Blueworld Hosting", blueworld.PeerName); + Assert.Equal("giganews.example", giganews.PathToken); + Assert.Equal("blueworld.example", blueworld.PathToken); + } + + [Fact] + public void Validate_PathToken_PreservesExactString_AndRejectsControlChars() + { + var preserved = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "Peer.Example")); + Assert.True(new TransitPeersOptionsValidator().Validate(null, preserved).Succeeded); + var snapshot = TransitConfigurationSnapshot.Create(preserved); + Assert.Equal("Peer.Example", snapshot.Peers["alpha"].PathToken); + + var empty = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "")); + Assert.True(new TransitPeersOptionsValidator().Validate(null, empty).Succeeded); + + var control = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "peer\tname")); + var result = new TransitPeersOptionsValidator().Validate(null, control); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PathToken", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData((long)int.MaxValue + 1)] + public void Validate_RejectsInvalidMaxSize(long maxSize) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: maxSize)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MaxSize", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_AcceptsMaxSizeBounds() + { + var min = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: 1)); + var max = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: TransitPeerOptions.MaxMaxSize)); + Assert.True(new TransitPeersOptionsValidator().Validate(null, min).Succeeded); + Assert.True(new TransitPeersOptionsValidator().Validate(null, max).Succeeded); + } + + [Fact] + public void Validate_RejectsUnknownMessageTypes() + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(messageTypes: ["default", "not-a-type"])); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MessageTypes", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsMissingConnectionLimits() + { + var options = TransitTestPeers.Dictionary( + "alpha", + new TransitPeerOptions { PeerName = "Alpha", AllowFrom = ["192.0.2.1"] }); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MaxIncomingConnections", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("MaxOutgoingConnections", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(-1)] + [InlineData(4097)] + public void Validate_RejectsOutOfRangeLimits(int value) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxIncoming: value, maxOutgoing: value)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData("ssl")] + [InlineData("plain")] + [InlineData("tls-start")] + public void Validate_RejectsInvalidSsl(string ssl) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: ssl)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("Ssl", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("")] + [InlineData("TLS")] + [InlineData("tls")] + [InlineData("STARTTLS")] + [InlineData("starttls")] + public void Validate_AcceptsCanonicalSslValues(string ssl) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: ssl)); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(TransitSslParser.TryParse(ssl, out _, out _)); + } + + [Theory] + [InlineData("news.example.net")] + [InlineData("192.0.2.10")] + [InlineData("2001:db8::10")] + [InlineData("[2001:db8::10]")] + [InlineData("2001:db8::10:119")] + public void Validate_RejectsInvalidConnectTo(string endpoint) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(connectTo: [endpoint])); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("ConnectTo", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("news.example.net:119")] + [InlineData("news.example.net:563")] + [InlineData("192.0.2.10:119")] + [InlineData("[2001:db8::10]:119")] + [InlineData("[2001:db8::10]:563")] + public void Validate_AcceptsExplicitConnectToPorts(string endpoint) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(connectTo: [endpoint])); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(TransitConnectEndpoint.TryParse(endpoint, out var parsed, out var error), error); + Assert.InRange(parsed!.Port, 1, 65535); + } + + [Fact] + public void Validate_RejectsInvalidAllowFromAndPatterns_WithoutLeakingPassword() + { + var options = TransitTestPeers.Dictionary( + "alpha", + TransitTestPeers.Peer( + allowFrom: ["not a host!"], + username: "peer", + password: "peer-secret", + patterns: "comp.[abc")); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("AllowFrom", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("Patterns", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static f => f.Contains("peer-secret", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsPartialCredentials_AllowsBothBlankOrBothSet() + { + var validator = new TransitPeersOptionsValidator(); + + var userOnly = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "peer", password: "")); + var userOnlyResult = validator.Validate(null, userOnly); + Assert.True(userOnlyResult.Failed); + Assert.Contains(userOnlyResult.Failures!, static f => f.Contains("Password", StringComparison.Ordinal)); + Assert.Contains(userOnlyResult.Failures!, static f => f.Contains("both be set or both be blank", StringComparison.Ordinal)); + + var passwordOnly = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "", password: "peer-secret")); + var passwordOnlyResult = validator.Validate(null, passwordOnly); + Assert.True(passwordOnlyResult.Failed); + Assert.Contains(passwordOnlyResult.Failures!, static f => f.Contains("Username", StringComparison.Ordinal)); + Assert.DoesNotContain(passwordOnlyResult.Failures!, static f => f.Contains("peer-secret", StringComparison.Ordinal)); + + var bothBlank = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "", password: "")); + Assert.True(validator.Validate(null, bothBlank).Succeeded); + + var bothSet = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "peer", password: "peer-secret")); + Assert.True(validator.Validate(null, bothSet).Succeeded); + Assert.True(TransitConfigurationSnapshot.Create(bothSet).Peers["alpha"].HasPeerCredentials); + } + + [Fact] + public void ProductionAppsettings_DoesNotOwnTransitPeers() + { + var path = FindProductionAppsettings(); + var config = new ConfigurationBuilder().AddJsonFile(path, optional: false).Build(); + + Assert.False(config.GetSection("Transit").Exists()); + Assert.Null(config["Nntpd:Transit:WantTrash"]); + Assert.Null(config["Nntpd:Transit:LogTrash"]); + Assert.Equal("8", config["Nntpd:Transit:StreamOutstandingArticleDepth"]); + Assert.Equal("4294967296", config["Nntpd:TransitQueueMemoryLimit"]); + + var peers = new TransitPeersOptions(); + config.GetSection("Transit").Bind(peers); + Assert.Empty(peers); + Assert.True(new TransitPeersOptionsValidator().Validate(null, peers).Succeeded); + + var leftover = new TransitLeftoverConfigurationValidator(config).Validate(null, new NntpdOptions()); + Assert.True(leftover.Succeeded); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.ReaderService", "VectorNNTPD.ReaderService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.ReaderService/VectorNNTPD.ReaderService.json."); + } + + [Fact] + public void Validate_RejectsOverlappingPrefixes() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"]), + ["two"] = TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("overlapping", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_RejectsSharedHostname() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(allowFrom: ["news.example.net"]), + ["two"] = TransitTestPeers.Peer(allowFrom: ["news.example.net"]), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("news.example.net", StringComparison.Ordinal)); + } + + [Fact] + public void ServiceBind_DoesNotRequireNestedPeersSection() + { + var json = """{ "Transit": { "alpha": { "MaxIncomingConnections": 1, "MaxOutgoingConnections": 0, "AllowFrom": [ "192.0.2.1" ] } } }"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var config = new ConfigurationBuilder().AddJsonStream(stream).Build(); + var services = new ServiceCollection(); + services.AddOptions().Bind(config.GetSection("Transit")); + using var provider = services.BuildServiceProvider(); + var bound = provider.GetRequiredService>().Value; + Assert.True(bound.ContainsKey("alpha")); + Assert.False(bound.ContainsKey("Peers")); + } + + private static TransitPeersOptions Bind(string json) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var config = new ConfigurationBuilder().AddJsonStream(stream).Build(); + var options = new TransitPeersOptions(); + config.GetSection("Transit").Bind(options); + return options; + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs new file mode 100644 index 00000000..75471318 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs @@ -0,0 +1,503 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.ReaderService.Authentication; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.Authentication; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.SessionState; +using VectorNNTPD.ReaderService.Tests.Session; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// AllowFrom identifies a connection as a named Transit peer before AUTHINFO. +/// READER AUTHINFO replaces that identity and must release the TransitPeerState slot. +/// +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class TransitReaderAdmissionReclassificationTests +{ + private static readonly IPAddress AllowFromClient = IPAddress.Parse("198.18.0.70"); + private const string PeerId = "usenet-ninja"; + + [Fact] + public async Task ReaderFromAllowFrom_DoesNotKeepTransitSlot() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var admission = new InMemorySessionStateTracker(); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store, admission); + var lease = await AdmitAsync(limiter, session); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("a", session.Authentication.Username); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + admission.TryAdmit("a", "other", AllowFromClient, sessionLimit: 1, srcIpLimit: 0)); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task GenuineTransit_WithoutAuthinfo_KeepsSlotAndEnforcesLimit() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var firstDuplex = new AuthDuplex(AllowFromClient); + var first = firstDuplex.CreateSession(store); + var lease1 = await AdmitAsync(limiter, first); + Assert.True(lease1.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + var firstRun = first.RunAsync(); + await firstDuplex.ReadGreetingAsync(); + Assert.Equal(PeerId, first.Authorization.TransitPeerName); + Assert.True(first.Authorization.AuthorizedTransit); + Assert.False(first.Authentication.IsAuthenticated); + + await using var blockedDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.71")); + var blocked = blockedDuplex.CreateSession(store); + var lease2 = await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked); + Assert.False(lease2.Admitted); + Assert.False(lease2.Lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await firstDuplex.QuitAsync(firstRun); + await lease1.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + + await using var reusedDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.72")); + var reused = reusedDuplex.CreateSession(store); + var lease3 = await AdmitAsync(limiter, reused); + Assert.True(lease3.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + await lease3.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task GenuineTransit_ReaderRole_PeerAuthinfoDoesNotAuthenticate_AndKeepsSlot() + { + var store = CreateStore(maxIncoming: 1, transitUsername: "feed", transitPassword: "peer-secret"); + var (limiter, tracker, _) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS peer-secret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + + // Reader role: peer credentials never authenticate and never reclassify the admission slot. + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Equal(PeerId, session.Authorization.TransitPeerName); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task ReadersFromAllowFrom_DoNotStarveTransitCapacity() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var admission = new InMemorySessionStateTracker(); + for (var i = 0; i < 3; i++) + { + await using var readerDuplex = new AuthDuplex(AllowFromClient); + var reader = readerDuplex.CreateSession(store, admission, username: "a", password: "a"); + var readerLease = await AdmitAsync(limiter, reader); + var readerRun = reader.RunAsync(); + await readerDuplex.ReadGreetingAsync(); + await readerDuplex.AuthenticateReaderAsync("a", "a"); + Assert.Null(reader.Authorization.TransitPeerName); + Assert.False(readerLease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + await readerDuplex.QuitAsync(readerRun); + await readerLease.DisposeAsync(); + } + + await using var transitDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.80")); + var transit = transitDuplex.CreateSession(store); + var transitLease = await AdmitAsync(limiter, transit); + Assert.True(transitLease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + await transitLease.DisposeAsync(); + } + + [Fact] + public async Task FailedReaderAuthinfo_KeepsTransitSlotUntilDisconnect() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER a"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + + Assert.False(session.Authentication.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedTransit); + Assert.Equal(PeerId, session.Authorization.TransitPeerName); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task AllowFromDisconnectBeforeAuthinfo_ReleasesSlotOnce() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, _) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task AmbiguousAllowFrom_DoesNotConsumeSlot_AndReaderAuthSucceeds() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["198.18.0.0/15"]), + ["two"] = TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["198.18.0.0/15"]), + }; + var store = new TransitConfigurationStore(); + store.Replace(TransitConfigurationSnapshot.Create(options)); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + Assert.Null(session.Authorization.TransitPeerName); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + session.AttachTransitAdmission(admitted.Lease); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.True(session.Authorization.AuthorizedReader); + Assert.Null(session.Authorization.TransitPeerName); + Assert.Equal(0, membership.ActiveCount("one", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.ActiveCount("two", TransitPeerStateTestFactory.NowMs())); + await duplex.QuitAsync(run); + } + + [Fact] + public async Task ReaderReclassification_RenewalDoesNotRecreateSlot() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + await tracker.RenewLeasesAsync(CancellationToken.None); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + } + + [Fact] + public async Task InFlightRenewal_CannotRecreateSlotAfterReaderReclassification() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + membership.NotifyRenewStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); + membership.BlockRenew = new(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + var renew = tracker.RenewLeasesAsync(CancellationToken.None).AsTask(); + await membership.NotifyRenewStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.Null(session.Authorization.TransitPeerName); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + membership.BlockRenew.TrySetResult(); + await renew.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, tracker.ReleaseCalls); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(1, tracker.ReleaseCalls); + } + + [Fact] + public async Task ReaderAuthinfoRelease_ConcurrentTeardownDispose_ReleasesOnce() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + membership.BlockRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER a"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS a"); + await WaitUntilAsync(() => session.Authorization.AuthorizedReader); + + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ReleaseCalls); + + var teardown = lease.DisposeAsync(); + membership.BlockRelease.TrySetResult(); + await teardown; + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal(1, tracker.ReleaseCalls); + Assert.Equal(1, membership.ReleaseCalls); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(1, tracker.ReleaseCalls); + } + + private static TransitConfigurationStore CreateStore( + int maxIncoming, + string transitUsername = "", + string transitPassword = "") + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + PeerId, + TransitTestPeers.Peer( + maxIncoming: maxIncoming, + allowFrom: ["198.18.0.0/15"], + username: transitUsername, + password: transitPassword))); + return store; + } + + private static async Task AdmitAsync( + TransitInboundConnectionLimiter limiter, + NntpSession session) + { + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(outcome.Admitted); + session.AttachTransitAdmission(outcome.Lease); + return outcome.Lease; + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(1, cts.Token); + } + } + + private sealed class AuthDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly IPAddress _clientIp; + + public AuthDuplex(IPAddress clientIp) => _clientIp = clientIp; + + public NntpSession CreateSession( + TransitConfigurationStore store, + ISessionStateTracker? admission = null, + string username = "a", + string password = "a") + { + var users = new MemoryNntpUserRecordStore(); + users.Add(MemoryNntpUserRecordStore.Create(username, password, sessionLimit: 1)); + var validator = new MySqlNntpCredentialValidator(users, NullLogger.Instance); + var provider = new CompositeNntpAuthenticationProvider( + DenyAllNntpAuthenticationProvider.Instance, + newsmasterUsername: null, + validator); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientIp, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: provider, + transitPeerAuthorization: TransitPeerAuthorization.CreateForStore(store), + sessionAdmission: admission); + } + + public async Task AuthenticateReaderAsync(string username, string password) + { + await WriteClientLineAsync($"AUTHINFO USER {username}"); + Assert.StartsWith("381 ", await ReadClientLineAsync(), StringComparison.Ordinal); + await WriteClientLineAsync($"AUTHINFO PASS {password}"); + Assert.StartsWith("281 ", await ReadClientLineAsync(), StringComparison.Ordinal); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("20", line, StringComparison.Ordinal); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task QuitAsync(Task run) + { + await WriteClientLineAsync("QUIT"); + await ReadClientLineAsync(); + await run; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public ConnectionClientIdentity ClientIdentity { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceivePolicyCommandTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceivePolicyCommandTests.cs new file mode 100644 index 00000000..118173e5 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceivePolicyCommandTests.cs @@ -0,0 +1,384 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.NntpDb; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// Role-isolation coverage: a Reader session never exposes the Transit receive commands, regardless of the +/// per-peer receive policy (size, patterns, article types, credentials) that would apply on the Transit service. +/// +/// +/// The receive-policy behavior itself (size min, receive patterns, article types, peer credentials) is covered by +/// the Transit service tests. Here the contract is that the Reader dispatcher answers +/// 502 Command not available on this server before any receive policy, authentication, or article read. +/// +[Collection("NntpCommandLoggerCollection")] +public sealed class TransitReceivePolicyCommandTests +{ + private const string CommandNotAvailable = "502 Command not available on this server"; + private const int GlobalMax = 5 * 1024 * 1024; + private const int OneMib = 1 * 1024 * 1024; + private const int TenMib = 10 * 1024 * 1024; + + private static readonly IPAddress PeerAddress = IPAddress.Parse("192.0.2.10"); + + public static IEnumerable ArticlePaths() + { + yield return [ReceivePath.SerialTakeThis]; + yield return [ReceivePath.PipelinedTakeThis]; + yield return [ReceivePath.IHave]; + } + + public static IEnumerable SizeCases() + { + foreach (var path in new[] { ReceivePath.SerialTakeThis, ReceivePath.PipelinedTakeThis, ReceivePath.IHave }) + { + yield return [path, (long)OneMib]; + yield return [path, (long)TenMib]; + yield return [path, 0L]; + } + } + + public static IEnumerable BlankCredentialCases() + { + yield return ["", "secret"]; + yield return ["feed", ""]; + yield return ["", ""]; + } + + [Theory] + [MemberData(nameof(SizeCases))] + public async Task MaxArticleBytes_PeerPolicy_DoesNotExposeReceiveCommandsOnReader(ReceivePath path, long peerMax) + { + var policy = Peer(maxSize: peerMax, sendPatterns: "!alt.test"); + await using var harness = await Harness.OpenAsync(policy, path); + Assert.Equal(peerMax, harness.Session.Authorization.TransitPeerPolicy!.MaxSize); + + await harness.AssertFeedCommandsUnavailableAsync(""); + } + + [Theory] + [MemberData(nameof(ArticlePaths))] + public async Task ReceivePatterns_PeerPolicy_DoNotExposeReceiveCommandsOnReader(ReceivePath path) + { + foreach (var patterns in new[] + { + "*", + "*,!unidata.*", + "news.*,!news.misc,*.misc", + "alt.*,@alt.binaries.warez,misc.*", + }) + { + await using var harness = await Harness.OpenAsync(Peer(receivePatterns: patterns, sendPatterns: "*"), path); + await harness.AssertFeedCommandsUnavailableAsync(""); + } + } + + [Theory] + [MemberData(nameof(ArticlePaths))] + public async Task ReceiveArticleTypes_PeerPolicy_DoNotExposeReceiveCommandsOnReader(ReceivePath path) + { + foreach (var receiveTypes in new[] { TransitMessageTypes.All, TransitMessageTypes.Default }) + { + await using var harness = await Harness.OpenAsync( + Peer(receiveTypes: receiveTypes, sendTypes: TransitMessageTypes.All), + path); + await harness.AssertFeedCommandsUnavailableAsync(""); + } + } + + [Fact] + public async Task Credentials_BothSet_ReaderAnswers502_NotAuthenticationRequired() + { + await using var harness = await Harness.OpenAsync(Peer(username: "feed", password: "secret"), ReceivePath.PipelinedTakeThis); + Assert.True(harness.Session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + Assert.False(harness.Session.Authorization.AuthorizedTransit); + + await harness.AssertFeedCommandsUnavailableAsync(""); + + await harness.Duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await harness.Duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.Duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("481 Authentication failed", await harness.Duplex.ReadClientLineAsync()); + Assert.False(harness.Session.Authentication.IsAuthenticated); + + await harness.AssertFeedCommandsUnavailableAsync(""); + } + + [Theory] + [MemberData(nameof(BlankCredentialCases))] + public async Task Credentials_EitherBlank_ReaderStillDoesNotExposeFeedCommands(string username, string password) + { + await using var harness = await Harness.OpenAsync(Peer(username: username, password: password), ReceivePath.PipelinedTakeThis); + Assert.False(harness.Session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + + await harness.AssertFeedCommandsUnavailableAsync(""); + } + + private static TransitPeerPolicy Peer( + long maxSize = TenMib, + string receivePatterns = "*", + string sendPatterns = "*", + TransitMessageTypes receiveTypes = TransitMessageTypes.All, + TransitMessageTypes sendTypes = TransitMessageTypes.All, + string username = "", + string password = "") + { + Assert.True(NewsfeedsPattern.TryParse(receivePatterns, out var receive, out var receiveError), receiveError); + Assert.True(NewsfeedsPattern.TryParse(sendPatterns, out var send, out var sendError), sendError); + return new TransitPeerPolicy( + "peer", + "Receive Peer", + maxIncomingConnections: 8, + maxOutgoingConnections: 0, + literalPrefixes: [IpPrefix.Host(PeerAddress)], + dnsHostnames: [], + connectTo: [], + username, + password, + TransitSslMode.None, + send!, + deferOnDuplicate: true, + pathToken: string.Empty, + maxSize, + sendTypes, + receive!, + receiveTypes, + sendMaxArticleBytes: 100, + sendUsername: "send-user", + sendPassword: "send-secret", + pathExclusions: ["send-only"]); + } + + private static TransitConfigurationSnapshot Snapshot(TransitPeerPolicy policy, long publicationId = 1) => + new( + new Dictionary(StringComparer.Ordinal) { [policy.Identifier] = policy }, + publicationId: publicationId); + + public enum ReceivePath + { + SerialTakeThis, + PipelinedTakeThis, + IHave, + } + + private sealed class FixedSharedPolicy : INntpArticlePolicySource + { + public FixedSharedPolicy(int maxArticleBytes) => MaxArticleBytes = maxArticleBytes; + + public int MaxArticleBytes { get; } + + public bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName) + { + maxArticleBytes = MaxArticleBytes; + siteName = "news.example"; + return true; + } + } + + private sealed class Harness : IAsyncDisposable + { + private readonly Task _run; + + private Harness(FeedDuplex duplex, NntpSession session, ArticleIngestionQueue queue, Task run) + { + Duplex = duplex; + Session = session; + Queue = queue; + _run = run; + } + + public FeedDuplex Duplex { get; } + + public NntpSession Session { get; } + + public ArticleIngestionQueue Queue { get; } + + public static async Task OpenAsync(TransitPeerPolicy policy, ReceivePath path) + { + var store = new TransitConfigurationStore(); + store.Replace(Snapshot(policy)); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 8, + MaxArticleBytes = 64 * 1024 * 1024, + }); + var duplex = await FeedDuplex.CreateAsync(); + var session = duplex.CreateSession(TransitPeerAuthorization.CreateForStore(store), queue); + session.SharedConfiguration = new FixedSharedPolicy(GlobalMax); + if (path == ReceivePath.SerialTakeThis) + { + session.SetMode(NntpSessionMode.Reader); + } + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + return new Harness(duplex, session, queue, run); + } + + /// + /// Sends the Transit-only receive commands. Each must be answered 502 by the role gate; no article + /// body is sent because the Reader never reads one, and nothing may reach the ingestion queue. + /// + /// + /// A Reader-only host registers , so + /// AuthorizedTransit is never true there. These tests inject a named peer to prove the peer policy + /// does not expose the commands. When the injected peer is already transit-authorized, CHECK and + /// stream-plane TAKETHIS are admitted by the session windows ahead of the dispatcher (an unreachable state + /// on a real Reader host), so only the dispatcher-routed commands are exercised; the all-commands case is + /// covered with the production Reader authorization by TransitReceiveSizeAccountingTests and + /// TransitDeferOnDuplicateCommandTests. + /// + public async Task AssertFeedCommandsUnavailableAsync(string id) + { + var commands = Session.Authorization.AuthorizedTransit + ? new[] { "MODE STREAM", "IHAVE " + id } + : new[] { "MODE STREAM", "CHECK " + id, "IHAVE " + id, "TAKETHIS " + id }; + foreach (var command in commands) + { + await Duplex.WriteClientLineAsync(command); + Assert.Equal(CommandNotAvailable, await Duplex.ReadClientLineAsync()); + } + + Assert.Equal(0, Queue.Count); + } + + public async ValueTask DisposeAsync() + { + await Duplex.WriteClientLineAsync("QUIT"); + _ = await Duplex.ReadClientLineAsync(); + await _run; + await Duplex.DisposeAsync(); + } + } + private sealed class FeedDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + + public static Task CreateAsync() => Task.FromResult(new FeedDuplex()); + + public NntpSession CreateSession(ITransitPeerAuthorization peers, IArticleIngestionQueue queue) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(PeerAddress, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(line + "\r\n")); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceiveSizeAccountingTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceiveSizeAccountingTests.cs new file mode 100644 index 00000000..b285522c --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitReceiveSizeAccountingTests.cs @@ -0,0 +1,372 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.ReaderService.ArticleIngestion; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.History; +using VectorNNTPD.ReaderService.Networking.Proxy; +using VectorNNTPD.ReaderService.Networking.Transport; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Session.CommandProcessor; +using VectorNNTPD.ReaderService.Tests.Fixtures; +using VectorNNTPD.ReaderService.Tests.TestDoubles; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +/// +/// Role isolation for the receive size scenarios: the Reader service answers MODE STREAM, IHAVE and TAKETHIS with +/// 502 Command not available on this server whatever the article size, so no article is read, nothing is +/// queued, and no history entry is written. The Transit service tests prove the size accounting itself. +/// +/// +/// Sessions use , which is what a Reader-only host registers. +/// Each scenario still builds its article and asserts the size precondition that defines it, so the scenario stays +/// pinned to the same wire/destuffed/canonical boundaries as the Transit tests. +/// +public sealed class TransitReceiveSizeAccountingTests +{ + private const int Limit = 1_000_000; + private const int CanonicalizationMaxArticleBytes = 5 * 1024 * 1024; + private const string CommandNotAvailable = "502 Command not available on this server"; + + private static readonly IPAddress Peer = IPAddress.Parse("198.18.0.82"); + + [Fact] + public async Task StuffedOverTheWire_WithinDestuffedLimit_ReceiveCommandsAreUnavailableOnReader() + { + var id = ""; + var stuffed = StuffedWithinDestuffedLimit(id, out var destuffedLength, out var canonicalSize); + Assert.True(destuffedLength <= Limit); + Assert.True(stuffed.Length > destuffedLength); + Assert.True(canonicalSize <= Limit); + + await AssertUnavailableOnEveryReaderPathAsync(id); + } + + [Fact] + public async Task DestuffedOverLimit_ReceiveCommandsAreUnavailableOnReader_AndStayFramed() + { + var id = ""; + var article = PlainOverLimit(id); + Assert.True(article.Length > Limit); + + await AssertUnavailableOnEveryReaderPathAsync(id); + } + + [Fact] + public async Task StuffedOverDestuffedLimit_ReceiveCommandsAreUnavailableOnReader() + { + var id = ""; + var stuffed = StuffedOverDestuffedLimit(id, out var destuffedLength); + Assert.True(destuffedLength > Limit); + Assert.True(stuffed.Length > destuffedLength); + + await AssertUnavailableOnEveryReaderPathAsync(id); + } + + [Fact] + public async Task DestuffedWithinLimit_CanonicalOverLimit_ReceiveCommandsAreUnavailableOnReader() + { + var id = ""; + _ = PlainWithinDestuffedButCanonicalOver(id, out var destuffedLength, out var canonicalSize); + Assert.True(destuffedLength <= Limit); + Assert.True(canonicalSize > Limit); + + await AssertUnavailableOnEveryReaderPathAsync(id); + } + + private static async Task AssertUnavailableOnEveryReaderPathAsync(string id) + { + foreach (var afterModeReader in new[] { false, true }) + { + await using var harness = await CommandHarness.StartAsync(); + var writes = harness.History.Writes.Count; + await harness.AssertReceiveCommandsUnavailableAsync(id, afterModeReader); + + Assert.Equal(0, harness.Queue.Count); + Assert.Equal(writes, harness.History.Writes.Count); + Assert.False(harness.History.ContainsLocal(HistoryDigest.FromMessageId(Encoding.ASCII.GetBytes(id)))); + } + } + private static byte[] StuffedWithinDestuffedLimit(string id, out int destuffedLength, out int canonicalSize) + { + var growth = CanonicalGrowth(id); + var headerLength = Encoding.ASCII.GetByteCount(CanonicalArticleText.Destuffed(id, string.Empty)); + var lines = (Limit - growth - headerLength) / 4; + Assert.True(lines > 0); + var body = new StringBuilder(lines * 3); + for (var i = 0; i < lines; i++) + { + body.Append(".\r\n"); + } + + var destuffed = CanonicalArticleText.Destuffed(id, body.ToString()); + destuffedLength = Encoding.ASCII.GetByteCount(destuffed); + canonicalSize = CanonicalLength(destuffed); + var stuffed = Encoding.ASCII.GetBytes(destuffed.Replace("\r\n.", "\r\n..", StringComparison.Ordinal)); + + // CanonicalV1 ArtData is stuffed, so canonical size >= stuffed wire size. A wire larger than Limit therefore + // cannot canonicalize within Limit; the article is sized so the stuffed canonical form fits (4 bytes per + // ".\r\n" line) while stuffing still adds one byte per line over the destuffed form. + Assert.True(stuffed.Length > destuffedLength); + Assert.True(stuffed.Length <= canonicalSize); + return stuffed; + } + + private static byte[] StuffedOverDestuffedLimit(string id, out int destuffedLength) + { + var headerLength = Encoding.ASCII.GetByteCount(CanonicalArticleText.Destuffed(id, string.Empty)); + var lines = ((Limit - headerLength) / 3) + 1; + var body = new StringBuilder(lines * 3); + for (var i = 0; i < lines; i++) + { + body.Append(".\r\n"); + } + + var destuffed = CanonicalArticleText.Destuffed(id, body.ToString()); + destuffedLength = Encoding.ASCII.GetByteCount(destuffed); + return Encoding.ASCII.GetBytes(destuffed.Replace("\r\n.", "\r\n..", StringComparison.Ordinal)); + } + + private static byte[] PlainOverLimit(string id) => + Encoding.ASCII.GetBytes(Sized(id, Limit + 2)); + + private static byte[] PlainWithinDestuffedButCanonicalOver( + string id, + out int destuffedLength, + out int canonicalSize) + { + var growth = CanonicalGrowth(id); + Assert.True(growth > 0); + var article = Sized(id, Limit - growth + 1); + destuffedLength = article.Length; + canonicalSize = CanonicalLength(article); + Assert.True(destuffedLength <= Limit); + Assert.True(canonicalSize > Limit); + return Encoding.ASCII.GetBytes(article); + } + + private static string Sized(string id, int destuffedLength) + { + var skeleton = CanonicalArticleText.Destuffed(id, string.Empty); + var bodyLength = destuffedLength - skeleton.Length; + Assert.True(bodyLength >= 2); + var body = new StringBuilder(bodyLength); + var left = bodyLength; + while (left > 0) + { + var content = Math.Min(60, left - 2); + if (left - (content + 2) == 1) + { + content--; + } + + Assert.InRange(content, 0, 60); + body.Append('x', content); + body.Append("\r\n"); + left -= content + 2; + } + + var article = CanonicalArticleText.Destuffed(id, body.ToString()); + Assert.Equal(destuffedLength, article.Length); + return article; + } + + private static int CanonicalGrowth(string id) + { + var probe = CanonicalArticleText.Destuffed(id, "probe\r\n"); + return CanonicalLength(probe) - Encoding.ASCII.GetByteCount(probe); + } + + private static int CanonicalLength(string destuffed) + { + var parser = new NntpArticleParser(NntpdOptions.FormatFqdn(1, "usenet.ninja")); + var bytes = Encoding.ASCII.GetBytes(destuffed); + var created = ArticleRecordIngress.TryCreateFromDestuffed( + parser, + bytes, + maxArticleBytes: CanonicalizationMaxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + Assert.True(created.IsAccepted, created.ParseFailure + " / " + created.MaterializeFailure); + return created.Record.ArtSize; + } + + private sealed class CommandHarness : IAsyncDisposable + { + private readonly Pipe _clientToServer; + private readonly Pipe _serverToClient; + private readonly NntpSession _session; + private readonly Task _run; + + private CommandHarness( + Pipe clientToServer, + Pipe serverToClient, + NntpSession session, + Task run, + HistoryDb history, + ArticleIngestionQueue queue) + { + _clientToServer = clientToServer; + _serverToClient = serverToClient; + _session = session; + _run = run; + History = history; + Queue = queue; + } + + public HistoryDb History { get; } + + public ArticleIngestionQueue Queue { get; } + + public static async Task StartAsync() + { + var clientToServer = new Pipe(new PipeOptions( + pauseWriterThreshold: 8 * 1024 * 1024, + resumeWriterThreshold: 4 * 1024 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false)); + var serverToClient = new Pipe(NntpPipeOptions.Create()); + var history = new HistoryDb( + new FakeRedisService(), + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = Limit, + }); + var connection = new PipeNntpConnection( + clientToServer.Reader, + serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(Peer, 40000))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: TransitPeerAuthorization.Disabled, + historyDb: history); + var run = session.RunAsync(); + var harness = new CommandHarness(clientToServer, serverToClient, session, run, history, queue); + Assert.StartsWith("2", await harness.ReadLineAsync(), StringComparison.Ordinal); + return harness; + } + + /// + /// Sends MODE STREAM, IHAVE and TAKETHIS (optionally after MODE READER). Each is answered 502 by the role gate; + /// no article body is written because the Reader never reads one, and DATE proves the stream stays framed. + /// + public async Task AssertReceiveCommandsUnavailableAsync(string id, bool afterModeReader) + { + if (afterModeReader) + { + await WriteCommandAsync("MODE READER\r\n"); + Assert.StartsWith("201 ", await ReadLineAsync(), StringComparison.Ordinal); + } + + foreach (var command in new[] { "MODE STREAM", $"IHAVE {id}", $"TAKETHIS {id}" }) + { + await WriteCommandAsync(command + "\r\n"); + Assert.Equal(CommandNotAvailable, await ReadLineAsync()); + } + + await WriteCommandAsync("DATE\r\n"); + Assert.StartsWith("111 ", await ReadLineAsync(), StringComparison.Ordinal); + } + public async Task WriteCommandAsync(string command) + { + var result = await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(command)); + Assert.False(result.IsCompleted); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadLineAsync() + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, timeout.Token); + Assert.NotNull(line); + return line; + } + + public async ValueTask DisposeAsync() + { + await WriteCommandAsync("QUIT\r\n"); + Assert.Equal("205 Connection closing", await ReadLineAsync()); + await _run; + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + + public PipeWriter Output { get; } = output; + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } = identity; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitTestPeers.cs b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitTestPeers.cs new file mode 100644 index 00000000..0497c9d8 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transit/TransitTestPeers.cs @@ -0,0 +1,102 @@ +using System.Net; +using VectorNNTPD.ReaderService.Configuration; +using VectorNNTPD.ReaderService.Session; +using VectorNNTPD.ReaderService.Transit; + +namespace VectorNNTPD.ReaderService.Tests.Transit; + +internal static class TransitTestPeers +{ + /// Canonical integration-test Transit identifier for the source-IP → AllowFrom path. + public const string DefaultPeerName = "test-peer"; + + /// Canonical integration-test display name (not a protocol argument). + public const string DefaultPeerDisplayName = "Test Peer"; + + public static TransitPeerOptions Peer( + int maxIncoming = 8, + int maxOutgoing = 1, + string[]? allowFrom = null, + string[]? connectTo = null, + string username = "", + string password = "", + string ssl = "", + string patterns = "*", + bool deferOnDuplicate = true, + string pathToken = "", + long maxSize = TransitPeerOptions.DefaultMaxSize, + string[]? messageTypes = null, + string peerName = DefaultPeerDisplayName) => + new() + { + PeerName = peerName, + MaxIncomingConnections = maxIncoming, + MaxOutgoingConnections = maxOutgoing, + AllowFrom = allowFrom ?? [], + ConnectTo = connectTo ?? [], + Username = username, + Password = password, + Ssl = ssl, + Patterns = patterns, + DeferOnDuplicate = deferOnDuplicate, + PathToken = pathToken, + MaxSize = maxSize, + MessageTypes = messageTypes ?? ["default"], + }; + + public static TransitPeersOptions Dictionary(string name, TransitPeerOptions peer) + { + return new TransitPeersOptions { [name] = peer }; + } + + public static TransitConfigurationSnapshot Snapshot(string name, TransitPeerOptions peer) => + TransitConfigurationSnapshot.Create(Dictionary(name, peer)); + + /// + /// Named-peer authorization whose AllowFrom is the supplied source addresses. + /// Sessions identify this peer only when the connection's effective client IP matches. + /// + public static TransitPeerAuthorization ForAllowFrom( + IPAddress source, + string name = DefaultPeerName, + int maxIncoming = 10, + int maxOutgoing = 0, + string username = "", + string password = "", + string patterns = "*", + bool deferOnDuplicate = true) => + ForAllowFrom( + [source], + name, + maxIncoming, + maxOutgoing, + username, + password, + patterns, + deferOnDuplicate); + + /// Named-peer authorization for one or more literal AllowFrom addresses. + public static TransitPeerAuthorization ForAllowFrom( + IReadOnlyList sources, + string name = DefaultPeerName, + int maxIncoming = 10, + int maxOutgoing = 0, + string username = "", + string password = "", + string patterns = "*", + bool deferOnDuplicate = true) + { + ArgumentNullException.ThrowIfNull(sources); + return TransitPeerAuthorization.CreateStatic( + Snapshot( + name, + Peer( + maxIncoming: maxIncoming, + maxOutgoing: maxOutgoing, + allowFrom: [.. sources.Select(static a => a.ToString())], + username: username, + password: password, + patterns: patterns, + deferOnDuplicate: deferOnDuplicate))); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpArticleClientTests.cs b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpArticleClientTests.cs new file mode 100644 index 00000000..f6d13d7a --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpArticleClientTests.cs @@ -0,0 +1,946 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.ReaderService.Transport.Vatp; + +namespace VectorNNTPD.ReaderService.Tests.Transport.Vatp; + +public sealed class VatpArticleClientTests +{ + /// Body larger than the production initial window so WINDOW replenishment is required. + private static int BodyLargerThanInitialWindow => + ArticleTransferLimits.Default.InitialStreamWindowBytes + (64 * 1024); + + [Fact] + public async Task Coalesced_multi_frame_transfer_succeeds() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CoalesceOutboundFrames; + var prepared = CreateArticle("", body: BuildLargeBody(96 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + } + + [Fact] + public async Task Coalesced_240KiB_article_matches_live_failure_scale() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CoalesceOutboundFrames; + var prepared = CreateArticle("", body: BuildLargeBody(240 * 1024)); + Assert.True(prepared.Record.ArtSize > VatpProtocol.DefaultMaxFramePayload); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + Assert.True(result.Record.ArtData.ToArray().AsSpan() + .SequenceEqual(prepared.Record.ArtData.ToArray())); + } + + [Fact] + public async Task Article_larger_than_initial_window_completes_with_window_pacing() + { + // Articles larger than the initial window must complete by refunding each accepted DATA chunk. + // A missing local replenish is a flow-control failure. + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + Assert.True(prepared.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + Assert.True(result.Record.ArtData.ToArray().AsSpan() + .SequenceEqual(prepared.Record.ArtData.ToArray())); + } + + [Fact] + public async Task Eight_concurrent_large_articles_share_connection_with_independent_windows() + { + await using var server = VatpLoopbackTestServer.Start(); + var articles = new PreparedArticle[8]; + for (var i = 0; i < articles.Length; i++) + { + var bodyBytes = i % 2 == 0 ? BodyLargerThanInitialWindow : 64 * 1024; + articles[i] = CreateArticle($"", body: BuildLargeBody(bodyBytes)); + server.Register(articles[i].Record, articles[i].SelectedDateHeaderName); + } + + Assert.Contains( + articles, + a => a.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + var tasks = articles.Select(a => + client.FetchArticleAsync(server.Host, + server.Port, a.RequestId, a.Record.ArtId, CancellationToken.None)) + .ToArray(); + var results = await Task.WhenAll(tasks); + + Assert.All(results, r => Assert.Equal(VatpFetchKind.Success, r.Kind)); + for (var i = 0; i < articles.Length; i++) + { + Assert.Equal(articles[i].Record.ArtId, results[i].Record.ArtId); + Assert.Equal(articles[i].Record.ArtSize, results[i].Record.ArtSize); + Assert.Equal(articles[i].Record.ArtHash, results[i].Record.ArtHash); + } + + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Cancel_one_large_stream_does_not_fail_sibling_large_stream() + { + await using var server = VatpLoopbackTestServer.Start(); + var keep = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + var cancel = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + Assert.True(keep.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + Assert.True(cancel.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register(keep.Record, keep.SelectedDateHeaderName); + server.Register( + cancel.Record, + cancel.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var keepTask = client.FetchArticleAsync( + server.Host, + server.Port, + keep.RequestId, + keep.Record.ArtId, + CancellationToken.None); + var cancelTask = client.FetchArticleAsync( + server.Host, + server.Port, + cancel.RequestId, + cancel.Record.ArtId, + cts.Token); + + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + var keepResult = await keepTask; + var cancelResult = await cancelTask; + Assert.Equal(VatpFetchKind.Success, keepResult.Kind); + Assert.Equal(keep.Record.ArtId, keepResult.Record.ArtId); + Assert.Equal(keep.Record.ArtSize, keepResult.Record.ArtSize); + Assert.True( + cancelResult.Kind is VatpFetchKind.Cancelled + or VatpFetchKind.ConnectionFailure + or VatpFetchKind.IncompleteOrMalformedArticle, + cancelResult.Kind.ToString()); + Assert.NotEqual(VatpFetchKind.Success, cancelResult.Kind); + Assert.Equal(default, cancelResult.Record); + + // Connection must remain usable for a subsequent large transfer. + var retry = CreateArticle("", body: BuildLargeBody(740 * 1024)); + server.Register(retry.Record, retry.SelectedDateHeaderName); + var retryResult = await client.FetchArticleAsync( + server.Host, + server.Port, + retry.RequestId, + retry.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retryResult.Kind); + Assert.Equal(retry.Record.ArtSize, retryResult.Record.ArtSize); + } + + [Fact] + public async Task Tiny_writes_reconstruct_partial_frames_across_reads() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.TinyOutboundWrites; + var prepared = CreateArticle("", body: BuildLargeBody(8 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + } + + [Fact] + public async Task Max_sized_single_data_frame_assembles_across_32KiB_reads() + { + // Live root cause: assembling one DefaultMaxFramePayload DATA across multiple + // 32 KiB socket reads must not trip the old one-frame accumulation guard. + await using var server = VatpLoopbackTestServer.Start(); + var bodyBytes = (int)VatpProtocol.DefaultMaxFramePayload - 256; + var prepared = CreateArticle("", body: BuildLargeBody(bodyBytes)); + Assert.True(prepared.Record.ArtSize > 32 * 1024); + Assert.True(prepared.Record.ArtSize <= VatpProtocol.DefaultMaxFramePayload); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + } + + [Fact] + public async Task Oversized_data_frame_still_rejected() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.OversizedDataFrame; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + } + + [Fact] + public async Task Hello_and_single_transfer_success() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + } + + [Fact] + public void Fin_without_end_does_not_complete_on_receive_stream() + { + var prepared = CreateArticle(""); + var stream = new ArticleTransferReceiveStream(1, prepared.RequestId, prepared.Record.ArtId); + var record = prepared.Record; + var meta = ArticleCanonicalTransferMeta.FromRecord(in record, prepared.SelectedDateHeaderName); + var metaBytes = VatpMetaCodec.Encode(in meta); + Assert.True(stream.TryAcceptMeta(metaBytes).Success); + var artData = prepared.Record.ArtData.ToArray(); + Assert.True(stream.TryAcceptData(artData, fin: true).Success); + Assert.Equal(ArticleTransferPhase.AwaitingEnd, stream.Phase); + Assert.False(stream.HasConsumableRecord); + Assert.False(stream.TryTakeRecord(out _)); + } + + [Fact] + public async Task Wrong_article_id_open_yields_remote_failure() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var wrongId = ArticleId.FromMessageId(""u8); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + wrongId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + } + + [Fact] + public async Task Eight_concurrent_streams_share_one_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var articles = new PreparedArticle[8]; + for (var i = 0; i < articles.Length; i++) + { + articles[i] = CreateArticle($""); + server.Register(articles[i].Record, articles[i].SelectedDateHeaderName); + } + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + var tasks = articles.Select(a => + client.FetchArticleAsync(server.Host, + server.Port, a.RequestId, a.Record.ArtId, CancellationToken.None)) + .ToArray(); + var results = await Task.WhenAll(tasks); + Assert.All(results, r => Assert.Equal(VatpFetchKind.Success, r.Kind)); + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Cancel_one_stream_returns_cancelled() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.FinWithoutEnd; + var prepared = CreateArticle("", body: BuildLargeBody(512 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + cts.Token); + Assert.True( + result.Kind is VatpFetchKind.Cancelled or VatpFetchKind.ConnectionFailure or VatpFetchKind.IncompleteOrMalformedArticle, + result.Kind.ToString()); + } + + [Fact] + public async Task Connection_loss_mid_data_fails_fetch() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + var prepared = CreateArticle("", body: BuildLargeBody(256 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task Pool_removes_dead_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + _ = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + } + + [Fact] + public async Task Tls_wrong_target_host_fails_without_test_callback() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = new VatpConnectionPool(NullLogger.Instance) + { + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + var client = new VatpArticleClient(pool, NullLogger.Instance); + var result = await client.FetchArticleAsync( + "wrong.host.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task Fin_without_end_over_tls_does_not_succeed() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.FinWithoutEnd; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task ConnectionLoss_MidMeta_FailsTransferCleanly() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidMeta; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + Assert.Equal(prepared.Record.ArtId, retry.Record.ArtId); + } + + [Fact] + public async Task ConnectionLoss_AfterFinBeforeEnd_FailsTransferCleanly() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectAfterFinBeforeEnd; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.True( + result.Kind is VatpFetchKind.ConnectionFailure or VatpFetchKind.IncompleteOrMalformedArticle, + result.Kind.ToString()); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + Assert.Equal(prepared.Record.ArtId, retry.Record.ArtId); + } + + [Fact] + public async Task Late_and_never_allocated_stream_frames_do_not_poison_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + server.EmitSpuriousFramesAfterTerminal = true; + var first = CreateArticle(""); + var second = CreateArticle(""); + server.Register(first.Record, first.SelectedDateHeaderName); + server.Register(second.Record, second.SelectedDateHeaderName); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var r1 = await client.FetchArticleAsync( + server.Host, + server.Port, + first.RequestId, + first.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, r1.Kind); + Assert.Equal(first.Record.ArtId, r1.Record.ArtId); + + // Spurious late END/FAIL for stream 1 and META/DATA/END for 0x7FFFFFFE must not + // inject state or kill the pooled connection. + var r2 = await client.FetchArticleAsync( + server.Host, + server.Port, + second.RequestId, + second.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, r2.Kind); + Assert.Equal(second.Record.ArtId, r2.Record.ArtId); + Assert.Equal(second.Record.ArtSize, r2.Record.ArtSize); + Assert.Equal(1, pool.AliveConnectionCount(server.Host, server.Port)); + + var openIds = server.ObservedOpenStreamIds.OrderBy(static id => id).ToArray(); + Assert.Equal(new uint[] { 1, 2 }, openIds); + Assert.DoesNotContain(0x7FFFFFFEu, openIds); + } + + [Fact] + public async Task Stream_ids_are_monotonic_and_not_reused_on_same_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + for (var i = 0; i < 4; i++) + { + var prepared = CreateArticle($""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, result.Kind); + } + + var ids = server.ObservedOpenStreamIds.OrderBy(static id => id).ToArray(); + Assert.Equal(new uint[] { 1, 2, 3, 4 }, ids); + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Active_stream_id_is_not_reused_while_sibling_is_held() + { + await using var server = VatpLoopbackTestServer.Start(); + var held = CreateArticle("", body: BuildLargeBody(64 * 1024)); + var other = CreateArticle(""); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register( + held.Record, + held.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + server.Register(other.Record, other.SelectedDateHeaderName); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var heldTask = client.FetchArticleAsync( + server.Host, + server.Port, + held.RequestId, + held.Record.ArtId, + cts.Token); + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var otherResult = await client.FetchArticleAsync( + server.Host, + server.Port, + other.RequestId, + other.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, otherResult.Kind); + + await cts.CancelAsync(); + var heldResult = await heldTask; + Assert.NotEqual(VatpFetchKind.Success, heldResult.Kind); + + var ids = server.ObservedOpenStreamIds.ToArray(); + Assert.Equal(2, ids.Length); + Assert.Contains(1u, ids); + Assert.Contains(2u, ids); + Assert.Equal(2, ids.Distinct().Count()); + } + + [Fact] + public async Task Cancelled_stream_spurious_frames_do_not_fail_sibling() + { + await using var server = VatpLoopbackTestServer.Start(); + server.EmitSpuriousFramesAfterTerminal = true; + var keep = CreateArticle("", body: BuildLargeBody(128 * 1024)); + var cancel = CreateArticle("", body: BuildLargeBody(128 * 1024)); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register(keep.Record, keep.SelectedDateHeaderName); + server.Register( + cancel.Record, + cancel.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var keepTask = client.FetchArticleAsync( + server.Host, + server.Port, + keep.RequestId, + keep.Record.ArtId, + CancellationToken.None); + var cancelTask = client.FetchArticleAsync( + server.Host, + server.Port, + cancel.RequestId, + cancel.Record.ArtId, + cts.Token); + + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + var keepResult = await keepTask; + var cancelResult = await cancelTask; + Assert.Equal(VatpFetchKind.Success, keepResult.Kind); + Assert.Equal(keep.Record.ArtId, keepResult.Record.ArtId); + Assert.NotEqual(VatpFetchKind.Success, cancelResult.Kind); + Assert.Equal(1, pool.AliveConnectionCount(server.Host, server.Port)); + } + + [Fact] + public async Task Meta_on_connection_stream_id_marks_connection_dead() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.MetaOnConnectionStreamId; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + } + + [Fact] + public async Task AcceptedDataBytes_ProtocolFailureBeforeStream_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + var client = CreateClient(server); + + var emptyRequest = await client.FetchArticleAsync( + server.Host, + server.Port, + Guid.Empty, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ProtocolFailure, emptyRequest.Kind); + Assert.Equal(0, emptyRequest.AcceptedDataBytes); + + var badEndpoint = await client.FetchArticleAsync( + "vatp" + "://" + "backfiller.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ProtocolFailure, badEndpoint.Kind); + Assert.Equal(0, badEndpoint.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureBeforeStream_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + var pool = new VatpConnectionPool(NullLogger.Instance) + { + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + var client = new VatpArticleClient(pool, NullLogger.Instance); + var result = await client.FetchArticleAsync( + "wrong.host.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureBeforeData_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidMeta; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureAfterData_IsPositive() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + var prepared = CreateArticle("", body: BuildLargeBody(256 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + Assert.True(prepared.Record.ArtData.Length > 64 * 1024); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(64 * 1024, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_RemoteFailureBeforeData_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var wrongId = ArticleId.FromMessageId(""u8); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + wrongId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_RemoteFailureAfterData_IsProbeLength() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DataThenRemoteFail; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_MetaRejected_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.InvalidMeta; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ShortFin_IsCopiedLength() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.ShortDataFin; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtSizeMismatch, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + Assert.NotEqual(prepared.Record.ArtSize, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_LaterDataRejectedBeforeCopy_KeepsFirstChunk() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DataThenSizeReject; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtSizeMismatch, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_HashMismatchAfterFullBody_IsArtSize() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CorruptArtHash; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtHashMismatch, result.ErrorCode); + Assert.Equal(prepared.Record.ArtSize, result.AcceptedDataBytes); + Assert.True(result.AcceptedDataBytes > VatpLoopbackTestServer.ProbePayloadBytes); + } + + [Fact] + public async Task AcceptedDataBytes_Success_IsArtSize() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.AcceptedDataBytes); + Assert.Equal(prepared.Record.ArtData.Length, result.AcceptedDataBytes); + } + + private static async Task FetchAsync(VatpLoopbackTestServer server, PreparedArticle prepared) + { + var client = CreateClient(server); + return await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + } + + private static PreparedArticle CreateProbeArticle(string messageId) + { + var prepared = CreateArticle(messageId); + Assert.True(prepared.Record.ArtData.Length > VatpLoopbackTestServer.ProbePayloadBytes); + Assert.Equal(prepared.Record.ArtData.Length, prepared.Record.ArtSize); + return prepared; + } + + private static VatpArticleClient CreateClient(VatpLoopbackTestServer server) => + new(CreatePool(server), NullLogger.Instance); + + private static VatpConnectionPool CreatePool(VatpLoopbackTestServer server) + { + var pool = new VatpConnectionPool( + NullLogger.Instance, + new VatpClientOptions + { + IoTimeout = TimeSpan.FromSeconds(2), + ConnectTimeout = TimeSpan.FromSeconds(5), + TlsHandshakeTimeout = TimeSpan.FromSeconds(5), + }) + { + ServerCertificateValidationCallback = static (_, _, _, _) => true, + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + return pool; + } + + private static PreparedArticle CreateArticle(string messageId, string body = "body\r\n") + { + var parser = new NntpArticleParser("backfiller.test"); + var destuffed = BuildDestuffed(messageId, body); + var created = ArticleRecordFactory.TryCreate(parser, destuffed); + Assert.True(created.IsAccepted, created.ParseFailure.ToString()); + return new PreparedArticle(Guid.NewGuid(), created.Record, created.SelectedDateHeaderName); + } + + private const string BodyLine = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"; + + private static string BuildLargeBody(int minimumBytes) + { + var builder = new System.Text.StringBuilder(minimumBytes + BodyLine.Length); + while (builder.Length < minimumBytes) + { + builder.Append(BodyLine); + } + + return builder.ToString(); + } + + private static byte[] BuildDestuffed(string messageId, string body) => + System.Text.Encoding.ASCII.GetBytes( + "Path: peer.example\r\n" + + "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" + + "Message-ID: " + messageId + "\r\n" + + "Newsgroups: alt.test\r\n" + + "From: user@example.test\r\n" + + "Subject: s\r\n" + + "\r\n" + + body); + + private static async Task WaitUntilAsync(Func predicate, TimeSpan timeout) + { + var deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; + while (Environment.TickCount64 < deadline) + { + if (predicate()) + { + return; + } + + await Task.Delay(20); + } + + Assert.Fail("Timed out waiting for condition."); + } + + private readonly record struct PreparedArticle( + Guid RequestId, + ArticleRecord Record, + NntpArticleHeaderName SelectedDateHeaderName); +} + +internal static class VatpConnectionPoolTestExtensions +{ + public static int ConnectionCountForTest(this VatpConnectionPool pool, string host, int port) => + pool.GetEndpointConnectionCount(host, port); + + public static int AliveConnectionCount(this VatpConnectionPool pool, string host, int port) => + pool.GetEndpointAliveConnectionCount(host, port); +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs new file mode 100644 index 00000000..0c8a9911 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs @@ -0,0 +1,661 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.ReaderService.Tests.Transport.Vatp; + +internal sealed class VatpLoopbackTestServer : IAsyncDisposable +{ + private readonly TcpListener _listener; + private readonly X509Certificate2 _certificate; + private readonly ConcurrentDictionary _articles = new(); + private readonly CancellationTokenSource _cts = new(); + private int _activeSessions; + private Task? _acceptLoop; + + private VatpLoopbackTestServer(TcpListener listener, X509Certificate2 certificate, int port) + { + _listener = listener; + _certificate = certificate; + Port = port; + Host = "backfiller.test"; + } + + public int Port { get; } + + public string Host { get; } + + public int ActiveSessions => Volatile.Read(ref _activeSessions); + + /// DATA prefix used by accepted-byte probe transfer modes. + internal const int ProbePayloadBytes = 64; + + internal TransferMode Mode { get; set; } = TransferMode.Complete; + + /// OPEN StreamIds observed on this listener (client-assigned). + public ConcurrentBag ObservedOpenStreamIds { get; } = new(); + + /// + /// When set, after a stream terminalizes (END or CANCEL) the server emits late/spurious + /// frames for the finished StreamId and for a never-allocated StreamId. + /// + public bool EmitSpuriousFramesAfterTerminal { get; set; } + + public static VatpLoopbackTestServer Start(X509Certificate2? certificate = null) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + certificate ??= VatpTestCertificates.CreateServerCertificate(); + var server = new VatpLoopbackTestServer(listener, certificate, port); + server._acceptLoop = server.AcceptLoopAsync(server._cts.Token); + return server; + } + + public void Register( + ArticleRecord record, + NntpArticleHeaderName selectedDateHeaderName, + TransferMode? modeOverride = null, + TaskCompletionSource? heldStarted = null) + { + _articles[record.ArtId] = new RegisteredArticle(record, selectedDateHeaderName, modeOverride, heldStarted); + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync().ConfigureAwait(false); + _listener.Stop(); + if (_acceptLoop is not null) + { + try + { + await _acceptLoop.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + _cts.Dispose(); + _certificate.Dispose(); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TcpClient tcp; + try + { + tcp = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: accept loop canceled during server stop + break; + } + + _ = RunSessionAsync(tcp, cancellationToken); + } + } + + private async Task RunSessionAsync(TcpClient tcp, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _activeSessions); + try + { + using var tcpDisposable = tcp; + // codeql[cs/local-not-disposed] SslStream ownership transfers to the VATP/TLS session fixture for connection lifetime. + var ssl = new SslStream(tcp.GetStream(), leaveInnerStreamOpen: false); + await ssl.AuthenticateAsServerAsync( + new SslServerAuthenticationOptions + { + ServerCertificate = _certificate, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }, + cancellationToken).ConfigureAwait(false); + var session = new Session( + ssl, + _articles, + () => Mode, + ObservedOpenStreamIds, + () => EmitSpuriousFramesAfterTerminal, + cancellationToken); + await session.RunAsync().ConfigureAwait(false); + await ssl.DisposeAsync().ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + Interlocked.Decrement(ref _activeSessions); + } + } + + internal enum TransferMode + { + Complete, + FinWithoutEnd, + DisconnectMidData, + DisconnectMidMeta, + DisconnectAfterFinBeforeEnd, + + /// + /// Writes META + all DATA frames + END as one contiguous socket write so the + /// client must parse multiple VATP frames from coalesced TCP/TLS reads. + /// + CoalesceOutboundFrames, + + /// + /// Writes each outbound transfer byte in 1-byte socket writes to force + /// partial-frame reconstruction across many reads. + /// + TinyOutboundWrites, + + /// + /// After META, sends a DATA frame whose declared payload exceeds the negotiated + /// maxFramePayload (must be rejected by the client). + /// + OversizedDataFrame, + + /// + /// Sends META + one DATA chunk, then holds the stream open until CANCEL/disconnect + /// so concurrent-cancellation tests can cancel mid-transfer deterministically. + /// + HoldAfterFirstData, + + /// + /// After accepting OPEN, writes a META frame on StreamId 0 (parser must reject). + /// + MetaOnConnectionStreamId, + + /// Sends a META payload the client must reject before any DATA. + InvalidMeta, + + /// Sends a short DATA frame with FIN so the client copies it and then fails. + ShortDataFin, + + /// Sends one DATA chunk, then a later DATA frame the client rejects before copying. + DataThenSizeReject, + + /// Sends one DATA chunk, then a stream FAIL. + DataThenRemoteFail, + + /// Sends a full body whose META ArtHash does not match, then END. + CorruptArtHash, + } + + private readonly record struct RegisteredArticle( + ArticleRecord Record, + NntpArticleHeaderName SelectedDateHeaderName, + TransferMode? ModeOverride, + TaskCompletionSource? HeldStarted); + + private sealed class Session( + SslStream stream, + ConcurrentDictionary articles, + Func mode, + ConcurrentBag observedOpenStreamIds, + Func emitSpuriousAfterTerminal, + CancellationToken cancellationToken) + { + private readonly Dictionary _streams = new(); + private readonly ArticleTransferLimits _limits = ArticleTransferLimits.Default; + private uint _maxFramePayload = VatpProtocol.DefaultMaxFramePayload; + private bool _clientHelloComplete; + + // Never allocated by monotonic NNTPD client ids starting at 1. + private const uint NeverAllocatedStreamId = 0x7FFFFFFEu; + + public async Task RunAsync() + { + var buffer = ArrayPool.Shared.Rent(VatpProtocol.HeaderLengthBytes + (int)VatpProtocol.DefaultMaxFramePayload); + var buffered = 0; + try + { + while (!cancellationToken.IsCancellationRequested) + { + var read = await stream.ReadAsync(buffer.AsMemory(buffered, buffer.Length - buffered), cancellationToken) + .ConfigureAwait(false); + if (read == 0) + { + return; + } + + buffered += read; + var consumed = 0; + while (consumed < buffered) + { + var sequence = new ReadOnlySequence(buffer, consumed, buffered - consumed); + var parsed = VatpFrameParser.ParseOneFrame(in sequence, _maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + return; + } + + consumed += (int)parsed.ConsumedBytes; + if (!await HandleFrameAsync(parsed.Frame!.Value).ConfigureAwait(false)) + { + return; + } + } + + if (consumed > 0) + { + buffer.AsSpan(consumed, buffered - consumed).CopyTo(buffer); + buffered -= consumed; + } + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private async Task HandleFrameAsync(VatpParsedFrame frame) + { + switch (frame.Header.Type) + { + case VatpFrameType.Hello: + return await HandleHelloAsync(frame).ConfigureAwait(false); + case VatpFrameType.Open: + return await HandleOpenAsync(frame).ConfigureAwait(false); + case VatpFrameType.Window: + await HandleWindowAsync(frame).ConfigureAwait(false); + return true; + case VatpFrameType.Cancel: + if (emitSpuriousAfterTerminal()) + { + await WriteSpuriousAfterTerminalAsync(frame.Header.StreamId).ConfigureAwait(false); + } + + _streams.Remove(frame.Header.StreamId); + return true; + default: + return true; + } + } + + private async Task HandleHelloAsync(VatpParsedFrame frame) + { + if (_clientHelloComplete + || !VatpHello.TryDecode(frame.Payload, out var hello, out _)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(VatpProtocol.ConnectionStreamId, VatpErrorCode.InvalidHello)) + .ConfigureAwait(false); + return false; + } + + _maxFramePayload = Math.Min(hello.MaxFramePayload, _limits.DefaultMaxFramePayload); + _clientHelloComplete = true; + await WriteFrameAsync(VatpFrameEncoder.EncodeHello(_maxFramePayload)).ConfigureAwait(false); + return true; + } + + private async Task HandleOpenAsync(VatpParsedFrame frame) + { + if (!VatpOpenPayload.TryDecode(frame.Payload, out var open, out _)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.InvalidFrameLength)) + .ConfigureAwait(false); + return true; + } + + if (!articles.TryGetValue(open.ArticleId, out var registered)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + return true; + } + + var record = registered.Record; + var meta = ArticleCanonicalTransferMeta.FromRecord(in record, registered.SelectedDateHeaderName); + var metaBytes = VatpMetaCodec.Encode(in meta); + var transferMode = registered.ModeOverride ?? mode(); + observedOpenStreamIds.Add(frame.Header.StreamId); + + if (transferMode == TransferMode.MetaOnConnectionStreamId) + { + var header = new byte[VatpProtocol.HeaderLengthBytes]; + VatpFrameHeader.Create( + VatpFrameType.Meta, + VatpProtocol.ConnectionStreamId, + (uint)metaBytes.Length, + flags: 0) + .WriteTo(header); + await stream.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(metaBytes, cancellationToken).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DisconnectMidMeta) + { + var metaFrame = VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes); + // Write only the 16-byte header so META never becomes usable. + await stream.WriteAsync(metaFrame.Header, cancellationToken).ConfigureAwait(false); + await stream.DisposeAsync().ConfigureAwait(false); + return false; + } + + if (transferMode == TransferMode.InvalidMeta) + { + var invalid = new byte[VatpProtocol.MetaPayloadLength]; + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, invalid)).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.CorruptArtHash) + { + var corrupt = new ArticleCanonicalTransferMeta( + meta.ArtHash ^ 1UL, + meta.ArtLines, + meta.ArtSize, + meta.SelectedDateHeaderName, + meta.Fields); + metaBytes = VatpMetaCodec.Encode(in corrupt); + } + + if (transferMode == TransferMode.ShortDataFin) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, record.ArtData.ToArray(), fin: true).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DataThenRemoteFail) + { + var artData = record.ArtData.ToArray(); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, artData, fin: false).ConfigureAwait(false); + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DataThenSizeReject) + { + var artData = record.ArtData.ToArray(); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, artData, fin: false).ConfigureAwait(false); + var overflow = new byte[artData.Length - VatpLoopbackTestServer.ProbePayloadBytes + 1]; + await WriteFrameAsync(VatpFrameEncoder.EncodeData(frame.Header.StreamId, overflow, fin: false)) + .ConfigureAwait(false); + return true; + } + + var send = new SendStream( + frame.Header.StreamId, + record.ArtData.ToArray(), + new ArticleTransferWindow(_limits.InitialStreamWindowBytes, _limits.MaxStreamCreditBytes), + transferMode); + _streams[frame.Header.StreamId] = send; + + if (transferMode == TransferMode.OversizedDataFrame) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)) + .ConfigureAwait(false); + await WriteOversizedDataFrameAsync(frame.Header.StreamId).ConfigureAwait(false); + return true; + } + + if (transferMode is TransferMode.CoalesceOutboundFrames or TransferMode.TinyOutboundWrites) + { + await WriteTransferBatchAsync(send, metaBytes).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.HoldAfterFirstData) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)) + .ConfigureAwait(false); + var firstChunk = (int)Math.Min( + Math.Min(send.SendWindow.Credit, _maxFramePayload), + send.ArtData.Length); + if (firstChunk <= 0 || !send.SendWindow.TryConsume(firstChunk)) + { + return true; + } + + send.SentBytes = firstChunk; + await WriteFrameAsync( + VatpFrameEncoder.EncodeData( + frame.Header.StreamId, + send.ArtData.AsMemory(0, firstChunk), + fin: false)) + .ConfigureAwait(false); + registered.HeldStarted?.TrySetResult(); + return true; + } + + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await PumpStreamAsync(send).ConfigureAwait(false); + return true; + } + + private async Task WriteTransferBatchAsync(SendStream sendStream, ReadOnlyMemory metaBytes) + { + var frames = new List(); + frames.Add(VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeMeta(sendStream.StreamId, metaBytes))); + + var offset = 0; + while (offset < sendStream.ArtData.Length) + { + var remaining = sendStream.ArtData.Length - offset; + var chunkSize = (int)Math.Min(_maxFramePayload, remaining); + var fin = offset + chunkSize >= sendStream.ArtData.Length; + var chunk = sendStream.ArtData.AsMemory(offset, chunkSize); + frames.Add(VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeData(sendStream.StreamId, chunk, fin))); + offset += chunkSize; + } + + frames.Add(VatpFrameEncoder.ToSingleBuffer(VatpFrameEncoder.EncodeEnd(sendStream.StreamId))); + + var total = 0; + foreach (var frame in frames) + { + total += frame.Length; + } + + var batch = new byte[total]; + var written = 0; + foreach (var frame in frames) + { + frame.AsSpan().CopyTo(batch.AsSpan(written)); + written += frame.Length; + } + + if (sendStream.TransferMode == TransferMode.TinyOutboundWrites) + { + for (var i = 0; i < batch.Length; i++) + { + await stream.WriteAsync(batch.AsMemory(i, 1), cancellationToken).ConfigureAwait(false); + } + } + else + { + await stream.WriteAsync(batch, cancellationToken).ConfigureAwait(false); + } + + sendStream.SentBytes = sendStream.ArtData.Length; + _streams.Remove(sendStream.StreamId); + } + + private async Task WriteOversizedDataFrameAsync(uint streamId) + { + var oversizedPayload = (int)_maxFramePayload + 1; + var header = new byte[VatpProtocol.HeaderLengthBytes]; + VatpFrameHeader.Create(VatpFrameType.Data, streamId, (uint)oversizedPayload, flags: 0) + .WriteTo(header); + var payload = new byte[oversizedPayload]; + payload.AsSpan().Fill(0x41); + await stream.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(payload, cancellationToken).ConfigureAwait(false); + } + + private async Task HandleWindowAsync(VatpParsedFrame frame) + { + if (!_streams.TryGetValue(frame.Header.StreamId, out var sendStream)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + return; + } + + Span windowBytes = stackalloc byte[VatpProtocol.WindowPayloadLength]; + frame.Payload.CopyTo(windowBytes); + if (!VatpControlPayload.TryDecodeWindow(windowBytes, out var credit)) + { + return; + } + + _ = sendStream.SendWindow.Add(credit); + await PumpStreamAsync(sendStream).ConfigureAwait(false); + } + + private async Task PumpStreamAsync(SendStream sendStream) + { + if (sendStream.TransferMode == TransferMode.HoldAfterFirstData) + { + return; + } + + while (sendStream.SentBytes < sendStream.ArtData.Length && sendStream.SendWindow.Credit > 0) + { + if (sendStream.TransferMode == TransferMode.DisconnectMidData && sendStream.SentBytes > 0) + { + await stream.DisposeAsync().ConfigureAwait(false); + return; + } + + var remaining = sendStream.ArtData.Length - sendStream.SentBytes; + var chunkSize = (int)Math.Min(Math.Min(sendStream.SendWindow.Credit, _maxFramePayload), remaining); + if (chunkSize <= 0) + { + return; + } + + if (!sendStream.SendWindow.TryConsume(chunkSize)) + { + return; + } + + sendStream.SentBytes += chunkSize; + var fin = sendStream.SentBytes >= sendStream.ArtData.Length; + var chunk = sendStream.ArtData.AsMemory(sendStream.SentBytes - chunkSize, chunkSize); + await WriteFrameAsync(VatpFrameEncoder.EncodeData(sendStream.StreamId, chunk, fin)).ConfigureAwait(false); + + if (fin && sendStream.TransferMode == TransferMode.FinWithoutEnd) + { + return; + } + + if (fin && sendStream.TransferMode == TransferMode.DisconnectAfterFinBeforeEnd) + { + await stream.DisposeAsync().ConfigureAwait(false); + return; + } + + if (fin && sendStream.TransferMode is TransferMode.Complete or TransferMode.CorruptArtHash) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(sendStream.StreamId)).ConfigureAwait(false); + _streams.Remove(sendStream.StreamId); + if (emitSpuriousAfterTerminal()) + { + await WriteSpuriousAfterTerminalAsync(sendStream.StreamId).ConfigureAwait(false); + } + } + } + } + + private async Task WriteSpuriousAfterTerminalAsync(uint terminalStreamId) + { + // Late frames for a stream the client already (or will imminently) remove. + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(terminalStreamId)).ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(terminalStreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(terminalStreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + + // Completely unallocated StreamId: must not create client stream state. + var junkMeta = new byte[VatpProtocol.MetaPayloadLength]; + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(NeverAllocatedStreamId, junkMeta)) + .ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeData(NeverAllocatedStreamId, new byte[] { 0x41 }, fin: true)) + .ConfigureAwait(false); + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(NeverAllocatedStreamId)).ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(NeverAllocatedStreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + } + + private async Task WriteProbeDataAsync(uint streamId, byte[] artData, bool fin) + { + if (artData.Length <= VatpLoopbackTestServer.ProbePayloadBytes) + { + throw new InvalidOperationException("Article is smaller than the accepted-data probe."); + } + + await WriteFrameAsync( + VatpFrameEncoder.EncodeData( + streamId, + artData.AsMemory(0, VatpLoopbackTestServer.ProbePayloadBytes), + fin)) + .ConfigureAwait(false); + } + + private async Task WriteFrameAsync(VatpFrameEncoder.EncodedFrame frame) + { + if (frame.Header.Length > 0) + { + await stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (frame.Payload.Length > 0) + { + await stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + } + + private sealed class SendStream( + uint streamId, + byte[] artData, + ArticleTransferWindow sendWindow, + TransferMode transferMode) + { + public uint StreamId { get; } = streamId; + + public byte[] ArtData { get; } = artData; + + public ArticleTransferWindow SendWindow { get; } = sendWindow; + + public TransferMode TransferMode { get; } = transferMode; + + public int SentBytes { get; set; } + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpTestCertificates.cs b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpTestCertificates.cs new file mode 100644 index 00000000..16ec0039 --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/Transport/Vatp/VatpTestCertificates.cs @@ -0,0 +1,34 @@ +using System.Net; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; + +namespace VectorNNTPD.ReaderService.Tests.Transport.Vatp; + +internal static class VatpTestCertificates +{ + internal static X509Certificate2 CreateServerCertificate(string commonName = "backfiller.test") + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + commonName, + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + request.CertificateExtensions.Add( + new X509KeyUsageExtension( + X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, + critical: true)); + var san = new SubjectAlternativeNameBuilder(); + san.AddDnsName("localhost"); + san.AddDnsName(commonName); + san.AddIpAddress(IPAddress.Loopback); + request.CertificateExtensions.Add(san.Build()); + using var created = request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow.AddDays(30)); + return X509CertificateLoader.LoadPkcs12( + created.Export(X509ContentType.Pfx, "test"), + "test", + X509KeyStorageFlags.Exportable | X509KeyStorageFlags.UserKeySet); + } +} diff --git a/tests/VectorNNTPD.ReaderService.Tests/VectorNNTPD.ReaderService.Tests.csproj b/tests/VectorNNTPD.ReaderService.Tests/VectorNNTPD.ReaderService.Tests.csproj new file mode 100644 index 00000000..0803031d --- /dev/null +++ b/tests/VectorNNTPD.ReaderService.Tests/VectorNNTPD.ReaderService.Tests.csproj @@ -0,0 +1,46 @@ + + + + net10.0 + false + true + VectorNNTPD.ReaderService.Tests + false + $(NoWarn);CS1591;CS0067 + x64 + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + diff --git a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs index 37730f94..f95fa62f 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs @@ -209,8 +209,10 @@ public async Task StopAsync() { await Starting.WaitAsync(TimeSpan.FromSeconds(30)); } - catch (Exception) + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) { + _ = ex.GetType(); // observed: expected during test teardown/cleanup // A startup failure is reported by the caller. Shutdown still runs below. } } @@ -241,7 +243,7 @@ private static Task CreateAsync(int port, bool holdListener, bool var session = new FleetSession(host, listener, publisher, factory, holdListener); var services = ordered.Select(service => service switch { - StorageNntpListenerService when holdListener => (IApplicationService)new HoldListener( + StorageNntpListenerService when holdListener => new HoldListener( listener, session.ListenerEntered, session._releaseListener), diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/DirectStoragePlacementPathTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/DirectStoragePlacementPathTests.cs index 8cde24c5..f262f8c1 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Ingress/DirectStoragePlacementPathTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/DirectStoragePlacementPathTests.cs @@ -93,16 +93,7 @@ private static string HeaderLine(ReadOnlySpan article, string name) private static int CountHop(string path, string hop) { var value = path.StartsWith("Path: ", StringComparison.Ordinal) ? path["Path: ".Length..] : path; - var count = 0; - foreach (var token in value.Split('!')) - { - if (string.Equals(token, hop, StringComparison.Ordinal)) - { - count++; - } - } - - return count; + return value.Split('!').Count(token => string.Equals(token, hop, StringComparison.Ordinal)); } } } diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs index c1b86eeb..dbfdaf42 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs @@ -3414,8 +3414,9 @@ public async ValueTask DisposeAsync() { await _worker.StopAsync(CancellationToken.None).ConfigureAwait(false); } - catch (InvalidOperationException) + catch (InvalidOperationException ex) { + _ = ex.GetType(); // observed: expected during test teardown/cleanup // Drain failure already recorded by the scenario. } diff --git a/tests/VectorNNTPD.StorageService.Tests/Logging/FileLoggingTests.cs b/tests/VectorNNTPD.StorageService.Tests/Logging/FileLoggingTests.cs index 545cf11f..52c54788 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Logging/FileLoggingTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Logging/FileLoggingTests.cs @@ -247,8 +247,10 @@ private static void TryDelete(string path) Directory.Delete(path, recursive: true); } } - catch + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) { + _ = ex.GetType(); // observed: expected during test teardown/cleanup // best-effort cleanup } } @@ -313,24 +315,20 @@ private static IEnumerable EnumerateChildSinks(object sink) { foreach (var field in sink.GetType().GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)) { - if (field.FieldType == typeof(ILogEventSink) || typeof(ILogEventSink).IsAssignableFrom(field.FieldType)) + if ((field.FieldType == typeof(ILogEventSink) || typeof(ILogEventSink).IsAssignableFrom(field.FieldType)) + && field.GetValue(sink) is { } child) { - if (field.GetValue(sink) is { } child) - { - yield return child; - } + yield return child; } else if (typeof(System.Collections.IEnumerable).IsAssignableFrom(field.FieldType) - && field.FieldType != typeof(string)) + && field.FieldType != typeof(string) + && field.GetValue(sink) is System.Collections.IEnumerable enumerable) { - if (field.GetValue(sink) is System.Collections.IEnumerable enumerable) + foreach (var item in enumerable) { - foreach (var item in enumerable) + if (item is ILogEventSink childSink) { - if (item is ILogEventSink childSink) - { - yield return childSink; - } + yield return childSink; } } } diff --git a/tests/VectorNNTPD.StorageService.Tests/Logging/SerilogLoggingTests.cs b/tests/VectorNNTPD.StorageService.Tests/Logging/SerilogLoggingTests.cs index cb8f9df2..0f5fe044 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Logging/SerilogLoggingTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Logging/SerilogLoggingTests.cs @@ -216,8 +216,10 @@ private static void TryDelete(string path) Directory.Delete(path, recursive: true); } } - catch + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) { + _ = ex.GetType(); // observed: expected during test teardown/cleanup // best-effort cleanup } } diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexBenchTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexBenchTests.cs index 6a8f0a12..47aeab53 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexBenchTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexBenchTests.cs @@ -376,12 +376,9 @@ private async Task MeasureSpoolCountAsync(int spoolCount, int rowsPerSpool) } ArticleIndexTestSupport.Delete(root); - foreach (var spool in spools) + foreach (var spool in spools.Where(static spool => spool is not null)) { - if (spool is not null) - { - ArticleIndexTestSupport.Delete(spool); - } + ArticleIndexTestSupport.Delete(spool); } } } diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticleIdentityGateTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticleIdentityGateTests.cs index c0a14896..0b315d42 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticleIdentityGateTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticleIdentityGateTests.cs @@ -11,22 +11,18 @@ public void Uncontended_acquire_removes_the_slot_and_a_second_dispose_is_a_no_op { var gate = new SpoolArticleIdentityGate(); var id = ArticleId.FromMessageId(""u8); - using (var lease = gate.Acquire(id, CancellationToken.None)) + using (gate.Acquire(id, CancellationToken.None)) { Assert.Equal(1, gate.Reservations(id)); } Assert.Equal(0, gate.Reservations(id)); + // codeql[cs/missed-using-statement] Intentional double-dispose contract exercise for SpoolArticleIdentityGate lease. var again = gate.Acquire(id, CancellationToken.None); - try - { - again.Dispose(); - } - finally - { - again.Dispose(); - } + again.Dispose(); + // codeql[cs/dispose-not-called-on-throw] Intentional second dispose verifies lease Dispose is a no-op. + again.Dispose(); Assert.Equal(0, gate.Reservations(id)); } diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs index d91da074..47b5205a 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs @@ -100,9 +100,7 @@ public async Task Pipeline_reports_managed_allocations_without_a_disk() articles[i] = BuildArticle(i, bodyBytes); } - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + PrepareForAllocationSample(); var gen0 = GC.CollectionCount(0); var gen1 = GC.CollectionCount(1); var gen2 = GC.CollectionCount(2); @@ -146,9 +144,7 @@ public async Task Allocation_breakdown_separates_rewrite_from_publish() } var path = new AcceptedArticlePathProcessor("cache01.usenet.ninja", NullLogger.Instance); - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + PrepareForAllocationSample(); var beforeRewrite = GC.GetTotalAllocatedBytes(precise: true); long rewrittenBytes = 0; for (var i = 0; i < count; i++) @@ -161,9 +157,7 @@ public async Task Allocation_breakdown_separates_rewrite_from_publish() var bundle = CreatePipeline(files: null, roots: null); var processor = bundle.Processor; var sink = bundle.Counting!; - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + PrepareForAllocationSample(); var beforePipeline = GC.GetTotalAllocatedBytes(precise: true); for (var i = 0; i < count; i++) { @@ -260,11 +254,19 @@ public void Warm_path_stage_allocations_are_reported() System.Buffers.ArrayPool.Shared.Return(buffer); } - private static long Measure(int count, Action action) + /// Forces a full GC cycle before allocation or gen-count sampling in benches. + private static void PrepareForAllocationSample() { + // codeql[cs/call-to-gc] Bench resets GC before allocation/throughput sampling. GC.Collect(); GC.WaitForPendingFinalizers(); + // codeql[cs/call-to-gc] Second collect after finalizers for stable gen counts. GC.Collect(); + } + + private static long Measure(int count, Action action) + { + PrepareForAllocationSample(); var before = GC.GetTotalAllocatedBytes(precise: true); for (var i = 0; i < count; i++) { @@ -379,9 +381,7 @@ private async Task RunWorkerPool( new SpoolPressure(), new SpoolPersistenceCounters(), TimeProvider.System); - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + PrepareForAllocationSample(); var gen0 = GC.CollectionCount(0); var gen1 = GC.CollectionCount(1); var gen2 = GC.CollectionCount(2); @@ -558,9 +558,7 @@ public void Filesystem_publish_reports_managed_allocations() Directory.CreateDirectory(Path.GetDirectoryName(final)!); } - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + PrepareForAllocationSample(); var gen0 = GC.CollectionCount(0); var gen1 = GC.CollectionCount(1); var gen2 = GC.CollectionCount(2); diff --git a/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementClient.cs b/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementClient.cs index 623a6190..36944dd1 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementClient.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementClient.cs @@ -150,6 +150,7 @@ public async ValueTask PlaceAsync( { return new ArticlePlacementResult(ArticlePlacementKind.Cancelled); } + // codeql[cs/catch-of-all-exceptions] Test process-boundary helper must tolerate any failure while releasing resources. catch (Exception ex) { return new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, DescribeTransportFailure(ex)); @@ -211,6 +212,7 @@ public async ValueTask DisposeAsync() if (_dialing.TryGetValue(key, out pending)) { + // Pending dial already owned by another waiter; join it below. } else if (_nextDial.TryGetValue(key, out var next) && _time.GetUtcNow() < next) { @@ -279,6 +281,7 @@ private async Task ReleaseDialAsync(string key, Task { await pending.ConfigureAwait(false); } + // codeql[cs/catch-of-all-exceptions] Test duplex teardown must complete pipes after any protocol or cancel fault. catch (Exception ex) { ExpectedException.Ignore(ex); diff --git a/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementConnection.cs b/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementConnection.cs index b74eaa72..2f62382a 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementConnection.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementConnection.cs @@ -530,11 +530,8 @@ private static (int Code, string Line) ParseStatus(byte[] lineBytes) private void FailPending(PendingTakeThis pending, ArticlePlacementResult result) { + // Complete without dequeuing: orphaned slots stay until reader or dispose clears under _pendingGate. pending.Completion.TrySetResult(result); - lock (_pendingGate) - { - // Leave orphaned slots; reader or dispose clears the queue. - } } private void CompleteAllPending(ArticlePlacementResult result) diff --git a/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeEngineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeEngineTests.cs new file mode 100644 index 00000000..20645970 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeEngineTests.cs @@ -0,0 +1,853 @@ +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Acme; + +public sealed class AcmeConfigurationTests +{ + internal const string TestPfxPassword = "unit-test-pfx-password"; + + [Fact] + public void Defaults_MatchSpecification() + { + var options = new NntpdOptions(); + Assert.Equal(NntpdOptions.DefaultAcmeDirectoryUrl, options.AcmeDirectoryUrl); + Assert.Equal(NntpdOptions.DefaultAcmeStateDir, options.AcmeStateDir); + Assert.Equal(30, options.AcmeRenewalThresholdDays); + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.Contains("staging", options.AcmeDirectoryUrl, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void MissingEmailAndPassword_WithTlsDisabled_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void MissingEmail_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = TestPfxPassword; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingPassword_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = string.Empty; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void WhitespaceOnlyPassword_WithTlsEnabled_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = " "; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void NonPositiveRenewalThreshold_FailsSharedConfiguration() + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 0, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ])); + Assert.Contains("acmerenewalthresholddays", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void RenewalThresholdAboveNinety_IsAcceptedBySharedConfiguration() + { + var configuration = NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 91, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ]); + Assert.Equal(91, configuration.AcmeRenewalThresholdDays); + } + + [Fact] + public void CustomDirectoryAndStateDir_Accepted() + { + var options = TestHostFactory.CreateValidOptions(); + options.AcmeDirectoryUrl = "https://acme.example.test/directory"; + options.AcmeStateDir = "var/acme-state"; + options.AcmeRenewalThresholdDays = 15; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void HttpDirectoryUrl_IsAcceptedBySharedConfiguration() + { + var configuration = NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "http://acme.example.test/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ]); + Assert.Equal("http://acme.example.test/directory", configuration.AcmeDirectoryUrl); + } + + [Fact] + public void EnvironmentVariable_PasswordName_IsExact() + { + Assert.Equal("VECTOR__ACMECERTIFICATEPASSWORD", NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + } +} + +public sealed class CertificateIdentitiesTests +{ + [Fact] + public void ForFqdn_ReturnsExactSanPair() + { + var ids = CertificateIdentities.ForFqdn("nntpd01.usenet.ninja"); + Assert.Equal(new[] { "nntpd01.usenet.ninja", "news.usenet.ninja" }, ids); + } + + [Fact] + public void NewsTypo_IsNotUsed() + { + Assert.DoesNotContain("ninaja", CertificateIdentities.NewsHostname, StringComparison.Ordinal); + Assert.Equal("news.usenet.ninja", CertificateIdentities.NewsHostname); + } +} + +public sealed class AccountStoreTests +{ + [Fact] + public void FirstSave_ThenLoad_ReusesAccount_AsPkcs8Der() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + Assert.Null(store.Load()); + + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + var state = new AcmeAccountState( + "https://acme.example/acme/acct/1", + NntpdOptions.DefaultAcmeDirectoryUrl, + key); + store.Save(state); + + var loaded = store.Load(); + Assert.NotNull(loaded); + Assert.Equal(state.AccountUri, loaded.AccountUri); + Assert.True(key.AsSpan().SequenceEqual(loaded.PrivateKeyDer)); + Assert.False(DerCrypto.LooksLikePem(loaded.PrivateKeyDer)); + Assert.False(File.ReadAllText(AcmePaths.AccountKeyPath(dir.Path)).Contains("BEGIN", StringComparison.Ordinal)); + } + + [Fact] + public void EnsureRegistered_ReusesExisting_WithoutCallingRegister() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + store.Save(new AcmeAccountState("https://acme.example/acme/acct/1", "https://dir.example/directory", key)); + + var registerCalls = 0; + var result = store.EnsureRegistered( + "https://dir.example/directory", + () => throw new InvalidOperationException("should not generate"), + _ => + { + registerCalls++; + return ("x", ""); + }); + + Assert.Equal(0, registerCalls); + Assert.Equal("https://acme.example/acme/acct/1", result.AccountUri); + } + + [Fact] + public void EnsureRegistered_FirstStartup_PersistsAccount() + { + using var dir = new TempAcmeDir(); + var store = new AccountStore(dir.Path); + using var rsa = RSA.Create(2048); + var key = rsa.ExportPkcs8PrivateKey(); + var registerCalls = 0; + + var result = store.EnsureRegistered( + "https://dir.example/directory", + () => key, + k => + { + registerCalls++; + Assert.True(k.AsSpan().SequenceEqual(key)); + return ("https://acme.example/acme/acct/9", "body"); + }); + + Assert.Equal(1, registerCalls); + Assert.Equal("https://acme.example/acme/acct/9", result.AccountUri); + Assert.False(File.Exists(AcmePaths.AccountPendingKeyPath(dir.Path))); + Assert.NotNull(store.Load()); + } + + [Fact] + public void IncompleteAccount_ThrowsWithoutDeleting() + { + using var dir = new TempAcmeDir(); + AcmePaths.EnsureStateLayout(dir.Path); + File.WriteAllBytes(AcmePaths.AccountKeyPath(dir.Path), [1, 2, 3]); + var store = new AccountStore(dir.Path); + Assert.Throws(() => store.Load()); + Assert.True(File.Exists(AcmePaths.AccountKeyPath(dir.Path))); + } +} + +public sealed class PfxStoreAndValidationTests +{ + private static readonly string[] RequiredDomains = ["nntpd01.usenet.ninja", "news.usenet.ninja"]; + + [Fact] + public void SaveAndLoad_UsesBinaryPfx_AndSurvivesRestart() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + + store.Save(material); + Assert.False(DerCrypto.LooksLikePem(material.PfxBytes)); + + var store2 = CreateStore(dir.Path); + var loaded = store2.Load(); + Assert.NotNull(loaded); + Assert.False(DerCrypto.LooksLikePem(loaded.PfxBytes)); + + var paths = store2.Paths(); + Assert.EndsWith("certificate.pfx", paths.PfxPath, StringComparison.OrdinalIgnoreCase); + var onDisk = File.ReadAllBytes(paths.PfxPath); + Assert.False(DerCrypto.LooksLikePem(onDisk)); + Assert.True(onDisk.Length > 0); + + using var cert = PfxCrypto.LoadCertificate(loaded.PfxBytes, AcmeConfigurationTests.TestPfxPassword); + Assert.True(cert.HasPrivateKey); + var sans = cert.Extensions.OfType().Single() + .EnumerateDnsNames().Select(n => n.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sans); + Assert.Contains("news.usenet.ninja", sans); + } + + [Fact] + public void IncorrectPassword_FailsValidation() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + "wrong-password", + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.StartsWith("parse_failed", status.Reason, StringComparison.Ordinal); + Assert.DoesNotContain("wrong-password", status.Reason, StringComparison.Ordinal); + } + + [Fact] + public void IncompleteGeneration_WithoutComplete_IsIgnored() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var genId = Guid.NewGuid().ToString("N"); + var genDir = AcmePaths.GenerationDir(dir.Path, "nntpd01.usenet.ninja", genId); + Directory.CreateDirectory(genDir); + var paths = AcmePaths.GenerationCertificatePaths(dir.Path, "nntpd01.usenet.ninja", genId); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + File.WriteAllBytes(paths.PfxPath, material.PfxBytes); + Assert.Null(store.Load()); + } + + [Fact] + public void FailedReplacement_PreservesKnownGood() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var first = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + store.Save(first); + var firstPaths = store.Paths(); + + var orphan = Guid.NewGuid().ToString("N"); + Directory.CreateDirectory(AcmePaths.GenerationDir(dir.Path, "nntpd01.usenet.ninja", orphan)); + store.Recover(); + Assert.Equal(firstPaths.PfxPath, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + [Fact] + public void InvalidPfxBytes_DoNotBecomeCurrent() + { + using var dir = new TempAcmeDir(); + var store = CreateStore(dir.Path); + var good = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + store.Save(good); + var currentBefore = store.Paths().PfxPath; + + Assert.ThrowsAny(() => + store.Save(new CertificateMaterial( + [1, 2, 3, 4], + RequiredDomains, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow.AddDays(30)))); + + Assert.Equal(currentBefore, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + [Fact] + public void Validate_WrongSan_Rejects() + { + var material = TestCertificateFactory.CreateMaterial( + ["wrong.example"], + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(60)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.Equal("missing_san", status.Reason); + } + + [Fact] + public void Validate_RenewalThreshold_Detected() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.True(status.Usable); + Assert.True(status.DueForRenewal); + } + + [Fact] + public void Validate_OutsideThreshold_NotDue() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.True(status.Usable); + Assert.False(status.DueForRenewal); + } + + [Fact] + public void Validate_Expired_Rejects() + { + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(-1), + notBefore: DateTimeOffset.UtcNow.AddDays(-40)); + var status = CertificateValidator.ValidatePfx( + material.PfxBytes, + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + Assert.False(status.Usable); + Assert.Equal("expired", status.Reason); + } + + private static CertificateStore CreateStore(string path) => + new(path, "nntpd01.usenet.ninja", AcmeConfigurationTests.TestPfxPassword, RequiredDomains, TimeSpan.FromDays(30)); +} + +public sealed class CertificateManagerTests +{ + private static readonly string[] RequiredDomains = ["nntpd01.usenet.ninja", "news.usenet.ninja"]; + + [Fact] + public async Task Ensure_ReusesValidCertificate_WithoutIssuance() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var material = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90)); + store.Save(material); + + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + var ensured = await manager.EnsureCertificateAsync(CancellationToken.None); + Assert.Equal(0, issuer.IssueCallCount); + Assert.True(Math.Abs((material.NotAfter - ensured.NotAfter).TotalSeconds) < 2); + Assert.Contains("news.usenet.ninja", ensured.Domains); + } + + [Fact] + public async Task Ensure_IssuesWhenAbsent() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + _ = await manager.EnsureCertificateAsync(CancellationToken.None); + Assert.Equal(1, issuer.IssueCallCount); + using var cert = manager.CreateTlsCertificate(); + Assert.True(cert.HasPrivateKey); + var sans = cert.Extensions.OfType().Single() + .EnumerateDnsNames().Select(n => n.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sans); + Assert.Contains("news.usenet.ninja", sans); + Assert.Equal(2, sans.Count); + Assert.True(File.Exists(store.Paths().PfxPath)); + } + + [Fact] + public async Task RenewIfDue_SkipsWhenNotDue() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + Assert.False(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(0, issuer.IssueCallCount); + } + + [Fact] + public async Task RenewIfDue_RenewsWithinThreshold() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10))); + var issuer = new FakeCertificateIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + Assert.True(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(1, issuer.IssueCallCount); + } + + [Fact] + public async Task FailedIssuance_PreservesPriorPfx() + { + using var dir = new TempAcmeDir(); + var store = new CertificateStore( + dir.Path, + "nntpd01.usenet.ninja", + AcmeConfigurationTests.TestPfxPassword, + RequiredDomains, + TimeSpan.FromDays(30)); + var prior = TestCertificateFactory.CreateMaterial( + RequiredDomains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(10)); + store.Save(prior); + var priorPath = store.Paths().PfxPath; + var issuer = new FakeCertificateIssuer { ThrowOnIssue = new AcmeOrderException("boom", "fail") }; + var manager = CreateManager(dir.Path, store, issuer); + + Assert.False(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(priorPath, store.Paths().PfxPath); + Assert.NotNull(store.Load()); + } + + private static CertificateManager CreateManager( + string stateDir, + CertificateStore store, + ICertificateIssuer issuer) => + new( + "nntpd01.usenet.ninja", + stateDir, + NntpdOptions.DefaultAcmeDirectoryUrl, + store, + issuer, + AcmeConfigurationTests.TestPfxPassword, + TimeSpan.FromDays(30), + NullLogger.Instance); +} + +public sealed class Dns01SolverTests +{ + [Fact] + public async Task PlaceWaitCleanup_CreatesAndRemovesTxt() + { + using var dir = new TempAcmeDir(); + var cf = new FakeCloudflareDnsClient(); + var resolver = new FakeTxtResolver(); + var solver = new Dns01Solver( + cf, + "zone-1", + resolver, + "nntpd01.usenet.ninja", + dir.Path, + propagationTimeout: TimeSpan.FromSeconds(2), + propagationInterval: TimeSpan.FromMilliseconds(10)); + + var spec = new Dns01ChallengeSpec("nntpd01.usenet.ninja", "validation-token"); + resolver.Set(spec.RecordName, spec.Validation); + + await solver.PlaceAsync([spec], CancellationToken.None); + Assert.Equal(1, cf.CreateCallCount); + var created = cf.Snapshot().Single(r => r.Type == CloudflareDnsRecordTypes.TXT); + Assert.Equal(Dns01Solver.ChallengeTtlSeconds, created.Ttl); + Assert.False(created.Proxied); + + await solver.WaitPropagatedAsync([spec], CancellationToken.None); + await solver.CleanupAsync(CancellationToken.None); + Assert.Equal(1, cf.DeleteCallCount); + Assert.Empty(cf.Snapshot()); + } + + [Fact] + public async Task CancellationDuringPlace_Propagates() + { + using var dir = new TempAcmeDir(); + var cf = new FakeCloudflareDnsClient(); + cf.OnMutate = async ct => + { + await Task.Delay(Timeout.Infinite, ct); + }; + var solver = new Dns01Solver(cf, "zone-1", new FakeTxtResolver(), "nntpd01.usenet.ninja", dir.Path); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(20)); + await Assert.ThrowsAnyAsync(() => + solver.PlaceAsync([new Dns01ChallengeSpec("nntpd01.usenet.ninja", "tok")], cts.Token)); + } +} + +public sealed class AcmeLifecycleServiceTests +{ + [Fact] + public async Task TlsDisabled_DoesNotInitializeAcme_OrRequireCertificate() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + options.AcmeStateDir = dir.Path; + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + + var factory = new AcmeComponentFactory( + Options.Create(options), + new FakeCloudflareDnsClient(), + new FakeHttpClientFactory(), + NullLoggerFactory.Instance); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.AcmeInitialized); + Assert.False(Directory.Exists(Path.Join(dir.Path, "account"))); + Assert.False(Directory.Exists(Path.Join(dir.Path, "live"))); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + } + + [Fact] + public async Task TlsEnabled_EnsureRuns_AndProviderAvailable() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = AcmeConfigurationTests.TestPfxPassword; + options.AcmeStateDir = dir.Path; + + var domains = CertificateIdentities.ForFqdn(options.Fqdn); + var store = new CertificateStore( + dir.Path, + options.Fqdn, + options.AcmeCertificatePassword, + domains, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial( + domains, + options.AcmeCertificatePassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + + var manager = new CertificateManager( + options.Fqdn, + dir.Path, + options.AcmeDirectoryUrl, + store, + new FakeCertificateIssuer(), + options.AcmeCertificatePassword, + TimeSpan.FromDays(30), + NullLogger.Instance); + + var factory = new InjectedManagerFactory(manager, Options.Create(options)); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.True(service.AcmeInitialized); + Assert.NotNull(manager.CurrentMaterial); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + } + + [Fact] + public async Task TlsEnabled_MissingCertificateAndFailingIssuer_FailsStartup() + { + using var dir = new TempAcmeDir(); + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 563; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = AcmeConfigurationTests.TestPfxPassword; + options.AcmeStateDir = dir.Path; + + var domains = CertificateIdentities.ForFqdn(options.Fqdn); + var manager = new CertificateManager( + options.Fqdn, + dir.Path, + options.AcmeDirectoryUrl, + new CertificateStore( + dir.Path, + options.Fqdn, + options.AcmeCertificatePassword, + domains, + TimeSpan.FromDays(30)), + new FakeCertificateIssuer { ThrowOnIssue = new AcmeOrderException("fail", "x") }, + options.AcmeCertificatePassword, + TimeSpan.FromDays(30), + NullLogger.Instance); + + var factory = new InjectedManagerFactory(manager, Options.Create(options)); + var service = new AcmeCertificateService( + Options.Create(options), + factory, + new TlsCertificateContextProvider(NullLogger.Instance), + new AcmeCertificateReadiness(), + NullLogger.Instance); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await service.DisposeAsync(); + } + + private sealed class InjectedManagerFactory : AcmeComponentFactory + { + private readonly CertificateManager _manager; + + public InjectedManagerFactory(CertificateManager manager, IOptions options) + : base( + options, + new FakeCloudflareDnsClient(), + new FakeHttpClientFactory(), + NullLoggerFactory.Instance) + { + _manager = manager; + } + + internal override CertificateManager? GetOrCreateManager() => _manager; + } +} + +internal sealed class FakeCertificateIssuer : ICertificateIssuer +{ + public int IssueCallCount { get; private set; } + + public Exception? ThrowOnIssue { get; set; } + + public Task IssueAsync( + IReadOnlyList domains, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + IssueCallCount++; + if (ThrowOnIssue is not null) + { + throw ThrowOnIssue; + } + + return Task.FromResult( + TestCertificateFactory.CreateMaterial( + domains, + AcmeConfigurationTests.TestPfxPassword, + notAfter: DateTimeOffset.UtcNow.AddDays(90))); + } +} + +internal sealed class FakeTxtResolver : IAuthoritativeTxtResolver +{ + private readonly Dictionary> _map = new(StringComparer.OrdinalIgnoreCase); + + public void Set(string name, params string[] values) => _map[name] = values; + + public Task> LookupTxtAsync(string name, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult( + _map.TryGetValue(name, out var values) ? values : Array.Empty()); + } +} + +internal sealed class FakeHttpClientFactory : IHttpClientFactory +{ + public HttpClient CreateClient(string name) => new(); +} + +internal sealed class TempAcmeDir : IDisposable +{ + public string Path { get; } = System.IO.Path.Join( + System.IO.Path.GetTempPath(), + "vectornntp-acme-" + Guid.NewGuid().ToString("N")); + + public TempAcmeDir() => Directory.CreateDirectory(Path); + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Best-effort test cleanup. + } + } +} + +internal static class TestCertificateFactory +{ + public static CertificateMaterial CreateMaterial( + IReadOnlyList dnsNames, + string password, + DateTimeOffset notAfter, + DateTimeOffset? notBefore = null) + { + var before = notBefore ?? DateTimeOffset.UtcNow.AddDays(-1); + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + dnsNames[0], + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + foreach (var name in dnsNames) + { + sanBuilder.AddDnsName(name); + } + + request.CertificateExtensions.Add(sanBuilder.Build()); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, + critical: false)); + + using var cert = request.CreateSelfSigned(before.UtcDateTime, notAfter.UtcDateTime); + var pfx = PfxCrypto.ExportPfx(cert, intermediateCertificates: [], password); + return new CertificateMaterial( + pfx, + dnsNames.Select(n => n.ToLowerInvariant()).OrderBy(n => n).ToArray(), + before, + notAfter); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeOrderReadinessTests.cs b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeOrderReadinessTests.cs new file mode 100644 index 00000000..f3c881a1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeOrderReadinessTests.cs @@ -0,0 +1,207 @@ +using VectorNNTPD.Common.Acme; + +namespace VectorNNTPD.TransitService.Tests.Acme; + +public sealed class AcmeOrderReadinessTests +{ + [Fact] + public void Evaluate_Pending_Continues() + { + var view = new AcmeOrderReadiness.OrderView( + "Pending", + [ + new AcmeOrderReadiness.AuthorizationView("nntpd01.usenet.ninja", "Pending", null, null, null), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null), + ]); + + var decision = AcmeOrderReadiness.Evaluate(view, out var diagnostic); + Assert.Equal(AcmeOrderReadiness.Decision.Continue, decision); + Assert.Null(diagnostic); + } + + [Fact] + public void Evaluate_Ready_IsReady() + { + var view = new AcmeOrderReadiness.OrderView( + "Ready", + [ + new AcmeOrderReadiness.AuthorizationView("nntpd01.usenet.ninja", "Valid", null, null, null), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null), + ]); + + Assert.Equal(AcmeOrderReadiness.Decision.Ready, AcmeOrderReadiness.Evaluate(view, out _)); + } + + [Fact] + public void Evaluate_InvalidAuthorization_SurfacesDnsProblem() + { + var view = new AcmeOrderReadiness.OrderView( + "Invalid", + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + "Invalid", + "urn:ietf:params:acme:error:dns", + "DNS problem: NXDOMAIN looking up TXT for _acme-challenge.nntpd01.usenet.ninja", + 400), + new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null), + ]); + + var decision = AcmeOrderReadiness.Evaluate(view, out var diagnostic); + Assert.Equal(AcmeOrderReadiness.Decision.Invalid, decision); + Assert.NotNull(diagnostic); + Assert.Contains("nntpd01.usenet.ninja", diagnostic, StringComparison.Ordinal); + Assert.Contains("urn:ietf:params:acme:error:dns", diagnostic, StringComparison.Ordinal); + Assert.Contains("NXDOMAIN", diagnostic, StringComparison.Ordinal); + Assert.Contains("type=", diagnostic, StringComparison.Ordinal); + Assert.Contains("detail=", diagnostic, StringComparison.Ordinal); + } + + [Fact] + public async Task WaitUntilReady_PendingThenReady_DoesNotThrow() + { + var polls = 0; + await AcmeOrderReadiness.WaitUntilReadyAsync( + _ => + { + polls++; + if (polls < 3) + { + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null)])); + } + + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Ready", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Valid", null, null, null)])); + }, + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask); + + Assert.True(polls >= 3); + } + + [Fact] + public async Task WaitUntilReady_Invalid_ThrowsAuthorizationInvalidWithDetails() + { + var ex = await Assert.ThrowsAsync(() => + AcmeOrderReadiness.WaitUntilReadyAsync( + _ => Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + "Invalid", + "urn:ietf:params:acme:error:dns", + "DNS problem: SERVFAIL", + 400), + ])), + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask)); + + Assert.Equal("authorization_invalid", ex.Category); + Assert.Contains("urn:ietf:params:acme:error:dns", ex.Message, StringComparison.Ordinal); + Assert.Contains("SERVFAIL", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("AcmeRequestException", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task WaitUntilReady_Timeout_ThrowsBoundedTimeout() + { + var delayCalls = 0; + var ex = await Assert.ThrowsAsync(() => + AcmeOrderReadiness.WaitUntilReadyAsync( + _ => Task.FromResult( + new AcmeOrderReadiness.OrderView( + "Pending", + [new AcmeOrderReadiness.AuthorizationView("news.usenet.ninja", "Pending", null, null, null)])), + timeout: TimeSpan.FromMilliseconds(50), + interval: TimeSpan.FromMilliseconds(10), + CancellationToken.None, + delayAsync: async (delay, ct) => + { + delayCalls++; + await Task.Delay(delay, ct); + })); + + Assert.Equal("authorization_timeout", ex.Category); + Assert.Contains("order=Pending", ex.Message, StringComparison.Ordinal); + Assert.True(delayCalls >= 1); + } + + [Fact] + public async Task WaitUntilReady_Cancellation_ThrowsPromptly() + { + using var cts = new CancellationTokenSource(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waitTask = AcmeOrderReadiness.WaitUntilReadyAsync( + async ct => + { + gate.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + return new AcmeOrderReadiness.OrderView("Pending", []); + }, + timeout: TimeSpan.FromMinutes(1), + interval: TimeSpan.FromMilliseconds(5), + cts.Token, + delayAsync: (delay, ct) => Task.Delay(delay, ct)); + + await gate.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waitTask); + } +} + +/// +/// Documents the required issue sequence: finalize must not run while the order is still Pending. +/// Exercised via which calls after challenge submit. +/// +public sealed class AcmeIssuanceSequenceTests +{ + [Fact] + public async Task ReadinessGate_PreventsFinalizeWhilePending_ThenAllowsWhenReady() + { + var generateAllowed = false; + var states = new Queue(["Pending", "Pending", "Ready"]); + + await AcmeOrderReadiness.WaitUntilReadyAsync( + _ => + { + var status = states.Dequeue(); + Assert.False(generateAllowed); + return Task.FromResult( + new AcmeOrderReadiness.OrderView( + status, + [ + new AcmeOrderReadiness.AuthorizationView( + "nntpd01.usenet.ninja", + status == "Ready" ? "Valid" : "Pending", + null, + null, + null), + new AcmeOrderReadiness.AuthorizationView( + "news.usenet.ninja", + status == "Ready" ? "Valid" : "Pending", + null, + null, + null), + ])); + }, + timeout: TimeSpan.FromSeconds(5), + interval: TimeSpan.FromMilliseconds(1), + CancellationToken.None, + delayAsync: (_, _) => Task.CompletedTask); + + generateAllowed = true; + Assert.Empty(states); + Assert.True(generateAllowed); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeProblemDiagnosticsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeProblemDiagnosticsTests.cs new file mode 100644 index 00000000..d30f7eb6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Acme/AcmeProblemDiagnosticsTests.cs @@ -0,0 +1,70 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Acme.Protocol; + +namespace VectorNNTPD.TransitService.Tests.Acme; + +public sealed class AcmeProblemDiagnosticsTests +{ + [Fact] + public void FormatProblemFields_PreservesStructuredFields() + { + var formatted = AcmeProblemDiagnostics.FormatProblemFields( + preface: null, + type: "urn:ietf:params:acme:error:orderNotReady", + status: 403, + identifier: "nntpd01.usenet.ninja", + detail: "Order's status (\"pending\") is not acceptable for finalization", + subproblems: null); + + Assert.Contains("type=urn:ietf:params:acme:error:orderNotReady", formatted, StringComparison.Ordinal); + Assert.Contains("status=403", formatted, StringComparison.Ordinal); + Assert.Contains("identifier=nntpd01.usenet.ninja", formatted, StringComparison.Ordinal); + Assert.Contains("detail=", formatted, StringComparison.Ordinal); + Assert.Contains("pending", formatted, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void FormatException_RedactsPemAndLongTokens() + { + var pem = "oops -----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY----- trailing"; + var ex = new AcmeCaException( + "server error", + "urn:ietf:params:acme:error:serverInternal", + pem + " " + new string('A', 100), + statusCode: 500); + var formatted = AcmeProblemDiagnostics.FormatException(ex); + Assert.Contains("[redacted-pem]", formatted, StringComparison.Ordinal); + Assert.Contains("[redacted-token]", formatted, StringComparison.Ordinal); + Assert.DoesNotContain("BEGIN PRIVATE KEY", formatted, StringComparison.Ordinal); + } + + [Fact] + public void FailureSanitizer_IncludesAcmeOrderDiagnosticMessage() + { + var ex = new AcmeOrderException( + "challenge_failed", + "type=urn:ietf:params:acme:error:dns status=400 identifier=nntpd01.usenet.ninja detail=NXDOMAIN"); + var sanitized = AcmeFailureSanitizer.Sanitize(ex); + Assert.Contains("challenge_failed", sanitized, StringComparison.Ordinal); + Assert.Contains("urn:ietf:params:acme:error:dns", sanitized, StringComparison.Ordinal); + Assert.Contains("nntpd01.usenet.ninja", sanitized, StringComparison.Ordinal); + Assert.DoesNotContain("PRIVATE KEY", sanitized, StringComparison.Ordinal); + } + + [Fact] + public void ChallengeFailedWrapper_UsesFormattedProtocolDetails_NotTypeNameOnly() + { + var protocol = new AcmeCaException( + "Error processing request", + "urn:ietf:params:acme:error:dns", + "DNS problem: NXDOMAIN looking up TXT", + statusCode: 400); + var wrapped = new AcmeOrderException("challenge_failed", AcmeProblemDiagnostics.FormatException(protocol)); + + Assert.Equal("challenge_failed", wrapped.Category); + Assert.Contains("type=urn:ietf:params:acme:error:dns", wrapped.Message, StringComparison.Ordinal); + Assert.DoesNotContain("challenge_failed: AcmeException", wrapped.Message, StringComparison.Ordinal); + Assert.DoesNotContain("challenge_failed: AcmeCaException", wrapped.Message, StringComparison.Ordinal); + Assert.Equal(wrapped.Message, AcmeFailureSanitizer.Sanitize(wrapped)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs b/tests/VectorNNTPD.TransitService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs new file mode 100644 index 00000000..214351c3 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Acme/NntpdAcmeFilesystemParityTests.cs @@ -0,0 +1,60 @@ +using VectorNNTPD.Common.Acme; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Acme; + +public sealed class NntpdAcmeFilesystemParityTests +{ + [Fact] + public void Nntpd_certificate_identity_uses_common_fqdn_paths() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + options.DnsSuffix = "usenet.ninja"; + var fqdn = CertificateIdentities.NormalizeFqdn(options.Fqdn); + Assert.Equal("nntpd01.usenet.ninja", fqdn); + Assert.Equal( + new[] { "nntpd01.usenet.ninja", CertificateIdentities.NewsHostname }, + CertificateIdentities.ForFqdn(fqdn)); + + const string certs = "certs-root"; + Assert.Equal( + Path.Join(certs, "live", fqdn, "current"), + AcmePaths.CurrentGenerationPointerPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, "gens"), + AcmePaths.GenerationsDir(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, "dns01"), + AcmePaths.Dns01RecoveryDir(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "journal", fqdn + ".json"), + AcmePaths.TransactionJournalPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "live", fqdn, ".issuance.lock"), + AcmePaths.IssuanceLockPath(certs, fqdn)); + Assert.Equal( + Path.Join(certs, "account", "private_key.der"), + AcmePaths.AccountKeyPath(certs)); + Assert.DoesNotContain( + Path.Join("live", "gens"), + AcmePaths.GenerationsDir(certs, fqdn), + StringComparison.Ordinal); + Assert.DoesNotContain( + Path.Join("live", "current"), + AcmePaths.CurrentGenerationPointerPath(certs, fqdn), + StringComparison.Ordinal); + } + + [Fact] + public void Nntpd_uses_shared_acme_account_environment_variable() + { + Assert.Equal("VECTOR__ACMEACCOUNT", AcmeCloudflareOptions.AcmeAccountEnvironmentVariable); + Assert.Equal( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + NntpdOptions.AcmeAccountEnvironmentVariable); + Assert.DoesNotContain("NNTPD", NntpdOptions.AcmeAccountEnvironmentVariable, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ApplicationVersionTests.cs b/tests/VectorNNTPD.TransitService.Tests/ApplicationVersionTests.cs new file mode 100644 index 00000000..5fc7205c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ApplicationVersionTests.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; +using System.Reflection; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests; + +public sealed class ApplicationVersionTests +{ + [Fact] + public void CompiledVersion_IsDateTimeFourPartNumeric_WithoutSourceRevision() + { + AssertDateTimeApplicationVersion(typeof(NntpdLoggingExtensions).Assembly); + } + + private static void AssertDateTimeApplicationVersion(Assembly assembly) + { + var version = assembly.GetName().Version; + Assert.NotNull(version); + Assert.Equal(1, version.Major); + Assert.Equal(0, version.Minor); + Assert.Equal((DateTime.Now.Date - new DateTime(2000, 1, 1)).Days, version.Build); + Assert.InRange(version.Revision, 0, 43199); + + var expected = $"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"; + Assert.Matches(@"^1\.0\.\d+\.\d+$", expected); + + var informational = assembly.GetCustomAttribute(); + Assert.NotNull(informational); + Assert.Equal(expected, informational.InformationalVersion); + Assert.DoesNotContain("+", informational.InformationalVersion, StringComparison.Ordinal); + Assert.DoesNotContain("git", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("commit", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("branch", informational.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("-", informational.InformationalVersion, StringComparison.Ordinal); + + var file = FileVersionInfo.GetVersionInfo(assembly.Location); + Assert.Equal(expected, file.FileVersion); + Assert.Equal(expected, file.ProductVersion); + + const string copyright = "© 2026 Chris Knipe "; + var copyrightAttribute = assembly.GetCustomAttribute(); + Assert.NotNull(copyrightAttribute); + Assert.Equal(copyright, copyrightAttribute.Copyright); + Assert.Equal(copyright, file.LegalCopyright); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs new file mode 100644 index 00000000..b681f872 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleIngestionQueueTests.cs @@ -0,0 +1,371 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class ArticleIngestionQueueTests +{ + [Fact] + public void DefaultMemoryLimit_IsOneGibibyte() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions()); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, queue.MemoryLimitBytes); + Assert.Equal(1_073_741_824L, queue.MemoryLimitBytes); + } + + [Fact] + public void ConfiguredMemoryLimit_IsHonoured() + { + var options = new NntpdOptions + { + TransitQueueMemoryLimit = 4096, + ArticleIngestion = new ArticleIngestionOptions(), + }; + var queue = new ArticleIngestionQueue(Options.Create(options)); + Assert.Equal(4096, queue.MemoryLimitBytes); + } + + [Fact] + public async Task ArticleSmallerThanBudget_IsAdmitted() + { + var article = Article("", InboundArticleProducer.IHave); + var queue = CreateQueue(article.Payload.Length + 16); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + Assert.Equal(article.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + } + + [Fact] + public async Task MultipleArticles_ConsumeBudget() + { + var a = Article(""); + var b = Article(""); + var c = Article(""); + var total = a.Payload.Length + b.Payload.Length + c.Payload.Length; + var queue = CreateQueue(total); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(c, CancellationToken.None)); + Assert.Equal(total, queue.QueuedBytes); + Assert.Equal(3, queue.Count); + Assert.Equal(total, queue.PeakQueuedBytes); + Assert.Equal(3, queue.PeakCount); + } + + [Fact] + public async Task Admission_BlocksWhenBudgetExhausted_AndResumesAfterRelease() + { + var full = Article(""); + var wait = Article(""); + var queue = CreateQueue(full.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(full, CancellationToken.None)); + + var blocked = queue.EnqueueAsync(wait, CancellationToken.None); + await WaitForWaitersAsync(queue, 1); + + Assert.False(blocked.IsCompleted); + Assert.Equal(full.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + + var first = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("", first!.MessageId); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Accepted, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(wait.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ConcurrentProducers_CannotOversubscribeBudget() + { + const int producers = 16; + const int admitted = 4; + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * admitted; + var queue = CreateQueue(limit); + var results = new ArticleEnqueueResult[producers]; + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = new Task[producers]; + for (var i = 0; i < producers; i++) + { + var index = i; + tasks[index] = Task.Run(async () => + { + await start.Task.ConfigureAwait(false); + results[index] = await queue + .EnqueueAsync(Article($""), CancellationToken.None) + .ConfigureAwait(false); + }); + } + + start.SetResult(); + + await WaitUntilAsync( + () => queue.Count == admitted && queue.DebugWaiterCount == producers - admitted, + TimeSpan.FromSeconds(5)); + + Assert.Equal(limit, queue.QueuedBytes); + Assert.Equal(admitted, queue.Count); + Assert.True(queue.QueuedBytes <= limit); + Assert.Equal(admitted, tasks.Count(static t => t.IsCompleted)); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + for (var i = 0; i < producers; i++) + { + Assert.NotNull(await queue.DequeueAsync(safety.Token)); + } + + await Task.WhenAll(tasks).WaitAsync(safety.Token); + Assert.All(results, static r => Assert.Equal(ArticleEnqueueResult.Accepted, r)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task ExactBoundaryArticle_Fits() + { + var edge = Article(""); + var queue = CreateQueue(edge.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(edge, CancellationToken.None)); + Assert.Equal(edge.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.False(queue.TryProbeCapacity()); + } + + [Fact] + public async Task ArticleExceedingBudget_IsRejected() + { + var big = Article(""); + var queue = CreateQueue(big.Payload.Length - 1); + Assert.Equal(ArticleEnqueueResult.Rejected, await queue.EnqueueAsync(big, CancellationToken.None)); + Assert.False(queue.TryEnqueue(Article(""))); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task FailedEnqueueAfterReservation_ReleasesBytes() + { + var article = Article(""); + var queue = CreateQueue(article.Payload.Length + 16); + queue.CompleteChannelWithoutMarkingUnavailableForTests(); + Assert.Equal(ArticleEnqueueResult.Unavailable, await queue.EnqueueAsync(article, CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task Cancellation_ReleasesReservation() + { + var held = Article(""); + var cancel = Article(""); + var queue = CreateQueue(held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(held, CancellationToken.None)); + + using var cts = new CancellationTokenSource(); + var blocked = queue.EnqueueAsync(cancel, cts.Token); + await WaitForWaitersAsync(queue, 1); + cts.Cancel(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ShutdownDrain_DoesNotLeakReservations() + { + var a = Article(""); + var b = Article(""); + var c = Article(""); + var queue = CreateQueue(a.Payload.Length + b.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + + var blocked = queue.EnqueueAsync(c, CancellationToken.None); + await WaitForWaitersAsync(queue, 1); + + queue.Complete(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await blocked.AsTask().WaitAsync(safety.Token)); + Assert.Equal(ArticleEnqueueResult.Unavailable, await queue.EnqueueAsync(Article(""), CancellationToken.None)); + + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.False(queue.IsAccepting); + } + + [Fact] + public async Task QueuedBytes_ReturnToZeroAfterConsume() + { + var a = Article(""); + var b = Article(""); + var total = a.Payload.Length + b.Payload.Length; + var queue = CreateQueue(total); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + Assert.Equal(total, queue.QueuedBytes); + Assert.NotNull(await queue.DequeueAsync(CancellationToken.None)); + Assert.NotNull(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(total, queue.PeakQueuedBytes); + } + + [Fact] + public void TryProbeCapacity_IsFalseWhenBudgetIsExhausted() + { + var full = Article(""); + var queue = CreateQueue(full.Payload.Length); + Assert.True(queue.TryProbeCapacity()); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(full)); + Assert.False(queue.TryProbeCapacity()); + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.Equal(full.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + } + + [Fact] + public void TryAdmit_DoesNotWait_WhenBudgetIsExhausted() + { + var held = Article(""); + var queue = CreateQueue(held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + var started = Environment.TickCount64; + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.True(Environment.TickCount64 - started < 250); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + } + + [Fact] + public void TryAdmit_RejectedArticle_DoesNotReserve() + { + var big = Article(""); + var queue = CreateQueue(big.Payload.Length - 1); + Assert.Equal(ArticleEnqueueResult.Rejected, queue.TryAdmit(big)); + Assert.Equal(0, queue.QueuedBytes); + Assert.True(queue.TryProbeCapacity()); + } + + [Fact] + public void TryAdmit_ConcurrentProducers_CannotOversubscribe() + { + const int producers = 16; + const int admitted = 4; + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * admitted; + var queue = CreateQueue(limit); + var results = new ArticleEnqueueResult[producers]; + Parallel.For(0, producers, i => + { + results[i] = queue.TryAdmit(Article($"")); + }); + + Assert.Equal(admitted, results.Count(static r => r == ArticleEnqueueResult.Accepted)); + Assert.Equal(producers - admitted, results.Count(static r => r == ArticleEnqueueResult.Full)); + Assert.Equal(limit, queue.QueuedBytes); + Assert.Equal(admitted, queue.Count); + Assert.True(queue.QueuedBytes <= limit); + } + + [Fact] + public void TryAdmit_RejectedAndFull_IncrementAdmissionFailures() + { + var fit = Article(""); + var huge = Article("", body: "body\r\n" + new string('Z', 80) + "\r\n"); + Assert.True(huge.Payload.Length > fit.Payload.Length); + var queue = CreateQueue(fit.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Rejected, queue.TryAdmit(huge)); + Assert.Equal(1, queue.AdmissionFailureCount); + + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(fit)); + Assert.Equal(ArticleEnqueueResult.Full, queue.TryAdmit(Article(""))); + Assert.Equal(2, queue.AdmissionFailureCount); + Assert.Equal(0, queue.AdmissionWaitTicks); + } + + [Fact] + public async Task Dequeue_IsFifo() + { + var sample = Article("<0@fifo>"); + var queue = CreateQueue(sample.Payload.Length * 20); + for (var i = 0; i < 10; i++) + { + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(Article($"<{i}@fifo>"))); + } + + for (var i = 0; i < 10; i++) + { + Assert.Equal($"<{i}@fifo>", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + } + + queue.Complete(); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ProducerIdentityAndOrder_RemainIntact() + { + var first = Article("", InboundArticleProducer.IHave); + var second = Article(""); + var queue = CreateQueue(first.Payload.Length + second.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(second, CancellationToken.None)); + + var dequeuedFirst = await queue.DequeueAsync(CancellationToken.None); + var dequeuedSecond = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("", dequeuedFirst!.MessageId); + Assert.Equal(InboundArticleProducer.IHave, dequeuedFirst.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, dequeuedFirst.Record.ParseStatus); + Assert.Equal("", dequeuedSecond!.MessageId); + Assert.Equal(InboundArticleProducer.TakeThis, dequeuedSecond.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, dequeuedSecond.Record.ParseStatus); + Assert.Equal(0, queue.QueuedBytes); + } + + private static ArticleIngestionQueue CreateQueue(long memoryLimit) => + new(new ArticleIngestionOptions(), memoryLimit); + + private static InboundArticle Article( + string messageId, + InboundArticleProducer producer = InboundArticleProducer.TakeThis, + string body = "body\r\n") => + CanonicalArticleText.CreateQueued(messageId, producer, body); + + private static Task WaitForWaitersAsync(ArticleIngestionQueue queue, int count) => + WaitUntilAsync(() => queue.DebugWaiterCount >= count, TimeSpan.FromSeconds(5)); + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + await Task.Delay(1, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs new file mode 100644 index 00000000..e0de96d7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleRecordIngressTests.cs @@ -0,0 +1,116 @@ +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class ArticleRecordIngressTests +{ + private static readonly NntpArticleParser Parser = new("nntpd01.usenet.ninja"); + + [Fact] + public void TryCreateFromDestuffed_ValidArticle_ReturnsCanonicalRecord() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("", "one\r\ntwo\r\n")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var record = created.Record; + Assert.Equal(ArticleParseStatus.CanonicalV1, record.ParseStatus); + Assert.Equal(record.ArtData.Length, record.ArtSize); + Assert.Equal(2, record.ArtLines); + Assert.NotEqual(ArticleType.None, record.ArtType); + Assert.True(record.MessageId.SequenceEqual(""u8)); + Assert.True(record.Fields.MessageId.IsPresent); + Assert.True(record.Fields.Newsgroups.IsPresent); + Assert.True(record.ArtData.Span.Slice(record.Fields.MessageId.Offset, record.Fields.MessageId.Length) + .SequenceEqual(""u8)); + } + + [Fact] + public void TryCreateFromStuffedWire_DestuffsOnce_ThenFactory() + { + var destuffed = CanonicalArticleText.Destuffed("", ".hidden\r\n"); + var stuffed = destuffed.Replace(".hidden\r\n", "..hidden\r\n", StringComparison.Ordinal); + var created = ArticleRecordIngress.TryCreateFromStuffedWire( + Parser, + Encoding.ASCII.GetBytes(stuffed), + 64 * 1024); + Assert.True(created.IsAccepted); + var text = Encoding.ASCII.GetString(created.Record.ArtData.Span); + Assert.Contains("\r\n\r\n..hidden\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("\r\n\r\n.hidden\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public void TryCreateFromDestuffed_MissingDate_DoesNotProduceRecord() + { + var destuffed = "Message-ID: \r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\n\r\nbody\r\n"u8.ToArray(); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.False(created.IsAccepted); + Assert.Equal(NntpArticleParseFailureCode.MissingOrInvalidDate, created.ParseFailure); + Assert.Equal(ArticleParseStatus.None, created.Record.ParseStatus); + Assert.Equal(0, created.Record.ArtSize); + } + + [Fact] + public void CreateQueued_PayloadAliasesArtData() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.True(inbound.Payload.Equals(created.Record.ArtData)); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.Equal(InboundArticleProducer.TakeThis, inbound.Producer); + } + + [Fact] + public void CreateQueued_IHave_IsCanonicalV1() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.IHave); + Assert.Equal(InboundArticleProducer.IHave, inbound.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(created.Record.ArtData)); + } + + [Fact] + public void CreateQueued_RejectsNonCanonicalRecord() + { + var destuffed = Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("")); + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, destuffed); + Assert.True(created.IsAccepted); + var invalid = new ArticleRecord( + created.Record.ArtId, + created.Record.ArtHash, + created.Record.ArtType, + created.Record.ArtLines, + created.Record.CanonicalUtc, + ArticleParseStatus.None, + created.Record.ArtData.ToArray(), + created.Record.Fields); + Assert.Throws(() => + ArticleRecordIngress.CreateQueued( + "", + invalid, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.Post)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs new file mode 100644 index 00000000..40beeaf8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/ArticleTypeClassifierTests.cs @@ -0,0 +1,117 @@ +using System.Text; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class ArticleTypeClassifierTests +{ + [Fact] + public void PlainText_IsDefault() + { + var type = ArticleTypeClassifier.Classify("From: a@b\r\nSubject: hi\r\n"u8, "hello\r\n"u8); + Assert.Equal(ArticleType.Default, type); + } + + [Fact] + public void YbeginLine_IsYEncodedAndBinary() + { + var type = ArticleTypeClassifier.Classify( + "Subject: part\r\n"u8, + "=ybegin line=128 size=12 name=a.bin\r\n"u8); + Assert.True(type.HasFlag(ArticleType.YEncoded)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void YbeginPart_IsYEncodedPartial() + { + var type = ArticleTypeClassifier.Classify( + default, + "=ybegin part=1 line=128 size=99 name=a.bin\r\n"u8); + Assert.True(type.HasFlag(ArticleType.YEncoded)); + Assert.True(type.HasFlag(ArticleType.Partial)); + } + + [Fact] + public void MimeAndBase64Headers_ClassifyWithoutBodyScan() + { + var type = ArticleTypeClassifier.Classify( + "Mime-Version: 1.0\r\nContent-Type: text/plain\r\nContent-Transfer-Encoding: base64\r\n"u8, + default); + Assert.True(type.HasFlag(ArticleType.Mime)); + Assert.True(type.HasFlag(ArticleType.Base64)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void UuencodeBegin_IsUuEncode() + { + var type = ArticleTypeClassifier.Classify(default, "begin 644 file.dat\r\n"u8); + Assert.True(type.HasFlag(ArticleType.UuEncode)); + Assert.True(type.HasFlag(ArticleType.Binary)); + } + + [Fact] + public void ContentLength_ParsesDestuffedHint() + { + Assert.Equal(12, ArticleTypeClassifier.TryParseContentLength("Content-Length: 12"u8)); + Assert.Equal(-1, ArticleTypeClassifier.TryParseContentLength("Subject: 12"u8)); + } + + [Fact] + public void YencSize_IsDecodedSizeNotWire() + { + Assert.Equal(12345, ArticleTypeClassifier.TryParseYencSize("=ybegin line=128 size=12345 name=a.bin"u8)); + Assert.Equal(-1, ArticleTypeClassifier.TryParseYencSize("=ybegin line=128 name=a.bin"u8)); + } + + [Fact] + public void YEncoded_MatchesTransitYencBit() + { + Assert.Equal((int)VectorNNTPD.TransitService.Configuration.TransitMessageTypes.Yenc, (int)ArticleType.YEncoded); + } + + [Fact] + public void CommonArticleTypeBits_MatchTransitMessageTypes() + { + Assert.Equal((int)TransitMessageTypes.None, (int)ArticleType.None); + Assert.Equal((int)TransitMessageTypes.Default, (int)ArticleType.Default); + Assert.Equal((int)TransitMessageTypes.Control, (int)ArticleType.Control); + Assert.Equal((int)TransitMessageTypes.Cancel, (int)ArticleType.Cancel); + Assert.Equal((int)TransitMessageTypes.Mime, (int)ArticleType.Mime); + Assert.Equal((int)TransitMessageTypes.Binary, (int)ArticleType.Binary); + Assert.Equal((int)TransitMessageTypes.UuEncode, (int)ArticleType.UuEncode); + Assert.Equal((int)TransitMessageTypes.Base64, (int)ArticleType.Base64); + Assert.Equal((int)TransitMessageTypes.Yenc, (int)ArticleType.YEncoded); + Assert.Equal((int)TransitMessageTypes.BommaNews, (int)ArticleType.BommaNews); + Assert.Equal((int)TransitMessageTypes.UniData, (int)ArticleType.UniData); + Assert.Equal((int)TransitMessageTypes.Multipart, (int)ArticleType.Multipart); + Assert.Equal((int)TransitMessageTypes.Html, (int)ArticleType.Html); + Assert.Equal((int)TransitMessageTypes.PostScript, (int)ArticleType.PostScript); + Assert.Equal((int)TransitMessageTypes.BinHex, (int)ArticleType.BinHex); + Assert.Equal((int)TransitMessageTypes.Partial, (int)ArticleType.Partial); + Assert.Equal((int)TransitMessageTypes.PgpMessage, (int)ArticleType.PgpMessage); + } + + [Fact] + public void IhaveDestuff_UsesCommonClassifier() + { + var article = IhaveArticleInterpreter.DestuffToArticle( + "Control: cancel \r\n\r\nbody\r\n"u8, + 64 * 1024); + Assert.True(article.Type.HasFlag(ArticleType.Control)); + Assert.True(article.Type.HasFlag(ArticleType.Cancel)); + } + + [Fact] + public void ArticleSize_IsDestuffedCompleteArticle() + { + var headers = Encoding.ASCII.GetBytes("Subject: hi\r\n"); + var body = Encoding.ASCII.GetBytes("body\r\n"); + var article = new Article(headers, body, ArticleType.Default); + Assert.Equal(headers.Length + 2 + body.Length, article.Size); + Assert.Equal("Subject: hi\r\n\r\nbody\r\n", Encoding.ASCII.GetString(article.Payload.Span)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs new file mode 100644 index 00000000..43df675b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IhaveArticleInterpreterTests.cs @@ -0,0 +1,113 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class IhaveArticleInterpreterTests +{ + [Fact] + public void DestuffToArticle_UnstuffsOnce_AndMatchesCanonical() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var article = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + Assert.Equal("Subject: d\r\n", Encoding.ASCII.GetString(article.Headers.Span)); + Assert.Equal(".foo\r\n", Encoding.ASCII.GetString(article.Body.Span)); + Assert.Equal("Subject: d\r\n\r\n.foo\r\n"u8.ToArray(), article.Payload.ToArray()); + Assert.Equal(article.Payload.Length, article.Size); + } + + [Fact] + public void DestuffToArticle_DoesNotDestuffTwice() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var once = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + var twice = IhaveArticleInterpreter.DestuffToArticle(once.Payload.Span, 64 * 1024); + Assert.Equal(".foo\r\n", Encoding.ASCII.GetString(once.Body.Span)); + Assert.Equal("foo\r\n", Encoding.ASCII.GetString(twice.Body.Span)); + } + + [Fact] + public void DestuffToArticle_Yenc_IsNotDecoded() + { + var wire = "Subject: y\r\n\r\n=ybegin line=128 size=4 name=a.bin\r\n=ypart begin=1 end=4\r\n)ab=\r\n=yend size=4\r\n"u8; + var article = IhaveArticleInterpreter.DestuffToArticle(wire, 64 * 1024); + Assert.True(article.Type.HasFlag(ArticleType.YEncoded)); + Assert.Contains("=ybegin", Encoding.ASCII.GetString(article.Body.Span), StringComparison.Ordinal); + } + + [Fact] + public void DestuffThenRestuff_PreservesDotStuffingOnOutbound() + { + var stuffed = "Subject: d\r\n\r\n..foo\r\n"u8.ToArray(); + var article = IhaveArticleInterpreter.DestuffToArticle(stuffed, 64 * 1024); + var wire = ArticleWireReconstructor.RestuffArticle(article.Payload.Span); + Assert.Equal("Subject: d\r\n\r\n..foo\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public async Task SpoolWriter_ConsumesCanonicalRecordsWithoutReparse() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = new ArticleIngestionOptions() }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + var ihave = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave, ".foo\r\n"); + var takeThis = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, ".foo\r\n"); + var post = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post, ".foo\r\n"); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(ihave, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(takeThis, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(post, CancellationToken.None)); + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(ArticleParseStatus.CanonicalV1, ihave.Record.ParseStatus); + Assert.Equal(ArticleParseStatus.CanonicalV1, takeThis.Record.ParseStatus); + Assert.Equal(ArticleParseStatus.CanonicalV1, post.Record.ParseStatus); + Assert.True(ihave.Payload.Equals(ihave.Record.ArtData)); + Assert.True(takeThis.Payload.Equals(takeThis.Record.ArtData)); + Assert.True(post.Payload.Equals(post.Record.ArtData)); + } + + [Fact] + public async Task SpoolWriter_CanonicalV1Record_IsNotReparsedOrCopied() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = new ArticleIngestionOptions() }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("", "one\r\n"))); + Assert.True(created.IsAccepted); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Record.ArtData.Equals(created.Record.ArtData)); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs new file mode 100644 index 00000000..52354be8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolNewsLogTests.cs @@ -0,0 +1,723 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Post-queue news disposition and INN news writer contracts. +public sealed class IncomingSpoolNewsLogTests +{ + [Fact] + public async Task NormalAcceptedArticle_ProducesPlus() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "alt.test"), + "", + news, + Catalogue("alt.test")); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, evt.Disposition); + Assert.True(evt.MessageId.Span.SequenceEqual(""u8)); + Assert.True(evt.Feed.IsEmpty); + Assert.True(evt.Sites.IsEmpty); + Assert.True(evt.Size > 0); + } + + [Fact] + public async Task AcceptedJunkArticle_ProducesJ() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "unknown.un.carried"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(Uncarried("unknown.un.carried"), Encoding.ASCII.GetString(evt.Reason.Span)); + } + + [Fact] + public async Task PeerOnlyCatalogueHit_ProducesJWithPeerOnlyReason() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "junk.local"), + "", + news, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + wantTrash: false, + logTrash: true); + + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(PeerOnlyReason("junk.local"), Encoding.ASCII.GetString(evt.Reason.Span)); + } + + [Fact] + public async Task NamedInboundFeed_IsCopiedOntoAcceptedEvent_AndFormattedWithoutSession() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + feed: "BlueWorldHosting"u8.ToArray()); + await RunWorkerAsync(inbound, news, Catalogue("alt.test")); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, evt.Disposition); + Assert.True(evt.Feed.Span.SequenceEqual("BlueWorldHosting"u8)); + Assert.Equal(inbound.Payload.Length, evt.Size); + var line = FormatNews(in evt); + Assert.Contains(" + BlueWorldHosting " + evt.Size + " ?", line, StringComparison.Ordinal); + Assert.DoesNotContain(" + ? ", line, StringComparison.Ordinal); + Assert.EndsWith(" ?\n", line, StringComparison.Ordinal); + } + + [Fact] + public async Task NamedInboundFeed_Junk_KeepsPeerAndOmitsOutboundSites() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.un.carried", + feed: "BlueWorldHosting"u8.ToArray()); + await RunWorkerAsync( + inbound, + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.True(evt.Feed.Span.SequenceEqual("BlueWorldHosting"u8)); + var line = FormatNews(in evt); + Assert.Contains(" j BlueWorldHosting " + evt.Size + " " + Uncarried("unknown.un.carried"), line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public async Task WantTrash_DoesNotRewriteOriginalNewsgroupsHeader() + { + var news = new RecordingNewsLogWriter(); + const string groups = "unknown.un.carried"; + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: groups), + "", + news, + Catalogue("alt.test")); + + var keep = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, keep.Disposition); + Assert.Equal(Uncarried(groups), Encoding.ASCII.GetString(keep.Reason.Span)); + } + + [Fact] + public async Task WantTrash_AppliesToIhaveUnknownGroup() + { + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.IHave, + newsgroups: "not.carried.here"); + await RunWorkerAsync( + inbound, + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var ihaveJunk = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Junk, ihaveJunk.Disposition); + Assert.Equal(Uncarried("not.carried.here"), Encoding.ASCII.GetString(ihaveJunk.Reason.Span)); + Assert.Contains("Newsgroups: not.carried.here"u8, inbound.Payload.Span); + } + + [Fact] + public async Task LogTrashFalse_SuppressesJunkLine_WithoutRejecting() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "unknown.group"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + Assert.Empty(news.Events); + Assert.Equal(0, news.WriteCalls); + } + + [Fact] + public async Task LogTrashFalse_StillLogsNormalAcceptedArticles() + { + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed("", newsgroups: "alt.test"), + "", + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + var stillPlus = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, stillPlus.Disposition); + Assert.True(stillPlus.Reason.IsEmpty); + } + + [Fact] + public async Task MessageId_IsActualArticleMessageId_NotArticleId() + { + const string id = ""; + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed(id, newsgroups: "alt.test")); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued(id, created, Identity(), DateTimeOffset.UtcNow, InboundArticleProducer.TakeThis), + news, + Catalogue("alt.test")); + + Assert.True(Assert.Single(news.Events).MessageId.Span.SequenceEqual(Encoding.ASCII.GetBytes(id))); + Assert.True(created.MessageId.SequenceEqual(Encoding.ASCII.GetBytes(id))); + Span artId = stackalloc byte[ArticleId.Length]; + created.ArtId.CopyTo(artId); + Assert.False(news.Events[0].MessageId.Span.SequenceEqual(artId)); + } + + [Fact] + public async Task TakeThisCanonicalRecord_IsNotReparsedToGenerateNews() + { + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, news, Catalogue("alt.test")); + + Assert.True(inbound.Record.ArtData.Equals(created.ArtData)); + Assert.True(news.Events[0].MessageId.Span.SequenceEqual(created.MessageId)); + Assert.True( + news.Events[0].MessageId.Span.SequenceEqual( + inbound.Record.ArtData.Span.Slice( + inbound.Record.Fields.MessageId.Offset, + inbound.Record.Fields.MessageId.Length))); + } + + [Fact] + public async Task Post_IsNeverClassifiedAsWantTrashJunk() + { + var news = new RecordingNewsLogWriter(); + var created = CreateRecord(CanonicalArticleText.Destuffed("", newsgroups: "unknown.group")); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.Post), + news, + Catalogue("alt.test"), + wantTrash: true, + logTrash: true); + + var post = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Accepted, post.Disposition); + Assert.NotEqual(NewsLogDisposition.Junk, post.Disposition); + Assert.True(post.Reason.IsEmpty); + } + + [Fact] + public async Task NewsEvent_IsProducedAfterEnqueue_NotDuringReceiveAdmission() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + Assert.Equal(0, news.WriteCalls); + + var writer = CreateWriter(queue, news, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(1, news.WriteCalls); + Assert.Equal(1, news.FlushCalls); + Assert.Equal(NewsLogDisposition.Accepted, Assert.Single(news.Events).Disposition); + } + + [Fact] + public async Task Shutdown_FlushesPendingNewsEntries() + { + using var dir = new TempDir(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using var news = new SerilogNewsLogWriter(configuration); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + LogDir = dir.Path, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + news.Dispose(); + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + } + + [Fact] + public async Task NewsWriterFailure_DoesNotReprocessOrSkipPersister() + { + var throwing = new ThrowingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed(""), + "", + throwing, + Catalogue("alt.test")); + + Assert.Equal(1, throwing.WriteCalls); + } + + [Fact] + public void Classify_CarriedNonJunkGroup_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "alt.test"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"))); + } + + [Fact] + public void Classify_UnknownGroup_WantTrash_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Equal(Uncarried("unknown.group"), reason); + } + + [Fact] + public void Classify_MultipleGroups_OneCarriedNonJunk_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group,alt.test,other.unknown"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Null(reason); + } + + [Fact] + public void Classify_MixedCarriedAndPeerOnly_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "alt.test,junk.local"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify( + inbound, + TrashOn(), + CatalogueWith( + ("alt.test", NewsgroupPostingStatus.Allowed), + ("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Null(reason); + } + + [Fact] + public void Classify_MixedUnknownAndPeerOnly_ReportsOnlyPeerOnlyGroups() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group,junk.local,other.unknown"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + TrashOn(), + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.local"), reason); + Assert.DoesNotContain("unknown.group", reason, StringComparison.Ordinal); + Assert.DoesNotContain("other.unknown", reason, StringComparison.Ordinal); + } + + [Fact] + public void Classify_MultiplePeerOnlyGroups_ReportsHeaderOrder() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.beta, junk.alpha"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith( + ("junk.alpha", NewsgroupPostingStatus.PeerOnly), + ("junk.beta", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.beta", "junk.alpha"), reason); + } + + [Fact] + public void Classify_MultipleUnknownGroups_WantTrash_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.one,unknown.two"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify(inbound, TrashOn(), Catalogue("alt.test"), out var reason)); + Assert.Equal(Uncarried("unknown.one", "unknown.two"), reason); + Assert.DoesNotContain("alt.test", reason, StringComparison.Ordinal); + } + + [Fact] + public void Classify_PeerOnlyJunkGroup_IsJunk() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.Equal( + NewsLogDisposition.Junk, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + out var reason)); + Assert.Equal(PeerOnlyReason("junk.local"), reason); + } + + [Fact] + public void Classify_WantTrashFalse_UnknownGroup_IsAccepted() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + Catalogue("alt.test"), + out var reason)); + Assert.Null(reason); + } + + [Fact] + public void TryCreateEvent_WantTrashUnknown_CarriesNewsgroupNotCarried() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.one,unknown.two"); + Assert.True( + IngressNewsDisposition.TryCreateEvent( + inbound, + TrashOn(), + Catalogue("alt.test"), + new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero), + out var evt)); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(Uncarried("unknown.one", "unknown.two"), Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.Equal( + $"Jan 5 00:00:00.000 j ? {evt.Size} newsgroup not carried: unknown.one, unknown.two\n", + FormatNews(in evt)); + } + + [Fact] + public void TryCreateEvent_PeerOnly_CarriesPeerOnlyReason() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.True( + IngressNewsDisposition.TryCreateEvent( + inbound, + new TransitOptions { WantTrash = false, LogTrash = true }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)), + new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero), + out var evt)); + Assert.Equal(NewsLogDisposition.Junk, evt.Disposition); + Assert.Equal(PeerOnlyReason("junk.local"), Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.Equal( + $"Jan 5 00:00:00.000 j ? {evt.Size} peer-only: junk.local\n", + FormatNews(in evt)); + } + + [Fact] + public void IsUncarriedWantTrashRejection_WantTrashFalse_UnknownGroup_IsTrue() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"); + Assert.True( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false, LogTrash = false }, + Catalogue("alt.test"), + out var reason)); + Assert.Equal(Uncarried("unknown.group"), reason); + } + + [Fact] + public void IsUncarriedWantTrashRejection_MultipleUnknown_ReportsHeaderOrder() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.two,unknown.one"); + Assert.True( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false, LogTrash = false }, + Catalogue("alt.test"), + out var reason)); + Assert.Equal(Uncarried("unknown.two", "unknown.one"), reason); + } + + [Fact] + public void IsUncarriedWantTrashRejection_WantTrashTrue_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.IHave, + newsgroups: "unknown.group"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = true }, + Catalogue("alt.test"))); + } + + [Fact] + public void IsUncarriedWantTrashRejection_PeerOnly_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "junk.local"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false }, + CatalogueWith(("junk.local", NewsgroupPostingStatus.PeerOnly)))); + } + + [Fact] + public void IsUncarriedWantTrashRejection_PostOrMissingCatalogue_IsFalse() + { + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.Post, + newsgroups: "unknown.group"); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + inbound, + new TransitOptions { WantTrash = false }, + Catalogue("alt.test"))); + Assert.False( + IngressNewsDisposition.IsUncarriedWantTrashRejection( + CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.group"), + new TransitOptions { WantTrash = false }, + catalogue: null)); + } + + [Fact] + public void Classify_WithoutCatalogue_IsAcceptedRatherThanInventedJunk() + { + var created = CreateRecord(CanonicalArticleText.Destuffed("", newsgroups: "unknown.group")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + Assert.Equal( + NewsLogDisposition.Accepted, + IngressNewsDisposition.Classify(inbound, new TransitOptions { WantTrash = true }, catalogue: null)); + } + + private static async Task RunTakeThisAsync( + string destuffed, + string messageId, + INewsLogWriter news, + INewsgroupCatalogue catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var created = CreateRecord(destuffed); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + messageId, + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + news, + catalogue, + wantTrash, + logTrash); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + INewsLogWriter news, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, news, catalogue, wantTrash, logTrash); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + INewsLogWriter news, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions(), + Transit = new TransitOptions { WantTrash = wantTrash, LogTrash = logTrash }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: catalogue); + + private static ArticleRecord CreateRecord(string destuffed) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(destuffed)); + Assert.True(created.IsAccepted); + return created.Record; + } + + private static string FormatNews(in NewsLogEvent evt) + { + var timestamp = evt.Timestamp == default + ? new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero) + : evt.Timestamp; + var buffer = new byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + return Encoding.ASCII.GetString(buffer.AsSpan(0, written)); + } + + private static string Uncarried(params string[] groups) => + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, groups); + + private static string PeerOnlyReason(params string[] groups) => + IngressNewsReasons.WithGroups(IngressNewsReasons.PeerOnly, groups); + + private static TransitOptions TrashOn() => new() { WantTrash = true, LogTrash = true }; + + private static INewsgroupCatalogue Catalogue(params string[] groups) => + CatalogueWith(groups.Select(static name => (name, NewsgroupPostingStatus.Allowed)).ToArray()); + + private static INewsgroupCatalogue CatalogueWith(params (string Name, NewsgroupPostingStatus Status)[] groups) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + groups.Select(static g => + new NewsgroupDefinition(g.Name, string.Empty, 2, 1, g.Status)).ToArray())); + + private static ConnectionClientIdentity Identity() => + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-news-stop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs new file mode 100644 index 00000000..2d6e318e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolOverviewHandoffTests.cs @@ -0,0 +1,298 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles.OverviewDb; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Ingestion-worker OverviewDB handoff success, requeue, and topology contracts. +[Collection(SerilogCollection.Name)] +public sealed class IncomingSpoolOverviewHandoffTests +{ + [Fact] + public async Task Topology_DeclaresOverviewDbClassicQueue_WithoutExchangeOrBinding() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + var overview = Assert.Single( + connection.QueueDeclarations, + static queue => queue.Name == OverviewDbTopology.QueueName); + Assert.Equal("overviewdb.queue", overview.Name); + Assert.True(overview.Durable); + Assert.False(overview.Exclusive); + Assert.False(overview.AutoDelete); + var arguments = overview.Arguments ?? new Dictionary(StringComparer.Ordinal); + Assert.True(arguments.TryGetValue(OverviewDbTopology.QueueTypeArgumentName, out var queueType)); + Assert.Equal(OverviewDbTopology.QuorumQueueType, queueType); + Assert.False(arguments.ContainsKey("x-message-ttl")); + Assert.False(arguments.ContainsKey("x-expires")); + Assert.Single(arguments); + Assert.DoesNotContain(connection.ExchangeDeclarations, static e => e.Name == OverviewDbTopology.QueueName); + Assert.DoesNotContain(connection.BindingDeclarations, static b => b.Queue == OverviewDbTopology.QueueName); + } + + [Fact] + public async Task PublishSuccess_CompletesWorkerAndPersistsArticle() + { + var overview = new RecordingOverviewDbHandoffPublisher(); + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(1, overview.AttemptCount); + var payload = Assert.Single(overview.Payloads); + var decoded = OverviewArticleV1Codec.Decode(payload); + Assert.True(Encoding.ASCII.GetBytes(inbound.MessageId).AsSpan().SequenceEqual(decoded.MessageId)); + Assert.Equal( + new[] { "alt.test", "rec.test" }.Select(static g => Encoding.ASCII.GetBytes(g)).ToList(), + decoded.Newsgroups); + Assert.Equal((uint)inbound.Record.ArtSize, decoded.Bytes); + Assert.Equal((uint)inbound.Record.ArtLines, decoded.Lines); + } + + [Fact] + public async Task Worker_RecordsOneItemOnPipelineMetrics() + { + var overview = new RecordingOverviewDbHandoffPublisher(); + var pipeline = new IngestionPipelineMetrics(); + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview, pipeline); + + var snapshot = pipeline.CaptureInterval(); + Assert.Equal(1, snapshot.WorkerItems); + Assert.Equal(1, snapshot.ToPublishStart.Count); + Assert.Equal(1, snapshot.Encode.Count); + Assert.Equal(1, snapshot.News.Count); + Assert.True(snapshot.BusyTicks > 0); + } + + [Fact] + public async Task PublishFailure_RequeuesUntilConfirm_ThenCompletes() + { + var overview = new RecordingOverviewDbHandoffPublisher { RemainingFailures = 1 }; + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public async Task UnroutableReturn_RequeuesSameInboundArticle_UntilConfirm() + { + var overview = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new InvalidOperationException( + "RabbitMQ returned the OverviewDB handoff as unroutable."), + }; + var inbound = CreateArticle(); + await RunWorkerAsync(inbound, overview); + + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public async Task PublishFailure_ArticlePersistsBeforeConfirm_AndWorkItemRetries() + { + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var overview = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + BlockOnFailure = block, + }; + var diagnostics = new SignalingFeedDiagnostics(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, overview, diagnostics); + await writer.StartAsync(CancellationToken.None); + var inbound = CreateArticle(); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var end = await diagnostics.Ended.Task.WaitAsync(cts.Token); + + // Article path continues after OverviewDB work enqueue; Rabbit confirm is background. + Assert.False(end.Persisted); + await overview.FirstAttempt.WaitAsync(cts.Token); + Assert.Equal(1, overview.AttemptCount); + Assert.Empty(overview.Payloads); + + block.TrySetResult(); + using var retryWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.AttemptCount < 2 || overview.Payloads.Count < 1) + { + retryWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, retryWait.Token); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + Assert.Equal(2, overview.AttemptCount); + Assert.Single(overview.Payloads); + } + + [Fact] + public void Worker_HasNoOverviewDbRpcOrDatabaseDependency() + { + var ctor = typeof(IncomingSpoolWriterService).GetConstructors().Single(); + var types = ctor.GetParameters().Select(static p => p.ParameterType).ToArray(); + Assert.Contains(typeof(IOverviewDbHandoffPublisher), types); + Assert.DoesNotContain(types, static t => t.Name.Contains("Persister", StringComparison.Ordinal)); + Assert.DoesNotContain(types, static t => t == typeof(IRabbitMqRpcChannel)); + Assert.DoesNotContain(types, static t => t.Name.Contains("NntpDb", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("MySql", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("Http", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(types, static t => t.Name.Contains("Grpc", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Host_RegistersOverviewDbHandoffPublisher_NotAnRpcClient() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var publisher = host.Services.GetRequiredService(); + Assert.IsType(publisher); + Assert.Same(IngestionPipelineMetrics.Shared, host.Services.GetRequiredService()); + Assert.IsNotType(publisher); + var services = host.Services.GetServices().Select(static s => s.GetType()).ToArray(); + Assert.DoesNotContain(services, static t => t.Name.Contains("OverviewDbClient", StringComparison.Ordinal)); + Assert.DoesNotContain(services, static t => t.Name.Contains("OverviewDbRpc", StringComparison.Ordinal)); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + RecordingOverviewDbHandoffPublisher overview, + IngestionPipelineMetrics? pipelineMetrics = null) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, overview, pipelineMetrics: pipelineMetrics); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await overview.Confirmed.WaitAsync(cts.Token); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + IOverviewDbHandoffPublisher overview, + IFeedDiagnostics? feedDiagnostics = null, + IngestionPipelineMetrics? pipelineMetrics = null) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + MaxPublishConcurrency = 1, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + feedDiagnostics: feedDiagnostics, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("rec.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)])), + overviewHandoff: overview, + pipelineMetrics: pipelineMetrics); + + private static InboundArticle CreateArticle() => + CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + body: "overview-body\r\n", + newsgroups: "alt.test,rec.test", + subject: "overview-subject", + from: "poster@example.test", + references: ""); + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + /// Signals spool-work completion for tests. + private sealed class SignalingFeedDiagnostics : IFeedDiagnostics + { + /// Completes when the worker finishes one spool-work attempt. + public TaskCompletionSource<(int Bytes, bool Persisted)> Ended { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + public bool IsEnabled => false; + + /// + public void OnRejected(string peerName, string remote) + { + } + + /// + public FeedSessionProbe? OnAccepted(NntpSession session) => null; + + /// + public void OnReleased(FeedSessionProbe? probe) + { + } + + /// + public void RecordTcpBytes(int bytes) + { + } + + /// + public void BeginSpoolWork() + { + } + + /// + public void EndSpoolWork(int bytes, bool persisted) => Ended.TrySetResult((bytes, persisted)); + + /// + public FeedDiagnosticsSnapshot CaptureSnapshot( + IArticleIngestionQueue queue, + TransitConfigurationStore store) => + FeedDiagnosticsSnapshot.Empty; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs new file mode 100644 index 00000000..d890d63a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IncomingSpoolPathSurveyTests.cs @@ -0,0 +1,299 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Post-queue Path-survey capture from canonical ArticleRecord.Path. +public sealed class IncomingSpoolPathSurveyTests +{ + [Fact] + public async Task CanonicalArticleRecordPath_IsWritten_WithoutSynthesis() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, news, paths, Catalogue("alt.test")); + + var observed = Assert.Single(paths.Paths); + Assert.True(observed.AsSpan().SequenceEqual(inbound.Record.Path)); + var line = Encoding.Latin1.GetString(FormatPath(observed)); + Assert.Equal("Path: " + Encoding.Latin1.GetString(inbound.Record.Path) + "\n", line); + Assert.DoesNotContain("giganews", line, StringComparison.Ordinal); + Assert.StartsWith("Path: ", line, StringComparison.Ordinal); + Assert.Equal(1, news.WriteCalls); + Assert.Equal(NewsLogDisposition.Accepted, Assert.Single(news.Events).Disposition); + } + + [Fact] + public async Task MultipleArticles_ProduceSequentialPathRecords() + { + var paths = new RecordingPathSurveyWriter(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(second, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(2, paths.WriteCalls); + Assert.True(paths.Paths[0].AsSpan().SequenceEqual(first.Record.Path)); + Assert.True(paths.Paths[1].AsSpan().SequenceEqual(second.Record.Path)); + Assert.Equal(1, paths.FlushCalls); + } + + [Fact] + public async Task JunkWithLogTrashFalse_StillWritesPath_AndOmitsNews() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + newsgroups: "unknown.un.carried"); + await RunWorkerAsync( + inbound, + news, + paths, + Catalogue("alt.test"), + wantTrash: true, + logTrash: false); + + Assert.Equal(0, news.WriteCalls); + var observed = Assert.Single(paths.Paths); + Assert.True(observed.AsSpan().SequenceEqual(inbound.Record.Path)); + } + + [Fact] + public async Task PathWriterFailure_DoesNotSkipNewsOrPersister() + { + var throwing = new ThrowingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + await RunTakeThisAsync( + CanonicalArticleText.Destuffed(""), + "", + news, + throwing, + Catalogue("alt.test")); + + Assert.Equal(1, throwing.WriteCalls); + Assert.Equal(1, news.WriteCalls); + } + + [Fact] + public async Task NewsWriterFailure_StillWritesPathAndPersists() + { + var throwingNews = new ThrowingNewsLogWriter(); + var paths = new RecordingPathSurveyWriter(); + var inbound = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis); + await RunWorkerAsync(inbound, throwingNews, paths, Catalogue("alt.test")); + + Assert.Equal(1, throwingNews.WriteCalls); + Assert.True(Assert.Single(paths.Paths).AsSpan().SequenceEqual(inbound.Record.Path)); + } + + [Fact] + public async Task PathIsNotWrittenUntilAfterOverviewConfirm() + { + var paths = new RecordingPathSurveyWriter(); + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + CancellationToken.None)); + Assert.Equal(0, paths.WriteCalls); + Assert.Equal(0, news.WriteCalls); + + var writer = CreateWriter(queue, news, paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(1, paths.WriteCalls); + Assert.Equal(1, news.WriteCalls); + } + + [Fact] + public async Task LargeSequentialInput_IsCountedWithoutRetainingTheStream() + { + var paths = new CountingPathSurveyWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 64 }); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + const int count = 32; + long expectedBytes = 0; + for (var i = 0; i < count; i++) + { + var inbound = CanonicalArticleText.CreateQueued( + $"", + InboundArticleProducer.TakeThis); + expectedBytes += inbound.Record.Path.Length; + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + + Assert.Equal(count, paths.WriteCalls); + Assert.Equal(expectedBytes, paths.Bytes); + Assert.Null(paths.GetType().GetProperty("Paths")); + } + + [Fact] + public async Task Shutdown_FlushesPathSurveyFile() + { + using var dir = new TempDir(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using var paths = new SerilogPathSurveyWriter(configuration); + var created = CreateRecord(CanonicalArticleText.Destuffed("")); + var inbound = ArticleRecordIngress.CreateQueued( + "", + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis); + var writer = CreateWriter(queue, new RecordingNewsLogWriter(), paths, Catalogue("alt.test")); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + paths.Dispose(); + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: " + Encoding.Latin1.GetString(inbound.Record.Path) + "\n", text); + } + + private static async Task RunTakeThisAsync( + string destuffed, + string messageId, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var created = CreateRecord(destuffed); + await RunWorkerAsync( + ArticleRecordIngress.CreateQueued( + messageId, + created, + Identity(), + DateTimeOffset.UtcNow, + InboundArticleProducer.TakeThis), + news, + paths, + catalogue, + wantTrash, + logTrash); + } + + private static async Task RunWorkerAsync( + InboundArticle inbound, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = CreateWriter(queue, news, paths, catalogue, wantTrash, logTrash); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(inbound, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static IncomingSpoolWriterService CreateWriter( + IArticleIngestionQueue queue, + INewsLogWriter news, + IPathSurveyWriter paths, + INewsgroupCatalogue? catalogue, + bool wantTrash = true, + bool logTrash = true) => + new( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions(), + Transit = new TransitOptions { WantTrash = wantTrash, LogTrash = logTrash }, + }), + NullLogger.Instance, + newsLog: news, + catalogue: catalogue, + pathSurvey: paths); + + private static ArticleRecord CreateRecord(string destuffed) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(destuffed)); + Assert.True(created.IsAccepted); + return created.Record; + } + + private static byte[] FormatPath(ReadOnlySpan path) + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + PathSurveyLineFormatter.Write(buffer, path); + return buffer; + } + + private static INewsgroupCatalogue Catalogue(params string[] groups) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + groups.Select(static name => + new NewsgroupDefinition(name, string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)).ToArray())); + + private static ConnectionClientIdentity Identity() => + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-inpaths-stop-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs new file mode 100644 index 00000000..0274f28f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionThroughputCeilingTests.cs @@ -0,0 +1,161 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// +/// Demonstrates that article workers are not serialized behind OverviewDB +/// publisher-confirm latency after the intermediate work-queue handoff, and that +/// the publisher stage itself pipelines confirms asynchronously. +/// +public sealed class IngestionThroughputCeilingTests +{ + private static readonly TimeSpan ConfirmLatency = TimeSpan.FromMilliseconds(18); + + [Fact] + public async Task ArticlePersistPath_IsNotSerializedBehindPublisherConfirmLatency() + { + const int articleCount = 32; + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = articleCount + 8 }, + transitQueueMemoryLimit: 64 * 1024 * 1024); + var overview = new RecordingOverviewDbHandoffPublisher + { + ConfirmDelay = ConfirmLatency, + OutstandingWindow = 100, + }; + using var done = new CountdownEvent(articleCount); + var paths = new CountingPathSurvey(done); + + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 4, + MaxWorkers = 4, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + OverviewDbPublisherBatchSize = 100, + ScaleIntervalSeconds = 3600, + }, + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed), + ])), + overviewHandoff: overview, + pathSurvey: paths); + + await writer.StartAsync(CancellationToken.None); + for (var i = 0; i < articleCount; i++) + { + var inbound = CanonicalArticleText.CreateQueued( + $"", + InboundArticleProducer.TakeThis, + body: "throughput-body\r\n", + newsgroups: "alt.test"); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync(inbound, CancellationToken.None)); + } + + var sw = Stopwatch.StartNew(); + Assert.True(done.Wait(TimeSpan.FromSeconds(5)), "Timed out waiting for Path survey."); + sw.Stop(); + + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(articleCount * ConfirmLatency.TotalMilliseconds / 2), + $"Article persist took {sw.Elapsed.TotalMilliseconds:F0} ms; expected well under serial confirm ceiling."); + + using var confirmWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.Payloads.Count < articleCount) + { + confirmWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, confirmWait.Token); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + Assert.Equal(articleCount, overview.Payloads.Count); + Assert.Equal(articleCount, paths.WriteCalls); + } + + [Fact] + public async Task PublisherStage_PipelinesConfirms_WithOutstandingWindow() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 16 * 1024 * 1024); + var overview = new RecordingOverviewDbHandoffPublisher + { + ConfirmDelay = ConfirmLatency, + OutstandingWindow = 100, + }; + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + OverviewDbPublisherBatchSize = 100, + OverviewDbPublisherShutdownSeconds = 5, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, overview, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + const int count = 40; + var sw = Stopwatch.StartNew(); + for (var i = 0; i < count; i++) + { + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem([(byte)i], $""), + CancellationToken.None)); + } + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (overview.Payloads.Count < count) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + sw.Stop(); + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(count * ConfirmLatency.TotalMilliseconds / 2), + $"Publisher stage took {sw.Elapsed.TotalMilliseconds:F0} ms; expected pipelined throughput."); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private sealed class CountingPathSurvey(CountdownEvent done) : IPathSurveyWriter + { + private int _writeCalls; + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + done.Signal(); + } + + public void Flush() + { + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs new file mode 100644 index 00000000..077ba472 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngestionWorkerPoolTests.cs @@ -0,0 +1,328 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Ingestion worker-pool scaling and lifecycle contracts. +public sealed class IngestionWorkerPoolTests +{ + [Fact] + public async Task Pool_StartsAtMinimumWorkers_AndNeverExceedsMaximum() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 2, + MaxWorkers = 3, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.1, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance, + samplePressure: static () => new IngestionPressureSnapshot(0, 0, 1, 0, 0)); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 2, TimeSpan.FromSeconds(2)); + Assert.Equal(2, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests( + new IngestionPressureSnapshot(1.0, 1, 1, 10, 1), + cts.Token); + Assert.Equal(3, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests( + new IngestionPressureSnapshot(1.0, 1, 1, 10, 1), + cts.Token); + Assert.Equal(3, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, pool.WorkerCount); + } + + [Fact] + public async Task Pool_ScalesDownOnlyAfterSustainedLowPressure_AndNotBelowMinimum() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 3, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.2, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 2, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 1, TimeSpan.FromSeconds(2)); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + Assert.Equal(3, pool.WorkerCount); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(3, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(2, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0, 0, 1, 0, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Pool_Hysteresis_ResetsWhenPressureIsInBand() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 64 * 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 4, + ScaleUpPressureThreshold = 0.6, + ScaleDownPressureThreshold = 0.2, + ScaleUpConsecutiveIntervals = 3, + ScaleDownConsecutiveIntervals = 3, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + static (_, _, _) => Task.CompletedTask, + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => pool.WorkerCount == 1, TimeSpan.FromSeconds(2)); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(0.4, 0, 1, 0, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + Assert.Equal(1, pool.WorkerCount); + Assert.Equal(0, pool.ScaleUpCount); + + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 0), cts.Token); + Assert.Equal(2, pool.WorkerCount); + Assert.Equal(1, pool.ScaleUpCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task MultipleWorkers_DrainQueueWithoutDuplicateOrLoss() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 1024 * 1024); + var processed = new ConcurrentBag(); + var options = new ArticleIngestionOptions + { + MinWorkers = 4, + MaxWorkers = 4, + ScaleIntervalSeconds = 3600, + }; + + for (var i = 0; i < 40; i++) + { + var article = CanonicalArticleText.CreateQueued( + $"<{i}@pool.test>", + InboundArticleProducer.TakeThis, + body: $"body-{i}\r\n"); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + using var cts = new CancellationTokenSource(); + var pool = new IngestionWorkerPool( + queue, + (article, _, _) => + { + processed.Add(article.MessageId); + return Task.CompletedTask; + }, + options, + NullLogger.Instance); + + var run = pool.RunAsync(cts.Token); + await WaitUntilAsync(() => processed.Count == 40, TimeSpan.FromSeconds(5)); + Assert.Equal(40, processed.Count); + Assert.Equal(40, processed.Distinct(StringComparer.Ordinal).Count()); + Assert.Equal(0, queue.Count); + + queue.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task GracefulShutdown_CancellationBetweenItems_PersistsArticlesWhenCountReadsEmpty() + { + var queue = new CountReportsEmptyQueue(); + var processed = new ConcurrentBag(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + for (var i = 0; i < 3; i++) + { + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync( + CanonicalArticleText.CreateQueued( + $"<{i}@shutdown.test>", + InboundArticleProducer.IHave, + "body\r\n"), + CancellationToken.None)); + } + + var pool = new IngestionWorkerPool( + queue, + async (article, _, _) => + { + if (entered.TrySetResult()) + { + await release.Task.ConfigureAwait(false); + } + + processed.Add(article.MessageId); + }, + options, + NullLogger.Instance); + + using var cts = new CancellationTokenSource(); + var run = pool.RunAsync(cts.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + queue.Complete(); + await cts.CancelAsync(); + release.TrySetResult(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(3, processed.Count); + Assert.Equal(3, processed.Distinct(StringComparer.Ordinal).Count()); + } + + [Fact] + public void PressureSnapshot_UsesByteUtilisationAndWaitingProducers() + { + var fromBytes = new IngestionPressureSnapshot(0.5, 50, 100, 3, 0); + Assert.Equal(0.5, fromBytes.Pressure); + + var waiting = IngestionPressureSnapshot.FromQueue( + new ArticleIngestionQueue(new ArticleIngestionOptions(), transitQueueMemoryLimit: 100)); + Assert.Equal(0, waiting.Pressure); + } + + /// + /// Queue whose stays 0 while accepted articles remain readable. + /// + /// + /// Models the approximate count window: an article can already be in the channel + /// before advances. Shutdown must not treat 0 as empty. + /// + private sealed class CountReportsEmptyQueue : IArticleIngestionQueue + { + private readonly ConcurrentQueue _items = new(); + private int _accepting = 1; + + public long MemoryLimitBytes => 1024 * 1024; + + public long QueuedBytes => 0; + + public long PeakQueuedBytes => 0; + + public int MaxArticleBytes => 1024 * 1024; + + public int Count => 0; + + public int PeakCount => 0; + + public bool IsAccepting => Volatile.Read(ref _accepting) != 0; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(article); + cancellationToken.ThrowIfCancellationRequested(); + if (!IsAccepting) + { + return new ValueTask(ArticleEnqueueResult.Unavailable); + } + + _items.Enqueue(article); + return new ValueTask(ArticleEnqueueResult.Accepted); + } + + public bool TryProbeCapacity() => IsAccepting; + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + ArgumentNullException.ThrowIfNull(article); + if (!IsAccepting) + { + return ArticleEnqueueResult.Unavailable; + } + + _items.Enqueue(article); + return ArticleEnqueueResult.Accepted; + } + + public bool TryEnqueue(InboundArticle article) => + TryAdmit(article) == ArticleEnqueueResult.Accepted; + + public void Complete() => Interlocked.Exchange(ref _accepting, 0); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_items.TryDequeue(out var article)) + { + return new ValueTask(article); + } + + return IsAccepting + ? throw new InvalidOperationException("Test queue was read while empty and still accepting.") + : new ValueTask((InboundArticle?)null); + } + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs new file mode 100644 index 00000000..2abc6685 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/IngressNewsReasonsTests.cs @@ -0,0 +1,104 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.Processing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Session.Commands.Posting; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class IngressNewsReasonsTests +{ + [Fact] + public void ForParseFailure_MapsSourceBackedCauses() + { + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.YEncDecodingFailed)); + Assert.Equal( + IngressNewsReasons.MessageIdInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.InvalidMessageId)); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.MissingOrInvalidDate)); + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForParseFailure(NntpArticleParseFailureCode.ArticleTooLarge)); + } + + [Fact] + public void ForExistingRejectDetail_MapsIhaveAndTakethisStrings() + { + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForExistingRejectDetail("rejected too large")); + Assert.Equal( + IngressNewsReasons.ArticleTypeNotPermitted, + IngressNewsReasons.ForExistingRejectDetail("rejected article type")); + Assert.Equal( + IngressNewsReasons.QueueCapacityExceeded, + IngressNewsReasons.ForExistingRejectDetail("rejected exceeds queue budget")); + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record YEncDecodingFailed")); + Assert.Equal( + IngressNewsReasons.MessageIdInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record InvalidMessageId")); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForExistingRejectDetail("rejected article record MissingOrInvalidDate")); + } + + [Fact] + public void ForPostingFailure_KeepsPostFilterClosed_AndMapsKnownCauses() + { + Assert.Equal( + "closed", + IngressNewsReasons.ForPostingFailure(new PostingFailure(PostingFailureCategory.PolicyRejected, "closed"))); + Assert.Equal( + IngressNewsReasons.ArticleTooLarge, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.ArticleTooLarge, "max article size exceeded"))); + Assert.Equal( + IngressNewsReasons.ArticleTypeNotPermitted, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PolicyRejected, "arttype-capability"))); + Assert.Equal( + IngressNewsReasons.QueueCapacityExceeded, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PersistenceFailure, nameof(ArticleEnqueueResult.Rejected)))); + Assert.Equal( + IngressNewsReasons.YEncodingInvalid, + IngressNewsReasons.ForPostingFailure( + new PostingFailure( + PostingFailureCategory.PolicyRejected, + NntpArticleParseFailureCode.YEncDecodingFailed.ToString()))); + Assert.Equal( + IngressNewsReasons.NewsgroupNotCarried, + IngressNewsReasons.ForPostingFailure( + new PostingFailure(PostingFailureCategory.PolicyRejected, "unknown newsgroup"))); + } + + [Fact] + public void WithGroups_JoinsAlreadySelectedNames_WithoutInventingThem() + { + Assert.Equal( + "newsgroup not carried", + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, [])); + Assert.Equal( + "newsgroup not carried: alt.example.foo", + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, ["alt.example.foo"])); + Assert.Equal( + "peer-only: alt.foo, alt.bar", + IngressNewsReasons.WithGroups(IngressNewsReasons.PeerOnly, ["alt.foo", "alt.bar"])); + } + + [Fact] + public void ForArticleRecord_UsesParseCodeNotEnumName() + { + var created = ArticleRecordCreateResult.RejectedParse(NntpArticleParseFailureCode.YEncDecodingFailed); + Assert.Equal(IngressNewsReasons.YEncodingInvalid, IngressNewsReasons.ForArticleRecord(in created)); + Assert.Equal( + IngressNewsReasons.DateInvalid, + IngressNewsReasons.ForMaterializeFailure(NntpArticleCanonicalFailureCode.MissingSelectedDateHeader)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs new file mode 100644 index 00000000..bd18743d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsLogLineFormatterTests.cs @@ -0,0 +1,419 @@ +using System.Text; +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Exact INN news line format from innd(8) LOGGING / ARTlog. +public sealed class NewsLogLineFormatterTests +{ + private static readonly DateTimeOffset ExampleTimestamp = + new(2026, 9, 28, 8, 18, 41, 398, TimeSpan.Zero); + + [Fact] + public void HasOutboundSiteField_IsTrueOnlyForAccepted() + { + Assert.True( + new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Junk, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Rejected, ""u8.ToArray()).HasOutboundSiteField); + Assert.False( + new NewsLogEvent(NewsLogDisposition.Moderated, ""u8.ToArray()).HasOutboundSiteField); + } + + [Fact] + public void Write_Accepted_NamedPeer_UsesInboundFeed_Size_AndOutboundPlaceholder() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + var line = Format(in evt, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + line); + Assert.DoesNotContain(" + ? ", line, StringComparison.Ordinal); + Assert.True(evt.HasOutboundSiteField); + } + + [Fact] + public void Write_Accepted_Giganews_EmitsOutboundPlaceholder() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + size: 793311); + Assert.Equal( + "Sep 28 08:18:41.398 + giganews 793311 ?\n", + Format(in evt, ExampleTimestamp)); + Assert.True(evt.HasOutboundSiteField); + } + + [Fact] + public void Write_Junk_Giganews_OmitsOutboundPlaceholder_ThenReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + reason: "newsgroup not carried: alt.binaries.encryptnzb.hotel"u8.ToArray(), + size: 741116); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal( + "Sep 28 08:18:41.398 j giganews 741116 newsgroup not carried: alt.binaries.encryptnzb.hotel\n", + line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_Giganews_OmitsOutboundPlaceholder_ThenReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 793311); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal( + "Sep 28 08:18:41.398 - giganews 793311 yEncoding invalid\n", + line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Moderated_Giganews_OmitsOutboundPlaceholder_AndHasNoReason() + { + var evt = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + timestamp: ExampleTimestamp, + size: 123456); + var line = Format(in evt, ExampleTimestamp); + Assert.Equal("Sep 28 08:18:41.398 m giganews 123456\n", line); + Assert.False(evt.HasOutboundSiteField); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmptySites_OnJunkRejectedModerated_DoesNotEmitOutboundPlaceholder() + { + var timestamp = ExampleTimestamp; + ReadOnlyMemory emptySites = ReadOnlyMemory.Empty; + var junk = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + reason: "newsgroup not carried: alt.binaries.encryptnzb.hotel"u8.ToArray(), + size: 741116); + var rejected = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 793311); + var moderated = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + emptySites, + timestamp, + size: 123456); + Assert.Equal( + "Sep 28 08:18:41.398 j giganews 741116 newsgroup not carried: alt.binaries.encryptnzb.hotel\n", + Format(in junk, timestamp)); + Assert.Equal( + "Sep 28 08:18:41.398 - giganews 793311 yEncoding invalid\n", + Format(in rejected, timestamp)); + Assert.Equal( + "Sep 28 08:18:41.398 m giganews 123456\n", + Format(in moderated, timestamp)); + Assert.DoesNotContain("?", Format(in junk, timestamp), StringComparison.Ordinal); + Assert.DoesNotContain("?", Format(in rejected, timestamp), StringComparison.Ordinal); + Assert.DoesNotContain("?", Format(in moderated, timestamp), StringComparison.Ordinal); + } + + [Fact] + public void Write_SitesOnJunkRejectedModerated_AreNotRendered() + { + var timestamp = ExampleTimestamp; + var sites = "peer.example"u8.ToArray(); + var junk = Format( + new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + var rejected = Format( + new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + var moderated = Format( + new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + "giganews"u8.ToArray(), + sites, + timestamp, + size: 1), + timestamp); + Assert.Equal("Sep 28 08:18:41.398 j giganews 1\n", junk); + Assert.Equal("Sep 28 08:18:41.398 - giganews 1\n", rejected); + Assert.Equal("Sep 28 08:18:41.398 m giganews 1\n", moderated); + Assert.DoesNotContain("peer.example", junk, StringComparison.Ordinal); + Assert.DoesNotContain("peer.example", rejected, StringComparison.Ordinal); + Assert.DoesNotContain("peer.example", moderated, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_NamedPeer_OmitsOutboundSites_ThenReason() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var id = ""u8.ToArray(); + var withoutReason = new NewsLogEvent( + NewsLogDisposition.Junk, + id, + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + Assert.Equal( + "Sep 27 00:04:04.293 j BlueWorldHosting 1584\n", + Format(in withoutReason, timestamp)); + Assert.DoesNotContain("?", Format(in withoutReason, timestamp), StringComparison.Ordinal); + + var withReason = new NewsLogEvent( + NewsLogDisposition.Junk, + id, + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: unknown.un.carried"u8.ToArray(), + size: 1584); + var line = Format(in withReason, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 j BlueWorldHosting 1584 newsgroup not carried: unknown.un.carried\n", + line); + Assert.DoesNotContain(" 1584 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_NamedPeer_OmitsOutboundSites_ThenReason() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray(), + size: 1584); + var line = Format(in evt, timestamp); + Assert.Equal( + "Sep 27 00:04:04.293 - BlueWorldHosting 1584 yEncoding invalid\n", + line); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + Assert.DoesNotContain("?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_IsSelfContained_DoesNotRequireSessionOrConnection() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "BlueWorldHosting"u8.ToArray(), + timestamp: timestamp, + size: 1584); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + Format(in evt, timestamp)); + } + + [Fact] + public void Write_Accepted_MatchesInnFieldOrder_FeedThenMessageId() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "?"u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:41.839 + ? 0 ?\n", line); + } + + [Fact] + public void Write_Junk_WithoutReason_DoesNotInventOne() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("newsgroup not carried", line, StringComparison.Ordinal); + Assert.DoesNotContain("peer-only", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_SerializesSuppliedReason_InInnFieldOrder() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: alt.example.foo"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 newsgroup not carried: alt.example.foo\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Junk_PeerOnly_SerializesSuppliedReason() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "peer-only: alt.example.foo"u8.ToArray()); + Assert.Equal( + "Jan 5 00:00:00.000 j ? 0 peer-only: alt.example.foo\n", + Format(in evt, timestamp)); + } + + [Fact] + public void Write_Junk_DoesNotDeriveGroupNames() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Junk, + ""u8.ToArray(), + timestamp: timestamp, + reason: "newsgroup not carried: supplied.group"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 newsgroup not carried: supplied.group\n", line); + Assert.DoesNotContain("Newsgroups", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_SerializesSuppliedUncarriedGroupReason() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + timestamp: timestamp, + responseCode: 437, + reason: "newsgroup not carried: alt.example.foo"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Jan 5 00:00:00.000 - ? 0 newsgroup not carried: alt.example.foo\n", line); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Moderated_UsesM_NotPlusOrUnknown() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Moderated, + ""u8.ToArray(), + timestamp: timestamp); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:41.839 m ? 0\n", line); + Assert.DoesNotContain(" + ", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.Equal((byte)'m', (byte)NewsLogDisposition.Moderated); + Assert.NotEqual((byte)'?', (byte)NewsLogDisposition.Moderated); + } + + [Fact] + public void Write_RejectedYEncoding_OmitsResponseCode_AndUsesInnFieldOrder() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + "<23k82@bar.net>"u8.ToArray(), + timestamp: timestamp, + responseCode: 439, + reason: "yEncoding invalid"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:54.638 - ? <23k82@bar.net> 0 yEncoding invalid\n", line); + Assert.DoesNotContain("439", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_Rejected_DoesNotRenderResponseCode() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Rejected, + ""u8.ToArray(), + timestamp: timestamp, + responseCode: 437, + reason: "article too large"u8.ToArray()); + var line = Format(in evt, timestamp); + Assert.Equal("Aug 25 13:37:54.638 - ? 0 article too large\n", line); + Assert.DoesNotContain("437", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_FutureSites_AreAppendedWithoutInventingThem() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new NewsLogEvent( + NewsLogDisposition.Accepted, + ""u8.ToArray(), + "news.server.fr"u8.ToArray(), + "a.peer other.server.org"u8.ToArray(), + timestamp); + var line = Format(in evt, timestamp); + Assert.Equal( + "Aug 25 13:37:41.839 + news.server.fr 0 a.peer other.server.org\n", + line); + } + + private static string Format(in NewsLogEvent evt, DateTimeOffset timestamp) + { + Span buffer = stackalloc byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + return Encoding.ASCII.GetString(buffer[..written]); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsTestConfiguration.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsTestConfiguration.cs new file mode 100644 index 00000000..da951e0d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/NewsTestConfiguration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +internal static class NewsTestConfiguration +{ + public static IConfiguration Create( + string directory, + string rollingInterval = "Day", + int retainedFileCountLimit = 14, + bool buffered = false, + long? fileSizeLimitBytes = null, + bool rollOnFileSizeLimit = false, + int bufferSize = 1000, + bool blockWhenFull = true) + { + Directory.CreateDirectory(directory); + return new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = directory, + [NntpdNewsLogging.SectionName + ":path"] = Path.Join(directory, "news-.log"), + [NntpdNewsLogging.SectionName + ":rollingInterval"] = rollingInterval, + [NntpdNewsLogging.SectionName + ":retainedFileCountLimit"] = retainedFileCountLimit.ToString(), + [NntpdNewsLogging.SectionName + ":buffered"] = buffered ? "true" : "false", + [NntpdNewsLogging.SectionName + ":rollOnFileSizeLimit"] = rollOnFileSizeLimit ? "true" : "false", + [NntpdNewsLogging.SectionName + ":fileSizeLimitBytes"] = fileSizeLimitBytes?.ToString(), + [NntpdNewsLogging.SectionName + ":bufferSize"] = bufferSize.ToString(), + [NntpdNewsLogging.SectionName + ":blockWhenFull"] = blockWhenFull ? "true" : "false", + }) + .Build(); + } + + public static string ReadNewsFile(string directory) + { + var files = Directory.GetFiles(directory, "news*") + .Where(static path => !path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (files.Length == 0) + { + return string.Empty; + } + + using var stream = new FileStream(files[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs new file mode 100644 index 00000000..305bf03a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbHandoffPublisherTests.cs @@ -0,0 +1,352 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// OverviewDB async publisher-confirm and AMQP property contracts. +public sealed class OverviewDbHandoffPublisherTests +{ + [Fact] + public async Task PublishAsync_SetsRequiredAmqpProperties_AndUsesDefaultExchange() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 10 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("overview-one"u8.ToArray(), ""), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("overview-two"u8.ToArray(), ""), CancellationToken.None); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Empty(connection.RpcChannels); + Assert.Empty(connection.PublishChannels); + var channel = Assert.Single(connection.AsyncConfirmPublishChannels); + Assert.Equal(2, channel.Publications.Count); + + Assert.All( + channel.Publications, + publication => + { + Assert.Equal(OverviewDbTopology.DefaultExchange, publication.Exchange); + Assert.Equal(OverviewDbTopology.QueueName, publication.RoutingKey); + Assert.Equal(nntpd.Fqdn, publication.AppId); + Assert.Equal(OverviewDbTopology.ExpirationMilliseconds, publication.Expiration); + Assert.True(publication.Persistent); + Assert.True(publication.Mandatory); + Assert.True(Guid.TryParse(publication.MessageId, out var messageId)); + Assert.NotEqual(Guid.Empty, messageId); + }); + + Assert.NotEqual(channel.Publications[0].MessageId, channel.Publications[1].MessageId); + Assert.Equal("overview-one"u8.ToArray(), channel.Publications[0].Body); + Assert.Equal("overview-two"u8.ToArray(), channel.Publications[1].Body); + } + + [Fact] + public void CreateHandoffProperties_SetsPerMessageExpirationAndSequenceHeader() + { + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + publishSequenceNumber: 42); + + Assert.Equal("2000", properties.Expiration); + Assert.Equal("11111111-1111-1111-1111-111111111111", properties.MessageId); + Assert.Equal("nntpd01.usenet.ninja", properties.AppId); + Assert.Equal(DeliveryModes.Persistent, properties.DeliveryMode); + Assert.True(properties.Persistent); + Assert.NotNull(properties.Headers); + // UInt64 is not a valid AMQP table value in RabbitMQ.Client 7.2.2. + Assert.IsType(properties.Headers[Constants.PublishSequenceNumberHeader]); + Assert.Equal("42", properties.Headers[Constants.PublishSequenceNumberHeader]); + } + + [Theory] + [InlineData(0UL)] + [InlineData(1UL)] + [InlineData(42UL)] + [InlineData(9_007_199_254_740_992UL)] // 2^53 — past float mantissa exactness + [InlineData(ulong.MaxValue)] + public void PublishSequenceHeader_RoundTripsLosslessThroughAmqpSupportedRepresentations( + ulong sequence) + { + var formatted = RabbitMqClientAsyncConfirmPublishChannel.FormatPublishSequenceNumberHeader( + sequence); + Assert.Equal(sequence.ToString(System.Globalization.CultureInfo.InvariantCulture), formatted); + + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + sequence); + Assert.True(OverviewDbHandoffPublisher.TryReadPublishSequence(properties, out var fromString)); + Assert.Equal(sequence, fromString); + + // Broker Basic.Return typically echoes longstr ('S') as byte[]. + var asBytes = System.Text.Encoding.ASCII.GetBytes(formatted); + Assert.True( + OverviewDbHandoffPublisher.TryParsePublishSequenceNumberHeader(asBytes, out var fromBytes)); + Assert.Equal(sequence, fromBytes); + + // RabbitMQ.Client HandleReturn also accepts long when the value fits. + if (sequence <= long.MaxValue) + { + Assert.True( + OverviewDbHandoffPublisher.TryParsePublishSequenceNumberHeader( + (long)sequence, + out var fromLong)); + Assert.Equal(sequence, fromLong); + } + } + + [Fact] + public void PublishSequenceHeader_UInt64IsNotUsedAsAmqpTableValue() + { + var properties = RabbitMqClientAsyncConfirmPublishChannel.CreateHandoffProperties( + "11111111-1111-1111-1111-111111111111", + "nntpd01.usenet.ninja", + OverviewDbTopology.ExpirationMilliseconds, + ulong.MaxValue); + + var header = Assert.IsType( + properties.Headers![Constants.PublishSequenceNumberHeader]); + Assert.Equal("18446744073709551615", header); + Assert.NotEqual(typeof(ulong), header.GetType()); + } + + [Fact] + public async Task BasicReturn_WithStringSequenceHeader_CorrelatesCorrectWorkItem() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var first = new OverviewDbWorkItem("one"u8.ToArray(), ""); + var second = new OverviewDbWorkItem("two"u8.ToArray(), ""); + await publisher.PublishAsync(first, CancellationToken.None); + await publisher.PublishAsync(second, CancellationToken.None); + + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + // Return only sequence 2 (string header via FakeRabbitMq); ack both. + await channel.RaiseReturnAsync(2, second.Payload); + await channel.RaiseAckAsync(1, multiple: false); + await channel.RaiseAckAsync(2, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(second, failed); + Assert.False(publisher.TryDequeuePublishFailure(out _)); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task PublishAsync_DoesNotWaitForConfirm_BeforeReturning() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + connection.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = true; + channel.AutoAckDelay = TimeSpan.FromMilliseconds(50); + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 100 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < 20; i++) + { + await publisher.PublishAsync( + new OverviewDbWorkItem([(byte)i], $"<{i}@test>"), + CancellationToken.None); + } + + sw.Stop(); + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(200), + $"PublishAsync serialized behind confirms: {sw.Elapsed.TotalMilliseconds:F0} ms"); + Assert.True(publisher.OutstandingCount > 0); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + } + + [Fact] + public async Task OutstandingWindow_BoundsUnconfirmedPublishes() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + connection.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = false; + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 2 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("a"u8.ToArray(), ""), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("b"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(2, publisher.OutstandingCount); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + await Assert.ThrowsAnyAsync( + () => publisher.PublishAsync(new OverviewDbWorkItem("c"u8.ToArray(), ""), cts.Token)); + + var channel = Assert.Single(connection.AsyncConfirmPublishChannels); + await channel.RaiseAckAsync(2, multiple: true); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task MultipleAck_SettlesAllSequencesUpToDeliveryTag() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 10 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("1"u8.ToArray(), "<1@test>"), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("2"u8.ToArray(), "<2@test>"), CancellationToken.None); + await publisher.PublishAsync(new OverviewDbWorkItem("3"u8.ToArray(), "<3@test>"), CancellationToken.None); + Assert.Equal(3, publisher.OutstandingCount); + + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseAckAsync(3, multiple: true); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task Nack_SurfacesFailureForRequeue() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var item = new OverviewDbWorkItem("nack"u8.ToArray(), ""); + await publisher.PublishAsync(item, CancellationToken.None); + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseNackAsync(1, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(item, failed); + Assert.Equal(0, publisher.OutstandingCount); + } + + [Fact] + public async Task BasicReturn_ThenAck_SurfacesFailure_NotSilentSuccess() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + var item = new OverviewDbWorkItem("ret"u8.ToArray(), ""); + await publisher.PublishAsync(item, CancellationToken.None); + var channel = Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + await channel.RaiseReturnAsync(1, item.Payload); + await channel.RaiseAckAsync(1, multiple: false); + + Assert.True(publisher.TryDequeuePublishFailure(out var failed)); + Assert.Same(item, failed); + Assert.False(publisher.TryDequeuePublishFailure(out _)); + } + + [Fact] + public async Task AbandonOutstanding_ReleasesWindow_WithoutInfiniteRetry() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = c => c.AutoAck = false; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 1 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + await publisher.PublishAsync(new OverviewDbWorkItem("x"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(1, publisher.OutstandingCount); + publisher.AbandonOutstanding(); + Assert.Equal(0, publisher.OutstandingCount); + + await publisher.PublishAsync(new OverviewDbWorkItem("y"u8.ToArray(), ""), CancellationToken.None); + Assert.Equal(1, publisher.OutstandingCount); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs new file mode 100644 index 00000000..c7243161 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublishConcurrencyTests.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Async outstanding-confirmation window contracts for OverviewDB publishing. +public sealed class OverviewDbPublishConcurrencyTests +{ + [Fact] + public async Task OutstandingWindow_AllowsPipelinedPublishes_OnSingleChannel() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + await rabbit.StartAsync(CancellationToken.None); + + factory.LastConnection!.ConfigureAsyncConfirmPublishChannel = channel => + { + channel.AutoAck = true; + channel.AutoAckDelay = TimeSpan.FromMilliseconds(18); + }; + + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.ArticleIngestion = new ArticleIngestionOptions { OverviewDbPublisherBatchSize = 100 }; + await using var publisher = new OverviewDbHandoffPublisher( + rabbit, + Options.Create(nntpd), + Options.Create(RabbitMqOptionsTests.CreateValid())); + + const int count = 50; + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < count; i++) + { + await publisher.PublishAsync( + new OverviewDbWorkItem([(byte)i], $"<{i}@test>"), + CancellationToken.None); + } + + sw.Stop(); + + // Serial confirm model would need >= 50 * 18ms = 900ms. Pipelined write path must be far lower. + Assert.True( + sw.Elapsed < TimeSpan.FromMilliseconds(count * 18 / 2), + $"Pipelined publish took {sw.Elapsed.TotalMilliseconds:F0} ms; expected well under serial confirm ceiling."); + + Assert.Single(factory.LastConnection.AsyncConfirmPublishChannels); + Assert.Equal(count, factory.LastConnection.AsyncConfirmPublishChannels[0].Publications.Count); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.OutstandingCount > 0) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs new file mode 100644 index 00000000..502dabeb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/OverviewDbPublisherStageTests.cs @@ -0,0 +1,340 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client.Exceptions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// OverviewDB intermediate work-queue and publisher-pool contracts. +public sealed class OverviewDbPublisherStageTests +{ + [Fact] + public async Task WorkQueue_BackpressuresWhenByteBudgetExhausted() + { + var queue = new OverviewDbWorkQueue(memoryLimitBytes: 10); + var first = new OverviewDbWorkItem(new byte[8], ""); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(first, CancellationToken.None)); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + var second = new OverviewDbWorkItem(new byte[8], ""); + await Assert.ThrowsAnyAsync( + () => queue.EnqueueAsync(second, cts.Token).AsTask()); + Assert.True(queue.WaitingProducerCount >= 0); + Assert.Equal(1, queue.Count); + } + + [Fact] + public async Task PublisherPool_PublishesAndAwaitsConfirm_WithoutArticleQueue() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + MaxPublishConcurrency = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool( + work, + publisher, + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("abc"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(1, publisher.AttemptCount); + Assert.Single(publisher.Payloads); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_RequeuesWorkItemOnPublishFailure() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher { RemainingFailures = 1 }; + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("retry"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(2, publisher.AttemptCount); + Assert.Single(publisher.Payloads); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_WireFormattingException_IsPublishFailure_NotQueueUnavailable() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new WireFormattingException( + "Value of type 'UInt64' cannot appear as table value"), + }; + var logger = new CollectingLogger(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, logger); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem("wire"u8.ToArray(), ""), + CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Payloads.Count < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + Assert.Equal(2, publisher.AttemptCount); + Assert.Contains( + logger.Messages, + m => m.Contains("OverviewDB RabbitMQ handoff failed", StringComparison.Ordinal)); + Assert.DoesNotContain( + logger.Messages, + m => m.Contains("queue is unavailable", StringComparison.Ordinal)); + Assert.Contains( + logger.Messages, + m => m.Contains("was requeued", StringComparison.Ordinal)); + + work.Complete(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_RequeueUnavailable_OnlyWhenWorkQueueUnavailable() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var publisher = new RecordingOverviewDbHandoffPublisher + { + RemainingFailures = 1, + TransientPublishException = new WireFormattingException("local serialize failure"), + BlockOnFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var logger = new CollectingLogger(); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, logger); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync( + new OverviewDbWorkItem("gone"u8.ToArray(), ""), + CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.AttemptCount < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + // Complete the work queue before the publish failure requeue runs. + work.Complete(); + publisher.BlockOnFailure.TrySetResult(); + + using var logWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!logger.Messages.Exists( + m => m.Contains("queue is unavailable", StringComparison.Ordinal))) + { + logWait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, logWait.Token); + } + + Assert.Contains( + logger.Messages, + m => m.Contains("OverviewDB RabbitMQ handoff failed", StringComparison.Ordinal)); + Assert.Contains( + logger.Messages, + m => m.Contains("queue is unavailable", StringComparison.Ordinal)); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_ScalesIndependentlyOfArticleWorkerBounds() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024); + var options = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + OverviewDbMinPublisherWorkers = 2, + OverviewDbMaxPublisherWorkers = 4, + ScaleUpPressureThreshold = 0.5, + ScaleDownPressureThreshold = 0.1, + ScaleUpConsecutiveIntervals = 1, + ScaleDownConsecutiveIntervals = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool( + work, + new RecordingOverviewDbHandoffPublisher(), + options, + NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal(2, pool.WorkerCount); + pool.ApplyScaleDecisionForTests(new IngestionPressureSnapshot(1, 1, 1, 1, 1), cts.Token); + Assert.Equal(3, pool.WorkerCount); + + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task PublisherPool_DrainsQueuedWorkOnComplete_BeforeExit() + { + var work = new OverviewDbWorkQueue(memoryLimitBytes: 1024 * 1024); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var publisher = new GatedOverviewDbHandoffPublisher(gate); + var options = new ArticleIngestionOptions + { + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }; + + using var cts = new CancellationTokenSource(); + var pool = new OverviewDbPublisherPool(work, publisher, options, NullLogger.Instance); + var run = pool.RunAsync(cts.Token); + + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("drain-a"u8.ToArray(), ""), CancellationToken.None)); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await work.EnqueueAsync(new OverviewDbWorkItem("drain-b"u8.ToArray(), ""), CancellationToken.None)); + + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (publisher.Started < 1) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, wait.Token); + } + + work.Complete(); + gate.TrySetResult(); + await cts.CancelAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, publisher.ConfirmCount); + Assert.Equal(0, work.Count); + } + + private sealed class GatedOverviewDbHandoffPublisher(TaskCompletionSource gate) : IOverviewDbHandoffPublisher + { + private int _started; + private int _confirms; + + public int Started => Volatile.Read(ref _started); + + public int ConfirmCount => Volatile.Read(ref _confirms); + + public int OutstandingCount => 0; + + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + Interlocked.Increment(ref _started); + await gate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _confirms); + } + + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + item = null!; + return false; + } + + public void AbandonOutstanding() + { + } + } + + private sealed class CollectingLogger : ILogger + { + private readonly object _gate = new(); + + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + lock (_gate) + { + Messages.Add(formatter(state, exception)); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs new file mode 100644 index 00000000..1b0dd7cd --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyLineFormatterTests.cs @@ -0,0 +1,78 @@ +using System.Text; +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// Exact INN-style Path-survey line from canonical ArticleRecord.Path bytes. +public sealed class PathSurveyLineFormatterTests +{ + [Fact] + public void Write_CopiesCanonicalPathBytes_WithPrefixAndLf() + { + var path = "news.usenet.ninja!nntpd01.usenet.ninja!peer.example"u8; + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + var written = PathSurveyLineFormatter.Write(buffer, path); + Assert.Equal(buffer.Length, written); + Assert.Equal("Path: news.usenet.ninja!nntpd01.usenet.ninja!peer.example\n", Encoding.Latin1.GetString(buffer)); + } + + [Fact] + public void Write_PreservesExactPathBytes_WithoutInventingHops() + { + var path = "foo!bar!not-for-mail"u8; + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + PathSurveyLineFormatter.Write(buffer, path); + var line = Encoding.Latin1.GetString(buffer); + Assert.Equal("Path: foo!bar!not-for-mail\n", line); + Assert.DoesNotContain("giganews", line, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd01", line, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmptyPath_FollowsArticleRecordEmptySpanSemantics() + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength([])]; + var written = PathSurveyLineFormatter.Write(buffer, []); + Assert.Equal(7, written); + Assert.Equal("Path: \n", Encoding.Latin1.GetString(buffer)); + } + + [Fact] + public void Write_MultipleCalls_AreIndependentSequentialRecords() + { + using var stream = new MemoryStream(); + WriteOne(stream, "alpha!one"u8); + WriteOne(stream, "beta!two"u8); + Assert.Equal("Path: alpha!one\nPath: beta!two\n", Encoding.Latin1.GetString(stream.ToArray())); + } + + [Fact] + public void Write_LargeSequentialInput_DoesNotMaterializeTheCompleteStream() + { + var path = "hop.example!not-for-mail"u8; + var required = PathSurveyLineFormatter.RequiredLength(path); + var buffer = new byte[required]; + const int count = 20_000; + using var stream = new FileStream( + Path.Join(Path.GetTempPath(), "vectornntp-inpaths-seq-" + Guid.NewGuid().ToString("N")), + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 64 * 1024, + FileOptions.DeleteOnClose); + for (var i = 0; i < count; i++) + { + var written = PathSurveyLineFormatter.Write(buffer, path); + stream.Write(buffer.AsSpan(0, written)); + } + + Assert.Equal((long)required * count, stream.Length); + } + + private static void WriteOne(Stream stream, ReadOnlySpan path) + { + var buffer = new byte[PathSurveyLineFormatter.RequiredLength(path)]; + var written = PathSurveyLineFormatter.Write(buffer, path); + stream.Write(buffer.AsSpan(0, written)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs new file mode 100644 index 00000000..d958c70f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PathSurveyTestConfiguration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +internal static class PathSurveyTestConfiguration +{ + public static IConfiguration Create( + string directory, + string rollingInterval = "Day", + int retainedFileCountLimit = 1, + bool buffered = false, + long? fileSizeLimitBytes = null, + bool rollOnFileSizeLimit = false, + int bufferSize = 1000, + bool blockWhenFull = true) + { + Directory.CreateDirectory(directory); + return new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = directory, + [NntpdPathSurveyLogging.SectionName + ":path"] = Path.Join(directory, "inpaths-.log"), + [NntpdPathSurveyLogging.SectionName + ":rollingInterval"] = rollingInterval, + [NntpdPathSurveyLogging.SectionName + ":retainedFileCountLimit"] = retainedFileCountLimit.ToString(), + [NntpdPathSurveyLogging.SectionName + ":buffered"] = buffered ? "true" : "false", + [NntpdPathSurveyLogging.SectionName + ":rollOnFileSizeLimit"] = rollOnFileSizeLimit ? "true" : "false", + [NntpdPathSurveyLogging.SectionName + ":fileSizeLimitBytes"] = fileSizeLimitBytes?.ToString(), + [NntpdPathSurveyLogging.SectionName + ":bufferSize"] = bufferSize.ToString(), + [NntpdPathSurveyLogging.SectionName + ":blockWhenFull"] = blockWhenFull ? "true" : "false", + }) + .Build(); + } + + public static string ReadInpathsFile(string directory) + { + var files = Directory.GetFiles(directory, "inpaths*") + .Where(static path => !path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (files.Length == 0) + { + return string.Empty; + } + + using var stream = new FileStream(files[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs new file mode 100644 index 00000000..10557735 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/PostPathSurveyDiscardTests.cs @@ -0,0 +1,151 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// +/// After Path survey the ingestion worker discards the article and does not +/// persist or place it. +/// +public sealed class PostPathSurveyDiscardTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.Parse("2026-10-04T00:00:00Z"); + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task AfterPathSurvey_DoesNotPersistSelectOrPlace(InboundArticleProducer producer) + { + var observed = await RunAsync( + CanonicalArticleText.CreateQueued($"", producer)); + + Assert.Equal(0, observed.Registry.ActiveCalls); + Assert.Equal(0, observed.Placement.Calls); + Assert.Equal(1, observed.Paths.WriteCalls); + Assert.Single(observed.Overview.Payloads); + Assert.Equal(0, observed.Queue.Count); + } + + [Fact] + public async Task SubsequentArticle_IsProcessedAfterTheFirstIsDiscarded() + { + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var observed = await RunAsync(first, second); + + Assert.Equal(0, observed.Registry.ActiveCalls); + Assert.Equal(0, observed.Placement.Calls); + Assert.Equal(2, observed.Paths.WriteCalls); + Assert.Equal(2, observed.Overview.Payloads.Count); + Assert.Equal(0, observed.Queue.Count); + } + + private static async Task RunAsync(params InboundArticle[] articles) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var placement = new CountingPlacement(); + var registry = new CountingRegistry(EligibleRegistry()); + var paths = new CountingPathSurveyWriter(); + var overview = new RecordingOverviewDbHandoffPublisher(); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = 1, + MaxWorkers = 1, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + ScaleIntervalSeconds = 3600, + }, + }), + NullLogger.Instance, + overviewHandoff: overview, + pathSurvey: paths, + placement: placement, + placementRegistry: registry); + + await writer.StartAsync(CancellationToken.None); + foreach (var article in articles) + { + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + queue.Complete(); + // Confirms are recorded after PublishAsync returns, while the worker is still + // running. Waiting here, before StopAsync, observes that signal. StopAsync's + // own budget must not be the observation window: once it expires, StopAsync + // abandons outstanding OverviewDB work and the confirm never arrives. + using var confirmed = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await overview.WaitUntilConfirmedAsync(articles.Length, confirmed.Token); + using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await writer.StopAsync(stop.Token); + return new Observed(queue, placement, registry, paths, overview); + } + + private static StorageServerRegistry EligibleRegistry() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.example", 10_000, 0, 10_000, Now, 563), + Now); + return registry; + } + + private sealed record Observed( + ArticleIngestionQueue Queue, + CountingPlacement Placement, + CountingRegistry Registry, + CountingPathSurveyWriter Paths, + RecordingOverviewDbHandoffPublisher Overview); + + private sealed class CountingPlacement : IArticlePlacementClient + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _calls); + return ValueTask.FromResult(new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + } + } + + private sealed class CountingRegistry(StorageServerRegistry inner) : IStorageServerRegistry + { + private int _activeCalls; + + public int ActiveCalls => Volatile.Read(ref _activeCalls); + + public void ApplyAdvertisement(StorageServerAdvertisement advertisement, DateTimeOffset receivedAtUtc) => + inner.ApplyAdvertisement(advertisement, receivedAtUtc); + + public bool TryGet(string fqdn, out StorageServerFleetEntry entry) => inner.TryGet(fqdn, out entry); + + public IReadOnlyList Snapshot() => inner.Snapshot(); + + public IReadOnlyList GetActive(DateTimeOffset utcNow) + { + Interlocked.Increment(ref _activeCalls); + return inner.GetActive(utcNow); + } + + public void ApplyLifecycle(StorageServerLifecycleAnnouncement announcement, DateTimeOffset receivedAtUtc) => + inner.ApplyLifecycle(announcement, receivedAtUtc); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/QueueDrain20Tests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/QueueDrain20Tests.cs new file mode 100644 index 00000000..1525e5c1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/QueueDrain20Tests.cs @@ -0,0 +1,438 @@ +using System.Collections.Concurrent; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +/// +/// True concurrent consumers. +/// Ownership is released inside dequeue; workers must not release again. +/// +public sealed class QueueDrain20Tests +{ + internal const int WorkerCount = 20; + + [Fact] + public async Task TwoWorkers_ConsumeEveryArticleExactlyOnce() + { + await DrainExactlyOnceAsync(workers: 2, articleCount: 100); + } + + [Fact] + public async Task TwentyWorkers_ConsumeEveryArticleExactlyOnce() + { + await DrainExactlyOnceAsync(workers: WorkerCount, articleCount: 200); + } + + [Fact] + public async Task TwentyWorkers_CallDequeueAsyncConcurrently_AndAllParticipate() + { + const int articleCount = 400; + var queue = CreateQueue(articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[WorkerCount]; + var workers = StartWorkers(queue, WorkerCount, seen, hits); + EnqueueUnique(queue, articleCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(articleCount, hits.Sum()); + Assert.True(hits.Count(static h => h > 0) > 1); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_WaitingThenEnqueue_AllExitAfterComplete() + { + const int articleCount = 200; + var queue = CreateQueue(articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[WorkerCount]; + var workers = StartWorkers(queue, WorkerCount, seen, hits); + + var ids = EnqueueUnique(queue, articleCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.True(ids.SetEquals(seen.Keys)); + Assert.Equal(articleCount, hits.Sum()); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task ConcurrentConsumers_EachWorkerReceivesIncreasingFifoSubsequence() + { + const int articleCount = 200; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount, idPrefix: "seq"); + var perWorker = new List[WorkerCount]; + for (var i = 0; i < WorkerCount; i++) + { + perWorker[i] = []; + } + + var workers = new Task[WorkerCount]; + for (var i = 0; i < WorkerCount; i++) + { + var workerIndex = i; + workers[workerIndex] = Task.Run(async () => + { + while (await queue.DequeueAsync(CancellationToken.None) is { } article) + { + var start = article.MessageId.IndexOf('.') + 1; + var end = article.MessageId.IndexOf('@'); + var seq = int.Parse(article.MessageId.AsSpan(start, end - start)); + perWorker[workerIndex].Add(seq); + } + }); + } + + queue.Complete(); + await WaitWorkersAsync(workers); + + var all = new HashSet(); + foreach (var ordered in perWorker.Select(static bag => bag.ToArray())) + { + for (var i = 1; i < ordered.Length; i++) + { + Assert.True(ordered[i] > ordered[i - 1], "per-worker dequeue subsequence must be increasing FIFO"); + } + + foreach (var seq in ordered) + { + Assert.True(all.Add(seq)); + } + } + + Assert.Equal(articleCount, all.Count); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_AccountingReturnsToZero_AndPeaksMatchPrefill() + { + const int articleCount = 80; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount); + var expectedBytes = queue.QueuedBytes; + + Assert.Equal(expectedBytes, queue.QueuedBytes); + Assert.Equal(articleCount, queue.Count); + Assert.Equal(expectedBytes, queue.PeakQueuedBytes); + Assert.Equal(articleCount, queue.PeakCount); + + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.Equal(expectedBytes, queue.PeakQueuedBytes); + Assert.Equal(articleCount, queue.PeakCount); + Assert.False(queue.IsAccepting); + } + + [Fact] + public async Task TwentyWorkers_CompleteEmptyQueue_UnblocksEveryWorker() + { + var queue = CreateQueue(1); + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + } + + [Fact] + public async Task TwentyWorkers_Cancellation_UnblocksWithoutReservationLeak() + { + var queue = CreateQueue(1); + using var cts = new CancellationTokenSource(); + var workers = StartWorkers(queue, WorkerCount, cancellationToken: cts.Token); + + await cts.CancelAsync(); + await WaitWorkersAsync(workers); + + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.True(queue.IsAccepting); + } + + [Fact] + public async Task CancelledConsumer_DoesNotCorruptAccounting_RemainingWorkersDrain() + { + const int articleCount = 120; + var queue = CreateQueue(articleCount); + EnqueueUnique(queue, articleCount); + + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + using var cancelled = new CancellationTokenSource(); + var tasks = new List(WorkerCount); + tasks.AddRange(StartWorkers(queue, 5, seen, cancellationToken: cancelled.Token)); + tasks.AddRange(StartWorkers(queue, 15, seen)); + + await cancelled.CancelAsync(); + queue.Complete(); + await WaitWorkersAsync(tasks.ToArray()); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task TwentyWorkers_CancelDuringDrain_DoesNotDoubleRelease() + { + const int articleCount = 100; + var queue = CreateQueue(articleCount); + var sample = Article(""); + EnqueueUnique(queue, articleCount); + + using var cts = new CancellationTokenSource(); + var consumed = new ConcurrentDictionary(StringComparer.Ordinal); + var workers = StartWorkers(queue, WorkerCount, consumed, cancellationToken: cts.Token); + await cts.CancelAsync(); + await WaitWorkersAsync(workers); + + var remainingBytes = queue.QueuedBytes; + var remainingCount = queue.Count; + Assert.True(remainingBytes >= 0); + Assert.True(remainingCount >= 0); + Assert.Equal(remainingCount * sample.Payload.Length, remainingBytes); + Assert.Equal(0, queue.DebugWaiterCount); + Assert.All(consumed.Values, static count => Assert.Equal(1, count)); + + queue.Complete(); + var leftover = 0; + while (await queue.DequeueAsync(CancellationToken.None) is not null) + { + leftover++; + } + + Assert.Equal(remainingCount, leftover); + Assert.Equal(articleCount, consumed.Count + leftover); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TwentyWorkers_CompleteAfterPartialPrefill_DrainsAndRejectsNewAdmits() + { + var queue = CreateQueue(5); + EnqueueUnique(queue, 5, idPrefix: "held"); + + var workers = StartWorkers(queue, WorkerCount); + queue.Complete(); + await WaitWorkersAsync(workers); + + Assert.Equal(ArticleEnqueueResult.Unavailable, queue.TryAdmit(Article(""))); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.False(queue.IsAccepting); + Assert.Null(await queue.DequeueAsync(CancellationToken.None)); + } + + [Fact] + public async Task ConcurrentProducersAndConsumers_CannotOversubscribeBudget() + { + var sample = Article(""); + var size = sample.Payload.Length; + var limit = size * 4L; + const int producers = 8; + const int consumers = 4; + const int perProducer = 8; + var queue = CreateWithByteLimit(limit); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var maxObserved = 0L; + var oversubscribed = 0; + using var observeCts = new CancellationTokenSource(); + var observer = Task.Run(async () => + { + while (!observeCts.Token.IsCancellationRequested) + { + var bytes = queue.QueuedBytes; + if (bytes > maxObserved) + { + maxObserved = bytes; + } + + if (bytes > limit) + { + Interlocked.Increment(ref oversubscribed); + } + + await Task.Yield(); + } + }); + + var consumerTasks = StartWorkers(queue, consumers, seen); + var producerTasks = new Task[producers]; + for (var p = 0; p < producers; p++) + { + var producer = p; + producerTasks[producer] = Task.Run(async () => + { + for (var i = 0; i < perProducer; i++) + { + var result = await queue.EnqueueAsync(Article($""), CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, result); + } + }); + } + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await Task.WhenAll(producerTasks).WaitAsync(safety.Token); + queue.Complete(); + await WaitWorkersAsync(consumerTasks); + await observeCts.CancelAsync(); + await observer.WaitAsync(safety.Token); + + Assert.Equal(0, oversubscribed); + Assert.True(maxObserved <= limit); + Assert.Equal(producers * perProducer, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + [Fact] + public async Task SuccessfulDequeue_ReleasesAccountingBeforeCallerOwnsArticle() + { + var article = Article(""); + var queue = CreateQueue(1); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(article)); + Assert.Equal(article.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + + var dequeued = await queue.DequeueAsync(CancellationToken.None); + Assert.Same(article, dequeued); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(article.Payload.Length, dequeued!.Payload.Length); + } + + private static async Task DrainExactlyOnceAsync(int workers, int articleCount) + { + var queue = CreateQueue(articleCount); + var ids = EnqueueUnique(queue, articleCount); + var seen = new ConcurrentDictionary(StringComparer.Ordinal); + var hits = new int[workers]; + + var tasks = StartWorkers(queue, workers, seen, hits); + queue.Complete(); + await WaitWorkersAsync(tasks); + + Assert.Equal(articleCount, seen.Count); + Assert.All(seen.Values, static count => Assert.Equal(1, count)); + Assert.True(ids.SetEquals(seen.Keys)); + Assert.Equal(articleCount, hits.Sum()); + Assert.True(hits.Count(static h => h > 0) >= 1); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + } + + private static ArticleIngestionQueue CreateQueue(int articleSlots) + { + ArgumentOutOfRangeException.ThrowIfNegative(articleSlots); + return new ArticleIngestionQueue(new ArticleIngestionOptions(), 64L * 1024 * 1024); + } + + private static ArticleIngestionQueue CreateWithByteLimit(long memoryLimit) => + new(new ArticleIngestionOptions(), memoryLimit); + + private static HashSet EnqueueUnique( + ArticleIngestionQueue queue, + int articleCount, + string idPrefix = "q20") + { + var ids = new HashSet(articleCount, StringComparer.Ordinal); + for (var i = 0; i < articleCount; i++) + { + var id = $"<{idPrefix}.{i:D3}@ex.com>"; + Assert.True(ids.Add(id)); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(Article(id))); + } + + return ids; + } + + private static Task[] StartWorkers( + IArticleIngestionQueue queue, + int workers, + ConcurrentDictionary? seen = null, + int[]? hits = null, + CancellationToken cancellationToken = default) + { + var tasks = new Task[workers]; + for (var i = 0; i < workers; i++) + { + var workerIndex = i; + tasks[workerIndex] = Task.Run( + () => DrainAsync(queue, seen, hits, workerIndex, cancellationToken), + CancellationToken.None); + } + + return tasks; + } + + private static async Task DrainAsync( + IArticleIngestionQueue queue, + ConcurrentDictionary? seen, + int[]? hits, + int workerIndex, + CancellationToken cancellationToken) + { + while (true) + { + InboundArticle? article; + try + { + article = await queue.DequeueAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + + if (article is null) + { + return; + } + + if (seen is not null) + { + seen.AddOrUpdate(article.MessageId, 1, static (_, count) => count + 1); + } + + if (hits is not null) + { + Interlocked.Increment(ref hits[workerIndex]); + } + } + } + + private static async Task WaitWorkersAsync(Task[] workers) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await Task.WhenAll(workers).WaitAsync(safety.Token); + } + + private static InboundArticle Article(string messageId) => + CanonicalArticleText.CreateQueued(messageId, InboundArticleProducer.IHave); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs new file mode 100644 index 00000000..8410930f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogNewsLogWriterTests.cs @@ -0,0 +1,189 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class SerilogNewsLogWriterTests +{ + [Fact] + public void ReadSettings_HonoursConfiguredOperationalProperties() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create( + dir.Path, + rollingInterval: "Hour", + retainedFileCountLimit: 3, + buffered: false, + fileSizeLimitBytes: 1_048_576, + rollOnFileSizeLimit: true, + bufferSize: 250, + blockWhenFull: false); + var settings = NntpdNewsLogging.ReadSettings(configuration); + Assert.Equal(Path.Join(dir.Path, "news-.log"), settings.Path); + Assert.Equal(RollingInterval.Hour, settings.RollingInterval); + Assert.Equal(3, settings.RetainedFileCountLimit); + Assert.False(settings.Buffered); + Assert.True(settings.RollOnFileSizeLimit); + Assert.Equal(1_048_576, settings.FileSizeLimitBytes); + Assert.Equal(250, settings.BufferSize); + Assert.False(settings.BlockWhenFull); + Assert.Null(settings.FlushToDiskInterval); + } + + [Fact] + public void BindResolvedNewsPath_UsesLogDir_AndDoesNotExposeOutputTemplate() + { + using var dir = new TempLogDir(); + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = dir.Path, + ["Serilog:News:path"] = "logs/news-.log", + ["Serilog:News:outputTemplate"] = "HACK {Message}", + }); + NntpdNewsLogging.BindResolvedNewsPath(configuration, dir.Path); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(dir.Path), configuration["Serilog:News:path"]); + var settings = NntpdNewsLogging.ReadSettings(configuration); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(dir.Path), settings.Path); + Assert.Equal("HACK {Message}", configuration["Serilog:News:outputTemplate"]); + } + + [Fact] + public void Write_UsesConfiguredPath_AndInnFormatter() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Flush(); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.Contains(" + ? 0 ?", text, StringComparison.Ordinal); + Assert.DoesNotContain("HACK", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_NamedPeer_EmitsInboundFeedSizeAndOutboundPlaceholder() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(new NewsLogEvent( + NewsLogDisposition.Accepted, + Encoding.ASCII.GetBytes(""), + Encoding.ASCII.GetBytes("BlueWorldHosting"), + size: 1584)); + writer.Flush(); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains( + " + BlueWorldHosting 1584 ?", + text, + StringComparison.Ordinal); + Assert.DoesNotContain(" + ? ", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_EmitsCompleteLines_AndDisposeFlushesThem() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite", buffered: true); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Write(Accepted("")); + } + + var text = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.EndsWith("\n", text, StringComparison.Ordinal); + Assert.Equal(2, text.Split('\n', StringSplitOptions.RemoveEmptyEntries).Length); + } + + [Fact] + public void DailyRolling_WritesDatedFileName() + { + using var dir = new TempLogDir(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Day"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Dispose(); + } + + var dated = Directory.GetFiles(dir.Path, "news-????????.log"); + Assert.Single(dated); + Assert.Contains("", NewsTestConfiguration.ReadNewsFile(dir.Path), StringComparison.Ordinal); + } + + [Fact] + public void ApplicationLogger_DoesNotReceiveNewsLines() + { + using var dir = new TempLogDir(); + var appSink = new CollectingSink(); + using var app = new LoggerConfiguration().WriteTo.Sink(appSink).CreateLogger(); + var configuration = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogNewsLogWriter(configuration)) + { + writer.Write(Accepted("")); + writer.Dispose(); + } + + app.Information("application diagnostic"); + Assert.DoesNotContain(appSink.Lines, static line => line.Contains("", StringComparison.Ordinal)); + Assert.Contains("", NewsTestConfiguration.ReadNewsFile(dir.Path), StringComparison.Ordinal); + } + + [Fact] + public void CustomNewsFileTypes_AreGone() + { + Assert.Null(Type.GetType("VectorNNTPD.TransitService.ArticleIngestion.InnNewsLogWriter, VectorNNTPD.TransitService")); + Assert.Null(Type.GetType("VectorNNTPD.TransitService.Logging.NntpdLogCompression, VectorNNTPD.TransitService")); + } + + private static NewsLogEvent Accepted(string messageId) => + new(NewsLogDisposition.Accepted, Encoding.ASCII.GetBytes(messageId)); + + private sealed class CollectingSink : ILogEventSink + { + public List Lines { get; } = []; + + public void Emit(LogEvent logEvent) => Lines.Add(logEvent.RenderMessage()); + } + + private sealed class TempLogDir : IDisposable + { + public TempLogDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-news-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs new file mode 100644 index 00000000..49b4b39e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/ArticleIngestion/SerilogPathSurveyWriterTests.cs @@ -0,0 +1,290 @@ +using System.IO.Compression; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.ArticleIngestion; + +public sealed class SerilogPathSurveyWriterTests +{ + [Fact] + public void ReadSettings_HonoursConfiguredOperationalProperties() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create( + dir.Path, + rollingInterval: "Hour", + retainedFileCountLimit: 1, + buffered: false, + fileSizeLimitBytes: 1_048_576, + rollOnFileSizeLimit: true, + bufferSize: 250, + blockWhenFull: false); + var settings = NntpdPathSurveyLogging.ReadSettings(configuration); + Assert.Equal(Path.Join(dir.Path, "inpaths-.log"), settings.Path); + Assert.Equal(RollingInterval.Hour, settings.RollingInterval); + Assert.Equal(1, settings.RetainedFileCountLimit); + Assert.False(settings.Buffered); + Assert.True(settings.RollOnFileSizeLimit); + Assert.Equal(1_048_576, settings.FileSizeLimitBytes); + Assert.Equal(250, settings.BufferSize); + Assert.False(settings.BlockWhenFull); + Assert.Null(settings.FlushToDiskInterval); + } + + [Fact] + public void BindResolvedPathSurveyPath_UsesLogDir_AndDoesNotExposeOutputTemplate() + { + using var dir = new TempLogDir(); + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = dir.Path, + ["Serilog:Inpaths:path"] = "logs/inpaths-.log", + ["Serilog:Inpaths:outputTemplate"] = "HACK {Message}", + }); + NntpdPathSurveyLogging.BindResolvedPathSurveyPath(configuration, dir.Path); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(dir.Path), configuration["Serilog:Inpaths:path"]); + var settings = NntpdPathSurveyLogging.ReadSettings(configuration); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(dir.Path), settings.Path); + Assert.Equal("HACK {Message}", configuration["Serilog:Inpaths:outputTemplate"]); + } + + [Fact] + public void Write_UsesConfiguredPath_AndPathFormatter() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("canonical.example!peer"u8); + writer.Flush(); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: canonical.example!peer\n", text); + Assert.DoesNotContain("HACK", text, StringComparison.Ordinal); + Assert.DoesNotContain("giganews", text, StringComparison.Ordinal); + } + + [Fact] + public void Write_PreservesExactCanonicalPathBytes() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + var path = "news.usenet.ninja!nntpd01.usenet.ninja!BlueWorldHosting!not-for-mail"u8; + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write(path); + writer.Dispose(); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: " + Encoding.Latin1.GetString(path) + "\n", text); + } + + [Fact] + public void Write_EmptyPath_EmitsPathPrefixAndNewline() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write([]); + writer.Dispose(); + } + + Assert.Equal("Path: \n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void Write_EmitsCompleteLines_AndDisposeFlushesThem() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite", buffered: true); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("one.example"u8); + writer.Write("two.example"u8); + } + + var text = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + Assert.Equal("Path: one.example\nPath: two.example\n", text); + } + + [Fact] + public void SizeBasedRoll_HandsCompletedFileToHandler_BeforeGzip() + { + using var dir = new TempLogDir(); + var handler = new RecordingCompletedPathSurveyFileHandler(); + var configuration = PathSurveyTestConfiguration.Create( + dir.Path, + rollingInterval: "Infinite", + retainedFileCountLimit: 1, + buffered: false, + fileSizeLimitBytes: 80, + rollOnFileSizeLimit: true); + using (var writer = new SerilogPathSurveyWriter( + configuration, + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 40; i++) + { + writer.Write("aaaaaaaaaaaaaaaaaaaaaaaa"u8); + } + + writer.Dispose(); + } + + Assert.NotEmpty(handler.Completed); + Assert.All(handler.GzipExistedAtHandoff, static existed => Assert.False(existed)); + Assert.All(handler.Completed, static path => Assert.False(File.Exists(path))); + Assert.Empty(Directory.GetFiles(dir.Path, "inpaths*.gz")); + Assert.Single(Directory.GetFiles(dir.Path, "inpaths*.log")); + } + + [Fact] + public void DailyRolling_WritesDatedFileName() + { + using var dir = new TempLogDir(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Day"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("day.example"u8); + writer.Dispose(); + } + + var dated = Directory.GetFiles(dir.Path, "inpaths-????????.log"); + Assert.Single(dated); + Assert.Equal("Path: day.example\n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void ApplicationLogger_DoesNotReceivePathLines() + { + using var dir = new TempLogDir(); + var appSink = new CollectingSink(); + using var app = new LoggerConfiguration().WriteTo.Sink(appSink).CreateLogger(); + var configuration = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var writer = new SerilogPathSurveyWriter(configuration)) + { + writer.Write("isolated.example"u8); + writer.Dispose(); + } + + app.Information("application diagnostic"); + Assert.DoesNotContain(appSink.Lines, static line => line.Contains("isolated.example", StringComparison.Ordinal)); + Assert.DoesNotContain(appSink.Lines, static line => line.StartsWith("Path: ", StringComparison.Ordinal)); + Assert.Equal("Path: isolated.example\n", PathSurveyTestConfiguration.ReadInpathsFile(dir.Path)); + } + + [Fact] + public void NewsLogger_DoesNotReceivePathSurveyLines() + { + using var dir = new TempLogDir(); + var pathConfig = PathSurveyTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + var newsConfig = NewsTestConfiguration.Create(dir.Path, rollingInterval: "Infinite"); + using (var paths = new SerilogPathSurveyWriter(pathConfig)) + using (var news = new SerilogNewsLogWriter(newsConfig)) + { + paths.Write("survey-only.example"u8); + news.Write(new NewsLogEvent(NewsLogDisposition.Accepted, Encoding.ASCII.GetBytes(""))); + paths.Dispose(); + news.Dispose(); + } + + var pathText = PathSurveyTestConfiguration.ReadInpathsFile(dir.Path); + var newsText = NewsTestConfiguration.ReadNewsFile(dir.Path); + Assert.Equal("Path: survey-only.example\n", pathText); + Assert.DoesNotContain("Path:", newsText, StringComparison.Ordinal); + Assert.Contains("", newsText, StringComparison.Ordinal); + Assert.DoesNotContain("", pathText, StringComparison.Ordinal); + Assert.DoesNotContain("survey-only.example", newsText, StringComparison.Ordinal); + } + + [Fact] + public void Writer_DoesNotRetainPathObservationsInACollection() + { + var fields = typeof(SerilogPathSurveyWriter).GetFields( + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain( + fields, + static field => + field.FieldType != typeof(Serilog.Core.Logger) + && typeof(System.Collections.IEnumerable).IsAssignableFrom(field.FieldType) + && field.FieldType != typeof(string)); + Assert.Single(fields); + } + + [Fact] + public void CreateLogger_UsesDistinctSourceContext() + { + Assert.Equal("VectorNNTPD.TransitService.Inpaths", NntpdPathSurveyLogging.SourceContext); + Assert.NotEqual(NntpdNewsLogging.SourceContext, NntpdPathSurveyLogging.SourceContext); + } + + [Fact] + public void ProductionAppsettings_DeclaresInpathsBesideNews() + { + var json = File.ReadAllText(FindProductionAppsettings()); + Assert.Contains("\"Inpaths\"", json, StringComparison.Ordinal); + Assert.Contains("logs/inpaths-.log", json, StringComparison.Ordinal); + Assert.Contains("logs/news-.log", json, StringComparison.Ordinal); + } + + private static string FindProductionAppsettings() + { + var dir = AppContext.BaseDirectory; + while (dir is not null) + { + var candidate = Path.Join(dir, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = Directory.GetParent(dir)?.FullName; + } + + throw new InvalidOperationException("Production VectorNNTPD.TransitService.json was not found."); + } + + private sealed class CollectingSink : ILogEventSink + { + public List Lines { get; } = []; + + public void Emit(LogEvent logEvent) => Lines.Add(logEvent.RenderMessage()); + } + + private sealed class TempLogDir : IDisposable + { + public TempLogDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "vectornntp-inpaths-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // best-effort + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs new file mode 100644 index 00000000..ea63050e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsClientTests.cs @@ -0,0 +1,691 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Client; + +public sealed class CloudflareDnsClientTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task ListRecordsAsync_ValidSinglePage_ReturnsRecords() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Equal("1", records[0].Id); + Assert.Single(handler.Requests); + } + + [Fact] + public async Task ListRecordsAsync_FollowsPagination_AndFiltersExactName() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false }, + { "id": "x", "type": "A", "name": "other.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.3", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Equal(2, records.Count); + Assert.All(records, r => Assert.Equal(Fqdn, r.Name)); + Assert.Equal(2, handler.Requests.Count); + Assert.Contains("page=2", handler.Requests[1].RequestUri!.Query, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, handler.Requests[0].RequestUri!.ToString()); + } + + [Fact] + public async Task ListRecordsAsync_MissingResultInfo_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": [] }"""); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.False(ex.IsOutcomeUncertain); + Assert.Contains("result_info", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ListRecordsAsync_MissingTotalPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [], + "result_info": { "page": 1, "per_page": 100, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task ListRecordsAsync_NonPositiveTotalPages_IsPermanentFailure(int totalPages) + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": {{totalPages}}, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ListRecordsAsync_TotalPagesChangesBetweenPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 3, "count": 1, "total_count": 3 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("total_pages changed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task ListRecordsAsync_PageMetadataMismatch_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [], + "result_info": { "page": 9, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("page does not match", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ListRecordsAsync_ExceedsMaxPages_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + var declaredTotal = CloudflareDnsClient.MaxListPages + 1; + for (var page = 1; page <= CloudflareDnsClient.MaxListPages; page++) + { + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [ { "id": "{{page}}", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.{{page}}", "ttl": 1, "proxied": false } ], + "result_info": { "page": {{page}}, "per_page": 1, "total_pages": {{declaredTotal}}, "count": 1, "total_count": {{declaredTotal}} } + } + """); + } + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("exceeded", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(CloudflareDnsClient.MaxListPages.ToString(System.Globalization.CultureInfo.InvariantCulture), ex.Message, StringComparison.Ordinal); + Assert.Equal(CloudflareDnsClient.MaxListPages, handler.Requests.Count); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ListRecordsAsync_CancellationDuringPagination_Propagates() + { + using var cts = new CancellationTokenSource(); + var handler = new ScriptedHttpMessageHandler + { + OnRequest = (_, requestIndex) => + { + if (requestIndex >= 1) + { + cts.Cancel(); + } + }, + }; + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "2", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } ], + "result_info": { "page": 2, "per_page": 1, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + await Assert.ThrowsAnyAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + } + + [Fact] + public async Task ListRecordsAsync_MatchingZoneId_IsAccepted() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": "zone-test" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var records = await CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Equal(ZoneId, records[0].ZoneId); + } + + [Fact] + public async Task ListRecordsAsync_OmittedZoneId_IsAcceptedPerApiContract() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var records = await CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Single(records); + Assert.Null(records[0].ZoneId); + } + + [Fact] + public async Task ListRecordsAsync_MismatchedZoneId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": "other-zone" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("other-zone", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\" \"")] + public async Task ListRecordsAsync_EmptyOrWhitespaceZoneId_IsPermanentFailure(string zoneJson) + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + $$""" + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false, "zone_id": {{zoneJson}} } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CreateRecordAsync_MismatchedZoneId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": { + "id": "new", + "type": "A", + "name": "nntpd01.usenet.ninja", + "content": "198.18.0.1", + "ttl": 1, + "proxied": false, + "zone_id": "other-zone" + } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.1", + }, + CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("zone_id", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Reconcile_PaginationIntegrityFailure_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.99", "ttl": 1, "proxied": false } ] }"""); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Put); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task Reconcile_ZoneMismatch_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.99", "ttl": 1, "proxied": false, "zone_id": "other-zone" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post || r.Method == HttpMethod.Put || r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task RemoveAll_PaginationIntegrityFailure_DoesNotMutate() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } ], + "result_info": { "page": 1, "per_page": 100, "count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Delete); + } + + [Fact] + public async Task Send_AddsBearerAuthorization_WithoutExposingKeyInExceptions() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.Unauthorized, + """ + { "success": false, "errors": [ { "code": 10000, "message": "Authentication error" } ], "result": null } + """); + + var client = CreateClient(handler); + var ex = await Assert.ThrowsAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Equal(401, ex.StatusCode); + Assert.True(ex.IsPermanentFailure); + Assert.Contains("Authentication error", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + Assert.Equal("Bearer", handler.Requests[0].Headers.Authorization!.Scheme); + Assert.Equal(TestHostFactory.TestCloudFlareApiKey, handler.Requests[0].Headers.Authorization!.Parameter); + } + + [Fact] + public async Task Send_UnsuccessfulSuccessFlag_IsFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { "success": false, "errors": [ { "code": 9109, "message": "Unauthorized to access requested resource" } ], "result": [] } + """); + + var client = CreateClient(handler); + var ex = await Assert.ThrowsAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Contains("Unauthorized to access requested resource", ex.Message, StringComparison.Ordinal); + Assert.Contains(9109, ex.CloudflareErrorCodes); + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task Send_RateLimit_RetriesThenSucceeds() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(1)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } } + """); + + var client = CreateClient(handler); + var records = await client.ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None); + + Assert.Empty(records); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task CreateUpdateDelete_UseExpectedPaths() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 1, "proxied": false } }"""); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.2", "ttl": 1, "proxied": false } }"""); + handler.EnqueueJson( + HttpStatusCode.OK, + """{ "success": true, "result": { "id": "new" } }"""); + + var client = CreateClient(handler); + var created = await client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.1", + }, + CancellationToken.None); + Assert.Equal("new", created.Id); + + await client.UpdateRecordAsync( + ZoneId, + "new", + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.2", + }, + CancellationToken.None); + + await client.DeleteRecordAsync(ZoneId, "new", CancellationToken.None); + + Assert.Equal(HttpMethod.Post, handler.Requests[0].Method); + Assert.Equal(HttpMethod.Put, handler.Requests[1].Method); + Assert.Equal(HttpMethod.Delete, handler.Requests[2].Method); + Assert.EndsWith($"/zones/{ZoneId}/dns_records", handler.Requests[0].RequestUri!.AbsolutePath, StringComparison.Ordinal); + Assert.EndsWith($"/zones/{ZoneId}/dns_records/new", handler.Requests[2].RequestUri!.AbsolutePath, StringComparison.Ordinal); + } + + [Fact] + public async Task Send_Cancellation_Propagates() + { + var handler = new ScriptedHttpMessageHandler + { + Delay = TimeSpan.FromSeconds(30), + }; + handler.EnqueueJson(HttpStatusCode.OK, """{ "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1 } }"""); + + var client = CreateClient(handler); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(20)); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + } + + [Fact] + public void RequireTotalPages_RejectsNullAndInvalidMetadata() + { + Assert.Throws(() => CloudflareDnsClient.RequireTotalPages(null, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = null }, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 0 }, expectedPage: 1)); + Assert.Throws(() => + CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 1, Page = 2 }, expectedPage: 1)); + + Assert.Equal(3, CloudflareDnsClient.RequireTotalPages(new CloudflareResultInfo { TotalPages = 3, Page = 1 }, expectedPage: 1)); + } + + [Fact] + public void EnsureRecordZoneMatches_AppliesOmissionAndMismatchPolicy() + { + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = null }, + ZoneId, + "List"); + + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = ZoneId }, + ZoneId, + "List"); + + var mismatch = Assert.Throws(() => + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = "other" }, + ZoneId, + "List")); + Assert.True(mismatch.IsPermanentFailure); + + var empty = Assert.Throws(() => + CloudflareDnsClient.EnsureRecordZoneMatches( + new CloudflareDnsRecord { Id = "1", ZoneId = " " }, + ZoneId, + "List")); + Assert.True(empty.IsPermanentFailure); + } + + private static CloudflareDnsClient CreateClient(ScriptedHttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + }; + + var options = Options.Create(TestHostFactory.CreateValidOptions()); + return new CloudflareDnsClient(httpClient, options, NullLogger.Instance); + } + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public TimeSpan Delay { get; set; } + + public Action? OnRequest { get; set; } + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri); + if (request.Headers.Authorization is { } auth) + { + copy.Headers.Authorization = auth; + } + + Requests.Add(copy); + OnRequest?.Invoke(request, Requests.Count - 1); + + if (Delay > TimeSpan.Zero) + { + await Task.Delay(Delay, cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return _responses.Dequeue(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs new file mode 100644 index 00000000..523e4018 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsDeadlineTests.cs @@ -0,0 +1,340 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Client; + +/// +/// Cancellation, shared operation budget, and per-request HTTP timeout regression coverage. +/// +public sealed class CloudflareDnsDeadlineTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task CallerCancellation_BeforeRequest_DoesNotStartHttpCall() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson(HttpStatusCode.OK, EmptyListJson()); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", cts.Token)); + + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task RateLimit_CancelDuringDelay_PreventsSubsequentRequest() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + handler.EnqueueJson(HttpStatusCode.OK, EmptyListJson()); + + var client = CreateClient(handler); + using var cts = new CancellationTokenSource(); + client.DelayAsync = (_, ct) => + { + cts.Cancel(); + return Task.Delay(TimeSpan.FromSeconds(60), ct); + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMinutes(2), cts.Token, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + + Assert.Single(handler.Requests); + } + + [Fact] + public void CreateRequestTimeoutCts_CapsByRemainingBudget() + { + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(150), CancellationToken.None, out var opToken); + using var cts = CloudflareDnsClient.CreateRequestTimeoutCts(opToken, out var sendToken); + + Assert.False(sendToken.IsCancellationRequested); + Assert.True(SpinWait.SpinUntil(() => sendToken.IsCancellationRequested, TimeSpan.FromSeconds(2))); + } + + [Fact] + public void CreateRequestTimeoutCts_ThrowsWhenBudgetAlreadyExpired() + { + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(1), CancellationToken.None, out var opToken); + Assert.True(SpinWait.SpinUntil(() => budget.Remaining <= TimeSpan.Zero, TimeSpan.FromSeconds(2))); + + Assert.ThrowsAny(() => + CloudflareDnsClient.CreateRequestTimeoutCts(opToken, out _)); + } + + [Fact] + public async Task List_RequestTimeout_IsNotUncertainMutation() + { + var handler = new StallingHttpMessageHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = Timeout.InfiniteTimeSpan, + }; + var client = new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMilliseconds(40), CancellationToken.None, out var opToken); + + // Budget expires → OCE (operation cancel), not CloudflareDnsException uncertain. + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + } + + [Fact] + public async Task Mutation_PerRequestTimeout_WithoutOperationCancel_IsUncertain() + { + var handler = new StallingHttpMessageHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + // Legacy HttpClient timeout still used by some tests; client also CancelAfter(PerRequestTimeout). + Timeout = TimeSpan.FromMilliseconds(40), + }; + var client = new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }, + CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.False(ex.IsPermanentFailure); + } + + [Fact] + public async Task Reconciler_DoesNotResetDeadlineAcrossAttempts() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("transient") { FailedOperation = "List" }, + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMilliseconds(80); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance) + { + DelayAsync = async (_, ct) => + { + await Task.Delay(TimeSpan.FromMilliseconds(5), ct).ConfigureAwait(false); + }, + }; + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.True(client.ListCallCount >= 1); + } + + [Fact] + public async Task Reconcile_CancelDuringVerification_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + + using var cts = new CancellationTokenSource(); + var listCount = 0; + client.TransformListedRecords = (type, records, callCount) => + { + listCount = callCount; + // After planning lists (2) and creates (0), verification lists start — cancel then. + if (callCount >= 3) + { + cts.Cancel(); + } + + return records; + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMinutes(2); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + + Assert.True(listCount >= 3); + } + + [Fact] + public async Task FailedStartCleanup_UsesDedicatedBudget_DoesNotMaskOriginal() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("original-startup-failure") + { + IsPermanentFailure = true, + FailedOperation = "List", + }, + TransformListedAllRecords = (_, callCount) => + { + if (callCount > 1) + { + throw new CloudflareDnsException("cleanup-still-failing") + { + IsPermanentFailure = true, + FailedOperation = "ListAll", + }; + } + + return []; + }, + }; + + var options = TestHostFactory.CreateValidOptions(); + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance), + new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Equal("original-startup-failure", ex.Message); + Assert.Equal(TimeSpan.FromSeconds(15), CloudflareDnsReconciliationService.FailedStartCleanupTimeout); + } + + [Fact] + public async Task NormalCleanup_HonorsCallerToken() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + return Task.CompletedTask; + }; + + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + await Assert.ThrowsAnyAsync(() => + reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token)); + } + + private static CloudflareDnsClient CreateClient(HttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = Timeout.InfiniteTimeSpan, + }; + return new CloudflareDnsClient( + http, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + } + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static string EmptyListJson() => + """{ "success": true, "result": [], "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } }"""; + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(new HttpRequestMessage(request.Method, request.RequestUri)); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } + + private sealed class StallingHttpMessageHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(HttpStatusCode.OK); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs new file mode 100644 index 00000000..323d742b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Client/CloudflareDnsHardeningTests.cs @@ -0,0 +1,394 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Client; + +/// +/// Hardening coverage: managed TTL policy budgets, 429/budget interaction, record validation, sanitized errors. +/// +public sealed class CloudflareDnsHardeningTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task RateLimit_RetryAfterExceedsBudget_CancelsWithoutSleepingPastDeadline() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMinutes(2)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + + var client = CreateClient(handler); + var delays = new List(); + client.DelayAsync = (delay, _) => + { + delays.Add(delay); + return Task.CompletedTask; + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromSeconds(5), CancellationToken.None, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + + Assert.Empty(delays); + } + + [Fact] + public async Task RateLimit_CallerCancellation_TakesPrecedence() + { + var handler = new ScriptedHttpMessageHandler(); + handler.Enqueue( + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + new HttpResponseMessage((HttpStatusCode)429) + { + Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)) }, + Content = new StringContent("""{"success":false,"errors":[{"code":1,"message":"rate"}]}""", Encoding.UTF8, "application/json"), + }); + + var client = CreateClient(handler); + using var callerCts = new CancellationTokenSource(); + client.DelayAsync = (_, ct) => + { + callerCts.Cancel(); + return Task.Delay(TimeSpan.FromSeconds(30), ct); + }; + + using var budget = CloudflareOperationBudget.Begin(TimeSpan.FromMinutes(2), callerCts.Token, out var opToken); + + await Assert.ThrowsAnyAsync(() => + client.ListRecordsAsync(ZoneId, Fqdn, "A", opToken)); + } + + [Fact] + public async Task Reconciler_SharedDeadline_CancelsBackoffWithoutSuccess() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("transient list failure") { FailedOperation = "List" }, + }; + + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareOperationTimeout = TimeSpan.FromMilliseconds(50); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance) + { + DelayAsync = (_, ct) => Task.Delay(TimeSpan.FromSeconds(30), ct), + }; + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + } + + [Fact] + public async Task FailureResponse_DoesNotEmbedRawBody() + { + const string marker = "RAW_BODY_SECRET_MARKER_SHOULD_NOT_APPEAR"; + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.BadRequest, + $$"""{ "success": false, "errors": [], "result": null, "extra": "{{marker}}" }"""); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.Equal(400, ex.StatusCode); + Assert.DoesNotContain(marker, ex.Message, StringComparison.Ordinal); + Assert.Contains("no Cloudflare error details returned", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task List_MissingSuccess_IsFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 0, "total_count": 0 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure || ex.StatusCode == 200); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task List_NonBooleanSuccess_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": 1, + "result": [], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task List_MissingRecordId_IsPermanentFailure_NoMutations() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300, "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var client = CreateClient(handler); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.All(handler.Requests, r => Assert.Equal(HttpMethod.Get, r.Method)); + } + + [Fact] + public async Task List_MissingTtl_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "proxied": false } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("ttl", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task List_MissingProxied_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300 } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("proxied", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task List_MalformedProxied_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "ttl": 300, "proxied": "yes" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListRecordsAsync(ZoneId, Fqdn, "A", CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task ListAll_MissingId_IsPermanentFailure() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "result": [ + { "type": "TXT", "name": "nntpd01.usenet.ninja", "content": "x" } + ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 1, "count": 1, "total_count": 1 } + } + """); + + var ex = await Assert.ThrowsAsync(() => + CreateClient(handler).ListAllRecordsForNameAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + } + + [Fact] + public async Task Reconcile_CreateUsesManagedTtl() + { + var client = new FakeCloudflareDnsClient(); + await new CloudflareDnsReconciler( + client, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + var created = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, created.Ttl); + Assert.Equal(false, created.Proxied); + Assert.Equal(1, client.CreateCallCount); + } + + [Fact] + public async Task FailedStartCleanup_Timeout_DoesNotClaimSuccess_PreservesOriginal() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + Proxied = false, + }); + client.OnMutate = async ct => + { + await Task.Delay(TimeSpan.FromSeconds(60), ct).ConfigureAwait(false); + }; + + var options = TestHostFactory.CreateValidOptions(); + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + + // Force reconcile to fail after ownership path by making list fail after first mutation attempt — + // simpler: inject permanent list failure then ensure cleanup uses 15s budget. + client.ListException = new CloudflareDnsException("boom") + { + IsPermanentFailure = true, + FailedOperation = "List", + }; + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Equal("boom", ex.Message); + Assert.Contains(client.Snapshot(), r => r.Id == "a1"); + } + + [Fact] + public void ManagedDnsPolicy_ConstantIs300() => + Assert.Equal(300, CloudflareManagedDnsPolicy.ManagedTtl); + + private static CloudflareDnsClient CreateClient(ScriptedHttpMessageHandler handler) + { + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + }; + return new CloudflareDnsClient( + httpClient, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + } + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + public void Enqueue(HttpResponseMessage response) => _responses.Enqueue(response); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri) + { + Content = request.Content, + }; + Requests.Add(copy); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs new file mode 100644 index 00000000..1b8885e9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsCleanupTests.cs @@ -0,0 +1,655 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Lifecycle; + +/// +/// Shutdown cleanup: remove every DNS record for the exact FQDN (all types), with lifecycle coverage. +/// +public sealed class CloudflareDnsCleanupTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task RemoveAll_DeletesAAndAaaa() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("aaaa", "AAAA", "2001:db8::10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RemoveAll_DeletesAllRecordTypesForExactFqdn() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("aaaa", "AAAA", "2001:db8::10"), + Record("cname", "CNAME", "target.example"), + Record("txt", "TXT", "v=spf1"), + Record("mx", "MX", "10 mail.example"), + Record("srv", "SRV", "1 1 443 target"), + Record("caa", "CAA", "0 issue \"letsencrypt.org\""), + Record("https", "HTTPS", "1 . alpn=h2"), + Record("svcb", "SVCB", "1 . alpn=h2")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + Assert.True(client.SuccessfulDeleteCount >= 9); + } + + [Fact] + public async Task RemoveAll_DeletesDuplicates() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("txt1", "TXT", "one"), + Record("txt2", "TXT", "two")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + Assert.Equal(4, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task ListAll_FollowsPagination_InHttpClient() + { + var handler = new ScriptedHttpMessageHandler(); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "errors": [], + "result": [ { "id": "1", "type": "A", "name": "nntpd01.usenet.ninja", "content": "198.18.0.1", "proxied": false, "ttl": 1 } ], + "result_info": { "page": 1, "per_page": 100, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + handler.EnqueueJson( + HttpStatusCode.OK, + """ + { + "success": true, + "errors": [], + "result": [ { "id": "2", "type": "TXT", "name": "nntpd01.usenet.ninja", "content": "x", "proxied": false, "ttl": 1 } ], + "result_info": { "page": 2, "per_page": 100, "total_pages": 2, "count": 1, "total_count": 2 } + } + """); + + var client = CreateHttpClient(handler); + var records = await client.ListAllRecordsForNameAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Equal(2, records.Count); + Assert.Contains(records, r => r.Type == "TXT"); + Assert.Equal(2, handler.Requests.Count); + Assert.DoesNotContain("type=", handler.Requests[0].RequestUri!.Query, StringComparison.Ordinal); + } + + [Fact] + public async Task RemoveAll_PreservesParentChildAndSimilarSuffixHostnames() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("ours", "A", "198.18.0.10"), + new CloudflareDnsRecord { Id = "parent", Type = "A", Name = "usenet.ninja", Content = "198.18.0.1" }, + new CloudflareDnsRecord { Id = "child", Type = "A", Name = "other.nntpd01.usenet.ninja", Content = "198.18.0.2" }, + new CloudflareDnsRecord { Id = "suffix", Type = "A", Name = "xnntpd01.usenet.ninja", Content = "198.18.0.3" }, + new CloudflareDnsRecord { Id = "peer", Type = "TXT", Name = "nntpd02.usenet.ninja", Content = "keep" }); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + var snap = client.Snapshot(); + Assert.DoesNotContain(snap, r => r.Id == "ours"); + Assert.Contains(snap, r => r.Id == "parent"); + Assert.Contains(snap, r => r.Id == "child"); + Assert.Contains(snap, r => r.Id == "suffix"); + Assert.Contains(snap, r => r.Id == "peer"); + } + + [Fact] + public async Task RemoveAll_PartialDelete_ThenRetry_Converges() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("transient delete") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + }; + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("txt", "TXT", "hello")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RemoveAll_TimeoutAfterDeleteApplied_RetryVerifiesGone() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("timeout after delete") + { + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + ApplyDeleteThenFailUncertain = true, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Empty(client.Snapshot()); + Assert.True(client.ListAllCallCount >= 2); + } + + [Fact] + public async Task RemoveAll_TimeoutWhenDeleteNotApplied_RetryDeletes() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("timeout no apply") + { + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + ApplyDeleteThenFailUncertain = false, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + + Assert.Empty(client.Snapshot()); + Assert.True(client.SuccessfulDeleteCount >= 1); + } + + [Fact] + public async Task RemoveAll_Retry_ReReadsActualState() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("fail once") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + DeleteFailRemaining = 1, + }; + client.Seed(Record("txt", "TXT", "x")); + + var listsBefore = 0; + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + Assert.True(client.ListAllCallCount > 2); + _ = listsBefore; + } + + [Fact] + public async Task RemoveAll_PermanentFailure_DoesNotRetryPointlessly() + { + var client = new FakeCloudflareDnsClient + { + ListAllException = new CloudflareDnsException("auth") + { + StatusCode = 401, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Equal(1, client.ListAllCallCount); + Assert.Equal(0, client.DeleteCallCount); + } + + [Fact] + public async Task RemoveAll_VerificationMismatch_FailsClosed() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10")); + client.TransformListedAllRecords = (records, callCount) => + { + // After deletes, verification lists still show a record. + if (client.SuccessfulDeleteCount > 0 && callCount > 1) + { + return + [ + Record("ghost", "TXT", "still-here"), + ]; + } + + return records; + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("cleanup completed", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task RemoveAll_RetryExhaustion_WithRemainingRecords_FailsClosed() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("always fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("a", "A", "198.18.0.10"), + Record("txt", "TXT", "remain")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.NotEmpty(client.Snapshot()); + } + + [Fact] + public async Task RemoveAll_CancellationDuringDelete_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10"), Record("txt", "TXT", "x")); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + return Task.CompletedTask; + }; + + await Assert.ThrowsAnyAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token)); + } + + [Fact] + public async Task RemoveAll_CancellationDuringBackoff_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("a", "A", "198.18.0.10")); + using var cts = new CancellationTokenSource(); + var failedOnce = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.OnMutate = _ => + { + failedOnce.TrySetResult(); + return Task.CompletedTask; + }; + + var task = CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, cts.Token); + await failedOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => task); + } + + [Fact] + public async Task RemoveAll_AlreadyAbsent_IsIdempotentSuccess() + { + var client = new FakeCloudflareDnsClient(); + await CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + Assert.Equal(0, client.DeleteCallCount); + Assert.True(client.ListAllCallCount >= 1); + } + + [Fact] + public async Task AmbiguousRecordName_FailsSafelyWithoutDeletingOthers() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("good", "A", "198.18.0.10")); + client.TransformListedAllRecords = (records, _) => + records.Append(new CloudflareDnsRecord + { + Id = "bad", + Type = "TXT", + Name = "nntpd01..usenet.ninja", + Content = "ambiguous", + }).ToArray(); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Contains("ambiguous", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task Lifecycle_Stop_RemovesAllRecordsForExactFqdn() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + options.GracefulShutdownTimeout = TimeSpan.FromSeconds(30); + + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("txt", "TXT", "preexisting"), + new CloudflareDnsRecord { Id = "child", Type = "A", Name = "other." + Fqdn, Content = "198.18.0.9" }); + + var dns = CreateService(options, client); + var other = new FakeApplicationService("listener"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), other], options); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Contains(client.Snapshot(), r => r.Type == "A"); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + Assert.Contains(client.Snapshot(), r => r.Id == "child"); + Assert.Equal(1, other.StopCount); + } + + [Fact] + public async Task Lifecycle_StopOrder_DnsCleanupRunsAfterOtherServices() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + var order = new List(); + + var dns = CreateService(options, client); + var trackingDns = new OrderedStopService(TestHostFactory.WrapDns(dns), order, "dns"); + var other = new FakeApplicationService( + "listener", + onStop: _ => + { + order.Add("listener"); + return Task.CompletedTask; + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([trackingDns, other], options); + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(["listener", "dns"], order); + } + + [Fact] + public async Task Lifecycle_PartialStartup_DnsRollbackCleansFqdn() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + + var dns = CreateService(options, client); + var failing = new FakeApplicationService( + "failing", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), failing], options); + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + } + + [Fact] + public async Task StartFailure_DuringReconcile_AttemptsCleanup_DoesNotClaimSuccessOnCleanupFail() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("aaaa fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + // Cleanup delete also fails — must not report cleanup success; original start failure surfaces. + DeleteException = new CloudflareDnsException("cleanup fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + var service = CreateService(options, client); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + // Ownership was never marked active; Stop should no-op. + await service.StopAsync(CancellationToken.None); + Assert.True(client.ListAllCallCount >= 1); + } + + [Fact] + public async Task Stop_WhenOwnershipInactive_DoesNotClaimCleanup() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a", "A", "198.18.0.10")); + var service = CreateService(options, client); + + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, client.DeleteCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task ReconcileAndCleanup_SameInstance_AreSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var max = 0; + var concurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref max, Math.Max(Volatile.Read(ref max), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + var reconcile = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + + var cleanup = reconciler.RemoveAllRecordsForFqdnAsync(ZoneId, Fqdn, CancellationToken.None); + await Task.Delay(30); + Assert.Equal(1, Volatile.Read(ref entered)); + + gate.TrySetResult(); + await Task.WhenAll(reconcile, cleanup); + Assert.Equal(1, max); + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, Fqdn)); + } + + [Fact] + public async Task RepeatedStop_IsSafe_AfterCleanup() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var client = new FakeCloudflareDnsClient(); + var service = CreateService(options, client); + + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + + Assert.DoesNotContain(client.Snapshot(), r => NamesEqual(r.Name, options.Fqdn)); + } + + [Fact] + public async Task ShutdownDeadline_PropagatesCancellation_WithoutClaimingRemoval() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + options.GracefulShutdownTimeout = TimeSpan.FromMilliseconds(50); + + var client = new FakeCloudflareDnsClient(); + var dns = CreateService(options, client); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns)], options); + await lifecycle.StartAsync(CancellationToken.None); + + client.OnMutate = async ct => + { + await Task.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); + }; + + // Stop should surface timeout/cancel from manager budget while cleanup is in progress. + await Assert.ThrowsAnyAsync(() => lifecycle.StopAsync(CancellationToken.None)); + } + + [Fact] + public void TryClassifyExactFqdn_RejectsAmbiguousAndNonExact() + { + Assert.True(CloudflareDnsReconciler.TryClassifyExactFqdn("NNTPD01.Usenet.Ninja.", "nntpd01.usenet.ninja", out var exact)); + Assert.True(exact); + + Assert.True(CloudflareDnsReconciler.TryClassifyExactFqdn("other.nntpd01.usenet.ninja", "nntpd01.usenet.ninja", out exact)); + Assert.False(exact); + + Assert.False(CloudflareDnsReconciler.TryClassifyExactFqdn("nntpd01..usenet.ninja", "nntpd01.usenet.ninja", out _)); + Assert.False(CloudflareDnsReconciler.TryClassifyExactFqdn(" ", "nntpd01.usenet.ninja", out _)); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static CloudflareDnsReconciliationService CreateService(NntpdOptions options, ICloudflareDnsClient client) + { + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + CreateReconciler(client), + NullLogger.Instance); + } + + private static CloudflareDnsClient CreateHttpClient(HttpMessageHandler handler) + { + var options = TestHostFactory.CreateValidOptions(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var http = new HttpClient(handler) { BaseAddress = new Uri("https://api.cloudflare.com/client/v4/") }; + return new CloudflareDnsClient( + http, + Options.Create(options), + NullLogger.Instance); + } + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static bool NamesEqual(string left, string right) => + string.Equals( + left.Trim().TrimEnd('.'), + right.Trim().TrimEnd('.'), + StringComparison.OrdinalIgnoreCase); + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } + + private sealed class OrderedStopService : IApplicationService + { + private readonly IApplicationService _inner; + private readonly List _order; + private readonly string _label; + + public OrderedStopService(IApplicationService inner, List order, string label) + { + _inner = inner; + _order = order; + _label = label; + } + + public string Name => _inner.Name; + public Task? Execution => _inner.Execution; + public Task StartAsync(CancellationToken cancellationToken) => _inner.StartAsync(cancellationToken); + + public async Task StopAsync(CancellationToken cancellationToken) + { + await _inner.StopAsync(cancellationToken).ConfigureAwait(false); + _order.Add(_label); + } + } + + private sealed class ScriptedHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses = new(); + + public List Requests { get; } = []; + + public void EnqueueJson(HttpStatusCode statusCode, string json) => + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + _responses.Enqueue(new HttpResponseMessage(statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var copy = new HttpRequestMessage(request.Method, request.RequestUri); + Requests.Add(copy); + cancellationToken.ThrowIfCancellationRequested(); + if (_responses.Count == 0) + { + throw new InvalidOperationException("No scripted HTTP response remaining."); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs new file mode 100644 index 00000000..97985027 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Lifecycle/CloudflareDnsReconciliationServiceTests.cs @@ -0,0 +1,253 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.TransitService.Acme; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Lifecycle; + +[Collection(SerilogCollection.Name)] +public sealed class CloudflareDnsReconciliationServiceTests +{ + [Fact] + public async Task StartAsync_WithNoEligibleAddresses_Fails() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(IPAddress.Loopback), + new FakeCloudflareDnsClient()); + + var ex = await Assert.ThrowsAsync(() => + service.StartAsync(CancellationToken.None)); + + Assert.Contains("No eligible IP addresses", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StartAsync_WhenReconcileFails_PropagatesFailure() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("boom"), + }; + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task Lifecycle_DnsReconciliationCompletesBeforeRunning() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient(); + var dnsService = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + client); + + var order = new List(); + var other = new FakeApplicationService("other") + { + SharedStartOrder = order, + }; + + var trackingDns = new TrackingApplicationService(TestHostFactory.WrapDns(dnsService), order, "dns"); + + await using var lifecycle = TestHostFactory.CreateLifecycle([trackingDns, other], options); + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Equal(["dns", "other"], order); + Assert.True(client.CreateCallCount >= 2); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Lifecycle_DnsFailure_PreventsRunning() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("list failed"), + }; + + var dnsService = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + var other = new FakeApplicationService("other"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dnsService), other], options); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(0, other.StartCount); + } + + [Fact] + public async Task Host_RegistersDnsReconciliation_BeforeOtherServices() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + var started = new List(); + builder.Services.AddSingleton( + new FakeApplicationService( + "app", + onStart: _ => + { + started.Add("app"); + return Task.CompletedTask; + })); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + // Shared configuration is published before Cloudflare so the zone id and FQDN exist. + Assert.Equal(typeof(NntpDbService), services[0].GetType()); + Assert.Equal(typeof(VectorNNTPD.TransitService.NntpDb.NntpSharedConfigurationService), services[1].GetType()); + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.TransitService.NntpDb.NntpSharedConfigurationService)); + AssertRegisteredBefore( + services, + typeof(VectorNNTPD.TransitService.NntpDb.NntpSharedConfigurationService), + typeof(CloudflareDnsReconciliationApplicationService)); + Assert.Equal( + typeof(CloudflareDnsReconciliationService).Assembly, + services.Single(static service => service.GetType() == typeof(CloudflareDnsReconciliationApplicationService)).GetType().Assembly); + + AssertRegisteredBefore(services, typeof(CloudflareDnsReconciliationApplicationService), typeof(VectorNNTPD.TransitService.Redis.RedisService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.Redis.RedisService), typeof(VectorNNTPD.Common.Messaging.RabbitMq.RabbitMqService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.Common.Messaging.RabbitMq.RabbitMqService), typeof(VectorNNTPD.TransitService.RabbitMq.RabbitMqTopologyService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.RabbitMq.RabbitMqTopologyService), typeof(VectorNNTPD.TransitService.RabbitMq.ArticleWork.ArticleWorkRpcService)); + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.TransitService.Newsgroups.NewsgroupCatalogueService)); + // Transit role: no Reader-only application services (moderator, post filter, email, session state). + AssertRegisteredBefore(services, typeof(NntpDbService), typeof(VectorNNTPD.TransitService.Newsgroups.NewsgroupCatalogueService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.Newsgroups.NewsgroupCatalogueService), typeof(VectorNNTPD.TransitService.History.HistoryWriteService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.History.HistoryWriteService), typeof(VectorNNTPD.TransitService.History.HistoryMaintenanceService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.History.HistoryMaintenanceService), typeof(IncomingSpoolWriterService)); + AssertRegisteredBefore(services, typeof(IncomingSpoolWriterService), typeof(VectorNNTPD.TransitService.Ninpaths.NinpathsProcessingService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.Ninpaths.NinpathsProcessingService), typeof(VectorNNTPD.TransitService.Transit.TransitDnsRefreshService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.Transit.TransitDnsRefreshService), typeof(VectorNNTPD.TransitService.Transit.TransitPeerStateService)); + AssertRegisteredBefore(services, typeof(VectorNNTPD.TransitService.Transit.TransitPeerStateService), typeof(NntpPlainListenerService)); + AssertRegisteredBefore(services, typeof(NntpPlainListenerService), typeof(AcmeCertificateApplicationService)); + AssertRegisteredBefore(services, typeof(AcmeCertificateApplicationService), typeof(NntpTlsListenerService)); + Assert.DoesNotContain(services, static s => s.GetType().Name == "AccountByteService"); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.TransitService.Moderation.ModeratorCatalogueService); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.TransitService.PostFilter.PostFilterPolicyService); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.TransitService.PostFilter.PostFilterRejectionEvidenceService); + Assert.DoesNotContain(services, static s => s is VectorNNTPD.TransitService.Email.EmailDeliveryService); + Assert.DoesNotContain(services, static s => s is SessionStateService); + + await host.StartAsync(); + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + Assert.Equal(["app"], started); + await host.StopAsync(); + } + + [Fact] + public async Task StartAsync_Cancellation_Propagates() + { + var options = TestHostFactory.CreateValidOptions(); + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var service = CreateService( + options, + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4), + client); + + await Assert.ThrowsAnyAsync(() => service.StartAsync(cts.Token)); + } + + private static void AssertRegisteredBefore(IApplicationService[] services, Type earlier, Type later) + { + var earlierIndex = -1; + var laterIndex = -1; + for (var i = 0; i < services.Length; i++) + { + var type = services[i].GetType(); + if (type == earlier && earlierIndex < 0) + { + earlierIndex = i; + } + else if (type == later && laterIndex < 0) + { + laterIndex = i; + } + } + + Assert.True(earlierIndex >= 0, $"{earlier.Name} must be registered."); + Assert.True(laterIndex >= 0, $"{later.Name} must be registered."); + Assert.True( + earlierIndex < laterIndex, + $"{earlier.Name} (index {earlierIndex}) must precede {later.Name} (index {laterIndex})."); + } + + private static CloudflareDnsReconciliationService CreateService( + NntpdOptions options, + ILocalIpAddressAssignee assignee, + ICloudflareDnsClient client) + { + var resolver = new BindAddressResolver(assignee, NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + } + + private sealed class TrackingApplicationService : IApplicationService + { + private readonly IApplicationService _inner; + private readonly List _order; + private readonly string _label; + + public TrackingApplicationService(IApplicationService inner, List order, string label) + { + _inner = inner; + _order = order; + _label = label; + } + + public string Name => _inner.Name; + + public Task? Execution => _inner.Execution; + + public async Task StartAsync(CancellationToken cancellationToken) + { + await _inner.StartAsync(cancellationToken).ConfigureAwait(false); + _order.Add(_label); + } + + public Task StopAsync(CancellationToken cancellationToken) => _inner.StopAsync(cancellationToken); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs new file mode 100644 index 00000000..5064fa84 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsAdversarialTests.cs @@ -0,0 +1,413 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Reconciliation; + +/// +/// Adversarial coverage for non-atomic Cloudflare mutations, uncertain outcomes, and fail-closed startup. +/// +public sealed class CloudflareDnsAdversarialTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task ASucceedsThenAaaaCreateFails_DoesNotReportSuccess_LeavesPartialRemoteState() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") { StatusCode = 500, IsOutcomeUncertain = true }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + Assert.True(client.SuccessfulCreateCount >= 1); + } + + [Fact] + public async Task AaaaSucceedsThenADeleteFails_DoesNotReportSuccess() + { + // Seed stale A that must be deleted after AAAA create path; fail deletes permanently. + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") { StatusCode = 500, IsOutcomeUncertain = true }, + }; + client.Seed( + Record("stale", "A", "198.18.0.99"), + Record("aaaa", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + // Desired A was created (create-before-delete) but stale may remain. + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task CreateSucceedsThenDeleteFails_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task DeleteWouldSucceedButCreateFailsFirst_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + // create-before-delete: create failed, so stale should still be present and desired absent. + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.DoesNotContain(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task UncertainCreateOutcome_AppliedRemotely_RetryConvergesAndVerifies() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + CreateFailTypeException = new CloudflareDnsException("timeout after apply") + { + IsOutcomeUncertain = true, + }, + ApplyCreateThenFailUncertain = true, + }; + + var reconciler = CreateReconciler(client); + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + Assert.Equal(2, snapshot.Count); + } + + [Fact] + public async Task UncertainOutcomeExhausted_FailsClosed_DoesNotClaimSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeException = new CloudflareDnsException("transport timeout") + { + IsOutcomeUncertain = true, + }, + // Do not apply: simulates a timeout where we cannot know whether Cloudflare wrote — + // here we model the "never observed as present" exhausted path by not mutating. + ApplyCreateThenFailUncertain = false, + }; + + var reconciler = CreateReconciler(client); + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.Contains("must not proceed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(client.Snapshot(), r => r.Content == "198.18.0.10"); + } + + [Fact] + public async Task RetryAfterPartialMutation_ReReadsAndConverges() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA failed"), + }; + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.5", "fd12:3456:789a::1"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "10.0.0.5"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + + client.CreateFailType = null; + client.CreateFailTypeException = null; + + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.5", "fd12:3456:789a::1"), CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "10.0.0.5"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "fd12:3456:789a::1"); + Assert.Equal(2, snapshot.Count); + } + + [Fact] + public async Task PrivateIpv4AndIpv6_ArePublishedExactly() + { + var client = new FakeCloudflareDnsClient(); + var reconciler = CreateReconciler(client); + var desired = Desired("10.0.0.8", "192.168.1.8", "172.16.0.8", "fd00::8"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Equal(4, snapshot.Count); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "10.0.0.8"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "192.168.1.8"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "172.16.0.8"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "fd00::8"); + } + + [Fact] + public async Task VerificationMismatch_NeverReportsSuccess() + { + var client = new AlwaysMismatchAfterMutationsClient(); + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("must not proceed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CancellationDuringReconcile_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + client.OnMutate = _ => + { + cts.Cancel(); + cts.Token.ThrowIfCancellationRequested(); + return Task.CompletedTask; + }; + + var reconciler = CreateReconciler(client); + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + } + + [Fact] + public async Task Startup_AfterDnsPartialFailure_DoesNotReachRunning_AndRollbackStopsDnsService() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA failed"), + }; + + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(IPAddress.Parse("10.0.0.9"), IPAddress.Parse("fd00::9")), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + var dns = new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance); + + var other = new FakeApplicationService("other"); + await using var lifecycle = TestHostFactory.CreateLifecycle([TestHostFactory.WrapDns(dns), other], options); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(0, other.StartCount); + Assert.DoesNotContain( + client.Snapshot(), + r => string.Equals( + r.Name.Trim().TrimEnd('.'), + options.Fqdn.Trim().TrimEnd('.'), + StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task MutationTimeout_IsMarkedUncertain_ByHttpClient() + { + var handler = new ScriptedTimeoutHandler(); + // codeql[cs/local-not-disposed] HttpClient ownership transfers to CloudflareDnsClient for the test lifetime. + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://api.cloudflare.com/client/v4/"), + Timeout = TimeSpan.FromMilliseconds(50), + }; + var client = new CloudflareDnsClient( + httpClient, + Options.Create(TestHostFactory.CreateValidOptions()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => + client.CreateRecordAsync( + ZoneId, + new CloudflareDnsRecordWriteRequest + { + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + }, + CancellationToken.None)); + + Assert.True(ex.IsOutcomeUncertain); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void BindAddressResolver_PublishesPrivateAddresses_ExcludesLinkLocal() + { + var privateV4 = IPAddress.Parse("10.1.2.3"); + var ula = IPAddress.Parse("fd12::1"); + var linkLocal = IPAddress.Parse("169.254.1.1"); + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(privateV4, ula, linkLocal), + NullLogger.Instance); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + var resolved = resolver.Resolve(options); + + Assert.Contains(privateV4, resolved.IPv4); + Assert.Contains(ula, resolved.IPv6); + Assert.DoesNotContain(linkLocal, resolved.All); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + /// + /// Lists empty during mutation planning, then returns a mismatched set on verification lists. + /// + private sealed class AlwaysMismatchAfterMutationsClient : ICloudflareDnsClient + { + private int _creates; + + public Task> ListRecordsAsync( + string zoneId, + string fqdn, + string type, + CancellationToken cancellationToken) + { + if (_creates == 0) + { + return Task.FromResult>([]); + } + + // After any create, always report a mismatched address so verification cannot succeed. + return Task.FromResult>( + [ + new CloudflareDnsRecord + { + Id = "mismatch", + Type = type, + Name = fqdn, + Content = type == "AAAA" ? "2001:db8::99" : "198.18.0.99", + }, + ]); + } + + public Task> ListAllRecordsForNameAsync( + string zoneId, + string fqdn, + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public Task CreateRecordAsync( + string zoneId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + _creates++; + return Task.FromResult(new CloudflareDnsRecord + { + Id = $"c{_creates}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + }); + } + + public Task UpdateRecordAsync( + string zoneId, + string recordId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) => + Task.FromResult(new CloudflareDnsRecord + { + Id = recordId, + Type = request.Type, + Name = request.Name, + Content = request.Content, + }); + + public Task DeleteRecordAsync(string zoneId, string recordId, CancellationToken cancellationToken) => + Task.CompletedTask; + } + + private sealed class ScriptedTimeoutHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json"), + Headers = { }, + }; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs new file mode 100644 index 00000000..593a7ca1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsFinalRegressionTests.cs @@ -0,0 +1,534 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Reconciliation; + +/// +/// Final regression coverage for staged create-all-then-delete, retry classification, and exact-set verify. +/// +public sealed class CloudflareDnsFinalRegressionTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task TransientFailure_ThenSuccessfulRetry_ReportsSuccessOnlyAfterVerify() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + CreateFailTypeException = new CloudflareDnsException("transient 502") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(2, client.Snapshot().Count); + Assert.True(client.CreateCallCount >= 2); + } + + [Fact] + public async Task PermanentAuthFailure_DoesNotRetryPointlessly() + { + var client = new FakeCloudflareDnsClient + { + ListException = new CloudflareDnsException("Authentication error") + { + StatusCode = 401, + IsPermanentFailure = true, + FailedOperation = "GET list", + }, + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + Assert.Equal(401, ex.StatusCode); + Assert.Equal(1, client.ListCallCount); + Assert.Equal(0, client.CreateCallCount); + } + + [Fact] + public async Task TimeoutAfterMutationApplied_RetryConvergesWithoutDuplicateSuccessClaim() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = true, + CreateFailTypeException = new CloudflareDnsException("timeout after apply") + { + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.Equal("198.18.0.10", client.Snapshot()[0].Content); + } + + [Fact] + public async Task TimeoutAfterMutationNotApplied_RetryCreatesThenVerifies() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = false, + CreateFailTypeException = new CloudflareDnsException("timeout no apply") + { + IsOutcomeUncertain = true, + }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.True(client.SuccessfulCreateCount >= 1); + } + + [Fact] + public async Task Retry_ReReadsActualState_DoesNotAssumePriorMutationFailed() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeRemaining = 1, + ApplyCreateThenFailUncertain = true, + CreateFailTypeException = new CloudflareDnsException("uncertain") { IsOutcomeUncertain = true }, + }; + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("10.0.0.1", "fd00::1"), CancellationToken.None); + + // Attempt 1: list A+AAAA; attempt 2: list A+AAAA again (+ verify lists). Must exceed one attempt. + Assert.True(client.ListCallCount > 4); + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task RetryExhaustion_AfterPartialChanges_FailsClosed() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA always fails") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(client.Snapshot(), r => r.Type == "A"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + } + + [Fact] + public async Task CreateFailure_BeforeAnyDeletion_LeavesStaleIntact() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") + { + StatusCode = 500, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.Equal(0, client.SuccessfulDeleteCount); + } + + [Fact] + public async Task DeleteFailure_AfterDesiredCreated_FailsClosed_KeepsSuperset() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("delete failed") + { + StatusCode = 500, + IsOutcomeUncertain = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task CrossFamily_CreatesAllMissingBeforeAnyDeletes() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(["Create:A", "Create:AAAA", "Delete", "Delete"], client.MutationOrder.Take(4).ToArray()); + Assert.DoesNotContain(client.Snapshot(), r => r.Content is "198.18.0.99" or "2001:db8::99"); + } + + [Fact] + public async Task AFailureWhileAaaaAlsoNeedsChanges_DoesNotDeleteYet() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "A", + CreateFailTypeException = new CloudflareDnsException("A create failed") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Equal(0, client.SuccessfulDeleteCount); + Assert.DoesNotContain(client.MutationOrder, m => m.StartsWith("Create:AAAA", StringComparison.Ordinal)); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + Assert.Contains(client.Snapshot(), r => r.Content == "2001:db8::99"); + } + + [Fact] + public async Task AaaaFailureAfterACreated_BeforeDeletes_KeepsStale() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99")); + + await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("198.18.0.10", "2001:db8::10"), + CancellationToken.None)); + + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.10"); + Assert.Equal(0, client.SuccessfulDeleteCount); + Assert.Contains(client.Snapshot(), r => r.Content == "198.18.0.99"); + } + + [Fact] + public async Task Ipv6ExpandedAndCompressedForms_NormalizeToSameDesiredSet() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("aaaa", "AAAA", "2001:db8::1")); + + var desired = new ResolvedBindAddresses( + [ + IPAddress.Parse("2001:0db8:0000:0000:0000:0000:0000:0001"), + ]); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task ProxiedDesiredRecord_IsUpdatedBeforeSuccess() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "proxied", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + }); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + Assert.False(Assert.Single(client.Snapshot()).Proxied); + } + + [Fact] + public async Task Verify_RejectsProxiedRecordEvenIfContentMatches() + { + var client = new FakeCloudflareDnsClient(); + client.TransformListedRecords = (_, records, _) => + records + .Select(r => new CloudflareDnsRecord + { + Id = r.Id, + Type = r.Type, + Name = r.Name, + Content = r.Content, + Ttl = r.Ttl, + Proxied = true, + }) + .ToArray(); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("proxied", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CancellationDuringBackoff_DoesNotReportSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }, + }; + using var cts = new CancellationTokenSource(); + var failedOnce = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var original = client.CreateException; + client.CreateException = new CloudflareDnsException("fail") + { + StatusCode = 502, + IsOutcomeUncertain = true, + }; + + // Signal after first create attempt by wrapping via OnMutate. + client.OnMutate = _ => + { + failedOnce.TrySetResult(); + return Task.CompletedTask; + }; + // CreateException throws after OnMutate — keep original fail. + client.CreateException = original; + + var task = CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token); + await failedOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => task); + } + + [Fact] + public async Task ConcurrentReconcile_SameInstance_IsSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var max = 0; + var concurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref max, Math.Max(Volatile.Read(ref max), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var first = reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + var second = reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + await Task.Delay(30); + Assert.Equal(1, Volatile.Read(ref entered)); + gate.TrySetResult(); + await Task.WhenAll(first, second); + Assert.Equal(1, max); + } + + [Fact] + public async Task ExternalModification_BetweenReadAndMutation_RetryConvergesOrFailsClosed() + { + var client = new FakeCloudflareDnsClient(); + var injected = 0; + client.OnMutate = _ => + { + // Between planned create and completion: external actor also publishes desired A. + if (Interlocked.Exchange(ref injected, 1) == 0) + { + client.Inject(Record("external", "A", "198.18.0.10")); + } + + return Task.CompletedTask; + }; + + // May succeed on retry after deduping, or exhaust if inject keeps happening — inject only once. + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Single(client.Snapshot()); + Assert.Equal("198.18.0.10", client.Snapshot()[0].Content); + } + + [Fact] + public async Task ExternalModification_DetectedOnFinalVerification_FailsClosed() + { + var client = new FakeCloudflareDnsClient(); + client.TransformListedRecords = (type, records, listCallCount) => + { + // After mutations (create call happened), inject an unexpected A on verification lists. + if (client.SuccessfulCreateCount > 0 + && string.Equals(type, "A", StringComparison.OrdinalIgnoreCase) + && listCallCount > 2) + { + return records + .Append(Record("external-stale", "A", "198.18.0.77")) + .ToArray(); + } + + return records; + }; + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("verification failed", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task PrivateAddresses_RemainPublished() + { + var client = new FakeCloudflareDnsClient(); + await CreateReconciler(client).ReconcileAsync( + ZoneId, + Fqdn, + Desired("10.0.0.8", "192.168.0.8", "fd00::8"), + CancellationToken.None); + + Assert.Equal(3, client.Snapshot().Count); + } + + [Fact] + public async Task NoEligibleAddresses_StartupFails() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + var service = new CloudflareDnsReconciliationService( + Options.Create(options), + new BindAddressResolver(new FakeLocalIpAddressAssignee(IPAddress.Loopback), NullLogger.Instance), + CreateReconciler(new FakeCloudflareDnsClient()), + NullLogger.Instance); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("No eligible IP addresses", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task DuplicateAndPagination_StillConverge() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10"), + Record("aaaa2", "AAAA", "2001:db8::10")); + + await CreateReconciler(client) + .ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None); + + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task PermanentValidationFailure_OnDelete_DoesNotRetryIndefinitely() + { + var client = new FakeCloudflareDnsClient + { + DeleteException = new CloudflareDnsException("validation error") + { + StatusCode = 400, + IsPermanentFailure = true, + }, + }; + client.Seed(Record("stale", "A", "198.18.0.99")); + + var ex = await Assert.ThrowsAsync(() => + CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.True(ex.IsPermanentFailure); + // Create once, delete once — no reconciler retry loop. + Assert.Equal(1, client.SuccessfulCreateCount); + Assert.Equal(1, client.DeleteCallCount); + Assert.DoesNotContain("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content) => + new() + { + Id = id, + Type = type, + Name = Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs new file mode 100644 index 00000000..fcc9ab1d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Cloudflare/Reconciliation/CloudflareDnsReconcilerTests.cs @@ -0,0 +1,383 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Cloudflare.Reconciliation; + +public sealed class CloudflareDnsReconcilerTests +{ + private const string ZoneId = "zone-test"; + private const string Fqdn = "nntpd01.usenet.ninja"; + + [Fact] + public async Task Reconcile_WhenRecordsAlreadyMatch_MakesNoMutations() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.UpdateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Equal(2, client.Snapshot().Count); + } + + [Fact] + public async Task Reconcile_MissingRecords_CreatesThem() + { + var client = new FakeCloudflareDnsClient(); + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Equal(2, client.CreateCallCount); + var snapshot = client.Snapshot(); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10" && r.Proxied == false); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10" && r.Proxied == false); + } + + [Fact] + public async Task Reconcile_StaleRecords_AreRemoved() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("stale-a", "A", "198.18.0.99"), + Record("stale-aaaa", "AAAA", "2001:db8::99"), + Record("other", "A", "198.18.0.50", name: "other.usenet.ninja")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Content is "198.18.0.99" or "2001:db8::99"); + Assert.Contains(snapshot, r => r.Name == "other.usenet.ninja" && r.Content == "198.18.0.50"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + } + + [Fact] + public async Task Reconcile_MixedDesiredAndStale_RetainsDesiredRemovesStale() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("keep", "A", "198.18.0.10"), + Record("stale", "A", "198.18.0.99")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Single(snapshot); + Assert.Equal("198.18.0.10", snapshot[0].Content); + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(1, client.DeleteCallCount); + } + + [Fact] + public async Task Reconcile_NoIpv4_RemovesAllARecords() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("2001:db8::10")]); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Type == "A"); + Assert.Contains(snapshot, r => r.Type == "AAAA" && r.Content == "2001:db8::10"); + } + + [Fact] + public async Task Reconcile_NoIpv6_RemovesAllAaaaRecords() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = new ResolvedBindAddresses([IPAddress.Parse("198.18.0.10")]); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.DoesNotContain(snapshot, r => r.Type == "AAAA"); + Assert.Contains(snapshot, r => r.Type == "A" && r.Content == "198.18.0.10"); + } + + [Fact] + public async Task Reconcile_EmptyDesired_ThrowsWithoutMutating() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(Record("a1", "A", "198.18.0.10")); + var reconciler = CreateReconciler(client); + + await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, new ResolvedBindAddresses([]), CancellationToken.None)); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Single(client.Snapshot()); + } + + [Fact] + public async Task Reconcile_DuplicateRecords_AreCollapsed() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + Record("a1", "A", "198.18.0.10"), + Record("a2", "A", "198.18.0.10"), + Record("aaaa1", "AAAA", "2001:db8::10"), + Record("aaaa2", "AAAA", "2001:db8::10")); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10", "2001:db8::10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + var snapshot = client.Snapshot(); + Assert.Equal(2, snapshot.Count); + Assert.Equal(2, client.DeleteCallCount); + } + + [Fact] + public async Task Reconcile_UnrelatedTxtRecord_RemainsUntouched() + { + var client = new FakeCloudflareDnsClient(); + client.Seed( + new CloudflareDnsRecord + { + Id = "txt1", + Type = "TXT", + Name = Fqdn, + Content = "v=spf1 -all", + }); + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + await reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + + Assert.Contains(client.Snapshot(), r => r.Type == "TXT" && r.Id == "txt1"); + } + + [Fact] + public async Task Reconcile_ApiFailure_SurfacesAndIsNotTreatedAsSuccess() + { + var client = new FakeCloudflareDnsClient + { + CreateException = new CloudflareDnsException("create failed") { StatusCode = 500 }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.Ordinal); + Assert.Contains("create failed", ex.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task Reconcile_PartialFailureAfterAUpdates_Surfaces() + { + var client = new FakeCloudflareDnsClient + { + CreateFailType = "AAAA", + CreateFailTypeException = new CloudflareDnsException("AAAA create failed") { StatusCode = 502 }, + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10", "2001:db8::10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.Ordinal); + Assert.Contains(client.Snapshot(), r => r.Type == "A" && r.Content == "198.18.0.10"); + Assert.DoesNotContain(client.Snapshot(), r => r.Type == "AAAA"); + } + + [Fact] + public async Task Reconcile_VerificationMismatch_Throws() + { + var client = new FakeCloudflareDnsClient(); + // Force verification lists to observe a mismatched set after mutations. + client.MutationsBeforeListFailure = 1; + client.FailListAfterMutationsException = new CloudflareDnsException( + "Cloudflare DNS verification failed for 'nntpd01.usenet.ninja': mismatched") + { + FailedOperation = "Verify", + }; + var reconciler = CreateReconciler(client); + + var ex = await Assert.ThrowsAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None)); + + Assert.Contains("failed after", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Reconcile_Cancellation_Propagates() + { + var client = new FakeCloudflareDnsClient(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var reconciler = CreateReconciler(client); + + await Assert.ThrowsAnyAsync(() => + reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), cts.Token)); + } + + [Fact] + public async Task Reconcile_ConcurrentCalls_AreSerialized() + { + var client = new FakeCloudflareDnsClient(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = 0; + var concurrent = 0; + var maxConcurrent = 0; + + client.OnMutate = async ct => + { + Interlocked.Increment(ref entered); + var now = Interlocked.Increment(ref concurrent); + Interlocked.Exchange(ref maxConcurrent, Math.Max(Volatile.Read(ref maxConcurrent), now)); + await gate.Task.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Decrement(ref concurrent); + }; + + var reconciler = CreateReconciler(client); + var desired = Desired("198.18.0.10"); + + var first = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await WaitUntilAsync(() => Volatile.Read(ref entered) >= 1); + + var second = reconciler.ReconcileAsync(ZoneId, Fqdn, desired, CancellationToken.None); + await Task.Delay(50); + Assert.Equal(1, Volatile.Read(ref entered)); + + gate.TrySetResult(); + await Task.WhenAll(first, second); + + Assert.Equal(1, maxConcurrent); + } + + [Fact] + public async Task Reconcile_ProxiedRecord_IsUpdatedToDnsOnly() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "proxied", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + }); + + var reconciler = CreateReconciler(client); + await reconciler.ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(false, updated.Proxied); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + } + + [Fact] + public async Task Reconcile_IncorrectTtl_IsUpdatedInPlace() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = false, + Ttl = 120, + }); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + Assert.Equal(false, updated.Proxied); + } + + [Fact] + public async Task Reconcile_IncorrectTtlAndProxied_UpdatesOnce() + { + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "a1", + Type = "A", + Name = Fqdn, + Content = "198.18.0.10", + Proxied = true, + Ttl = 60, + }); + + await CreateReconciler(client).ReconcileAsync(ZoneId, Fqdn, Desired("198.18.0.10"), CancellationToken.None); + + Assert.Equal(1, client.UpdateCallCount); + var updated = Assert.Single(client.Snapshot()); + Assert.Equal(CloudflareManagedDnsPolicy.ManagedTtl, updated.Ttl); + Assert.Equal(false, updated.Proxied); + } + + private static CloudflareDnsReconciler CreateReconciler(ICloudflareDnsClient client) => + new(client, Options.Create(TestHostFactory.CreateValidOptions()), NullLogger.Instance); + + private static ResolvedBindAddresses Desired(params string[] addresses) => + new(addresses.Select(IPAddress.Parse)); + + private static CloudflareDnsRecord Record(string id, string type, string content, string? name = null) => + new() + { + Id = id, + Type = type, + Name = name ?? Fqdn, + Content = content, + Proxied = false, + Ttl = CloudflareManagedDnsPolicy.ManagedTtl, + }; + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + + await Task.Delay(10); + } + + Assert.Fail("Condition was not met within the safety timeout."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/ControlOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/ControlOptionsTests.cs new file mode 100644 index 00000000..8294366b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/ControlOptionsTests.cs @@ -0,0 +1,320 @@ +using System.Reflection; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +public sealed class ControlOptionsTests +{ + private const string AioeFingerprint = "22031AAC51E7C7FD664F1D8090DF6C712322A7F8"; + private const string AioeFingerprintGrouped = "2203 1AAC 51E7 C7FD 664F 1D80 90DF 6C71 2322 A7F8"; + private const string BigEightNewsgroups = "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*"; + private const int SourceAuthorityCount = 112; + private const int SourceAuthorizationCount = 317; + + [Fact] + public void Bind_ControlSection_Succeeds() + { + var options = Bind( + """ + { + "Control": { + "PgpAuthorities": { + "Source": { + "LastModified": "2023-08-05" + }, + "Authorities": [ + { "Name": "AIOE", "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8" } + ] + } + } + } + """); + + Assert.Equal("2023-08-05", options.PgpAuthorities.Source.LastModified); + Assert.Single(options.PgpAuthorities.Authorities); + Assert.Equal("AIOE", options.PgpAuthorities.Authorities[0].Name); + Assert.True(new ControlOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_PgpAuthorities_Succeeds() + { + var options = Bind( + """ + { + "Control": { + "PgpAuthorities": { + "Source": { + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl", + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl" + }, + "Authorities": [] + } + } + } + """); + + Assert.Equal("InnUrl", options.PgpAuthorities.Source.RetrievedFrom); + Assert.Equal( + "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl", + options.PgpAuthorities.Source.Url); + Assert.Equal( + "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + options.PgpAuthorities.Source.InnUrl); + Assert.Empty(options.PgpAuthorities.Authorities); + } + + [Fact] + public void Bind_KnownAuthority_HasExpectedFingerprint() + { + var options = BindProduction(); + var aioe = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "AIOE"); + Assert.Equal(AioeFingerprint, aioe.KeyFingerprint); + Assert.Equal("usenet@aioe.org", aioe.Contact); + Assert.Equal("http://news.aioe.org/hierarchy/aioe.txt", aioe.KeyUrl); + } + + [Fact] + public void FingerprintNormalization_PreservesHexBytes() + { + Assert.Equal(AioeFingerprint, PgpAuthorityFingerprint.Normalize(AioeFingerprintGrouped)); + Assert.Equal(AioeFingerprint, PgpAuthorityFingerprint.Normalize(AioeFingerprint.ToLowerInvariant())); + Assert.True(PgpAuthorityFingerprint.TryNormalize(AioeFingerprintGrouped, out var normalized)); + Assert.Equal(AioeFingerprint, normalized); + + var sourceDigits = new string(AioeFingerprintGrouped.Where(static c => !char.IsWhiteSpace(c)).ToArray()); + Assert.Equal(sourceDigits, normalized, ignoreCase: true); + Assert.False(PgpAuthorityFingerprint.TryNormalize("not-a-fingerprint", out _)); + Assert.False(PgpAuthorityFingerprint.TryNormalize("220", out _)); + Assert.False(PgpAuthorityFingerprint.TryNormalize(" ", out _)); + } + + [Fact] + public void Bind_PostConfigure_NormalizesGroupedFingerprint() + { + var configuration = LoadJson( + """ + { + "Control": { + "PgpAuthorities": { + "Authorities": [ + { "Name": "AIOE", "KeyFingerprint": "2203 1AAC 51E7 C7FD 664F 1D80 90DF 6C71 2322 A7F8" } + ] + } + } + } + """); + + var services = new ServiceCollection(); + services + .AddOptions() + .Bind(configuration.GetSection(ControlOptions.SectionName)) + .PostConfigure(static options => + { + // codeql[cs/linq/missed-where] TryNormalize out-params; foreach preserves mutate-in-place clarity. + foreach (var authority in options.PgpAuthorities.Authorities) + { + if (PgpAuthorityFingerprint.TryNormalize(authority.KeyFingerprint, out var normalized)) + { + authority.KeyFingerprint = normalized; + } + } + }); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(AioeFingerprint, options.PgpAuthorities.Authorities[0].KeyFingerprint); + } + + [Fact] + public void Bind_MultipleHierarchyRules_RemainDistinct() + { + var options = BindProduction(); + var de = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "DE"); + var deAlt = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "DE.ALT"); + Assert.NotSame(de, deAlt); + Assert.Equal("de.*", de.Authorizations[0].Newsgroups); + Assert.Equal("de.alt.*", deAlt.Authorizations[0].Newsgroups); + + var bigEight = Assert.Single( + options.PgpAuthorities.Authorities, + a => a.Name == "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK"); + Assert.Equal(3, bigEight.Authorizations.Length); + Assert.All(bigEight.Authorizations, rule => Assert.Equal(BigEightNewsgroups, rule.Newsgroups)); + Assert.Equal( + ["checkgroups", "newgroup", "rmgroup"], + bigEight.Authorizations.Select(static r => r.Message).ToArray()); + + var maus = Assert.Single(options.PgpAuthorities.Authorities, a => a.Name == "MAUS"); + Assert.Equal(6, maus.Authorizations.Length); + Assert.Equal(2, maus.Authorizations.Count(static r => r.Message == "checkgroups")); + Assert.Contains(maus.Authorizations, static r => r.From == "guenter@gst0hb.hb.provi.de"); + Assert.Contains(maus.Authorizations, static r => r.From == "guenter@gst0hb.north.de"); + } + + [Fact] + public void Validate_MissingOptionalMetadata_DoesNotFail() + { + var options = new ControlOptions + { + PgpAuthorities = new PgpAuthoritiesOptions + { + Authorities = + [ + new PgpAuthorityOptions { Name = "LOCAL-TEST" }, + ], + }, + }; + + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + } + + [Fact] + public void CatalogueTypesAndProductionJson_HaveNoPrivateKeyOrPassphraseConfiguration() + { + var secretFragments = new[] + { + "PrivateKey", + "Passphrase", + "Password", + "SecretKey", + "Armored", + "BEGIN PGP PRIVATE", + }; + + Type[] catalogueTypes = + [ + typeof(ControlOptions), + typeof(PgpAuthoritiesOptions), + typeof(PgpAuthorityOptions), + typeof(PgpAuthoritySourceOptions), + typeof(PgpAuthorityAuthorizationOptions), + ]; + + foreach (var type in catalogueTypes) + { + foreach (var property in type.GetProperties(BindingFlags.Instance | BindingFlags.Public)) + { + Assert.DoesNotContain( + secretFragments, + fragment => property.Name.Contains(fragment, StringComparison.OrdinalIgnoreCase)); + } + } + + using var document = JsonDocument.Parse(File.ReadAllText(FindProductionAppsettings())); + var controlJson = document.RootElement.GetProperty("Control").GetRawText(); + foreach (var fragment in secretFragments) + { + Assert.DoesNotContain(fragment, controlJson, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public void Validate_EmptyAuthorityConfiguration_Succeeds() + { + Assert.True(new ControlOptionsValidator().Validate(null, new ControlOptions()).Succeeded); + Assert.True( + new ControlOptionsValidator() + .Validate(null, new ControlOptions { PgpAuthorities = new PgpAuthoritiesOptions() }) + .Succeeded); + + var omitted = Bind("{}"); + Assert.Empty(omitted.PgpAuthorities.Authorities); + Assert.True(new ControlOptionsValidator().Validate(null, omitted).Succeeded); + + var emptySection = Bind("""{ "Control": { } }"""); + Assert.Empty(emptySection.PgpAuthorities.Authorities); + Assert.True(new ControlOptionsValidator().Validate(null, emptySection).Succeeded); + } + + [Fact] + public void ProductionAppsettings_BindsCompleteCatalogue() + { + var options = BindProduction(); + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + + Assert.Equal("2023-08-05", options.PgpAuthorities.Source.LastModified); + Assert.Equal("InnUrl", options.PgpAuthorities.Source.RetrievedFrom); + Assert.Equal(SourceAuthorityCount, options.PgpAuthorities.Authorities.Length); + Assert.Equal( + SourceAuthorizationCount, + options.PgpAuthorities.Authorities.Sum(static a => a.Authorizations.Length)); + Assert.Equal(55, options.PgpAuthorities.Authorities.Count(static a => a.KeyFingerprint is not null)); + Assert.All( + options.PgpAuthorities.Authorities.Where(static a => a.KeyFingerprint is not null), + static a => + { + Assert.True(PgpAuthorityFingerprint.TryNormalize(a.KeyFingerprint, out var normalized)); + Assert.Equal(normalized, a.KeyFingerprint); + }); + } + + [Fact] + public void Validate_RejectsNonHexadecimalFingerprint() + { + var options = new ControlOptions + { + PgpAuthorities = new PgpAuthoritiesOptions + { + Authorities = + [ + new PgpAuthorityOptions { Name = "AIOE", KeyFingerprint = "not-hex" }, + ], + }, + }; + + var result = new ControlOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + private static ControlOptions BindProduction() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(FindProductionAppsettings(), optional: false) + .Build(); + var options = new ControlOptions(); + configuration.GetSection(ControlOptions.SectionName).Bind(options); + return options; + } + + private static ControlOptions Bind(string json) + { + var options = new ControlOptions(); + LoadJson(json).GetSection(ControlOptions.SectionName).Bind(options); + return options; + } + + private static IConfigurationRoot LoadJson(string json) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + return new ConfigurationBuilder() + .AddJsonStream(stream) + .Build(); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/EmailOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/EmailOptionsTests.cs new file mode 100644 index 00000000..662ef86a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/EmailOptionsTests.cs @@ -0,0 +1,179 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +public sealed class EmailOptionsTests +{ + [Fact] + public void Disabled_EmptyHost_IsValid() + { + var options = Bind( + """ + { + "Email": { + "Enabled": false + } + } + """); + Assert.False(options.Enabled); + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void ProductionAppsettings_IsDisabledAndValid() + { + var path = FindProductionAppsettings(); + var configuration = new ConfigurationBuilder() + .AddJsonFile(path, optional: false) + .Build(); + var options = new EmailOptions(); + configuration.GetSection(EmailOptions.SectionName).Bind(options); + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + Assert.False(options.Enabled); + Assert.Equal("noreply@usenet.ninja", options.DefaultFrom); + Assert.Equal(SmtpSecurityMode.StartTls, options.Smtp.Security); + Assert.True(options.Smtp.RequireTlsForAuthentication); + Assert.True(string.IsNullOrEmpty(options.Smtp.Password)); + Assert.Equal(EmailSpoolOptions.DefaultDirectory, options.Spool.Directory); + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var email = document.RootElement.GetProperty("Email"); + var smtp = email.GetProperty("Smtp"); + Assert.False(smtp.TryGetProperty("EhloHostname", out _)); + Assert.False(smtp.TryGetProperty("DangerousAcceptAnyServerCertificate", out _)); + Assert.False(email.TryGetProperty("Queue", out _)); + Assert.True(email.TryGetProperty("Spool", out _)); + } + + [Fact] + public void Enabled_RequiresHostAndFrom() + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = string.Empty, + Smtp = new SmtpOptions { Host = string.Empty, Port = 587 }, + }; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + var failures = result.Failures ?? []; + Assert.Contains(failures, static f => f.Contains("Host", StringComparison.Ordinal)); + Assert.Contains(failures, static f => f.Contains("DefaultFrom", StringComparison.Ordinal)); + Assert.DoesNotContain(failures, static f => f.Contains("secret", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Enabled_AuthWithoutTls_FailsUnlessExplicit() + { + var options = ValidEnabled(); + options.Smtp.Username = "user"; + options.Smtp.Password = "password"; + options.Smtp.Security = SmtpSecurityMode.None; + options.Smtp.RequireTlsForAuthentication = true; + Assert.False(new EmailOptionsValidator().Validate(null, options).Succeeded); + + options.Smtp.RequireTlsForAuthentication = false; + Assert.True(new EmailOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void UsernameWithoutPassword_FailsWithoutEchoingSecret() + { + var options = ValidEnabled(); + options.Smtp.Username = "user"; + options.Smtp.Password = string.Empty; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + Assert.DoesNotContain(result.Failures ?? [], static f => f.Contains("hunter2", StringComparison.Ordinal)); + } + + [Fact] + public void InvalidPortAndScanInterval_Fail() + { + var options = ValidEnabled(); + options.Smtp.Port = 0; + options.Spool.ScanInterval = TimeSpan.Zero; + var result = new EmailOptionsValidator().Validate(null, options); + Assert.False(result.Succeeded); + } + + [Fact] + public void Options_InvalidEnabledHost_FailsOnStart() + { + var services = new ServiceCollection(); + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes( + """ + { + "Email": { + "Enabled": true, + "DefaultFrom": "noreply@example.com", + "Smtp": { "Host": "", "Port": 587, "Security": "StartTls" } + } + } + """)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + services.AddSingleton(configuration); + services + .AddOptions() + .BindConfiguration(EmailOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, EmailOptionsValidator>(); + using var provider = services.BuildServiceProvider(); + Assert.Throws( + () => provider.GetRequiredService>().Value); + } + + private static EmailOptions ValidEnabled() => + new() + { + Enabled = true, + DefaultFrom = "noreply@example.com", + EnvelopeSender = "noreply@example.com", + Smtp = new SmtpOptions + { + Host = "smtp.example.com", + Port = 587, + Security = SmtpSecurityMode.StartTls, + }, + }; + + private static EmailOptions Bind(string json) + { + var options = new EmailOptions(); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + new ConfigurationBuilder() + .AddJsonStream(stream) + .Build() + .GetSection(EmailOptions.SectionName) + .Bind(options); + options.Smtp ??= new SmtpOptions(); + options.Spool ??= new EmailSpoolOptions(); + return options; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/ModerationOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/ModerationOptionsTests.cs new file mode 100644 index 00000000..8e77c034 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/ModerationOptionsTests.cs @@ -0,0 +1,132 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +public sealed class ModerationOptionsTests +{ + [Fact] + public void Bind_EmptySection_IsValid() + { + var options = Bind( + """ + { + "Moderation": { + "Moderators": [] + } + } + """); + + Assert.Empty(options.Moderators); + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_OmittedSection_IsValid() + { + var options = Bind("""{ "Nntpd": { "ServerId": 1 } }"""); + Assert.Empty(options.Moderators); + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Bind_LeftoverModerators_FailsValidation() + { + var options = Bind( + """ + { + "Moderation": { + "Moderators": [ + { + "Pattern": "comp.example.*", + "Address": "moderator@example.com", + "Username": "moderator-example" + } + ] + } + } + """); + + var result = new ModerationOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("nntpmoderators", result.FailureMessage, StringComparison.Ordinal); + } + + [Fact] + public void Bind_ProductionAppsettings_KeepsSourceProvenanceWithoutStaticList() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(FindProductionAppsettings(), optional: false) + .Build(); + var options = new ModerationOptions(); + configuration.GetSection(ModerationOptions.SectionName).Bind(options); + options.Moderators ??= []; + Assert.True(new ModerationOptionsValidator().Validate(null, options).Succeeded); + Assert.Equal("https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", options.Source.InnUrl); + Assert.Equal("InnUrl", options.Source.RetrievedFrom); + Assert.Equal("https://github.com/InterNetNews/inn/blob/main/samples/moderators", options.Source.Url); + Assert.Empty(options.Moderators); + } + + [Fact] + public void Options_LeftoverModerators_FailsOnStart() + { + var services = new ServiceCollection(); + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes( + """ + { + "Moderation": { + "Moderators": [ + { "Pattern": "comp.example.*", "Address": "mod@example.com", "Username": "mod" } + ] + } + } + """)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + services.AddSingleton(configuration); + services + .AddOptions() + .BindConfiguration(ModerationOptions.SectionName) + .ValidateOnStart(); + services.AddSingleton, ModerationOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var resolve = () => provider.GetRequiredService>().Value; + Assert.Throws(resolve); + } + + private static ModerationOptions Bind(string json) + { + var options = new ModerationOptions(); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + new ConfigurationBuilder() + .AddJsonStream(stream) + .Build() + .GetSection(ModerationOptions.SectionName) + .Bind(options); + options.Moderators ??= []; + return options; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs new file mode 100644 index 00000000..b004896e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdAcmeConfigurationOwnershipTests.cs @@ -0,0 +1,315 @@ +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +/// +/// Proves NNTPD-owned ACME state binds from Nntpd, that the ACME account +/// email and certificate password are not supplied by VECTOR__ACMEACCOUNT or +/// VECTOR__ACMECERTIFICATEPASSWORD, and that relative AcmeStateDir resolves +/// through Common +/// against . +/// +[Collection("WorkingDirectory")] +public sealed class NntpdAcmeConfigurationOwnershipTests +{ + private const string NntpdDirectoryUrl = "https://acme-staging-v02.api.letsencrypt.org/directory"; + private const string SharedAcmeAccount = "shared-acme@example.test"; + private const int NntpdRenewalDays = 14; + private const string NntpdStateDir = "certs/"; + private const string RootDirectoryUrl = "https://root-must-not-bind.example/directory"; + private const string RootEmail = "root-must-not-bind@example.test"; + private const string RootStateDir = "root-must-not-bind/"; + private const string TestCertificatePassword = "unit-test-nntpd-pfx-password"; + + [Fact] + public void Nntpd_json_does_not_own_the_acme_directory_url() + { + var json = File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"))); + Assert.DoesNotContain("\"AcmeDirectoryUrl\"", json, StringComparison.Ordinal); + } + + [Fact] + public void Shared_acme_account_email_is_not_supplied_by_vector_environment() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.NotEqual(SharedAcmeAccount, options.AcmeEmail); + Assert.Equal( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + "VECTOR__ACMEACCOUNT"); + } + + [Fact] + public void Nntpd_json_does_not_own_the_acme_renewal_threshold() + { + var json = File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"))); + Assert.DoesNotContain("\"AcmeRenewalThresholdDays\"", json, StringComparison.Ordinal); + } + + [Fact] + public void Nntpd_acme_state_dir_survives_configuration_binding() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + Assert.DoesNotContain("root-must-not-bind", options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Root_acme_values_cannot_overwrite_nntpd_owned_settings() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + var acme = host.Services.GetRequiredService>().Value; + + Assert.Equal(string.Empty, options.AcmeEmail); + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + + Assert.NotEqual(RootEmail, options.AcmeEmail); + Assert.NotEqual(SharedAcmeAccount, options.AcmeEmail); + Assert.NotEqual(RootStateDir, options.AcmeStateDir); + + Assert.Same(options, acme); + } + + [Fact] + public void Vector_acme_certificate_password_is_not_supplied_by_the_environment_variable() + { + var previous = Environment.GetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + TestCertificatePassword); + + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.NotEqual(TestCertificatePassword, options.AcmeCertificatePassword); + } + finally + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + previous); + } + } + + [Fact] + public void Nntpd_does_not_require_an_application_specific_acme_password_variable() + { + Assert.Equal( + "VECTOR__ACMECERTIFICATEPASSWORD", + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + Assert.Equal( + AcmeCloudflareOptions.AcmeCertificatePasswordEnvironmentVariable, + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + Assert.Null( + typeof(NntpdOptions).GetField( + "AcmeCertificatePasswordEnvironmentVariable", + BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)); + Assert.DoesNotContain( + "NNTPD__", + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + StringComparison.OrdinalIgnoreCase); + + var previousShared = Environment.GetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable); + var previousNntpd = Environment.GetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD"); + try + { + Environment.SetEnvironmentVariable( + NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, + TestCertificatePassword); + Environment.SetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD", "nntpd-specific-must-not-be-required"); + + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + var rabbit = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.AcmeCertificatePassword); + Assert.NotEqual(TestCertificatePassword, options.AcmeCertificatePassword); + Assert.NotEqual("nntpd-specific-must-not-be-required", options.AcmeCertificatePassword); + Assert.Null(rabbit.Username); + Assert.Null(rabbit.Password); + } + finally + { + Environment.SetEnvironmentVariable(NntpdOptions.AcmeCertificatePasswordEnvironmentVariable, previousShared); + Environment.SetEnvironmentVariable("NNTPD__ACMECERTIFICATEPASSWORD", previousNntpd); + } + } + + [Fact] + public void Relative_nntpd_acme_state_dir_resolves_under_appcontext_base_directory() + { + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + var expected = ExpectedResolvedStateDir(AppContext.BaseDirectory); + Assert.Equal(expected, options.AcmeStateDir); + Assert.StartsWith( + Path.GetFullPath(AppContext.BaseDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + options.AcmeStateDir, + StringComparison.OrdinalIgnoreCase); + Assert.Contains( + "ContentRootPath = AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "Program.cs"))), + StringComparison.Ordinal); + } + + [Fact] + public void Changing_process_working_directory_does_not_change_resolved_acme_state_directory() + { + var previous = Environment.CurrentDirectory; + var scratch = Directory.CreateTempSubdirectory("nntpd-acme-cwd-"); + try + { + Environment.CurrentDirectory = scratch.FullName; + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + var expected = ExpectedResolvedStateDir(AppContext.BaseDirectory); + var cwdResolved = Path.GetFullPath(NntpdStateDir.Trim()); + + Assert.Equal(expected, options.AcmeStateDir); + Assert.NotEqual(cwdResolved, options.AcmeStateDir); + Assert.DoesNotContain( + Path.GetFullPath(scratch.FullName).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + options.AcmeStateDir, + StringComparison.OrdinalIgnoreCase); + } + finally + { + Environment.CurrentDirectory = previous; + scratch.Delete(recursive: true); + } + } + + [Fact] + public void Nntpd_and_backfiller_use_the_same_common_acme_path_resolution_mechanism() + { + using var host = CreateHost(contentRootPath: AppContext.BaseDirectory); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(ExpectedResolvedStateDir(AppContext.BaseDirectory), options.AcmeStateDir); + + var nntpdHosting = File.ReadAllText(FindRepoFile( + Path.Join("src", "VectorNNTPD.TransitService", "Hosting", "NntpdServiceCollectionExtensions.cs"))); + var backFillerHosting = File.ReadAllText(FindRepoFile( + Path.Join("src", "VectorNNTPD.BackFillerService", "Hosting", "BackFillerServiceCollectionExtensions.cs"))); + + Assert.Contains("ApplicationLocalPath.ResolveApplicationLocalPath", nntpdHosting, StringComparison.Ordinal); + Assert.Contains("ApplicationLocalPath.ResolveApplicationLocalPath", backFillerHosting, StringComparison.Ordinal); + Assert.Contains("AppContext.BaseDirectory", nntpdHosting, StringComparison.Ordinal); + Assert.Contains("AppContext.BaseDirectory", backFillerHosting, StringComparison.Ordinal); + } + + private static string ExpectedResolvedStateDir(string contentRootPath) => + ApplicationLocalPath.ResolveApplicationLocalPath(NntpdStateDir, contentRootPath); + + private static IHost CreateHost(string? contentRootPath = null) + { + // VECTOR__ACMEACCOUNT overlays in-memory ACMEACCOUNT. Isolate from the + // operator environment so these tests prove the shared-account contract. + var previousAccount = Environment.GetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable); + Environment.SetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + SharedAcmeAccount); + try + { + return BuildHost(contentRootPath); + } + finally + { + Environment.SetEnvironmentVariable( + AcmeCloudflareOptions.AcmeAccountEnvironmentVariable, + previousAccount); + } + } + + private static IHost BuildHost(string? contentRootPath) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.TransitService.Tests", + ContentRootPath = contentRootPath ?? AppContext.BaseDirectory, + EnvironmentName = Environments.Development, + }); + + builder.Configuration.AddInMemoryCollection(CreateConfigurationPairs()); + builder.Configuration.AddVectorEnvironmentVariables(); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + TestHostFactory.IsolateTransit(builder.Services); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static Dictionary CreateConfigurationPairs() + { + return new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:AcmeDirectoryUrl"] = NntpdDirectoryUrl, + [$"{NntpdOptions.SectionName}:AcmeEmail"] = "nntpd-section-must-not-bind@example.test", + [$"{NntpdOptions.SectionName}:AcmeRenewalThresholdDays"] = "14", + [$"{NntpdOptions.SectionName}:AcmeStateDir"] = NntpdStateDir, + [AcmeCloudflareOptions.AcmeAccountConfigurationKey] = SharedAcmeAccount, + ["AcmeDirectoryUrl"] = RootDirectoryUrl, + ["AcmeEmail"] = RootEmail, + ["AcmeRenewalThresholdDays"] = "30", + ["AcmeStateDir"] = RootStateDir, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d", + [$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja", + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }; + } + + private static string FindRepoFile(string relativePath) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var candidate = Path.Join(directory.FullName, relativePath); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException($"Could not locate {relativePath}."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdApplicationLocalPathTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdApplicationLocalPathTests.cs new file mode 100644 index 00000000..c775b01c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdApplicationLocalPathTests.cs @@ -0,0 +1,282 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +/// +/// Proves NNTPD application-local paths resolve against +/// , not the process working directory +/// or an IDE/project content root. +/// +[Collection("WorkingDirectory")] +public sealed class NntpdApplicationLocalPathTests +{ + [Fact] + public void Relative_certs_resolves_beneath_application_base() + { + using var host = CreateHost(decoyContentRoot: CreateTempDir("nntpd-decoy-content-")); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("certs/"), options.AcmeStateDir); + Assert.StartsWith(Normalized(AppContext.BaseDirectory), options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Resolved_certs_is_not_based_on_process_working_directory() + { + var previous = Environment.CurrentDirectory; + var cwd = CreateTempDir("nntpd-cwd-certs-"); + try + { + Environment.CurrentDirectory = cwd; + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("certs/"), options.AcmeStateDir); + Assert.NotEqual(Path.GetFullPath("certs/"), options.AcmeStateDir); + Assert.DoesNotContain(Normalized(cwd), options.AcmeStateDir, StringComparison.OrdinalIgnoreCase); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(cwd); + } + } + + [Fact] + public void Changing_working_directory_does_not_change_resolved_certs_or_logs() + { + var previous = Environment.CurrentDirectory; + var first = CreateTempDir("nntpd-cwd-a-"); + var second = CreateTempDir("nntpd-cwd-b-"); + try + { + Environment.CurrentDirectory = first; + using var hostA = CreateHost(); + var certsA = hostA.Services.GetRequiredService>().Value.AcmeStateDir; + Environment.CurrentDirectory = second; + using var hostB = CreateHost(); + var certsB = hostB.Services.GetRequiredService>().Value.AcmeStateDir; + Assert.Equal(certsA, certsB); + Assert.Equal(Expected("certs/"), certsB); + Assert.Equal( + NntpdFileLogging.ResolveDirectory("logs/"), + ApplicationLocalPath.ResolveApplicationLocalPath("logs/", AppContext.BaseDirectory)); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(first); + TryDelete(second); + } + } + + [Fact] + public void Explicit_absolute_paths_remain_absolute() + { + var absoluteCerts = CreateTempDir("nntpd-abs-certs-"); + var absoluteLogs = CreateTempDir("nntpd-abs-logs-"); + try + { + using var host = CreateHost( + extra: + [ + new KeyValuePair($"{NntpdOptions.SectionName}:AcmeStateDir", absoluteCerts), + new KeyValuePair($"{NntpdOptions.SectionName}:LogDir", absoluteLogs), + ]); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected(absoluteCerts), options.AcmeStateDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath(absoluteLogs, AppContext.BaseDirectory), + NntpdFileLogging.ResolveDirectory(absoluteLogs)); + } + finally + { + TryDelete(absoluteCerts); + TryDelete(absoluteLogs); + } + } + + [Fact] + public void Explicit_relative_paths_resolve_beneath_application_base() + { + using var host = CreateHost( + extra: [new KeyValuePair($"{NntpdOptions.SectionName}:AcmeStateDir", "nested-certs/")]); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(Expected("nested-certs/"), options.AcmeStateDir); + Assert.Equal( + Expected("state-logs/"), + NntpdFileLogging.ResolveDirectory("state-logs/")); + } + + [Fact] + public void Nntpd_uses_the_same_common_path_resolution_as_backfiller() + { + using var host = CreateHost(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal( + AcmeCloudflareOptionsValidator.ResolveAcmeStateDir("certs/", AppContext.BaseDirectory), + options.AcmeStateDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath("certs/", AppContext.BaseDirectory), + options.AcmeStateDir); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "Hosting", "NntpdServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "Logging", "NntpdFileLogging.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "ApplicationLocalPath.ResolveApplicationLocalPath", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "Email", "FilesystemEmailSpool.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.TransitService", "Hosting", "NntpdServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + Assert.Contains( + "AppContext.BaseDirectory", + File.ReadAllText(FindRepoFile(Path.Join("src", "VectorNNTPD.BackFillerService", "Hosting", "BackFillerServiceCollectionExtensions.cs"))), + StringComparison.Ordinal); + } + + [Fact] + public void Log_directory_creation_uses_application_base_not_working_directory() + { + var previous = Environment.CurrentDirectory; + var cwd = CreateTempDir("nntpd-log-cwd-"); + var baseDir = CreateTempDir("nntpd-log-base-"); + try + { + Environment.CurrentDirectory = cwd; + using var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = "logs/", + [$"{NntpdOptions.SectionName}:ApplicationName"] = "VectorNNTPD.TransitService", + }); + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + configuration[pair.Key] = pair.Value; + } + + NntpdFileLogging.BindResolvedFilePath(configuration, baseDir); + + var expected = ApplicationLocalPath.ResolveApplicationLocalPath("logs/", baseDir); + Assert.True(Directory.Exists(expected)); + Assert.False(Directory.Exists(Path.Join(cwd, "logs"))); + Assert.Equal(expected, NntpdFileLogging.ResolveDirectory("logs/", baseDir)); + } + finally + { + Environment.CurrentDirectory = previous; + TryDelete(cwd); + TryDelete(baseDir); + } + } + + private static string Expected(string configured) => + ApplicationLocalPath.ResolveApplicationLocalPath(configured, AppContext.BaseDirectory); + + private static string Normalized(string path) => + Path.GetFullPath(path).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + + private static IHost CreateHost( + string? decoyContentRoot = null, + IEnumerable>? extra = null) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.TransitService.Tests", + ContentRootPath = decoyContentRoot ?? AppContext.BaseDirectory, + EnvironmentName = Environments.Development, + }); + + var pairs = new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:AcmeStateDir"] = "certs/", + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d", + [$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja", + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }; + if (extra is not null) + { + foreach (var pair in extra) + { + pairs[pair.Key] = pair.Value; + } + } + + builder.Configuration.AddInMemoryCollection(pairs); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + TestHostFactory.IsolateTransit(builder.Services); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static string CreateTempDir(string prefix) => + Directory.CreateTempSubdirectory(prefix).FullName; + + private static void TryDelete(string path) + { + try + { + Directory.Delete(path, recursive: true); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private static string FindRepoFile(string relativePath) + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + var candidate = Path.Join(directory.FullName, relativePath); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + throw new FileNotFoundException($"Could not locate {relativePath}."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdOptionsValidatorTests.cs b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdOptionsValidatorTests.cs new file mode 100644 index 00000000..940c8b37 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Configuration/NntpdOptionsValidatorTests.cs @@ -0,0 +1,1907 @@ +using System.ComponentModel.DataAnnotations; +using System.Globalization; +using System.Net; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.TransitService.Tests.Configuration; + +public sealed class NntpdOptionsValidatorTests +{ + private static NntpdOptionsValidator CreateValidator(ILocalIpAddressAssignee? assignee = null) => + new(assignee ?? new FakeLocalIpAddressAssignee(assignAll: true)); + + [Fact] + public void Validate_Rejects_OutstandingStoreLimitOutsideOneOrTwo() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(2, options.ArticleIngestion.MaxOutstandingStoresPerConnection); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.ArticleIngestion.MaxOutstandingStoresPerConnection = 1; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.ArticleIngestion.MaxOutstandingStoresPerConnection = 64; + var result = CreateValidator().Validate(null, options); + Assert.False(result.Succeeded); + Assert.Contains("MaxOutstandingStoresPerConnection", result.FailureMessage, StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_ForDefaultHistoryTime() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(TimeSpan.FromHours(2), options.HistoryTime); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForConfiguredHistoryTime() + { + var options = TestHostFactory.CreateValidOptions(); + options.HistoryTime = TimeSpan.FromHours(6); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void CheckPipelineDepth_IsNotAnOptionsProperty() + { + Assert.Null(typeof(NntpdOptions).GetProperty("CheckPipelineDepth")); + Assert.Equal(16, VectorNNTPD.TransitService.Session.CheckPipeline.Depth); + } + + [Fact] + public void BindConfiguration_IgnoresCheckPipelineDepthKey() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:ServerId"] = "1", + ["Nntpd:BindAddress:0"] = "127.0.0.1", + ["Nntpd:CheckPipelineDepth"] = "64", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Null(typeof(NntpdOptions).GetProperty("CheckPipelineDepth")); + Assert.Equal(16, VectorNNTPD.TransitService.Session.CheckPipeline.Depth); + } + + [Fact] + public void Validate_Succeeds_ForDefaultIdleTime() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultIdleTime, options.IdleTime); + Assert.Equal(300, new NntpdOptions().IdleTime); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForConfiguredIdleTime() + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = 120; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Validate_Fails_ForIdleTimeBelowOneSecond(int idleTime) + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = idleTime; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.IdleTime), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForIdleTimeAboveOneDay() + { + var options = TestHostFactory.CreateValidOptions(); + options.IdleTime = NntpdOptions.MaxIdleTime + 1; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.IdleTime), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursIdleTimeSeconds() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:IdleTime"] = "120", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(120, options.IdleTime); + } + + [Fact] + public void Validate_Succeeds_ForDefaultMaxArticleSize() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultMaxArticleSize, options.MaxArticleSize); + Assert.Equal(5_242_880, new NntpdOptions().MaxArticleSize); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Validate_Fails_ForNonPositiveMaxArticleSize(int size) + { + var options = TestHostFactory.CreateValidOptions(); + options.MaxArticleSize = size; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MaxArticleSize), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForMaxArticleSizeAboveCeiling() + { + var options = TestHostFactory.CreateValidOptions(); + options.MaxArticleSize = NntpdOptions.MaxMaxArticleSize + 1; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MaxArticleSize), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursMaxArticleSize() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:MaxArticleSize"] = "1048576", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(1_048_576, options.MaxArticleSize); + } + + [Fact] + public void Validate_Succeeds_ForDefaultMailComplaintsTo() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(NntpdOptions.DefaultMailComplaintsTo, options.MailComplaintsTo); + Assert.Equal("abuse@usenet.ninja", new NntpdOptions().MailComplaintsTo); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_WhenTop1000IsMissingOrEmpty() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Empty(options.Top1000); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.Top1000 = []; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + options.Top1000 = ["", " ", "\t"]; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForTop1000Mailboxes_IgnoringWhitespace() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["top1000@anthologeek.net", " ", "ops@usenet.ninja"]; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_ForInvalidTop1000Mailbox() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["not-a-mailbox"]; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.Top1000), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursTop1000() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:Top1000:0"] = "top1000@anthologeek.net", + ["Nntpd:Top1000:1"] = "ops@usenet.ninja", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(["top1000@anthologeek.net", "ops@usenet.ninja"], options.Top1000); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("not-a-mailbox")] + [InlineData("abuse@localhost")] + public void Validate_Fails_ForInvalidMailComplaintsTo(string value) + { + var options = TestHostFactory.CreateValidOptions(); + options.MailComplaintsTo = value; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(nameof(NntpdOptions.MailComplaintsTo), StringComparison.Ordinal)); + } + + [Fact] + public void BindConfiguration_HonoursMailComplaintsTo() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:MailComplaintsTo"] = "ops@usenet.ninja", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal("ops@usenet.ninja", options.MailComplaintsTo); + } + + [Fact] + public void Validate_Succeeds_ForDefaultXTraceKeyWhenConfigured() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + Assert.True(XTraceKeyParser.TryDecode(options.XTraceKey, out var key)); + Assert.Equal(32, key!.Length); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("not-a-key")] + [InlineData("0123456789abcdef")] + public void Validate_Fails_ForInvalidXTraceKey(string value) + { + var options = TestHostFactory.CreateValidOptions(); + options.XTraceKey = value; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.XTraceKeyConfigurationKey, StringComparison.Ordinal)); + var joined = NntpdOptionsValidator.JoinFailures(result); + if (!string.IsNullOrWhiteSpace(value)) + { + Assert.DoesNotContain(value.Trim(), joined, StringComparison.Ordinal); + } + } + + [Fact] + public void Validate_Fails_ForInvalidXTracePreviousKey() + { + var options = TestHostFactory.CreateValidOptions(); + options.XTracePreviousKey = "short"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.XTracePreviousKeyConfigurationKey, StringComparison.Ordinal)); + Assert.DoesNotContain("short", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_WhenNewsmasterCredentialsArePaired() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenOnlyNewsmasterUserIsSet() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains(NntpdOptions.NewsmasterPasswordConfigurationKey, StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenOnlyNewsmasterPasswordIsSet() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + var joined = NntpdOptionsValidator.JoinFailures(result); + Assert.Contains(NntpdOptions.NewsmasterUserConfigurationKey, joined, StringComparison.Ordinal); + Assert.DoesNotContain("unit-test-newsmaster-password", joined, StringComparison.Ordinal); + } + + [Fact] + public void BindConfiguration_HonoursXTraceKey() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:XTraceKey"] = TestHostFactory.TestXTraceKey, + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + } + + [Fact] + public void Validate_Fails_ForHistoryTimeBelowOneSecond() + { + var options = TestHostFactory.CreateValidOptions(); + options.HistoryTime = TimeSpan.Zero; + Assert.True(CreateValidator().Validate(null, options).Failed); + } + + [Fact] + public void Validate_Succeeds_ForValidDefaults() + { + var result = CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()); + Assert.True(result.Succeeded); + } + + [Fact] + public void FeedDiagnostics_DefaultsAreOff() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.False(options.FeedDiagnostics.Enabled); + Assert.Equal(FeedDiagnosticsOptions.DefaultIntervalSeconds, options.FeedDiagnostics.IntervalSeconds); + Assert.True(options.FeedDiagnostics.IncludeSessions); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void SpeedTest_DefaultsAreDiagnosticSized() + { + var options = TestHostFactory.CreateValidOptions(); + Assert.Equal(SpeedTestOptions.DefaultMaxDurationSeconds, options.SpeedTest.MaxDurationSeconds); + Assert.Equal(SpeedTestOptions.DefaultMaxBytes, options.SpeedTest.MaxBytes); + Assert.Equal(SpeedTestOptions.DefaultMaxConcurrent, options.SpeedTest.MaxConcurrent); + Assert.Equal(SpeedTestOptions.DefaultMaxConcurrentPerPeer, options.SpeedTest.MaxConcurrentPerPeer); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Theory] + [InlineData(0, 4096, 2, 1)] + [InlineData(61, 4096, 2, 1)] + [InlineData(10, 512, 2, 1)] + [InlineData(10, 4096, 0, 1)] + [InlineData(10, 4096, 9, 1)] + [InlineData(10, 4096, 2, 0)] + [InlineData(10, 4096, 2, 5)] + public void SpeedTest_InvalidLimits_FailValidation( + int duration, + long maxBytes, + int concurrent, + int perPeer) + { + var options = TestHostFactory.CreateValidOptions(); + options.SpeedTest = new SpeedTestOptions + { + MaxDurationSeconds = duration, + MaxBytes = maxBytes, + MaxConcurrent = concurrent, + MaxConcurrentPerPeer = perPeer, + }; + Assert.True(CreateValidator().Validate(null, options).Failed); + } + + [Fact] + public void TransitQueueMemoryLimit_DefaultsToOneGibibyte() + { + Assert.Equal(1_073_741_824L, NntpdOptions.DefaultTransitQueueMemoryLimit); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, new NntpdOptions().TransitQueueMemoryLimit); + Assert.Equal(NntpdOptions.DefaultTransitQueueMemoryLimit, TestHostFactory.CreateValidOptions().TransitQueueMemoryLimit); + Assert.True(CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()).Succeeded); + } + + [Fact] + public void BindConfiguration_HonoursTransitQueueMemoryLimit() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:TransitQueueMemoryLimit"] = "2097152", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(2_097_152L, options.TransitQueueMemoryLimit); + var valid = TestHostFactory.CreateValidOptions(); + valid.TransitQueueMemoryLimit = options.TransitQueueMemoryLimit; + Assert.True(CreateValidator().Validate(null, valid).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(long.MinValue)] + public void Validate_Fails_ForNonPositiveTransitQueueMemoryLimit(long limit) + { + var options = TestHostFactory.CreateValidOptions(); + options.TransitQueueMemoryLimit = limit; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("TransitQueueMemoryLimit", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Succeeds_ForMinimumTransitQueueMemoryLimit() + { + var options = TestHostFactory.CreateValidOptions(); + options.TransitQueueMemoryLimit = 1; + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Succeeds_ForDefaultIngestionWorkerPoolSettings() + { + var options = TestHostFactory.CreateValidOptions(); + var ingestion = options.ArticleIngestion; + Assert.Equal(ArticleIngestionOptions.DefaultMinWorkers, ingestion.MinWorkers); + Assert.Equal(ArticleIngestionOptions.DefaultMaxWorkers, ingestion.MaxWorkers); + Assert.Equal(ArticleIngestionOptions.DefaultMaxPublishConcurrency, ingestion.MaxPublishConcurrency); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbWorkQueueMemoryLimit, + ingestion.OverviewDbWorkQueueMemoryLimit); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbMinPublisherWorkers, + ingestion.OverviewDbMinPublisherWorkers); + Assert.Equal( + ArticleIngestionOptions.DefaultOverviewDbMaxPublisherWorkers, + ingestion.OverviewDbMaxPublisherWorkers); + Assert.True(CreateValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenOverviewDbMaxPublisherWorkersLessThanMin() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.OverviewDbMinPublisherWorkers = 4; + options.ArticleIngestion.OverviewDbMaxPublisherWorkers = 2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + "OverviewDbMaxPublisherWorkers", + NntpdOptionsValidator.JoinFailures(result), + StringComparison.Ordinal); + } + + [Fact] + public void Validate_Fails_WhenMaxWorkersLessThanMinWorkers() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.MinWorkers = 4; + options.ArticleIngestion.MaxWorkers = 2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("MaxWorkers", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void Validate_Fails_WhenScaleUpThresholdNotGreaterThanScaleDown() + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.ScaleUpPressureThreshold = 0.2; + options.ArticleIngestion.ScaleDownPressureThreshold = 0.2; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("ScaleUpPressureThreshold", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(513)] + public void Validate_Fails_ForInvalidMinWorkers(int minWorkers) + { + var options = TestHostFactory.CreateValidOptions(); + options.ArticleIngestion.MinWorkers = minWorkers; + options.ArticleIngestion.MaxWorkers = Math.Max(minWorkers, 1); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("MinWorkers", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Fact] + public void BindConfiguration_HonoursIngestionWorkerPoolSettings() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:ArticleIngestion:MinWorkers"] = "3", + ["Nntpd:ArticleIngestion:MaxWorkers"] = "12", + ["Nntpd:ArticleIngestion:MaxPublishConcurrency"] = "8", + ["Nntpd:ArticleIngestion:OverviewDbWorkQueueMemoryLimit"] = "536870912", + ["Nntpd:ArticleIngestion:OverviewDbMinPublisherWorkers"] = "3", + ["Nntpd:ArticleIngestion:OverviewDbMaxPublisherWorkers"] = "10", + ["Nntpd:ArticleIngestion:ScaleIntervalSeconds"] = "5", + ["Nntpd:ArticleIngestion:ScaleUpPressureThreshold"] = "0.55", + ["Nntpd:ArticleIngestion:ScaleDownPressureThreshold"] = "0.15", + ["Nntpd:ArticleIngestion:ScaleUpConsecutiveIntervals"] = "4", + ["Nntpd:ArticleIngestion:ScaleDownConsecutiveIntervals"] = "6", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal(3, options.ArticleIngestion.MinWorkers); + Assert.Equal(12, options.ArticleIngestion.MaxWorkers); + Assert.Equal(8, options.ArticleIngestion.MaxPublishConcurrency); + Assert.Equal(536870912, options.ArticleIngestion.OverviewDbWorkQueueMemoryLimit); + Assert.Equal(3, options.ArticleIngestion.OverviewDbMinPublisherWorkers); + Assert.Equal(10, options.ArticleIngestion.OverviewDbMaxPublisherWorkers); + Assert.Equal(5, options.ArticleIngestion.ScaleIntervalSeconds); + Assert.Equal(0.55, options.ArticleIngestion.ScaleUpPressureThreshold); + Assert.Equal(0.15, options.ArticleIngestion.ScaleDownPressureThreshold); + Assert.Equal(4, options.ArticleIngestion.ScaleUpConsecutiveIntervals); + Assert.Equal(6, options.ArticleIngestion.ScaleDownConsecutiveIntervals); + + var valid = TestHostFactory.CreateValidOptions(); + valid.ArticleIngestion = options.ArticleIngestion; + Assert.True(CreateValidator().Validate(null, valid).Succeeded); + } + + [Fact] + public void LogDir_DefaultsToLogsSlash() + { + Assert.Equal("logs/", NntpdOptions.DefaultLogDir); + Assert.Equal(NntpdOptions.DefaultLogDir, new NntpdOptions().LogDir); + Assert.Equal(NntpdOptions.DefaultLogDir, TestHostFactory.CreateValidOptions().LogDir); + Assert.True(CreateValidator().Validate(null, TestHostFactory.CreateValidOptions()).Succeeded); + } + + [Fact] + public void Validate_Fails_ForEmptyLogDir() + { + var options = TestHostFactory.CreateValidOptions(); + options.LogDir = " "; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains(nameof(NntpdOptions.LogDir), StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_ForEmptyApplicationName() + { + var options = TestHostFactory.CreateValidOptions(); + options.ApplicationName = " "; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForTooShortShutdownTimeout() + { + var options = TestHostFactory.CreateValidOptions(); + options.GracefulShutdownTimeout = TimeSpan.FromMilliseconds(100); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForTooShortStartupTimeout() + { + var options = TestHostFactory.CreateValidOptions(); + options.StartupTimeout = TimeSpan.FromMilliseconds(10); + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_ForInvalidWatchdogIntervalFraction() + { + var options = TestHostFactory.CreateValidOptions(); + options.Systemd = new SystemdOptions { WatchdogIntervalFraction = 1.5 }; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(4)] + [InlineData(8)] + [InlineData(16)] + public void Validate_Succeeds_ForValidStreamOutstandingArticleDepth(int depth) + { + var options = TestHostFactory.CreateValidOptions(); + options.Transit.StreamOutstandingArticleDepth = depth; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(3)] + [InlineData(17)] + [InlineData(0)] + [InlineData(64)] + public void Validate_Fails_ForInvalidStreamOutstandingArticleDepth(int depth) + { + var options = TestHostFactory.CreateValidOptions(); + options.Transit.StreamOutstandingArticleDepth = depth; + var result = CreateValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("StreamOutstandingArticleDepth", StringComparison.Ordinal)); + } + + [Fact] + public void Transit_WantTrashAndLogTrash_DefaultTrue() + { + var transit = new TransitOptions(); + Assert.True(transit.WantTrash); + Assert.True(transit.LogTrash); + var options = TestHostFactory.CreateValidOptions(); + Assert.True(options.Transit.WantTrash); + Assert.True(options.Transit.LogTrash); + } +} + +public sealed class NntpdConfigurationTests +{ + [Fact] + public void Bind_StandardizedPascalCaseNames_FromConfiguration() + { + // Isolated JSON (not the working-tree appsettings) so bind defaults and PascalCase names are asserted. + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(string.Empty, options.CloudFlareApiKey); + Assert.Equal("usenet.ninja", options.DnsSuffix); + Assert.Equal(1, options.ServerId); + Assert.Equal("transit01.usenet.ninja", options.Fqdn); + } + + [Fact] + public void Bind_ConsumesNntpdSectionBindSettings() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "127.0.0.1" ], + "BindPort": 1199, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_NntpdTransit_RejectsDatabaseOwnedWantTrashAndLogTrash() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "Transit": { + "LogTrash": false, + "WantTrash": true + } + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = Assert.Throws( + () => _ = host.Services.GetRequiredService>().Value); + Assert.Contains("nntptransitglobalrevision", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Bind_NntpdTransit_KeepsStreamOutstandingArticleDepth() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "Transit": { + "StreamOutstandingArticleDepth": 12 + } + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(12, options.Transit.StreamOutstandingArticleDepth); + Assert.True(options.Transit.WantTrash); + Assert.True(options.Transit.LogTrash); + } + + [Fact] + public void Bind_DoesNotConsumeRootLevelBindSettings() + { + var json = """ + { + "BindAddress": [ "203.0.113.10" ], + "BindPort": 2119, + "BindPortTls": 2563, + "Nntpd": { + "BindAddress": [ "127.0.0.1" ], + "BindPort": 1199, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json, forceTlsDisabled: false); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_RootLevelBindSettingsAlone_DoNotReplaceNntpdDefaults() + { + var json = """ + { + "BindAddress": [ "203.0.113.10" ], + "BindPort": 2119, + "BindPortTls": 2563, + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json, forceTlsDisabled: false); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + } + + [Fact] + public void Bind_OverlayPreservesNntpdBindPortTlsAgainstRoot() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "127.0.0.1", + [$"{NntpdOptions.SectionName}:BindPort"] = "1199", + [$"{NntpdOptions.SectionName}:BindPortTls"] = "5633", + ["BindAddress:0"] = "203.0.113.10", + ["BindPort"] = "2119", + ["BindPortTls"] = "2563", + }) + .Build(); + + var options = new NntpdOptions(); + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(1199, options.BindPort); + Assert.Equal(5633, options.BindPortTls); + Assert.True(options.IsTlsListenerEnabled); + } + + [Fact] + public void Bind_HasNoDedicatedBindEnvironmentVariableConstants() + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy; + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindAddressEnvironmentVariable", flags)); + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindPortEnvironmentVariable", flags)); + Assert.Null(typeof(AcmeCloudflareOptions).GetField("BindPortTlsEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindAddressEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindPortEnvironmentVariable", flags)); + Assert.Null(typeof(NntpdOptions).GetField("BindPortTlsEnvironmentVariable", flags)); + } + + [Fact] + public void Bind_HonoursConfiguredLogDir() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Nntpd:LogDir"] = "D:/nntp-logs/", + }) + .Build(); + var options = new NntpdOptions(); + configuration.GetSection("Nntpd").Bind(options); + Assert.Equal("D:/nntp-logs/", options.LogDir); + var valid = TestHostFactory.CreateValidOptions(); + valid.LogDir = options.LogDir; + var validator = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)); + Assert.True(validator.Validate(null, valid).Succeeded); + } + + [Fact] + public void ProductionAppsettings_DeclaresIdleTimeDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal(NntpdOptions.DefaultIdleTime, doc.RootElement.GetProperty("Nntpd").GetProperty("IdleTime").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DeclaresMaxArticleSizeDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal( + NntpdOptions.DefaultMaxArticleSize, + doc.RootElement.GetProperty("Nntpd").GetProperty("MaxArticleSize").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DeclaresMailComplaintsToDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal( + NntpdOptions.DefaultMailComplaintsTo, + doc.RootElement.GetProperty("Nntpd").GetProperty("MailComplaintsTo").GetString()); + } + + [Fact] + public void ProductionAppsettings_DeclaresTop1000Recipients() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var top = doc.RootElement.GetProperty("Nntpd").GetProperty("Top1000"); + Assert.Equal(2, top.GetArrayLength()); + Assert.Equal("top1000@anthologeek.net", top[0].GetString()); + Assert.Equal("cknipe@opticnetworks.net", top[1].GetString()); + } + + [Fact] + public void ProductionAppsettings_DoesNotDeclareXTraceKey() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("XTraceKey", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("XTracePreviousKey", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("NewsmasterUser", out _)); + Assert.False(doc.RootElement.GetProperty("Nntpd").TryGetProperty("NewsmasterPassword", out _)); + } + + [Fact] + public void ProductionAppsettings_DeclaresBindSettingsUnderNntpdOnly() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var root = doc.RootElement; + var nntpd = root.GetProperty("Nntpd"); + + Assert.False(root.TryGetProperty("BindAddress", out _)); + Assert.False(root.TryGetProperty("BindPort", out _)); + Assert.False(root.TryGetProperty("BindPortTls", out _)); + Assert.Equal("198.18.0.66", nntpd.GetProperty("BindAddress")[0].GetString()); + Assert.Equal("2c0f:f030:1442:501:198:18:0:66", nntpd.GetProperty("BindAddress")[1].GetString()); + Assert.Equal(1199, nntpd.GetProperty("BindPort").GetInt32()); + Assert.Equal(5633, nntpd.GetProperty("BindPortTls").GetInt32()); + } + + [Fact] + public void ProductionAppsettings_DoesNotDeclareSharedConfigColumns() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + var root = doc.RootElement; + var nntpd = root.GetProperty("Nntpd"); + + Assert.False(root.TryGetProperty("CloudFlareZoneId", out _)); + Assert.False(root.TryGetProperty("DnsSuffix", out _)); + Assert.False(root.TryGetProperty("AcmeDirectoryUrl", out _)); + Assert.False(root.TryGetProperty("AcmeRenewalThresholdDays", out _)); + Assert.False(nntpd.TryGetProperty("CloudFlareZoneId", out _)); + Assert.False(nntpd.TryGetProperty("DnsSuffix", out _)); + Assert.False(nntpd.TryGetProperty("AcmeDirectoryUrl", out _)); + Assert.False(nntpd.TryGetProperty("AcmeRenewalThresholdDays", out _)); + } + + [Fact] + public void Bind_ConsumesNntpdZoneIdAndDnsSuffix() + { + var json = """ + { + "CloudFlareZoneId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "DnsSuffix": "root-must-not-bind.example", + "Nntpd": { + "BindAddress": [ "*" ], + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "example.test", + "ServerId": 7 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal("example.test", options.DnsSuffix); + Assert.Equal("transit07.example.test", options.Fqdn); + } + + [Fact] + public void ProductionAppsettings_DeclaresLogDirDefault() + { + var path = FindProductionAppsettings(); + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.Equal("logs/", doc.RootElement.GetProperty("Nntpd").GetProperty("LogDir").GetString()); + Assert.Equal( + "Information", + doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[0] + .GetProperty("Args").GetProperty("restrictedToMinimumLevel").GetString()); + Assert.Equal( + "Debug", + doc.RootElement.GetProperty("Serilog").GetProperty("MinimumLevel") + .GetProperty("Override").GetProperty("VectorNNTPD.TransitService").GetString()); + var async = doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[1]; + Assert.Equal("Async", async.GetProperty("Name").GetString()); + Assert.Equal(50000, async.GetProperty("Args").GetProperty("bufferSize").GetInt32()); + Assert.True(async.GetProperty("Args").GetProperty("blockWhenFull").GetBoolean()); + Assert.Equal( + "Debug", + async.GetProperty("Args").GetProperty("configure")[0].GetProperty("Args") + .GetProperty("restrictedToMinimumLevel").GetString()); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } + + [Fact] + public void OldUnderscoreSeparatedNames_AreNotBound() + { + // Underscore-separated legacy keys are not aliases. (DnsSuffix vs dnssuffix is the same + // case-insensitive configuration key and is therefore not covered here.) + var json = """ + { + "Nntpd": { + "bind_address": [ "198.18.0.66" ], + "bind_port": 1199, + "bind_port_tls": 5633, + "cloudflare_api_key": "should-not-bind", + "cloudflare_zone_id": "should-not-bind-zone", + "server_id": 42, + "ServerId": 7, + "DnsSuffix": "example.test", + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .PostConfigure(static o => + { + if (o.BindAddress is null || o.BindAddress.Length == 0) + { + o.BindAddress = ["*"]; + } + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.Equal(7, options.ServerId); + Assert.Equal("example.test", options.DnsSuffix); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(TestHostFactory.TestCloudFlareApiKey, options.CloudFlareApiKey); + Assert.NotEqual("should-not-bind", options.CloudFlareApiKey); + Assert.NotEqual("should-not-bind-zone", options.CloudFlareZoneId); + } + + [Fact] + public void EnvironmentVariables_UseExactCloudFlareNames() + { + const string envKey = "env-override-cloudflare-api-key"; + + var previousKey = Environment.GetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable); + + try + { + Environment.SetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, envKey); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:ServerId"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:CloudFlareApiKey"] = "from-json-should-be-overridden", + [$"{NntpdOptions.SectionName}:CloudFlareZoneId"] = "5811a29d39a0732afb5f160c9b137c3d", + }) + .AddVectorEnvironmentVariables() + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .BindConfiguration(NntpdOptions.SectionName) + .Configure(static (options, config) => + { + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme(options, config); + }) + .PostConfigure(static options => + { + options.AcmeEmail = string.Empty; + options.AcmeCertificatePassword = string.Empty; + options.CloudFlareApiKey = string.Empty; + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.NotEqual(envKey, options.CloudFlareApiKey); + Assert.NotEqual("from-json-should-be-overridden", options.CloudFlareApiKey); + Assert.Equal(string.Empty, options.CloudFlareApiKey); + Assert.Equal("5811a29d39a0732afb5f160c9b137c3d", options.CloudFlareZoneId); + Assert.Equal(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, "VECTOR__CLOUDFLAREAPIKEY"); + } + finally + { + Environment.SetEnvironmentVariable(NntpdOptions.CloudFlareApiKeyEnvironmentVariable, previousKey); + } + } + + [Fact] + public void ValidationErrors_DoNotExposeApiKey() + { + const string secret = "super-secret-cloudflare-api-key-value"; + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = secret; + options.BindPort = 0; + + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + + Assert.True(result.Failed); + var joined = NntpdOptionsValidator.JoinFailures(result); + Assert.DoesNotContain(secret, joined, StringComparison.Ordinal); + Assert.False(NntpdOptionsValidator.ContainsSecret(joined, secret)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("\t")] + public void CloudFlareApiKey_BlankOrMissing_DoesNotFailOptionsValidation(string? apiKey) + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = apiKey!; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void CloudFlareApiKey_ValidNonblank_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = TestHostFactory.TestCloudFlareApiKey; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("\t")] + public void CloudFlareZoneId_BlankOrMissing_FailsSharedConfiguration(string? zoneId) + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + zoneId, + "usenet.ninja"), + ])); + Assert.Contains("cloudflarezoneid", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void CloudFlareZoneId_ValidNonblank_Succeeds() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareZoneId = "5811a29d39a0732afb5f160c9b137c3d"; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingCloudFlareApiKey_DoesNotFailOptionsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.CloudFlareApiKey = " "; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void MissingCloudFlareZoneId_FailsSharedConfiguration() + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + string.Empty, + "usenet.ninja"), + ])); + Assert.Contains("cloudflarezoneid", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Defaults_ApplyWhenListenerSettingsOmitted() + { + var json = """ + { + "Nntpd": { + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1, + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .PostConfigure(static o => + { + if (o.BindAddress is null || o.BindAddress.Length == 0) + { + o.BindAddress = ["*"]; + } + }) + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.Equal(["*"], options.BindAddress); + Assert.Equal(119, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.False(options.IsTlsListenerEnabled); + Assert.Equal("usenet.ninja", options.DnsSuffix); + } + + [Fact] + public void MissingServerId_FailsValidationAndIsDistinctFromZero() + { + var missing = TestHostFactory.CreateValidOptions(); + missing.ServerId = null; + var missingResult = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, missing); + Assert.True(missingResult.Failed); + Assert.Contains("required", NntpdOptionsValidator.JoinFailures(missingResult), StringComparison.OrdinalIgnoreCase); + + var zero = TestHostFactory.CreateValidOptions(); + zero.ServerId = 0; + var zeroResult = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, zero); + Assert.True(zeroResult.Failed); + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(zeroResult), StringComparison.Ordinal); + + Assert.Null(missing.ServerId); + Assert.Equal(0, zero.ServerId); + } + + [Fact] + public void MissingServerId_FromConfiguration_FailsStartup() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "usenet.ninja" + } + } + """; + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(jsonStream) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + services.AddOptions() + .Bind(configuration.GetSection(NntpdOptions.SectionName)) + .ValidateDataAnnotations() + .ValidateOnStart(); + services.AddSingleton, NntpdOptionsValidator>(); + + using var provider = services.BuildServiceProvider(); + var ex = Assert.Throws( + () => _ = provider.GetRequiredService>().Value); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("198.18.0.66")] + [InlineData("2001:db8::1")] + public void BindAddress_AssignedAddresses_AreAccepted(string address) + { + var ip = IPAddress.Parse(address); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [address]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(ip)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Fact] + public void BindAddress_MultipleConfiguredAddresses_AreAllValidated() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["198.18.0.66", "2001:db8::1"]; + var assignee = new FakeLocalIpAddressAssignee( + IPAddress.Parse("198.18.0.66"), + IPAddress.Parse("2001:db8::1")); + var result = new NntpdOptionsValidator(assignee).Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void BindAddress_AssignedPrivateAndPublic_AreAccepted() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["10.0.0.5", "203.0.113.10"]; + var assignee = new FakeLocalIpAddressAssignee( + IPAddress.Parse("10.0.0.5"), + IPAddress.Parse("203.0.113.10")); + var result = new NntpdOptionsValidator(assignee).Validate(null, options); + Assert.True(result.Succeeded); + } + + [Fact] + public void BindAddress_UnassignedAddress_IsRejected() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["198.18.0.66"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(false)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("not assigned", NntpdOptionsValidator.JoinFailures(result), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void BindAddress_InvalidSyntax_IsRejected() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["not-an-ip"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData("*")] + [InlineData("0.0.0.0")] + [InlineData("::")] + public void BindAddress_Wildcards_AreAcceptedWithoutNicAssignment(string wildcard) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [wildcard]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(false)) + .Validate(null, options); + Assert.True(result.Succeeded, NntpdOptionsValidator.JoinFailures(result)); + } + + [Theory] + [InlineData(1)] + [InlineData(119)] + [InlineData(65535)] + public void BindPort_ValidRange_Succeeds(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPort = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(65536)] + public void BindPort_InvalidValues_Fail(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPort = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void BindPortTls_UnsetAndZero_DisableTls() + { + var unset = TestHostFactory.CreateValidOptions(); + unset.BindPortTls = default; + Assert.Equal(0, unset.BindPortTls); + Assert.False(unset.IsTlsListenerEnabled); + Assert.True( + new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, unset) + .Succeeded); + + var explicitZero = TestHostFactory.CreateValidOptions(); + explicitZero.BindPortTls = 0; + Assert.False(explicitZero.IsTlsListenerEnabled); + Assert.True( + new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, explicitZero) + .Succeeded); + } + + [Theory] + [InlineData(1)] + [InlineData(563)] + [InlineData(65535)] + public void BindPortTls_NonZero_EnablesTlsConfiguration(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = port; + options.AcmeEmail = "ops@example.org"; + options.AcmeCertificatePassword = "unit-test-pfx-password"; + Assert.True(options.IsTlsListenerEnabled); + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(-1)] + [InlineData(65536)] + public void BindPortTls_OutOfRange_Fails(int port) + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = port; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(1, "nntpd01.usenet.ninja")] + [InlineData(8, "nntpd08.usenet.ninja")] + [InlineData(9, "nntpd09.usenet.ninja")] + [InlineData(10, "nntpd10.usenet.ninja")] + [InlineData(99, "nntpd99.usenet.ninja")] + public void Fqdn_UsesExactFormat_WithoutDotBeforeId(int serverId, string expected) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + options.DnsSuffix = "usenet.ninja"; + + Assert.Equal(expected, options.Fqdn); + Assert.Equal(NntpdOptions.ApplicationPrefix, "nntpd"); + Assert.Equal( + ApplicationFqdn.Build(NntpdOptions.ApplicationPrefix, serverId, "usenet.ninja"), + options.Fqdn); + Assert.Equal( + ApplicationFqdn.Build(NntpdOptions.ApplicationPrefix, serverId, "usenet.ninja"), + NntpdOptions.FormatFqdn(serverId, "usenet.ninja")); + Assert.NotEqual( + ApplicationFqdn.Build("backfiller", serverId, "usenet.ninja"), + options.Fqdn); + Assert.DoesNotContain("nntpd.", options.Fqdn, StringComparison.Ordinal); + Assert.StartsWith($"nntpd{serverId:00}.", options.Fqdn, StringComparison.Ordinal); + + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(1)] + [InlineData(99)] + [InlineData(100)] + [InlineData(255)] + public void ServerId_ValidBoundaries_Succeed(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(256)] + public void ServerId_OutOfRange_Fails(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + + [Theory] + [InlineData(null, (int)ServerIdValidationStatus.Missing, false)] + [InlineData(-1, (int)ServerIdValidationStatus.OutOfRange, false)] + [InlineData(0, (int)ServerIdValidationStatus.OutOfRange, false)] + [InlineData(1, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(8, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(99, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(100, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(255, (int)ServerIdValidationStatus.Valid, true)] + [InlineData(256, (int)ServerIdValidationStatus.OutOfRange, false)] + public void ServerId_DelegatesNumericRulesToCommon( + int? serverId, + int expectedStatus, + bool expectedValid) + { + Assert.Equal((ServerIdValidationStatus)expectedStatus, ServerIdRules.Classify(serverId)); + + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + + Assert.Equal(expectedValid, result.Succeeded); + if (!expectedValid) + { + Assert.Contains("1–255", NntpdOptionsValidator.JoinFailures(result), StringComparison.Ordinal); + } + } + + [Fact] + public void ServerId_RangeAttribute_UsesSharedCommonBounds() + { + var range = typeof(NntpdOptions) + .GetProperty(nameof(NntpdOptions.ServerId)) + ?.GetCustomAttribute(); + Assert.NotNull(range); + Assert.Equal(ServerIdRules.MinimumInclusive, Convert.ToInt32(range.Minimum, CultureInfo.InvariantCulture)); + Assert.Equal(ServerIdRules.MaximumInclusive, Convert.ToInt32(range.Maximum, CultureInfo.InvariantCulture)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("-bad.example")] + [InlineData("bad..example")] + [InlineData("bad_label.example")] + public void DnsSuffix_Invalid_FailsSharedConfiguration(string suffix) + { + var ex = Assert.Throws(() => NntpSharedConfigurationReader.Validate( + [ + new NntpSharedConfigurationCandidate( + 1024, + "news.example", + null, + "https://acme.example.test/directory", + 14, + "0123456789abcdef0123456789abcdef", + suffix), + ])); + Assert.Contains("dnssuffix", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fqdn_CannotBeIndependentlyConfiguredOrOverridden() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": 119, + "BindPortTls": 0, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "DnsSuffix": "usenet.ninja", + "ServerId": 8, + "Fqdn": "evil.example.com" + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + + Assert.Equal(8, options.ServerId); + Assert.Equal("transit08.usenet.ninja", options.Fqdn); + Assert.DoesNotContain("evil", options.Fqdn, StringComparison.Ordinal); + } + + [Fact] + public async Task HostStart_MissingCloudFlareApiKey_PublishesTheSharedSnapshot() + { + var port = TestHostFactory.GetFreeTcpPort(); + var json = $$""" + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": {{port}}, + "CloudFlareApiKey": "", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + await host.StartAsync(); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, options.CloudFlareApiKey); + Assert.NotEqual(string.Empty, options.CloudFlareApiKey); + await host.StopAsync(); + } + + [Fact] + public void HostStart_MissingCloudFlareZoneId_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": " ", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(" ", options.CloudFlareZoneId); + } + + [Fact] + public async Task HostStart_MissingServerId_FailsBeforeRunning() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d" + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + Assert.Contains(NntpdOptions.ServerIdEnvironmentVariable, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.TestCloudFlareApiKey, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task HostStart_ValidMandatorySettings_ReachesRunning() + { + var port = TestHostFactory.GetFreeTcpPort(); + var json = $$""" + { + "Nntpd": { + "BindAddress": [ "*" ], + "BindPort": {{port}}, + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + await host.StartAsync(); + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + await host.StopAsync(); + } + + [Fact] + public void CloudFlareApiKey_NullFromConfiguration_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": null, + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.Equal(string.Empty, options.CloudFlareApiKey); + } + + [Fact] + public void CloudFlareZoneId_NullFromConfiguration_DoesNotFailOptionsValidation() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": null, + "ServerId": 1 + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var options = host.Services.GetRequiredService>().Value; + Assert.True(string.IsNullOrWhiteSpace(options.CloudFlareZoneId)); + } + + [Fact] + public void ServerId_NullFromConfiguration_FailsStartup() + { + var json = """ + { + "Nntpd": { + "BindAddress": [ "*" ], + "CloudFlareApiKey": "unit-test-cloudflare-api-key", + "CloudFlareZoneId": "5811a29d39a0732afb5f160c9b137c3d", + "ServerId": null + } + } + """; + + using var host = CreateEmptyNntpdHost(json); + var ex = Assert.Throws( + () => _ = host.Services.GetRequiredService>().Value); + Assert.Contains(nameof(NntpdOptions.ServerId), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void EnvironmentVariables_DocumentedNames_AreExact() + { + Assert.Equal("VECTOR__CLOUDFLAREAPIKEY", NntpdOptions.CloudFlareApiKeyEnvironmentVariable); + Assert.Equal("VECTOR__CLOUDFLAREZONEID", NntpdOptions.CloudFlareZoneIdEnvironmentVariable); + Assert.Equal("NNTPD__SERVERID", NntpdOptions.ServerIdEnvironmentVariable); + Assert.Equal("NNTPD__XTRACEKEY", NntpdOptions.XTraceKeyEnvironmentVariable); + Assert.Equal("NNTPD__XTRACEPREVIOUSKEY", NntpdOptions.XTracePreviousKeyEnvironmentVariable); + Assert.Equal("NNTPD__NEWSMASTERUSER", NntpdOptions.NewsmasterUserEnvironmentVariable); + Assert.Equal("NNTPD__NEWSMASTERPASSWORD", NntpdOptions.NewsmasterPasswordEnvironmentVariable); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.ServerIdEnvironmentVariable)); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.XTraceKeyEnvironmentVariable)); + Assert.False(VectorEnvironment.IsCanonicalName(NntpdOptions.NewsmasterPasswordEnvironmentVariable)); + } + + [Fact] + public void ApplicationSpecificSettings_BindFromNntpdSection_NotVectorRootKeys() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Nntpd:ServerId"] = "12", + ["Nntpd:XTraceKey"] = TestHostFactory.TestXTraceKey, + ["ServerId"] = "99", + ["XTraceKey"] = new string('a', 64), + ["NewsmasterPassword"] = "vector-must-not-bind", + }) + .Build(); + + var options = new NntpdOptions(); + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + AcmeCloudflareOptions.OverlaySharedFromRoot(options, configuration); + + Assert.Equal(12, options.ServerId); + Assert.Equal(TestHostFactory.TestXTraceKey, options.XTraceKey); + Assert.True(string.IsNullOrEmpty(options.NewsmasterPassword)); + } + + private static IHost CreateEmptyNntpdHost(string json, bool forceTlsDisabled = true) + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings + { + ApplicationName = "VectorNNTPD.TransitService.Tests", + EnvironmentName = Environments.Development, + }); + + using var jsonStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + builder.Configuration.AddJsonStream(jsonStream); + TestHostFactory.PromoteSharedKeysToRoot(builder.Configuration); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = + TestHostFactory.TestXTraceKey, + }); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.PostConfigure(options => + { + if (forceTlsDisabled) + { + options.BindPortTls = 0; + } + + options.AcmeEmail = string.Empty; + }); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }); + builder.Services.AddSingleton(); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static IHost CreateConfiguredHost( + IReadOnlyDictionary? configuration = null, + ILocalIpAddressAssignee? assignee = null) + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = + TestHostFactory.TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindAddress:1"] = null, + [$"{NntpdOptions.SectionName}:LogDir"] = TestHostFactory.NewTestLogDir(), + ["Redis:Host:0"] = "127.0.0.1", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = + TestHostFactory.TestNntpDbConnectionString, + }); + + if (configuration is not null) + { + builder.Configuration.AddInMemoryCollection(configuration); + } + + builder.Services.AddSingleton( + assignee ?? new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.PostConfigure(static options => + { + options.BindAddress = ["*"]; + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + }); + TestHostFactory.IsolateTransit(builder.Services); + + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs b/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs new file mode 100644 index 00000000..2a713fd8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleDisposalTests.cs @@ -0,0 +1,286 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Core.Lifecycle; + +[Collection(SerilogCollection.Name)] +public sealed class ApplicationLifecycleDisposalTests +{ + [Fact] + public async Task HostStopThenDispose_DoesNotThrowObjectDisposedException() + { + using var host = CreateHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await host.StopAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + // DI disposal previously reproduced ObjectDisposedException from DisposeAsync → StopAsync. + var dispose = Record.ExceptionAsync(async () => + { + if (host is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + host.Dispose(); + } + }); + + Assert.Null(await dispose); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartStopDispose_CompletesWithoutObjectDisposedException() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + var dispose = await Record.ExceptionAsync(async () => await lifecycle.DisposeAsync()); + Assert.Null(dispose); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_AfterNormalStop_IsIdempotent() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + await lifecycle.DisposeAsync(); + await lifecycle.DisposeAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_WhileRunning_StopsServicesOnce() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await lifecycle.DisposeAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task DisposeAsync_SurfacesShutdownFailure_WhenDisposeTriggersStop() + { + var stopAttempts = 0; + var service = new FakeApplicationService( + "bad", + onStop: _ => + { + Interlocked.Increment(ref stopAttempts); + throw new InvalidOperationException("dispose-stop-fail"); + }); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + async () => await lifecycle.DisposeAsync()); + Assert.Contains(ex.InnerExceptions, e => e.Message == "dispose-stop-fail"); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, stopAttempts); + + // Subsequent dispose is idempotent and does not rethrow or re-stop. + await lifecycle.DisposeAsync(); + Assert.Equal(1, stopAttempts); + } + + [Fact] + public async Task ConcurrentStopRequests_ShareSingleShutdown() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "svc", + onStop: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + var lifecycle = TestHostFactory.CreateLifecycle( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await lifecycle.StartAsync(CancellationToken.None); + + var stop1 = lifecycle.StopAsync(CancellationToken.None); + await entered.Task; + var stop2 = lifecycle.StopAsync(CancellationToken.None); + var stop3 = lifecycle.StopAsync(CancellationToken.None); + + release.TrySetResult(); + await Task.WhenAll(stop1, stop2, stop3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + + await lifecycle.DisposeAsync(); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StopAsync_AfterDispose_WhenAlreadyStopped_DoesNotThrow() + { + var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + await lifecycle.DisposeAsync(); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task ConcurrentStopAndDispose_DoNotThrowOrDoubleStopServices() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + await lifecycle.StartAsync(CancellationToken.None); + + var stop = lifecycle.StopAsync(CancellationToken.None); + var dispose = lifecycle.DisposeAsync().AsTask(); + + await Task.WhenAll(stop, dispose); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + + await lifecycle.DisposeAsync(); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task ConcurrentDisposeRequests_AreSafe() + { + var service = new FakeApplicationService("svc"); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + await lifecycle.StartAsync(CancellationToken.None); + + var d1 = lifecycle.DisposeAsync().AsTask(); + var d2 = lifecycle.DisposeAsync().AsTask(); + var d3 = lifecycle.DisposeAsync().AsTask(); + + await Task.WhenAll(d1, d2, d3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task ShutdownFailure_RemainsObservable_OnStopAsync() + { + var service = new FakeApplicationService( + "bad", + onStop: _ => throw new InvalidOperationException("stop-fail")); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StopAsync(CancellationToken.None)); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-fail"); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartupFailure_FollowedByDispose_Succeeds() + { + var service = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("start-fail")); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + + await lifecycle.DisposeAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task ShutdownCancellation_Propagates() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "slow", + onStop: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await lifecycle.StartAsync(CancellationToken.None); + + using var cts = new CancellationTokenSource(); + var stop = lifecycle.StopAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => stop); + } + + [Fact] + public void HostWithSerilog_DoesNotRegisterMicrosoftConsoleProvider() + { + using var host = CreateHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.DoesNotContain(host.Services.GetLoggerProviders(), static p => p is ConsoleLoggerProvider); + Assert.Empty(host.Services.GetLoggerProviders()); + } + + private static IHost CreateHost(Action? configureServices = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs b/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs new file mode 100644 index 00000000..6c7dc959 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Core/Lifecycle/ApplicationLifecycleTests.cs @@ -0,0 +1,189 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Tests.Core.Lifecycle; + +public sealed class ApplicationLifecycleTests +{ + [Fact] + public async Task StartAsync_TransitionsCreatedToRunning() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + Assert.Equal(ApplicationState.Created, lifecycle.State); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Equal(1, service.StartCount); + } + + [Fact] + public async Task StopAsync_TransitionsRunningToStopped() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StartAsync_WhenAlreadyRunning_ThrowsInvalidOperation() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Contains("Invalid application lifecycle transition", ex.Message, StringComparison.Ordinal); + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task ConcurrentStart_SecondCallerFailsPredictably() + { + var releaseStart = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var enteredStart = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + enteredStart.TrySetResult(); + await releaseStart.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var first = lifecycle.StartAsync(CancellationToken.None); + await enteredStart.Task; + + var second = lifecycle.StartAsync(CancellationToken.None); + + releaseStart.TrySetResult(); + await first; + + await Assert.ThrowsAsync(() => second); + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task RepeatedStop_IsIdempotent() + { + var service = new FakeApplicationService("a"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + + var stop1 = lifecycle.StopAsync(CancellationToken.None); + var stop2 = lifecycle.StopAsync(CancellationToken.None); + var stop3 = lifecycle.StopAsync(CancellationToken.None); + + await Task.WhenAll(stop1, stop2, stop3); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task StopAsync_BeforeStart_TransitionsToStopped() + { + await using var lifecycle = TestHostFactory.CreateLifecycle([]); + + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartupFailure_EndsInStopped_AndRethrows() + { + var ok = new FakeApplicationService("ok"); + var bad = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([ok, bad]); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Equal("boom", ex.Message); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, ok.StartCount); + Assert.Equal(1, ok.StopCount); + Assert.Equal(0, bad.StartCount); + } + + [Fact] + public async Task StartupCancellation_PropagatesAndStopsCleanly() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var cts = new CancellationTokenSource(); + + var start = lifecycle.StartAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task State_IsVisibleAcrossThreads() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var start = lifecycle.StartAsync(CancellationToken.None); + await entered.Task; + + var observed = await Task.Run(() => lifecycle.State); + Assert.Equal(ApplicationState.Starting, observed); + + release.TrySetResult(); + await start; + Assert.Equal(ApplicationState.Running, lifecycle.State); + } + + [Fact] + public async Task UnexpectedServiceTermination_IsObservable() + { + var service = new FakeApplicationService("a", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + service.CompleteExecution(new InvalidOperationException("service crashed")); + + await Assert.ThrowsAsync( + () => lifecycle.WaitAsync(CancellationToken.None)); + Assert.True(lifecycle.UnexpectedTermination.IsCompletedSuccessfully); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Core/Services/ApplicationServiceManagerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Core/Services/ApplicationServiceManagerTests.cs new file mode 100644 index 00000000..7cad2bdd --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Core/Services/ApplicationServiceManagerTests.cs @@ -0,0 +1,371 @@ +using System.Diagnostics; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Tests.Core.Services; + +public sealed class ApplicationServiceManagerTests +{ + [Fact] + public async Task StartAsync_StartsServicesInRegistrationOrder() + { + var order = new List(); + var a = new FakeApplicationService("a") { SharedStartOrder = order }; + var b = new FakeApplicationService("b") { SharedStartOrder = order }; + var c = new FakeApplicationService("c") { SharedStartOrder = order }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + await manager.StartAsync(CancellationToken.None); + + Assert.Equal(["a", "b", "c"], order); + Assert.Equal(3, manager.StartedServices.Count); + } + + [Fact] + public async Task StopAsync_StopsServicesInReverseOrder() + { + var startOrder = new List(); + var stopOrder = new List(); + var a = new FakeApplicationService("a") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var b = new FakeApplicationService("b") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var c = new FakeApplicationService("c") { SharedStartOrder = startOrder, SharedStopOrder = stopOrder }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + Assert.Equal(["a", "b", "c"], startOrder); + Assert.Equal(["c", "b", "a"], stopOrder); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task StartupFailure_RollsBackStartedServices() + { + var stopOrder = new List(); + var a = new FakeApplicationService("a") { SharedStopOrder = stopOrder }; + var b = new FakeApplicationService( + "b", + onStart: _ => throw new InvalidOperationException("fail-b")) + { + SharedStopOrder = stopOrder, + }; + var c = new FakeApplicationService("c") { SharedStopOrder = stopOrder }; + var manager = TestHostFactory.CreateServiceManager([a, b, c]); + + var ex = await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + + Assert.Equal("fail-b", ex.Message); + Assert.Equal(1, a.StartCount); + Assert.Equal(1, a.StopCount); + Assert.Equal(0, b.StartCount); + Assert.Equal(0, c.StartCount); + Assert.Equal(["a"], stopOrder); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task ShutdownFailure_IsAggregatedAndSurfaced() + { + var a = new FakeApplicationService( + "a", + onStop: _ => throw new InvalidOperationException("stop-a")); + var b = new FakeApplicationService( + "b", + onStop: _ => throw new InvalidOperationException("stop-b")); + var manager = TestHostFactory.CreateServiceManager([a, b]); + + await manager.StartAsync(CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + Assert.Equal(2, ex.InnerExceptions.Count); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-b"); + Assert.Contains(ex.InnerExceptions, e => e.Message == "stop-a"); + } + + [Fact] + public async Task CancellationDuringStartup_RollsBackAndThrows() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var a = new FakeApplicationService("a"); + var b = new FakeApplicationService( + "b", + onStart: async ct => + { + entered.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + var manager = TestHostFactory.CreateServiceManager([a, b]); + using var cts = new CancellationTokenSource(); + + var start = manager.StartAsync(cts.Token); + await entered.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(1, a.StartCount); + Assert.Equal(1, a.StopCount); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task CancellationDuringShutdown_Throws() + { + var enteredStop = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var a = new FakeApplicationService("a"); + var b = new FakeApplicationService( + "b", + onStop: async ct => + { + enteredStop.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct); + }); + var manager = TestHostFactory.CreateServiceManager([a, b]); + + await manager.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(); + + var stop = manager.StopAsync(cts.Token); + await enteredStop.Task; + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => stop); + + // Reverse order: b canceled first; a must still receive a best-effort abort and leave tracking. + Assert.Empty(manager.StartedServices); + Assert.Equal(1, b.StopAttemptCount); + Assert.Equal(1, a.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_ThrowsTimeoutException() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50)); + var manager = TestHostFactory.CreateServiceManager([service], options); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task GracefulShutdownTimeout_ClearsTracking_AndStillAttemptsEarlierServices() + { + // Reverse stop order: slow (started second) consumes the overall budget first; cooperative + // (started first) must still be offered a stop attempt and leave StartedServices empty. + var cooperative = new FakeApplicationService("cooperative"); + var slow = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50)); + var manager = TestHostFactory.CreateServiceManager([cooperative, slow], options); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + + Assert.Empty(manager.StartedServices); + Assert.Equal(1, slow.StopAttemptCount); + Assert.Equal(1, cooperative.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_IsOverallBudget_NotPerService() + { + // Two non-cooperative stops must not receive a refreshed full timeout each. + var first = new FakeApplicationService("first") { HoldStopUntilReleased = true }; + var second = new FakeApplicationService("second") { HoldStopUntilReleased = true }; + var timeout = TimeSpan.FromMilliseconds(80); + var options = TestHostFactory.CreateOptions(gracefulShutdownTimeout: timeout); + var manager = TestHostFactory.CreateServiceManager([first, second], options); + + await manager.StartAsync(CancellationToken.None); + + var sw = Stopwatch.StartNew(); + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + sw.Stop(); + + Assert.Empty(manager.StartedServices); + // Per-service refresh would take ~2× timeout; overall budget should stay near one window. + Assert.True( + sw.Elapsed < timeout + TimeSpan.FromMilliseconds(250), + $"Shutdown elapsed {sw.Elapsed} exceeded overall budget expectation for timeout {timeout}."); + Assert.Equal(1, second.StopAttemptCount); + Assert.Equal(1, first.StopAttemptCount); + } + + [Fact] + public async Task ShutdownException_RemovesServiceFromTracking() + { + var a = new FakeApplicationService( + "a", + onStop: _ => throw new InvalidOperationException("stop-a")); + var manager = TestHostFactory.CreateServiceManager([a]); + + await manager.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + } + + [Fact] + public async Task DuplicateStart_ThrowsInvalidOperation() + { + var manager = TestHostFactory.CreateServiceManager([new FakeApplicationService("a")]); + + await manager.StartAsync(CancellationToken.None); + await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task ConcurrentLifecycleOperations_AreRejected() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "a", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + var manager = TestHostFactory.CreateServiceManager([service]); + + var start = manager.StartAsync(CancellationToken.None); + await entered.Task; + + await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + release.TrySetResult(); + await start; + } + + [Fact] + public async Task UnexpectedExecutionFault_RaisesEvent() + { + var service = new FakeApplicationService("a", withExecution: true); + var manager = TestHostFactory.CreateServiceManager([service]); + var observed = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + manager.UnexpectedServiceTermination += (_, args) => observed.TrySetResult(args); + + await manager.StartAsync(CancellationToken.None); + service.CompleteExecution(new InvalidOperationException("crash")); + + var args = await observed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal("a", args.ServiceName); + Assert.False(args.CompletedNormally); + Assert.IsType(args.Exception); + } + + [Fact] + public async Task ConcurrentStopAsync_IsRejected() + { + var service = new FakeApplicationService("a") + { + HoldStopUntilReleased = true, + }; + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMinutes(1))); + + await manager.StartAsync(CancellationToken.None); + + var stop1 = manager.StopAsync(CancellationToken.None); + await service.StopEntered.Task; + + await Assert.ThrowsAsync( + () => manager.StopAsync(CancellationToken.None)); + + service.StopGate.TrySetResult(); + await stop1; + + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + Assert.Equal(1, service.StopCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_NonCooperativeStop_KeepsAwaitingUntilReturn_ThenTimesOut() + { + var service = new FakeApplicationService("stubborn") + { + HoldStopUntilReleased = true, + IgnoreStopCancellation = true, + }; + var timeout = TimeSpan.FromMilliseconds(80); + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: timeout)); + + await manager.StartAsync(CancellationToken.None); + + var stop = manager.StopAsync(CancellationToken.None); + await service.StopEntered.Task; + + // Manager must not abandon the in-flight StopAsync when the budget elapses. + await Assert.ThrowsAsync(() => stop.WaitAsync(timeout + TimeSpan.FromMilliseconds(200))); + Assert.Single(manager.StartedServices); + + service.StopGate.TrySetResult(); + + await Assert.ThrowsAsync(() => stop); + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + Assert.Equal(1, service.StopCount); + + // Repeated stop must not re-invoke a service already removed from tracking. + await manager.StopAsync(CancellationToken.None); + Assert.Equal(1, service.StopAttemptCount); + } + + [Fact] + public async Task GracefulShutdownTimeout_DoesNotDoubleStop_AfterTimeoutClearsTracking() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + var manager = TestHostFactory.CreateServiceManager( + [service], + TestHostFactory.CreateOptions(gracefulShutdownTimeout: TimeSpan.FromMilliseconds(50))); + + await manager.StartAsync(CancellationToken.None); + await Assert.ThrowsAsync(() => manager.StopAsync(CancellationToken.None)); + Assert.Empty(manager.StartedServices); + Assert.Equal(1, service.StopAttemptCount); + + await manager.StopAsync(CancellationToken.None); + Assert.Equal(1, service.StopAttemptCount); + } + + [Fact] + public async Task ServicesAreNotStartedTwice_AfterSuccessfulStartAndStop() + { + var service = new FakeApplicationService("a"); + var manager = TestHostFactory.CreateServiceManager([service]); + + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + // After full stop, started list is empty but duplicate start is blocked by design + // until we allow restart. Phase 0 treats restart as unsupported on the same manager + // only when started list is non-empty; empty allows start again. + await manager.StartAsync(CancellationToken.None); + await manager.StopAsync(CancellationToken.None); + + Assert.Equal(2, service.StartCount); + Assert.Equal(2, service.StopCount); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/1 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/1 new file mode 100644 index 00000000..0a464332 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/1 @@ -0,0 +1,11 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Test article +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Organization: Leading space tab + folded header and trailing space. +Message-ID: +Xref: news.example.com example.test:1 + +This is a test message to example.test in native format. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/2 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/2 new file mode 100644 index 00000000..35e5862d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/2 @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.config +Subject: Test article #2 +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:54 -0800 +Message-ID: +Xref: news.example.com example.config:1 + +This is a test message to example.config in native format. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/3 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/3 new file mode 100644 index 00000000..a61360c5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/3 @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test,example.config +Subject: Test article #3 with a | | tab +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com example.test:2 example.config:2 + +This is an example crossposted message. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/4 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/4 new file mode 100644 index 00000000..136c0f48 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/4 @@ -0,0 +1,20 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: This + is an aggressively obnoxious article + for + testing + + overview +Date: Tue, 18 May 2004 01:43:21 -0700 +Expires: Wed, 19 May 2004 01:43:21 -0700 +Message-ID: +References: + +Xref: news.example.com example.test:3 +From: I + Am + User + +This is a header with complex headers for testing overview generation +using makehistory. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/5 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/5 new file mode 100644 index 00000000..ae6f8384 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/5 @@ -0,0 +1,26 @@ +T: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +The^Previous@Header#Is&Exactly*1000+Characters=Long: Whee! +Subject: Torture test for article parsing +TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT: +X-Test: The previous header is also exactly 1000 characters long and has + a colon-space but no content + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + and the previous continuation line is exactly 1000 characters long +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Expires: Wed, 19 May 2004 01:43:21 -0700 +Message-ID: +References: + +Xref: news.example.com example.test:4 +From: I + Am + User +X-Test: This header has the same name as a previous header + . + and has a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +. +nosense in it to torture the article parser. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/6 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/6 new file mode 100644 index 00000000..3cea95e7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/6 @@ -0,0 +1,137 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Long test article +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com example.test:5 + +This is an example article that's longer than 8192 bytes, used to test +large article rejection in article parsing. + +Each of these lines but the last is 60 characters, becoming 63 characters +with dot escaping. Everything before these lines is exactly 607 bytes. +Therefore, this article is exactly 8193 bytes long in wire format, but +less than that in native format. + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +.................... diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/7 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/7 new file mode 100644 index 00000000..3a976bf8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/7 @@ -0,0 +1,16 @@ +.Actually^Valid: Strangely enough +Subject: Wire format testing +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Message-ID: +X-Test: This header has + . + a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +. +nosense in it to torture the article parser. +.. +... +. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/LICENSE.ISC.txt b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/LICENSE.ISC.txt new file mode 100644 index 00000000..2fd0df6c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/LICENSE.ISC.txt @@ -0,0 +1,22 @@ +Copyright (c) 2004-2026 +by Internet Systems Consortium, Inc. ("ISC") +Copyright (c) 1991, 1994-2003 +by The Internet Software Consortium and Rich Salz + +This code is derived from software contributed to the Internet Software +Consortium by Rich Salz. + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Source: InterNetNews/inn LICENSE at commit 6843efceb8a15774d23e4e2892d08daf54574314 +https://github.com/InterNetNews/inn/blob/6843efceb8a15774d23e4e2892d08daf54574314/LICENSE diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/README.md b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/README.md new file mode 100644 index 00000000..80a81e94 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/README.md @@ -0,0 +1,45 @@ +# INN article test corpus (byte-exact fixtures) + +## Provenance + +| Field | Value | +|-------|-------| +| Repository | [InterNetNews/inn](https://github.com/InterNetNews/inn) | +| Source path | `tests/data/articles` | +| Upstream ref | `main` | +| Upstream commit | `6843efceb8a15774d23e4e2892d08daf54574314` | +| Retrieval date (UTC) | 2026-09-22 | +| Upstream commit date (UTC) | 2026-09-11 | + +Fixtures were imported from the GitHub archive of that exact commit: + +`https://github.com/InterNetNews/inn/archive/6843efceb8a15774d23e4e2892d08daf54574314.zip` + +Paths under `inn-6843efceb8a15774d23e4e2892d08daf54574314/tests/data/articles/` were copied **byte-for-byte**. No line-ending conversion, text decoding, or content normalization was applied. + +Machine-readable per-file hashes and byte characteristics: + +- `inn-articles.manifest.json` (same directory) + +## License / attribution + +INN as a whole (and unmarked code/data in the repository) is covered by the ISC-style license in the upstream `LICENSE` file: + +> Copyright (c) 2004-2026 by Internet Systems Consortium, Inc. ("ISC") +> Copyright (c) 1991, 1994-2003 by The Internet Software Consortium and Rich Salz +> +> Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. +> +> THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Full upstream license text: + +No separate license/notice file was present inside `tests/data/articles` itself at this commit. + +## Framing note (VectorNNTP TAKETHIS experiment) + +These fixtures are INN parser/spool test inputs. Most numeric and `bad-*` files are **LF-only spool articles** and do **not** include an NNTP multiline terminator (`.\r\n` / `\r\n.\r\n`). + +The `wire-*` files are wire-oriented. At this commit, only `wire-7` contains the five-byte `\r\n.\r\n` delimiter and ends with a `.\r\n` terminator suitable for a complete multiline framing benchmark. + +Do **not** treat filename categories as NNTP validity by themselves; use the manifest byte characteristics and framing classification. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-empty b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-empty new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-empty @@ -0,0 +1 @@ + diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-empty b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-empty new file mode 100644 index 00000000..99e9698d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-empty @@ -0,0 +1,4 @@ +Test: This article has a bad header +From: + +Empty header but with a colon-space. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-nospc b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-nospc new file mode 100644 index 00000000..0cea9203 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-nospc @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test:<-he: re + +No space before colon. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-space b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-space new file mode 100644 index 00000000..c0a66b15 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-space @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test : <-here + +Space before colon. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-trunc b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-trunc new file mode 100644 index 00000000..c2359d66 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-hdr-trunc @@ -0,0 +1,4 @@ +Test: This article has a bad header +Test: + +Empty header with no colon-space. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-cont b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-cont new file mode 100644 index 00000000..e9e9b29e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-cont @@ -0,0 +1,12 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Article with too long of a header +T: Testing + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + Testing +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:1 + +This is an article with a header continuation line exceeding MAXARTLINELENGTH. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-hdr b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-hdr new file mode 100644 index 00000000..a133a639 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-long-hdr @@ -0,0 +1,10 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Article with too long of a header +T: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:1 + +This is an article with a header line exceeding MAXARTLINELENGTH. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-msgid b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-msgid new file mode 100644 index 00000000..a98e22ef --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-msgid @@ -0,0 +1,8 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Test article with stray CR | | and no message ID +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Xref: news.example.com example.test:4 + +This is a test message to example.test with a stray CR and no message ID. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-body b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-body new file mode 100644 index 00000000..8aba3720 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-body @@ -0,0 +1,6 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Bad article with no body +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-header b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-header new file mode 100644 index 00000000..b7dec89b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-no-header @@ -0,0 +1,2 @@ + +This is a test article that contains no header. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-body b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-body new file mode 100644 index 00000000..5f6da45b Binary files /dev/null and b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-body differ diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-header b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-header new file mode 100644 index 00000000..09782f23 Binary files /dev/null and b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-nul-header differ diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-subj b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-subj new file mode 100644 index 00000000..0f4e75a1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/bad-subj @@ -0,0 +1,8 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: +Xref: news.example.com example.test:3 + +This is a test message to example.test with no Subject header. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/inn-articles.manifest.json b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/inn-articles.manifest.json new file mode 100644 index 00000000..867bc24a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/inn-articles.manifest.json @@ -0,0 +1,436 @@ +{ + "repository": "InterNetNews/inn", + "sourcePath": "tests/data/articles", + "upstreamRef": "main", + "upstreamCommit": "6843efceb8a15774d23e4e2892d08daf54574314", + "retrievalDateUtc": "2026-09-22", + "fileCount": 25, + "totalBytes": 19544, + "files": [ + { + "relativePath": "1", + "length": 346, + "sha256": "a4ab547e5f1c1de730f4cf93a20a22e680c0ab2e1b7416cf71393133889bb0e8", + "upstreamFileName": "1", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "2", + "length": 285, + "sha256": "4d57fe8c5ba8193ebaf650e4a295a1af3738c311fb037c9e45784e38640f84aa", + "upstreamFileName": "2", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "3", + "length": 309, + "sha256": "88a3622ad30b41a131b7f0a85e28381dadb51dec803f65e351054a67f765e2e6", + "upstreamFileName": "3", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "4", + "length": 484, + "sha256": "e0c8efc04310d40053bdac08a637b7e9f9b3e0b80fb6c34fc53a1b80bc7d6bab", + "upstreamFileName": "4", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "5", + "length": 3810, + "sha256": "9f0ff1b216d75d25feb0d8e0d7cd181bed84d65447878e58b92c306c96634ad8", + "upstreamFileName": "5", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "6", + "length": 7932, + "sha256": "7db1d0ab9df4fa9318c4f3f2ad344969c034bcba83844c159f2cd0bcf022bbea", + "upstreamFileName": "6", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "7", + "length": 387, + "sha256": "723d2e133a19b3ea1d321b76728a1643d738d8252e952b9e397e29e7ec76a2ef", + "upstreamFileName": "7", + "categoryByName": "numeric", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-empty", + "length": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "upstreamFileName": "bad-empty", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-empty", + "length": 81, + "sha256": "72b802aa141b9da402098d7752fb5df9937a69e87a0522bc2ba98620c94e0250", + "upstreamFileName": "bad-hdr-empty", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-nospc", + "length": 74, + "sha256": "5d0b224e4e96b2f7721216ee79b4b6a8384798b6cc21f93f079c33a8625ad42a", + "upstreamFileName": "bad-hdr-nospc", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-space", + "length": 71, + "sha256": "949c2a4e92d9776c3d341140a7b9d0dc5f129920c855b2e01b0d550f03e1c10b", + "upstreamFileName": "bad-hdr-space", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-hdr-trunc", + "length": 77, + "sha256": "2c774bced25643912ed6e76f170caf7ced866adb474839b359fb639a114be7d4", + "upstreamFileName": "bad-hdr-trunc", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": true + }, + { + "relativePath": "bad-long-cont", + "length": 1349, + "sha256": "08d6c7f7567826d9c21e6f984574f824f1d984554ddbd262616ea3afaaee73c1", + "upstreamFileName": "bad-long-cont", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-long-hdr", + "length": 1314, + "sha256": "ae69d5b5b0468df45cff370a9ec973bf15f6020431ad11ca5fb8f15dc34acfcc", + "upstreamFileName": "bad-long-hdr", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-msgid", + "length": 293, + "sha256": "d2197e7be69357add2b527fd91e014e5cfd74f31de257be15d94ffac30e51d87", + "upstreamFileName": "bad-msgid", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-no-body", + "length": 191, + "sha256": "f3503b1b140a4a5d962db445d2efcea6e89082b7b186553ca8c428cbe415ddbd", + "upstreamFileName": "bad-no-body", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-no-header", + "length": 49, + "sha256": "e06feff8243eeb2365e3fa0a1eaa4d37db3cd6c1cce26c96e3af45f2bca0295e", + "upstreamFileName": "bad-no-header", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-nul-body", + "length": 310, + "sha256": "6ebe35ae053f2b298cace44b8fc947b9f7a57a7fac0ea2a6b7b420afff738ee2", + "upstreamFileName": "bad-nul-body", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-nul-header", + "length": 303, + "sha256": "8c58d98396c666a09e8dabdbe6c90ec36a2f4100c570811f063159ba07b00ca6", + "upstreamFileName": "bad-nul-header", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "bad-subj", + "length": 267, + "sha256": "db59026c63d0bf3ebc5a7e6d043e218cc0fc7e6a204c8459c1d4e98eb3936266", + "upstreamFileName": "bad-subj", + "categoryByName": "bad", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-7", + "length": 411, + "sha256": "3bb2fe13f24d179db4a79847f7c93cf98359a0ad9991bebe8c37b63b93a44a02", + "upstreamFileName": "wire-7", + "categoryByName": "wire", + "framingClassification": "complete_multiline_wire", + "containsCrlf": true, + "containsLfOnly": false, + "containsCrOnly": true, + "containsNul": false, + "containsLeadingDotLines": true, + "containsDotStuffedLines": true, + "endsWithDotCrlfTerminator": true, + "containsFiveByteCrlfDotCrlf": true, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-no-body", + "length": 191, + "sha256": "f3503b1b140a4a5d962db445d2efcea6e89082b7b186553ca8c428cbe415ddbd", + "upstreamFileName": "wire-no-body", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-strange", + "length": 477, + "sha256": "0bf53004be197496191b2fc25c3332d35aee1b59cd550c8b92f6f18a3d28ec8e", + "upstreamFileName": "wire-strange", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": true, + "containsLfOnly": true, + "containsCrOnly": true, + "containsNul": true, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + }, + { + "relativePath": "wire-truncated", + "length": 115, + "sha256": "1e37f7cf1606ee6bbd3424ebcd47ba9c994d57d2af7c9011829bd617cd2ed92f", + "upstreamFileName": "wire-truncated", + "categoryByName": "wire", + "framingClassification": "wire_incomplete_or_nonstandard", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": true + }, + { + "relativePath": "xref", + "length": 417, + "sha256": "f4587f5fb11e872de275548347eeb7ebb5211ddc54913ca43d82d2f5207ea67a", + "upstreamFileName": "xref", + "categoryByName": "other", + "framingClassification": "spool_or_non_wire_no_nntp_terminator", + "containsCrlf": false, + "containsLfOnly": true, + "containsCrOnly": false, + "containsNul": false, + "containsLeadingDotLines": false, + "containsDotStuffedLines": false, + "endsWithDotCrlfTerminator": false, + "containsFiveByteCrlfDotCrlf": false, + "looksTruncatedByName": false + } + ] +} \ No newline at end of file diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-7 b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-7 new file mode 100644 index 00000000..6fe8c626 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-7 @@ -0,0 +1,17 @@ +..Actually^Valid: Strangely enough +Subject: Wire format testing +Path: news.example.com!not-for-mail +Newsgroups: example.test +Date: Tue, 18 May 2004 01:43:21 -0700 +Message-ID: +X-Test: This header has + . + a continuation line that's almost an article terminator + +This is an article | | with all sorts of random +.. +nosense in it to torture the article parser. +... +.... +.. +. diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-no-body b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-no-body new file mode 100644 index 00000000..8aba3720 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-no-body @@ -0,0 +1,6 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test +Subject: Bad article with no body +From: user@example.com +Date: Sat, 06 Mar 2004 21:39:44 -0800 +Message-ID: diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-strange b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-strange new file mode 100644 index 00000000..3c0731a9 Binary files /dev/null and b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-strange differ diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-truncated b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-truncated new file mode 100644 index 00000000..c1667121 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/wire-truncated @@ -0,0 +1,5 @@ +Path: foo!bar +From: example@example.com +Subject: Test +Message-ID: +Date: Mon, 23 Dec 2002 16:00:04 \ No newline at end of file diff --git a/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/xref b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/xref new file mode 100644 index 00000000..6af82e6c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Data/InnArticles/xref @@ -0,0 +1,9 @@ +Path: news.example.com!not-for-mail +Newsgroups: example.test,example.config +Subject: Test article #3 with a | | tab +From: user@example.com +Date: Sat, 06 Mar 2004 21:40:54 -0800 +Message-ID: +Xref: news.example.com blah example:test1:2 example.test1:4 example.test2:-1 example.test3:2 example.test3 example.test2:1roger example.test2: + +This is a test of Xref parsing for message cancellation. diff --git a/tests/VectorNNTPD.TransitService.Tests/Diagnostics/FeedDiagnosticsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Diagnostics/FeedDiagnosticsTests.cs new file mode 100644 index 00000000..09c70db4 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Diagnostics/FeedDiagnosticsTests.cs @@ -0,0 +1,477 @@ +using System.IO.Pipelines; +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Transit; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Diagnostics; + +public sealed class FeedDiagnosticsTests +{ + [Fact] + public void Disabled_IsNoOp() + { + var feed = NullFeedDiagnostics.Instance; + Assert.False(feed.IsEnabled); + feed.OnRejected("giganews", "1.2.3.4:119"); + Assert.Null(feed.OnAccepted(CreateSession("192.0.2.10"))); + feed.OnReleased(null); + var snapshot = feed.CaptureSnapshot( + DisabledArticleIngestionQueue.Instance, + new TransitConfigurationStore()); + Assert.Same(FeedDiagnosticsSnapshot.Empty, snapshot); + } + + [Fact] + public void AcceptRelease_AdjustsActiveAndPeak() + { + var hub = new FeedDiagnosticsHub(); + var first = hub.OnAccepted(CreateSession("192.0.2.10")); + var second = hub.OnAccepted(CreateSession("192.0.2.11")); + Assert.NotNull(first); + Assert.NotNull(second); + + var store = CreateStore(maxIncoming: 10); + var mid = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + var peer = Assert.Single(mid.Peers); + Assert.Equal(TransitTestPeers.DefaultPeerName, peer.PeerName); + Assert.Equal(2, peer.ActiveConnections); + Assert.Equal(2, peer.PeakConnections); + Assert.Equal(2, peer.AcceptedConnections); + Assert.Equal(0, peer.RejectedConnections); + Assert.Equal(10, peer.ConfiguredMaxConnections); + + hub.OnReleased(first); + var after = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + peer = Assert.Single(after.Peers); + Assert.Equal(1, peer.ActiveConnections); + Assert.Equal(2, peer.PeakConnections); + Assert.Equal(2, peer.AcceptedConnections); + } + + [Fact] + public void Reject_IncrementsWithoutActivating() + { + var hub = new FeedDiagnosticsHub(); + for (var i = 0; i < 10; i++) + { + Assert.NotNull(hub.OnAccepted(CreateSession("192.0.2.10"))); + } + + hub.OnRejected(TransitTestPeers.DefaultPeerName, "192.0.2.99:1"); + var snapshot = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, CreateStore(maxIncoming: 10)); + var peer = Assert.Single(snapshot.Peers); + Assert.Equal(10, peer.ActiveConnections); + Assert.Equal(10, peer.PeakConnections); + Assert.Equal(10, peer.AcceptedConnections); + Assert.Equal(1, peer.RejectedConnections); + } + + [Fact] + public void Release_DoesNotGoNegative() + { + var hub = new FeedDiagnosticsHub(); + var probe = hub.OnAccepted(CreateSession("192.0.2.10")); + hub.OnReleased(probe); + hub.OnReleased(probe); + var snapshot = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, CreateStore(maxIncoming: 10)); + var peer = Assert.Single(snapshot.Peers); + Assert.Equal(0, peer.ActiveConnections); + Assert.Equal(1, peer.PeakConnections); + } + + [Fact] + public void Peak_IsMonotonic() + { + var hub = new FeedDiagnosticsHub(); + var a = hub.OnAccepted(CreateSession("192.0.2.10")); + var b = hub.OnAccepted(CreateSession("192.0.2.11")); + var c = hub.OnAccepted(CreateSession("192.0.2.12")); + hub.OnReleased(a); + hub.OnReleased(b); + _ = hub.OnAccepted(CreateSession("192.0.2.13")); + var snapshot = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, CreateStore(maxIncoming: 10)); + Assert.Equal(3, Assert.Single(snapshot.Peers).PeakConnections); + Assert.Equal(2, Assert.Single(snapshot.Peers).ActiveConnections); + Assert.NotNull(c); + } + + [Fact] + public void PerPeerIsolation_DoesNotShareSlots() + { + var hub = new FeedDiagnosticsHub(); + var store = new TransitConfigurationStore(); + store.Replace( + new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + ["alpha"] = TransitConfigurationSnapshot.CreatePeer( + "alpha", + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10"], peerName: "A")), + ["beta"] = TransitConfigurationSnapshot.CreatePeer( + "beta", + TransitTestPeers.Peer(maxIncoming: 8, allowFrom: ["198.51.100.1"], peerName: "B")), + })); + + var alphaAuth = TransitPeerAuthorization.CreateForStore(store); + var alpha = hub.OnAccepted(CreateSession(alphaAuth, IPAddress.Parse("192.0.2.10"))); + hub.OnRejected("alpha", "192.0.2.10:2"); + var beta = hub.OnAccepted(CreateSession(alphaAuth, IPAddress.Parse("198.51.100.1"))); + Assert.NotNull(alpha); + Assert.NotNull(beta); + + var snapshot = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + var alphaRow = Assert.Single(snapshot.Peers, p => p.PeerName == "alpha"); + var betaRow = Assert.Single(snapshot.Peers, p => p.PeerName == "beta"); + Assert.Equal(1, alphaRow.ActiveConnections); + Assert.Equal(1, alphaRow.RejectedConnections); + Assert.Equal(1, betaRow.ActiveConnections); + Assert.Equal(0, betaRow.RejectedConnections); + } + + [Fact] + public void Probe_ArticleAndHistoryCounters() + { + var probe = new FeedSessionProbe("192.0.2.10:40000", "giganews"); + probe.RecordCommand(); + probe.SetState(FeedSessionState.Receiving); + probe.RecordArticleReceived(768_000, 1_000); + probe.RecordHistory(HistoryLookupResult.Unseen, 500); + probe.RecordQueue(ArticleEnqueueResult.Accepted, 200, 768_000); + probe.RecordArticleCompleted(duplicate: false, responseTicks: 100); + + Assert.Equal(1, probe.Commands); + Assert.Equal(1, probe.ArticlesReceived); + Assert.Equal(1, probe.ArticlesCompleted); + Assert.Equal(768_000, probe.ArticleBytes); + Assert.Equal(1, probe.HistoryMisses); + Assert.Equal(0, probe.HistoryHits); + Assert.Equal(1, probe.QueueAccepted); + Assert.Equal(768_000, probe.QueueAdmittedBytes); + Assert.Equal(FeedSessionState.Idle, probe.State); + Assert.True(probe.ReceiveTime > TimeSpan.Zero || probe.ArticleBytes == 768_000); + } + + [Fact] + public void Probe_StateTransitions_DoNotCorruptCounters() + { + var probe = new FeedSessionProbe("192.0.2.10:40000", "giganews"); + probe.RecordArticleReceived(100, 10); + probe.SetState(FeedSessionState.WaitingHistory); + probe.SetState(FeedSessionState.WaitingQueue); + probe.SetState(FeedSessionState.Completing); + probe.SetState(FeedSessionState.Idle); + probe.SetState(FeedSessionState.Receiving); + + Assert.Equal(1, probe.ArticlesReceived); + Assert.Equal(100, probe.ArticleBytes); + Assert.Equal(0, probe.ArticlesCompleted); + Assert.Equal(0, probe.HistoryHits); + Assert.Equal(0, probe.HistoryMisses); + Assert.Equal(0, probe.QueueAccepted); + Assert.Equal(FeedSessionState.Receiving, probe.State); + + probe.RecordArticleCompleted(duplicate: false, responseTicks: 1); + Assert.Equal(1, probe.ArticlesReceived); + Assert.Equal(1, probe.ArticlesCompleted); + Assert.Equal(100, probe.ArticleBytes); + Assert.Equal(FeedSessionState.Idle, probe.State); + } + + [Fact] + public void Snapshot_IsInternallyConsistent() + { + var hub = new FeedDiagnosticsHub(); + var session = CreateSession("192.0.2.10"); + var probe = hub.OnAccepted(session); + Assert.NotNull(probe); + session.FeedProbe = probe; + probe.RecordArticleReceived(1000, 10); + probe.RecordHistory(HistoryLookupResult.Seen, 5); + probe.RecordArticleCompleted(duplicate: true, responseTicks: 1); + + var snapshot = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, CreateStore(maxIncoming: 50)); + var peer = Assert.Single(snapshot.Peers); + var row = Assert.Single(snapshot.Sessions); + Assert.Equal(peer.ActiveConnections, snapshot.Sessions.Count); + Assert.Equal(probe.ArticlesCompleted, peer.ArticlesCompleted); + Assert.Equal(probe.ArticleBytes, peer.ArticleBytes); + Assert.Equal(probe.ArticlesCompleted, row.ArticlesCompleted); + Assert.Equal("192.0.2.10:40000", row.Remote); + Assert.Equal(1, snapshot.HistoryHits); + Assert.Equal(0, snapshot.HistoryMisses); + } + + [Fact] + public void Formatter_DoesNotLeakSecretsOrPayloads() + { + var snapshot = new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = TimeSpan.FromSeconds(10), + Peers = + [ + new FeedPeerSnapshot + { + PeerName = "giganews", + ConfiguredMaxConnections = 50, + ActiveConnections = 50, + PeakConnections = 50, + AcceptedConnections = 50, + RejectedConnections = 0, + ArticlesCompleted = 12, + ArticleBytes = 9_000_000, + }, + ], + Sessions = + [ + new FeedSessionSnapshot + { + Remote = "69.80.99.14:119", + PeerName = "giganews", + Age = TimeSpan.FromMinutes(3), + State = FeedSessionState.Receiving, + ArticlesCompleted = 4, + ArticleBytes = 3_000_000, + }, + ], + IntervalArticles = 12, + IntervalBytes = 9_000_000, + }; + + var text = FeedDiagnosticsFormatter.Format(snapshot, includeSessions: true); + Assert.Contains("giganews", text, StringComparison.Ordinal); + Assert.Contains("69.80.99.14:119", text, StringComparison.Ordinal); + Assert.DoesNotContain("password", text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("AUTHINFO", text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("@", text, StringComparison.Ordinal); + Assert.DoesNotContain("<", text, StringComparison.Ordinal); + Assert.DoesNotContain("Message-ID", text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("secret", text, StringComparison.OrdinalIgnoreCase); + Assert.Contains("stage routing status=not_implemented", text, StringComparison.Ordinal); + Assert.Contains("stage rx_app", text, StringComparison.Ordinal); + Assert.Contains("stage ingest", text, StringComparison.Ordinal); + Assert.Contains("stage process", text, StringComparison.Ordinal); + Assert.Contains("stage spool", text, StringComparison.Ordinal); + } + + [Fact] + public void RateCalculator_ByteDelta_ProducesCorrectMbps() + { + var elapsed = TimeSpan.FromSeconds(2); + var mbps = FeedRateCalculator.MegabitsPerSecond(125_000_000, elapsed); + Assert.Equal(500d, mbps, precision: 6); + Assert.Equal(12.5d, FeedRateCalculator.PerSecond(25, elapsed), precision: 6); + } + + [Fact] + public void RateCalculator_ZeroElapsed_IsZero() + { + Assert.Equal(0, FeedRateCalculator.PerSecond(1_000, TimeSpan.Zero)); + Assert.Equal(0, FeedRateCalculator.MegabitsPerSecond(1_000_000, TimeSpan.Zero)); + Assert.Equal(0, FeedRateCalculator.PerSecond(1_000, TimeSpan.FromMilliseconds(-1))); + } + + [Fact] + public void RateCalculator_CounterReset_DoesNotProduceNegativeRates() + { + Assert.Equal(0, FeedRateCalculator.NonNegativeDelta(10, 20)); + Assert.Equal(0, FeedRateCalculator.PerSecond(FeedRateCalculator.NonNegativeDelta(10, 20), TimeSpan.FromSeconds(1))); + Assert.Equal(0, FeedRateCalculator.MegabitsPerSecond(FeedRateCalculator.NonNegativeDelta(5, 50), TimeSpan.FromSeconds(1))); + } + + [Fact] + public void RateCalculator_ZeroActivity_IsZero() + { + Assert.Equal(0, FeedRateCalculator.NonNegativeDelta(42, 42)); + Assert.Equal(0, FeedRateCalculator.PerSecond(0, TimeSpan.FromSeconds(5))); + Assert.Equal(0, FeedRateCalculator.MegabitsPerSecond(0, TimeSpan.FromSeconds(5))); + } + + [Fact] + public void Snapshot_IntervalDeltas_AreMonotonicAndAggregated() + { + var hub = new FeedDiagnosticsHub(); + var session = CreateSession("192.0.2.10"); + var probe = hub.OnAccepted(session); + Assert.NotNull(probe); + probe.RecordCommand(); + probe.RecordArticleReceived(1_000, 1); + probe.RecordQueue(ArticleEnqueueResult.Accepted, 1, 1_000); + probe.RecordArticleCompleted(duplicate: false, responseTicks: 1); + hub.RecordTcpBytes(2_000); + hub.BeginSpoolWork(); + hub.EndSpoolWork(1_000, persisted: true); + + var store = CreateStore(maxIncoming: 50); + var first = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + Assert.Equal(1, first.IntervalArticles); + Assert.Equal(1, first.IntervalArticlesReceived); + Assert.Equal(1, first.IntervalCommands); + Assert.Equal(1_000, first.IntervalBytes); + Assert.Equal(2_000, first.IntervalTcpBytes); + Assert.Equal(1_000, first.IntervalQueuedBytes); + Assert.Equal(1, first.IntervalQueuedArticles); + Assert.Equal(1, first.IntervalSpoolArticles); + Assert.Equal(1_000, first.IntervalSpoolBytes); + Assert.Equal(1, first.ActiveSessions); + Assert.Equal(0, first.SpoolActiveWorkers); + + var second = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + Assert.Equal(0, second.IntervalArticles); + Assert.Equal(0, second.IntervalArticlesReceived); + Assert.Equal(0, second.IntervalCommands); + Assert.Equal(0, second.IntervalBytes); + Assert.Equal(0, second.IntervalTcpBytes); + Assert.Equal(0, second.IntervalQueuedBytes); + Assert.Equal(0, second.IntervalSpoolArticles); + Assert.Equal(0, second.ArticlesPerSecond); + Assert.Equal(0, second.TcpMegabitsPerSecond); + + probe.RecordCommand(); + probe.RecordArticleReceived(3_000, 1); + probe.RecordQueue(ArticleEnqueueResult.Accepted, 1, 3_000); + probe.RecordArticleCompleted(duplicate: false, responseTicks: 1); + hub.RecordTcpBytes(4_000); + hub.BeginSpoolWork(); + hub.EndSpoolWork(3_000, persisted: true); + + var third = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, store); + Assert.Equal(1, third.IntervalArticles); + Assert.Equal(1, third.IntervalArticlesReceived); + Assert.Equal(1, third.IntervalCommands); + Assert.Equal(3_000, third.IntervalBytes); + Assert.Equal(4_000, third.IntervalTcpBytes); + Assert.Equal(3_000, third.IntervalQueuedBytes); + Assert.Equal(1, third.IntervalQueuedArticles); + Assert.Equal(1, third.IntervalSpoolArticles); + Assert.Equal(3_000, third.IntervalSpoolBytes); + var row = Assert.Single(third.Sessions); + Assert.Equal(1, row.IntervalCommands); + Assert.Equal(1, row.IntervalArticlesReceived); + Assert.Equal(3_000, row.IntervalBytes); + } + + [Fact] + public void Snapshot_Rates_UseIntervalElapsed() + { + var snapshot = new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = TimeSpan.FromSeconds(5), + Peers = [], + Sessions = [], + IntervalArticles = 100, + IntervalArticlesReceived = 100, + IntervalBytes = 62_500_000, + IntervalCommands = 250, + IntervalTcpBytes = 62_500_000, + IntervalQueuedBytes = 62_500_000, + IntervalSpoolBytes = 31_250_000, + }; + + Assert.Equal(20d, snapshot.ArticlesPerSecond, precision: 6); + Assert.Equal(50d, snapshot.CommandsPerSecond, precision: 6); + Assert.Equal(100d, snapshot.TcpMegabitsPerSecond, precision: 6); + Assert.Equal(100d, snapshot.IngestMegabitsPerSecond, precision: 6); + Assert.Equal(100d, snapshot.ProcessMegabitsPerSecond, precision: 6); + Assert.Equal(50d, snapshot.SpoolMegabitsPerSecond, precision: 6); + Assert.Equal(0, FeedRateCalculator.MegabitsPerSecond(62_500_000, TimeSpan.Zero)); + } + + [Fact] + public void FeedDiagnosticsOptions_InvalidInterval_FailsValidation() + { + var options = TestHostFactory.CreateValidOptions(); + options.FeedDiagnostics = new FeedDiagnosticsOptions { IntervalSeconds = 0 }; + var validator = new NntpdOptionsValidator(new VectorNNTPD.TransitService.Tests.TestDoubles.FakeLocalIpAddressAssignee(true)); + Assert.True(validator.Validate(null, options).Failed); + } + + private static TransitConfigurationStore CreateStore(int maxIncoming) + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: maxIncoming, allowFrom: ["192.0.2.0/24"]))); + return store; + } + + private static NntpSession CreateSession(string address) => + CreateSession( + TransitPeerAuthorization.CreateForStore(CreateStore(maxIncoming: 50)), + IPAddress.Parse(address)); + + private static NntpSession CreateSession(ITransitPeerAuthorization peers, IPAddress source) + { + var input = new Pipe(); + var output = new Pipe(); + var connection = new ProbeConnection(input.Reader, output.Writer, source); + return new NntpSession( + connection, + NullLogger.Instance, + transitPeerAuthorization: peers); + } + + private sealed class ProbeConnection(PipeReader input, PipeWriter output, IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = + ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs new file mode 100644 index 00000000..1b82ff9b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Diagnostics/IngestionPipelineMetricsTests.cs @@ -0,0 +1,56 @@ +using System.Diagnostics; +using VectorNNTPD.TransitService.Diagnostics; + +namespace VectorNNTPD.TransitService.Tests.Diagnostics; + +/// Bucketed ingestion-pipeline histogram contracts used by one-minute telemetry. +public sealed class IngestionPipelineMetricsTests +{ + [Fact] + public void CaptureInterval_ReportsCountAverageAndP95() + { + var metrics = new IngestionPipelineMetrics(); + for (var i = 0; i < 94; i++) + { + metrics.RecordNews(TimeSpan.FromMilliseconds(1)); + } + + for (var i = 0; i < 6; i++) + { + metrics.RecordNews(TimeSpan.FromMilliseconds(100)); + } + + var snapshot = metrics.CaptureInterval(); + Assert.Equal(100, snapshot.News.Count); + Assert.Equal(100, snapshot.News.P95Ms); + Assert.InRange(snapshot.News.AvgMs, 4, 12); + Assert.True(snapshot.News.MaxMs >= 90); + + var empty = metrics.CaptureInterval(); + Assert.Equal(0, empty.News.Count); + Assert.Equal(0, empty.News.P95Ms); + } + + [Fact] + public void ConfirmBusyPercent_IsConfirmSumOverBusyTime() + { + var metrics = new IngestionPipelineMetrics(); + var confirmTicks = (long)(0.040 * Stopwatch.Frequency); + RecordAgo(metrics.RecordPublishAndConfirm, TimeSpan.FromMilliseconds(40)); + metrics.AddBusyTicks((long)(0.100 * Stopwatch.Frequency)); + metrics.AddIdleTicks((long)(0.100 * Stopwatch.Frequency)); + metrics.RecordWorkerItem(Stopwatch.GetTimestamp() - confirmTicks); + + var snapshot = metrics.CaptureInterval(); + Assert.Equal(1, snapshot.WorkerItems); + Assert.InRange(snapshot.BusyPercent, 40, 60); + Assert.InRange(snapshot.ConfirmBusyPercent, 30, 50); + Assert.Equal(1.0, snapshot.ArticlesPerSecond(TimeSpan.FromSeconds(1)), precision: 6); + } + + private static void RecordAgo(Action record, TimeSpan ago) + { + var start = Stopwatch.GetTimestamp() - (long)(ago.TotalSeconds * Stopwatch.Frequency); + record(start); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/EmailDeliveryServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/EmailDeliveryServiceTests.cs new file mode 100644 index 00000000..3156f025 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/EmailDeliveryServiceTests.cs @@ -0,0 +1,270 @@ +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Email.Smtp; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class EmailDeliveryServiceTests +{ + [Fact] + public async Task Disabled_DoesNotStartWorkerOrTouchSmtp() + { + using var harness = new EmailSpoolHarness(enabled: false); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Null(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, transport.Calls); + Assert.False(Directory.Exists(harness.Directory)); + } + + [Fact] + public async Task Worker_DeliversPreExistingEmlAndDeletesIt() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.Single(harness.EmlFiles()); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls == 1); + await service.StopAsync(CancellationToken.None); + Assert.Equal(1, transport.Calls); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + } + + [Fact] + public async Task Worker_DoesNotDeliverDeliveredOrFailedFiles() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var deliveredId = EmailSpoolFileName.NewId(); + var failedId = EmailSpoolFileName.NewId(); + await File.WriteAllBytesAsync( + Path.Join(harness.Directory, deliveredId + ".delivered"), + EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + await File.WriteAllBytesAsync( + Path.Join(harness.Directory, "failed", failedId + ".eml"), + EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.NotNull(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, transport.Calls); + Assert.True(File.Exists(Path.Join(harness.Directory, deliveredId + ".delivered"))); + Assert.True(File.Exists(Path.Join(harness.Directory, "failed", failedId + ".eml"))); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task TransientFailure_RetainsEml() + { + using var harness = new EmailSpoolHarness( + smtp: new SmtpOptions + { + Host = "127.0.0.1", + MaxAttempts = 1, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Transient, "4xx") { StatusCode = 451 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls >= 1 && harness.EmlFiles().Length == 1); + await service.StopAsync(CancellationToken.None); + Assert.Single(harness.EmlFiles()); + Assert.Empty(harness.FailedFiles()); + } + + [Fact] + public async Task PermanentFailure_MovesToFailed() + { + using var harness = new EmailSpoolHarness(); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Permanent, "5xx") { StatusCode = 550 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => transport.Calls == 1 && harness.FailedFiles().Length == 1); + await service.StopAsync(CancellationToken.None); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task Transient_IsRetriedThenSucceeds() + { + using var harness = new EmailSpoolHarness( + smtp: new SmtpOptions + { + Host = "127.0.0.1", + MaxAttempts = 3, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }); + var transport = new ScriptedSmtpTransport + { + Failures = [new SmtpException(SmtpFailureKind.Transient, "4xx") { StatusCode = 451 }], + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Accepted, (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + await WaitUntilAsync(() => transport.Calls == 2 && harness.EmlFiles().Length == 0); + await service.StopAsync(CancellationToken.None); + Assert.Equal(2, transport.Calls); + } + + [Fact] + public async Task Shutdown_StopsAcceptingAndRetainsUndelivered() + { + using var harness = new EmailSpoolHarness(); + var transport = new ScriptedSmtpTransport(); + var service = harness.CreateDelivery(transport); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.False(harness.Spool.IsAccepting); + Assert.Equal( + EmailEnqueueStatus.Unavailable, + (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task Cancellation_RetainsEml() + { + using var harness = new EmailSpoolHarness(); + await harness.CreateService().SendAsync(EmailServiceTests.TextMessage()); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = new ScriptedSmtpTransport + { + BeforeSend = async cancellationToken => + { + started.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + }; + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.True(harness.EmlFiles().Length + harness.WorkFiles().Length == 1); + harness.Spool.RecoverClaims(); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task Worker_DeliversThroughFakeSmtp() + { + await using var smtp = new FakeSmtpServer(); + using var harness = new EmailSpoolHarness(smtp: smtp.ClientOptions(SmtpSecurityMode.None)); + var transport = SmtpTransportTests.Create(harness.Options.Smtp); + var service = harness.CreateDelivery(transport); + await service.StartAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Accepted, (await harness.CreateService().SendAsync(EmailServiceTests.TextMessage())).Status); + await WaitUntilAsync(() => smtp.LastMessage is not null); + await service.StopAsync(CancellationToken.None); + Assert.Contains("Subject: hello", System.Text.Encoding.ASCII.GetString(smtp.LastMessage!), StringComparison.Ordinal); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task TwoWorkers_DoNotDeliverTheSameFile() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slow = new ScriptedSmtpTransport + { + BeforeSend = async _ => + { + firstStarted.TrySetResult(); + await release.Task; + }, + }; + var fast = new ScriptedSmtpTransport(); + var a = harness.CreateDelivery(slow); + var b = harness.CreateDelivery(fast); + await a.StartAsync(CancellationToken.None); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await b.StartAsync(CancellationToken.None); + Assert.Equal(0, fast.Calls); + release.TrySetResult(); + await WaitUntilAsync(() => slow.Calls == 1); + await a.StopAsync(CancellationToken.None); + await b.StopAsync(CancellationToken.None); + Assert.Equal(1, slow.Calls + fast.Calls); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public void RetryDelay_IsBounded() + { + var smtp = new SmtpOptions + { + InitialRetryDelay = TimeSpan.FromSeconds(2), + MaximumRetryDelay = TimeSpan.FromSeconds(5), + }; + var delay = EmailRetryDelay.Compute(smtp, attempt: 8, new Random(1)); + Assert.True(delay <= TimeSpan.FromSeconds(5)); + Assert.True(delay >= TimeSpan.Zero); + } + + private static async Task WaitUntilAsync(Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (!predicate()) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException(); + } + + await Task.Delay(20); + } + } +} + +internal sealed class ScriptedSmtpTransport : ISmtpTransport +{ + public List Failures { get; init; } = []; + + public Func? BeforeSend { get; init; } + + public int Calls { get; private set; } + + public async Task SendAsync(EmailWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + if (BeforeSend is not null) + { + await BeforeSend(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + if (Failures.Count > 0) + { + var failure = Failures[0]; + Failures.RemoveAt(0); + throw failure; + } + + return new SmtpSendResult + { + Succeeded = true, + Recipients = [new SmtpRecipientResult(item.Recipients[0], true, 250, "OK")], + DataStatusCode = 250, + }; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/EmailServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/EmailServiceTests.cs new file mode 100644 index 00000000..774540c6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/EmailServiceTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class EmailServiceTests +{ + [Fact] + public async Task SendAsync_Disabled_DoesNotCreateSpoolFile() + { + using var harness = new EmailSpoolHarness(enabled: false); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Disabled, result.Status); + Assert.False(Directory.Exists(harness.Directory)); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_Enabled_AcceptsWithoutSmtp() + { + using var harness = new EmailSpoolHarness(); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Accepted, result.Status); + Assert.Contains("spool", result.Detail, StringComparison.Ordinal); + Assert.Single(harness.EmlFiles()); + var bytes = await File.ReadAllBytesAsync(harness.EmlFiles()[0]); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("envelope@example.com", item!.EnvelopeSender.Address); + Assert.NotEqual(0, item.EncodedMessage.Length); + } + + [Fact] + public async Task SendAsync_Injection_IsRejectedWithoutSpoolFile() + { + using var harness = new EmailSpoolHarness(); + var result = await harness.CreateService().SendAsync(TextMessage(subject: "x\r\nBcc: evil@example.com")); + Assert.Equal(EmailEnqueueStatus.Rejected, result.Status); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_StoppedSpool_IsUnavailable() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.StopAccepting(); + var result = await harness.CreateService().SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Unavailable, result.Status); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task SendAsync_SpoolWriteFailure_ReturnsFailed() + { + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-email-failed-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = "from@example.com", + EnvelopeSender = "envelope@example.com", + Smtp = new SmtpOptions { Host = "127.0.0.1", Port = 25 }, + Spool = new EmailSpoolOptions { Directory = blocker }, + }; + var spool = new FilesystemEmailSpool(Options.Create(options), NullLogger.Instance); + var service = new EmailService( + spool, + new Rfc5322MessageEncoder(), + Options.Create(options), + NullLogger.Instance); + var result = await service.SendAsync(TextMessage()); + Assert.Equal(EmailEnqueueStatus.Failed, result.Status); + } + finally + { + File.Delete(blocker); + } + } + + [Fact] + public void ChannelEmailQueue_IsGone() + { + Assert.Null(typeof(EmailService).Assembly.GetType("VectorNNTPD.TransitService.Email.ChannelEmailQueue")); + Assert.Null(typeof(EmailService).Assembly.GetType("VectorNNTPD.TransitService.Email.IEmailQueue")); + } + + internal static EmailMessage TextMessage(string subject = "hello") => + new() + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Subject = subject, + Body = "body\r\n"u8.ToArray(), + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/EmailSpoolHarness.cs b/tests/VectorNNTPD.TransitService.Tests/Email/EmailSpoolHarness.cs new file mode 100644 index 00000000..cf6888ca --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/EmailSpoolHarness.cs @@ -0,0 +1,101 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Email.Smtp; + +namespace VectorNNTPD.TransitService.Tests.Email; + +internal sealed class EmailSpoolHarness : IDisposable +{ + public EmailSpoolHarness(bool enabled = true, SmtpOptions? smtp = null) + { + Directory = Path.Join( + Path.GetTempPath(), + "vectornntp-email-spool", + Guid.NewGuid().ToString("N")); + Options = new EmailOptions + { + Enabled = enabled, + DefaultFrom = "from@example.com", + EnvelopeSender = "envelope@example.com", + Smtp = smtp ?? new SmtpOptions + { + Host = "127.0.0.1", + Port = 25, + Security = SmtpSecurityMode.None, + MaxAttempts = 1, + InitialRetryDelay = TimeSpan.Zero, + MaximumRetryDelay = TimeSpan.Zero, + }, + Spool = new EmailSpoolOptions + { + Directory = Directory, + ScanInterval = TimeSpan.FromMilliseconds(20), + ShutdownTimeout = TimeSpan.FromSeconds(2), + }, + }; + Spool = new FilesystemEmailSpool(Microsoft.Extensions.Options.Options.Create(Options), NullLogger.Instance); + } + + public string Directory { get; } + + public EmailOptions Options { get; } + + public FilesystemEmailSpool Spool { get; } + + public EmailService CreateService() => + new( + Spool, + new Rfc5322MessageEncoder(), + Microsoft.Extensions.Options.Options.Create(Options), + NullLogger.Instance); + + public EmailDeliveryService CreateDelivery(ISmtpTransport transport) => + new( + Spool, + transport, + Microsoft.Extensions.Options.Options.Create(Options), + NullLogger.Instance); + + public string[] EmlFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.eml") + : []; + + public string[] WorkFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.wrk") + : []; + + public string[] TmpFiles() => + System.IO.Directory.Exists(Directory) + ? System.IO.Directory.GetFiles(Directory, "*.tmp") + : []; + + public string[] FailedFiles() + { + var failed = Path.Join(Directory, "failed"); + return System.IO.Directory.Exists(failed) + ? System.IO.Directory.GetFiles(failed, "*.eml") + : []; + } + + public void Dispose() + { + try + { + if (System.IO.Directory.Exists(Directory)) + { + System.IO.Directory.Delete(Directory, recursive: true); + } + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (UnauthorizedAccessException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/FilesystemEmailSpoolTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/FilesystemEmailSpoolTests.cs new file mode 100644 index 00000000..cf3f7f4b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/FilesystemEmailSpoolTests.cs @@ -0,0 +1,220 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class FilesystemEmailSpoolTests +{ + [Fact] + public async Task WriteAsync_CreatesDirectoryAndAtomicEml() + { + using var harness = new EmailSpoolHarness(); + Assert.False(Directory.Exists(harness.Directory)); + var path = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(Directory.Exists(harness.Directory)); + Assert.True(File.Exists(path)); + Assert.EndsWith(".eml", path, StringComparison.OrdinalIgnoreCase); + Assert.Empty(harness.TmpFiles()); + var name = Path.GetFileNameWithoutExtension(path); + Assert.True(EmailSpoolFileName.IsId(name)); + Assert.DoesNotContain("hello", name, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("example.com", name, StringComparison.OrdinalIgnoreCase); + var bytes = await File.ReadAllBytesAsync(path); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("envelope@example.com", item!.EnvelopeSender.Address); + Assert.Contains("Subject: hello", Encoding.ASCII.GetString(item.EncodedMessage.Span), StringComparison.Ordinal); + Assert.StartsWith(EmailSpoolRecord.Magic, Encoding.ASCII.GetString(bytes), StringComparison.Ordinal); + } + + [Fact] + public async Task TmpFiles_AreNeverClaimed() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var tmp = Path.Join(harness.Directory, EmailSpoolFileName.NewId() + ".tmp"); + await File.WriteAllBytesAsync(tmp, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + Assert.True(File.Exists(tmp)); + } + + [Fact] + public void NewId_IsOpaqueAndUnique() + { + var ids = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < 64; i++) + { + var id = EmailSpoolFileName.NewId(); + Assert.True(EmailSpoolFileName.IsId(id)); + Assert.True(ids.Add(id)); + } + + Assert.False(EmailSpoolFileName.IsId("not-an-id")); + Assert.False(EmailSpoolFileName.IsId(new string('A', 32))); + } + + [Fact] + public async Task Collision_DoesNotOverwriteExistingEml() + { + using var harness = new EmailSpoolHarness(); + var first = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var original = await File.ReadAllBytesAsync(first); + var second = await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.NotEqual(first, second); + Assert.Equal(original, await File.ReadAllBytesAsync(first)); + Assert.Equal(2, harness.EmlFiles().Length); + } + + [Fact] + public void RecoverClaims_RenamesWorkBackToEml() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var wrk = Path.Join(harness.Directory, id + ".wrk"); + File.WriteAllBytes(wrk, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + harness.Spool.RecoverClaims(); + Assert.False(File.Exists(wrk)); + Assert.True(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + } + + [Fact] + public async Task Restart_DiscoversPreExistingEml() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + var claim = restarted.TryClaimNext(); + Assert.NotNull(claim); + Assert.Equal("to@example.com", claim!.Item.Recipients[0].Address); + restarted.Release(claim); + Assert.Single(harness.EmlFiles()); + } + + [Fact] + public async Task ConcurrentClaim_IsExclusive() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claims = new EmailSpoolClaim?[8]; + Parallel.For(0, claims.Length, i => + { + claims[i] = harness.Spool.TryClaimNext(); + }); + Assert.Equal(1, claims.Count(static claim => claim is not null)); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.WorkFiles()); + } + + [Fact] + public async Task MalformedFile_IsQuarantined() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var bad = Path.Join(harness.Directory, EmailSpoolFileName.NewId() + ".eml"); + await File.WriteAllTextAsync(bad, "not a spool file"); + Assert.Null(harness.Spool.TryClaimNext()); + Assert.False(File.Exists(bad)); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task DeliveredMarker_IsNeverClaimedRecoveredOrRetransmitted() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var delivered = Path.Join(harness.Directory, id + ".delivered"); + await File.WriteAllBytesAsync(delivered, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + harness.Spool.RecoverClaims(); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + Assert.Null(restarted.TryClaimNext()); + Assert.True(File.Exists(delivered)); + Assert.False(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + } + + [Fact] + public async Task FailedDirectory_IsNeverClaimedOrRequeued() + { + using var harness = new EmailSpoolHarness(); + harness.Spool.EnsureDirectory(); + var id = EmailSpoolFileName.NewId(); + var failed = Path.Join(harness.Directory, "failed", id + ".eml"); + await File.WriteAllBytesAsync(failed, EmailSpoolRecord.Serialize(SmtpTransportTests.Item())); + Assert.Null(harness.Spool.TryClaimNext()); + harness.Spool.RecoverClaims(); + var restarted = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + restarted.RecoverClaims(); + Assert.Null(restarted.TryClaimNext()); + Assert.True(File.Exists(failed)); + Assert.False(File.Exists(Path.Join(harness.Directory, id + ".eml"))); + Assert.Empty(harness.EmlFiles()); + Assert.Single(harness.FailedFiles()); + } + + [Fact] + public async Task Complete_DeletesWorkFile_WithoutDeliveredMarker() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claim = harness.Spool.TryClaimNext(); + Assert.NotNull(claim); + harness.Spool.Complete(claim!); + Assert.False(File.Exists(claim!.WorkPath)); + Assert.False(File.Exists(Path.Join(harness.Directory, claim.Id + ".delivered"))); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + Assert.Null(harness.Spool.TryClaimNext()); + } + + [Fact] + public async Task Complete_DeleteFailure_LeavesDeliveredMarker() + { + using var harness = new EmailSpoolHarness(); + await harness.Spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None); + var claim = harness.Spool.TryClaimNext(); + Assert.NotNull(claim); + harness.Spool.CompleteWorkFileDelete = _ => throw new IOException("injected work-file delete failure"); + harness.Spool.Complete(claim!); + var delivered = Path.Join(harness.Directory, claim!.Id + ".delivered"); + Assert.True(File.Exists(delivered)); + Assert.False(File.Exists(claim.WorkPath)); + Assert.Empty(harness.EmlFiles()); + Assert.Empty(harness.WorkFiles()); + Assert.Null(harness.Spool.TryClaimNext()); + } + + [Fact] + public async Task WriteAsync_WhenDirectoryIsAFile_Fails() + { + using var harness = new EmailSpoolHarness(); + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-email-spool-blocker-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + harness.Options.Spool.Directory = blocker; + var spool = new FilesystemEmailSpool( + Options.Create(harness.Options), + NullLogger.Instance); + await Assert.ThrowsAnyAsync( + () => spool.WriteAsync(SmtpTransportTests.Item(), CancellationToken.None)); + } + finally + { + File.Delete(blocker); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/Rfc5322MessageEncoderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/Rfc5322MessageEncoderTests.cs new file mode 100644 index 00000000..24ea79a3 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/Rfc5322MessageEncoderTests.cs @@ -0,0 +1,115 @@ +using System.Text; +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class Rfc5322MessageEncoderTests +{ + private readonly Rfc5322MessageEncoder _encoder = new(); + + [Fact] + public void Encode_UsesCrlfAndOmitsBcc() + { + var message = Create( + subject: "Hello", + body: "line1\nline2", + bcc: new EmailAddress("hidden@example.com")); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("From: from@example.com\r\n", text, StringComparison.Ordinal); + Assert.Contains("To: to@example.com\r\n", text, StringComparison.Ordinal); + Assert.Contains("Subject: Hello\r\n", text, StringComparison.Ordinal); + Assert.Contains("Cc: cc@example.com\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("Bcc:", text, StringComparison.Ordinal); + Assert.Contains("line1\r\nline2", text, StringComparison.Ordinal); + Assert.DoesNotContain("\nline1", text.Replace("\r\n", string.Empty, StringComparison.Ordinal), StringComparison.Ordinal); + } + + [Fact] + public void Encode_EncodesNonAsciiSubject() + { + var message = Create(subject: "café"); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("=?UTF-8?B?", text, StringComparison.Ordinal); + Assert.DoesNotContain("café", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_RejectsHeaderInjection() + { + Assert.Throws(() => + _encoder.Encode(Create(subject: "ok\r\nBcc: evil@example.com"))); + Assert.Throws(() => + new EmailHeader("X-Injected\r\nBcc", "x").Validate()); + Assert.Throws(() => + new EmailHeader("X-Good", "bad\r\nBcc: evil@example.com").Validate()); + Assert.Throws(() => + new EmailAddress("ok@example.com", "bad\r\nFrom: evil")); + } + + [Fact] + public void Encode_CustomHeadersAndNewsTransmission() + { + var message = new EmailMessage + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Subject = "group mid", + Body = "From: poster@example.com\r\n\r\nbody\r\n"u8.ToArray(), + ContentType = "application/news-transmission; usage=moderate", + Headers = [new EmailHeader("X-Moderated-Newsgroup", "comp.example")], + }; + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("Content-Type: application/news-transmission; usage=moderate\r\n", text, StringComparison.Ordinal); + Assert.Contains("X-Moderated-Newsgroup: comp.example\r\n", text, StringComparison.Ordinal); + Assert.Contains("usage=moderate", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_QuotedPrintableFor8BitBody() + { + var message = Create(bodyBytes: "naïve"u8.ToArray()); + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("Content-Transfer-Encoding: quoted-printable", text, StringComparison.Ordinal); + Assert.Contains("=C3=AF", text, StringComparison.Ordinal); + } + + [Fact] + public void Encode_AttachmentIsMultipartBase64() + { + var message = Create(); + message = new EmailMessage + { + From = message.From, + To = message.To, + Subject = message.Subject, + Body = message.Body, + Attachments = [new EmailAttachment("note.txt", "hi"u8.ToArray(), "text/plain")], + }; + var text = Encoding.ASCII.GetString(_encoder.Encode(message).Span); + Assert.Contains("multipart/mixed", text, StringComparison.Ordinal); + Assert.Contains("filename=\"note.txt\"", text, StringComparison.Ordinal); + Assert.Contains("aGk=", text, StringComparison.Ordinal); + } + + [Fact] + public void EnvelopeRecipients_AreDistinct() + { + var message = Create(bcc: new EmailAddress("to@example.com")); + Assert.Equal(2, message.EnvelopeRecipients().Count); + } + + private static EmailMessage Create( + string subject = "Subject", + string body = "body\r\n", + byte[]? bodyBytes = null, + EmailAddress? bcc = null) => + new() + { + From = new EmailAddress("from@example.com"), + To = [new EmailAddress("to@example.com")], + Cc = [new EmailAddress("cc@example.com")], + Bcc = bcc is null ? [] : [bcc], + Subject = subject, + Body = bodyBytes ?? Encoding.ASCII.GetBytes(body), + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/SmtpAuthTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpAuthTests.cs new file mode 100644 index 00000000..29b49eb1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpAuthTests.cs @@ -0,0 +1,130 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Email.Smtp; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class SmtpAuthTests +{ + [Fact] + public async Task AuthPlain_SucceedsAfterTls() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var logger = new MemoryLogger(); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create( + smtp, + FakeSmtpServer.TrustAnchors(server.Certificate!), + logger); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.Authenticated); + Assert.DoesNotContain(server.RawLines, static l => l.Contains("s3cret", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Messages, static m => m.Contains("s3cret", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Messages, static m => m.Contains("user", StringComparison.Ordinal)); + Assert.DoesNotContain(server.Commands, static c => c.Contains("PLAIN", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task AuthLogin_SucceedsWhenPlainMissing() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthLogin = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.Authenticated); + Assert.DoesNotContain(server.RawLines, static l => l.Contains("s3cret", StringComparison.Ordinal)); + } + + [Fact] + public async Task AuthFailure_IsNotRetriedByTransport() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "right", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "wrong"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Authentication, ex.Kind); + } + + [Fact] + public async Task Auth_NotAttemptedBeforeTlsWhenRequired() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "s3cret", + Certificate = FakeSmtpServer.CreateSelfSignedCertificate(), + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "s3cret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(server.Certificate!)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + var commands = server.Commands.ToList(); + var startTls = commands.FindIndex(static c => c.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)); + var auth = commands.FindIndex(static c => c.Equals("AUTH", StringComparison.OrdinalIgnoreCase)); + Assert.True(auth > startTls); + } + + [Fact] + public async Task Credentials_AreNotCopiedIntoHeaders() + { + var message = EmailServiceTests.TextMessage(); + var encoded = System.Text.Encoding.ASCII.GetString(new Rfc5322MessageEncoder().Encode(message).Span); + Assert.DoesNotContain("AUTH", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Password", encoded, StringComparison.Ordinal); + } +} + +internal sealed class MemoryLogger : ILogger +{ + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/SmtpProtocolTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpProtocolTests.cs new file mode 100644 index 00000000..aa3e139a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpProtocolTests.cs @@ -0,0 +1,78 @@ +using System.Text; +using VectorNNTPD.TransitService.Email.Smtp; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class SmtpProtocolTests +{ + [Theory] + [InlineData("250 OK", 250, false, "OK")] + [InlineData("250-PIPELINING", 250, true, "PIPELINING")] + [InlineData("354", 354, false, "")] + public void ParseReplyLine_Structured(string line, int code, bool continuation, string text) + { + Assert.True(SmtpResponseReader.TryParseReplyLine( + Encoding.ASCII.GetBytes(line), + out var parsed, + out var cont, + out var body)); + Assert.Equal(code, parsed); + Assert.Equal(continuation, cont); + Assert.Equal(text, body); + } + + [Theory] + [InlineData("OK")] + [InlineData("2OK")] + [InlineData("25X OK")] + [InlineData("250XOK")] + public void ParseReplyLine_RejectsMalformed(string line) + { + Assert.False(SmtpResponseReader.TryParseReplyLine(Encoding.ASCII.GetBytes(line), out _, out _, out _)); + } + + [Fact] + public void Capabilities_ParseExtensions() + { + var response = new SmtpResponse( + 250, + [ + "test.example", + "STARTTLS", + "AUTH PLAIN LOGIN", + "SIZE 1000", + "8BITMIME", + "SMTPUTF8", + ], + enhancedStatus: null, + text: "ehlo"); + var caps = SmtpCapabilities.Parse(response); + Assert.True(caps.StartTls); + Assert.True(caps.EightBitMime); + Assert.True(caps.SmtpUtf8); + Assert.Equal(1000, caps.MaxSize); + Assert.True(caps.SupportsAuth("PLAIN")); + Assert.True(caps.SupportsAuth("LOGIN")); + Assert.False(caps.SupportsAuth("CRAM-MD5")); + } + + [Fact] + public async Task DataEncoder_DotStuffsAndTerminates() + { + using var stream = new MemoryStream(); + await SmtpDataEncoder.WriteStuffedAsync( + stream, + ".hidden\r\nnormal\r\n"u8.ToArray(), + CancellationToken.None); + var text = Encoding.ASCII.GetString(stream.ToArray()); + Assert.Equal("..hidden\r\nnormal\r\n.\r\n", text); + } + + [Fact] + public void EnhancedStatus_IsSplit() + { + Assert.True(SmtpResponseReader.TrySplitEnhanced("2.0.0 OK", out var enhanced, out var rest)); + Assert.Equal("2.0.0", enhanced); + Assert.Equal("OK", rest); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTlsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTlsTests.cs new file mode 100644 index 00000000..5051e373 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTlsTests.cs @@ -0,0 +1,116 @@ +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email.Smtp; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class SmtpTlsTests +{ + [Fact] + public async Task ImplicitTls_ValidCertificate_SucceedsWithoutPlaintext() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + Assert.True(certificate.HasPrivateKey); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create( + server.ClientOptions(SmtpSecurityMode.ImplicitTls), + FakeSmtpServer.TrustAnchors(certificate)); + var result = await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(result.Succeeded); + Assert.True(server.StartedTls); + Assert.False(server.SawPlaintextSmtpBeforeTls); + } + + [Fact] + public async Task ImplicitTls_UntrustedCertificate_Fails() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create(server.ClientOptions(SmtpSecurityMode.ImplicitTls)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + } + + [Fact] + public async Task ImplicitTls_HostnameMismatch_Fails() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate("wrong.example", includeLoopbackIp: false); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + ImplicitTls = true, + Certificate = certificate, + }); + var transport = SmtpTransportTests.Create( + server.ClientOptions(SmtpSecurityMode.ImplicitTls), + FakeSmtpServer.TrustAnchors(certificate)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + } + + [Fact] + public async Task StartTls_SecondEhloAndNoPlaintextAuth() + { + var certificate = FakeSmtpServer.CreateSelfSignedCertificate(); + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions + { + AdvertiseStartTls = true, + AdvertiseAuthPlain = true, + ExpectedUsername = "user", + ExpectedPassword = "secret", + Certificate = certificate, + }); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls, "user", "secret"); + var transport = SmtpTransportTests.Create(smtp, FakeSmtpServer.TrustAnchors(certificate)); + await transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None); + Assert.True(server.StartedTls); + Assert.True(server.Authenticated); + var commands = server.Commands.ToList(); + var ehlo = commands.Count(static c => c.StartsWith("EHLO", StringComparison.OrdinalIgnoreCase)); + Assert.True(ehlo >= 2); + var startTlsIndex = commands.FindIndex(static c => c.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)); + var authIndex = commands.FindIndex(static c => c.Equals("AUTH", StringComparison.OrdinalIgnoreCase)); + Assert.True(startTlsIndex >= 0); + Assert.True(authIndex > startTlsIndex); + } + + [Fact] + public async Task StartTls_Required_FailsWhenNotAdvertised() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { AdvertiseStartTls = false }); + var transport = SmtpTransportTests.Create(server.ClientOptions(SmtpSecurityMode.StartTls)); + var ex = await Assert.ThrowsAsync( + () => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Tls, ex.Kind); + Assert.DoesNotContain(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + } + + [Fact] + public async Task StartTls_DoesNotDowngradeToPlaintext() + { + await using var server = new FakeSmtpServer(); + var smtp = server.ClientOptions(SmtpSecurityMode.StartTls); + var transport = SmtpTransportTests.Create(smtp); + await Assert.ThrowsAsync(() => transport.SendAsync(SmtpTransportTests.Item(), CancellationToken.None)); + Assert.False(server.StartedTls); + Assert.DoesNotContain(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + } + + [Fact] + public void SmtpOptions_HasNoCertificateBypassOrEhloHostname() + { + Assert.Null(typeof(SmtpOptions).GetProperty("DangerousAcceptAnyServerCertificate")); + Assert.Null(typeof(SmtpOptions).GetProperty("EhloHostname")); + Assert.Null(typeof(EmailOptions).GetProperty("Queue")); + Assert.NotNull(typeof(EmailOptions).GetProperty(nameof(EmailOptions.Spool))); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTransportTests.cs b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTransportTests.cs new file mode 100644 index 00000000..9bd94df9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Email/SmtpTransportTests.cs @@ -0,0 +1,186 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Email.Smtp; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Email; + +public sealed class SmtpTransportTests +{ + [Fact] + public async Task Plain_SendsMailRcptDataQuit() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var result = await transport.SendAsync(Item(), CancellationToken.None); + Assert.True(result.Succeeded); + Assert.Contains(server.Commands, static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal)); + Assert.Contains(server.Commands, static c => c.StartsWith("RCPT TO:", StringComparison.Ordinal)); + Assert.Contains(server.Commands, static c => c.Equals("DATA", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(server.Commands, static c => c.Equals("QUIT", StringComparison.OrdinalIgnoreCase)); + Assert.NotNull(server.LastMessage); + Assert.Contains("Subject: hello", System.Text.Encoding.ASCII.GetString(server.LastMessage), StringComparison.Ordinal); + } + + [Fact] + public async Task MultipleRecipients_OneConnection() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var item = Item(); + item = new EmailWorkItem + { + Message = item.Message, + EncodedMessage = item.EncodedMessage, + EnvelopeSender = item.EnvelopeSender, + Recipients = + [ + new EmailAddress("a@example.com"), + new EmailAddress("b@example.com"), + ], + }; + var result = await transport.SendAsync(item, CancellationToken.None); + Assert.True(result.Succeeded); + Assert.Equal(2, result.Recipients.Count); + Assert.Equal(1, server.Commands.Count(static c => c.StartsWith("MAIL FROM:", StringComparison.Ordinal))); + } + + [Fact] + public async Task Recipient5xx_IsPermanent() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { DefaultRcptCode = 550 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Permanent, ex.Kind); + } + + [Fact] + public async Task Recipient4xx_IsTransient() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { DefaultRcptCode = 450 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Transient, ex.Kind); + } + + [Fact] + public async Task MixedRecipients_ArePartial() + { + var options = new FakeSmtpServerOptions { DefaultRcptCode = 250 }; + options.RcptCodes["bad@example.com"] = 550; + await using var server = new FakeSmtpServer(options); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var item = Item(); + item = new EmailWorkItem + { + Message = item.Message, + EncodedMessage = item.EncodedMessage, + EnvelopeSender = item.EnvelopeSender, + Recipients = + [ + new EmailAddress("to@example.com"), + new EmailAddress("bad@example.com"), + ], + }; + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(item, CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Partial, ex.Kind); + } + + [Fact] + public async Task DataDotStuffing_IsApplied() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var encoded = "From: from@example.com\r\nTo: to@example.com\r\nSubject: x\r\n\r\n.secret\r\n"u8.ToArray(); + await transport.SendAsync(Item(encoded), CancellationToken.None); + var received = System.Text.Encoding.ASCII.GetString(server.LastMessage!); + Assert.Contains(".secret", received, StringComparison.Ordinal); + } + + [Fact] + public async Task ServerCloseAfterGreeting_IsNetworkFailure() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { CloseAfterGreeting = true }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Network, ex.Kind); + } + + [Fact] + public async Task OversizedMessage_UsesSizeLimit() + { + await using var server = new FakeSmtpServer(new FakeSmtpServerOptions { Size = 16 }); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + var ex = await Assert.ThrowsAsync(() => transport.SendAsync(Item(), CancellationToken.None)); + Assert.Equal(SmtpFailureKind.Message, ex.Kind); + } + + [Fact] + public async Task Cancellation_Propagates() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => transport.SendAsync(Item(), cts.Token)); + } + + [Fact] + public async Task MultilineEhlo_IsConsumed() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + await transport.SendAsync(Item(), CancellationToken.None); + Assert.Contains(server.Commands, static c => c.StartsWith("EHLO", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Ehlo_UsesApplicationFqdn() + { + await using var server = new FakeSmtpServer(); + var transport = Create(server.ClientOptions(SmtpSecurityMode.None)); + await transport.SendAsync(Item(), CancellationToken.None); + Assert.Contains( + server.Commands, + static c => c.Equals("EHLO nntpd01.usenet.ninja", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + server.Commands, + static c => c.Contains("localhost", StringComparison.OrdinalIgnoreCase)); + } + + internal static SmtpTransport Create( + SmtpOptions smtp, + IReadOnlyCollection? extraTrustedRoots = null, + ILogger? logger = null) + { + var options = new EmailOptions + { + Enabled = true, + DefaultFrom = "from@example.com", + Smtp = smtp, + }; + return new SmtpTransport( + Options.Create(options), + Options.Create(new NntpdOptions { ServerId = 1, DnsSuffix = "usenet.ninja" }), + logger ?? NullLogger.Instance, + extraTrustedRoots); + } + + internal static EmailWorkItem Item(ReadOnlyMemory? encoded = null) + { + var message = EmailServiceTests.TextMessage(); + var bytes = encoded ?? new Rfc5322MessageEncoder().Encode(message); + return new EmailWorkItem + { + Message = message, + EncodedMessage = bytes, + EnvelopeSender = new EmailAddress("envelope@example.com"), + Recipients = [new EmailAddress("to@example.com")], + }; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/CanonicalArticleText.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/CanonicalArticleText.cs new file mode 100644 index 00000000..29f6b3ff --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/CanonicalArticleText.cs @@ -0,0 +1,85 @@ +using System.Net; +using System.Text; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Networking.Proxy; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Destuffed article text that Common ArticleRecordFactory accepts. +/// +internal static class CanonicalArticleText +{ + /// Date header value accepted by NewsDateParser. + public const string Date = "Fri, 23 Aug 2024 07:30:10 +0000"; + + /// Builds destuffed headers + body without an NNTP terminator. + /// Message-ID value including angle brackets. + /// Destuffed body, typically CRLF-terminated. + /// Newsgroups header value. + /// Destuffed article text. + public static string Destuffed( + string messageId, + string body = "body\r\n", + string newsgroups = "alt.test", + string subject = "ingress-test", + string from = "user@example.test", + string? references = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ArgumentNullException.ThrowIfNull(body); + ArgumentException.ThrowIfNullOrWhiteSpace(newsgroups); + ArgumentNullException.ThrowIfNull(subject); + ArgumentException.ThrowIfNullOrWhiteSpace(from); + var referencesHeader = string.IsNullOrEmpty(references) + ? string.Empty + : "References: " + references + "\r\n"; + return + "Path: peer.example\r\n" + + "Date: " + Date + "\r\n" + + "Message-ID: " + messageId + "\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "From: " + from + "\r\n" + + "Subject: " + subject + "\r\n" + + referencesHeader + + "\r\n" + + body; + } + + /// NNTP stuffed wire for without the terminator. + public static string Stuffed( + string messageId, + string destuffedBody = "body\r\n", + string newsgroups = "alt.test") => + Destuffed(messageId, destuffedBody, newsgroups) + .Replace("\r\n.", "\r\n..", StringComparison.Ordinal); + + /// Builds a CanonicalV1 queue item for tests. + public static InboundArticle CreateQueued( + string messageId, + InboundArticleProducer producer, + string body = "body\r\n", + string newsgroups = "alt.test", + ReadOnlyMemory feed = default, + string subject = "ingress-test", + string from = "user@example.test", + string? references = null) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed( + new NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(Destuffed(messageId, body, newsgroups, subject, from, references))); + if (!created.IsAccepted) + { + throw new InvalidOperationException("Test article must be CanonicalV1."); + } + + return ArticleRecordIngress.CreateQueued( + messageId, + created.Record, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + DateTimeOffset.UtcNow, + producer, + feed); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/ControllableTimeProvider.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/ControllableTimeProvider.cs new file mode 100644 index 00000000..83526ec0 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/ControllableTimeProvider.cs @@ -0,0 +1,189 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Test that advances monotonically and fires +/// callbacks so +/// completes without real waits. +/// +internal sealed class ControllableTimeProvider : TimeProvider +{ + private readonly object _gate = new(); + private readonly List _timers = []; + private long _ticks; + + /// + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + /// + public override long GetTimestamp() + { + lock (_gate) + { + return _ticks; + } + } + + /// + public override DateTimeOffset GetUtcNow() + { + lock (_gate) + { + return DateTimeOffset.UnixEpoch.AddTicks(_ticks); + } + } + + /// Gets whether any callback is currently scheduled. + public bool HasScheduledTimers + { + get + { + lock (_gate) + { + return _timers.Count > 0; + } + } + } + + /// + /// Gets whether a timer is waiting for its due instant. + /// A fired one-shot timer is not pending, so this becomes true again only when the next delay is registered. + /// + public bool HasPendingTimer + { + get + { + lock (_gate) + { + foreach (var timer in _timers) + { + if (timer.IsPending) + { + return true; + } + } + + return false; + } + } + } + + /// Advances virtual time and invokes timers that are now due. + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfLessThan(delta, TimeSpan.Zero); + lock (_gate) + { + _ticks += delta.Ticks; + } + + // Invoke due timers synchronously so tests do not depend on ThreadPool + // scheduling. Callbacks may start a new Delay; only timers already due + // at the current virtual time are collected on each pass. + for (var pass = 0; pass < 32; pass++) + { + List due = []; + lock (_gate) + { + var now = _ticks; + // codeql[cs/linq/missed-where] CollectIfDue has side effects under lock; no LINQ filter. + foreach (var timer in _timers) + { + if (timer.CollectIfDue(now)) + { + due.Add(timer); + } + } + } + + if (due.Count == 0) + { + return; + } + + foreach (var timer in due) + { + timer.Invoke(); + } + } + } + + /// + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ArgumentNullException.ThrowIfNull(callback); + var entry = new TimerEntry(this, callback, state); + lock (_gate) + { + _timers.Add(entry); + entry.Schedule(_ticks, dueTime, period); + } + + return entry; + } + + private void Remove(TimerEntry entry) + { + lock (_gate) + { + _timers.Remove(entry); + } + } + + private sealed class TimerEntry : ITimer + { + private readonly ControllableTimeProvider _owner; + private readonly TimerCallback _callback; + private readonly object? _state; + private long? _dueTicks; + private TimeSpan _period = Timeout.InfiniteTimeSpan; + + public TimerEntry(ControllableTimeProvider owner, TimerCallback callback, object? state) + { + _owner = owner; + _callback = callback; + _state = state; + } + + public void Schedule(long nowTicks, TimeSpan dueTime, TimeSpan period) + { + _period = period; + _dueTicks = dueTime == Timeout.InfiniteTimeSpan ? null : nowTicks + dueTime.Ticks; + } + + public bool IsPending => _dueTicks is not null; + + public bool CollectIfDue(long nowTicks) + { + if (_dueTicks is not { } due || due > nowTicks) + { + return false; + } + + _dueTicks = _period <= TimeSpan.Zero || _period == Timeout.InfiniteTimeSpan + ? null + : nowTicks + _period.Ticks; + + return true; + } + + public void Invoke() => _callback(_state); + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + lock (_owner._gate) + { + Schedule(_owner._ticks, dueTime, period); + } + + return true; + } + + public void Dispose() => _owner.Remove(this); + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs new file mode 100644 index 00000000..682bd717 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlModeratorIntegrationFixture.cs @@ -0,0 +1,183 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.NntpDb; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Seeds uniquely prefixed nntpmoderators rows on a live MySQL target and +/// deletes only those rows. Unconfigured runs do not open a connection. +/// +public sealed class MySqlModeratorIntegrationFixture : IAsyncLifetime +{ + private readonly List _insertedIds = []; + private NntpDbService? _nntpDb; + + public bool IsConfigured { get; private set; } + + public string RunId { get; } = Guid.NewGuid().ToString("N"); + + public string PatternPrefix => "test.vnntp.modcat." + RunId; + + public MySqlNntpModeratorRepository? Repository { get; private set; } + + public SeededModeratorRow Static { get; private set; } = null!; + + public SeededModeratorRow Template { get; private set; } = null!; + + public SeededModeratorRow Disabled { get; private set; } = null!; + + public SeededModeratorRow SharedFirst { get; private set; } = null!; + + public SeededModeratorRow SharedSecond { get; private set; } = null!; + + public SeededModeratorRow Boundary { get; private set; } = null!; + + public string SharedPattern => PatternPrefix + ".shared.*"; + + public string LongAccountName { get; } = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"; + + public string LongModeratorAddress { get; } = new string('a', 308) + "@example.org"; + + public async Task InitializeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is null) + { + return; + } + + IsConfigured = true; + var options = Options.Create(NntpDbIntegration.CreateOptions(connectionString)); + _nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + options, + NullLogger.Instance); + await _nntpDb.StartAsync(CancellationToken.None); + Repository = new MySqlNntpModeratorRepository( + _nntpDb, + NullLogger.Instance); + + Static = await InsertAsync( + connectionString, + PatternPrefix + ".static.*", + "moderator@example.org", + "TESTMOD01", + enabled: true); + Template = await InsertAsync( + connectionString, + PatternPrefix + ".template.*", + "moderator-%s@example.org", + "TESTMOD02", + enabled: true); + Disabled = await InsertAsync( + connectionString, + PatternPrefix + ".disabled.*", + "disabled@example.org", + "TESTMOD03", + enabled: false); + SharedFirst = await InsertAsync( + connectionString, + SharedPattern, + "shared-first@example.org", + "TESTMOD04", + enabled: true); + SharedSecond = await InsertAsync( + connectionString, + SharedPattern, + "shared-second@example.org", + "TESTMOD05", + enabled: true); + Boundary = await InsertAsync( + connectionString, + CreateBoundaryPattern(), + LongModeratorAddress, + LongAccountName, + enabled: true); + } + + public async Task DisposeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is not null && _insertedIds.Count > 0) + { + await CleanupAsync(connectionString).ConfigureAwait(false); + } + + if (_nntpDb is not null) + { + await _nntpDb.DisposeAsync().ConfigureAwait(false); + } + } + + public IReadOnlyList Ours(IReadOnlyList rows) + { + ArgumentNullException.ThrowIfNull(rows); + return rows.Where(row => row.GroupPattern.StartsWith(PatternPrefix, StringComparison.Ordinal)).ToArray(); + } + + private string CreateBoundaryPattern() + { + var prefix = PatternPrefix + ".long."; + var remaining = 255 - prefix.Length - 1; + return prefix + new string('x', remaining) + "*"; + } + + private async Task InsertAsync( + string connectionString, + string pattern, + string address, + string account, + bool enabled) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + "INSERT INTO nntpmoderators " + + "(group_pattern, moderator_address, account_name, is_enabled) " + + "VALUES (@pattern, @address, @account, @enabled); " + + "SELECT LAST_INSERT_ID();"; + command.Parameters.AddWithValue("@pattern", pattern); + command.Parameters.AddWithValue("@address", address); + command.Parameters.AddWithValue("@account", account); + command.Parameters.AddWithValue("@enabled", enabled ? "Y" : "N"); + var id = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + _insertedIds.Add(id); + return new SeededModeratorRow(id, pattern, address, account, enabled); + } + + private async Task CleanupAsync(string connectionString) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (_insertedIds.Count > 0) + { + await using var byId = connection.CreateCommand(); + var names = new string[_insertedIds.Count]; + for (var i = 0; i < _insertedIds.Count; i++) + { + names[i] = "@id" + i.ToString(CultureInfo.InvariantCulture); + byId.Parameters.AddWithValue(names[i], _insertedIds[i]); + } + + byId.CommandText = "DELETE FROM nntpmoderators WHERE moderator_id IN (" + string.Join(", ", names) + ")"; + await byId.ExecuteNonQueryAsync(); + } + + await using var byPrefix = connection.CreateCommand(); + byPrefix.CommandText = "DELETE FROM nntpmoderators WHERE group_pattern LIKE @prefix"; + byPrefix.Parameters.AddWithValue("@prefix", PatternPrefix + "%"); + await byPrefix.ExecuteNonQueryAsync(); + } +} + +/// One row inserted by . +public sealed record SeededModeratorRow( + long ModeratorId, + string GroupPattern, + string ModeratorAddress, + string AccountName, + bool Enabled); diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs new file mode 100644 index 00000000..5647a9fb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/MySqlPostFilterPolicyIntegrationFixture.cs @@ -0,0 +1,664 @@ +using System.Globalization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MySqlConnector; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.PostFilter; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Applies the official PostFilter DDL when missing and isolates test revisions +/// on a live MySQL target. Unconfigured runs do not open a connection. +/// +public sealed class MySqlPostFilterPolicyIntegrationFixture : IAsyncLifetime +{ + private NntpDbService? _nntpDb; + private long? _savedCurrentRevision; + private readonly List _revisions = []; + + public bool IsConfigured { get; private set; } + + public string? SkipReason { get; private set; } + + public long RevisionBase { get; private set; } = + 9_000_000_000L + (Environment.TickCount64 % 1_000_000L) * 100L; + + public MySqlPostFilterPolicyRepository? Repository { get; private set; } + + public async Task InitializeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is null) + { + SkipReason = NntpDbIntegration.SkipReason; + return; + } + + try + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (await HasColumnAsync(connection, "nntppostfilterpolicy", "policy_id") + && !await HasTableAsync(connection, "nntppostfiltercurrent")) + { + SkipReason = + "NntpDB still has the pre-revision PostFilter singleton. Apply docs/schema/postfilter.sql."; + return; + } + + if (!await HasTableAsync(connection, "nntppostfiltercurrent")) + { + foreach (var statement in PostFilterSchemaScript.ReadStatements()) + { + await using var command = connection.CreateCommand(); + command.CommandText = statement; + await command.ExecuteNonQueryAsync(); + } + } + else + { + try + { + await EnsureExtendedSchemaAsync(connection); + } + catch (MySqlException ex) + { + SkipReason = FormatPrepareFailure(ex); + return; + } + } + + _savedCurrentRevision = await ReadCurrentRevisionAsync(connection); + if (_savedCurrentRevision is { } saved && saved >= RevisionBase) + { + RevisionBase = saved + 1_000; + } + + IsConfigured = true; + } + catch (Exception ex) when (ex is not InvalidOperationException) + { + SkipReason = FormatPrepareFailure(ex); + return; + } + + var options = Options.Create(NntpDbIntegration.CreateOptions(connectionString)); + _nntpDb = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + options, + NullLogger.Instance); + await _nntpDb.StartAsync(CancellationToken.None); + Repository = new MySqlPostFilterPolicyRepository( + _nntpDb, + NullLogger.Instance); + } + + public async Task DisposeAsync() + { + var connectionString = NntpDbIntegration.TryGetConnectionString(); + if (connectionString is not null && _revisions.Count > 0) + { + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + if (_savedCurrentRevision is { } saved) + { + await using var restore = connection.CreateCommand(); + restore.CommandText = + "UPDATE nntppostfiltercurrent SET revision = @revision WHERE policy_id = 1"; + restore.Parameters.AddWithValue("@revision", saved); + try + { + await restore.ExecuteNonQueryAsync(); + } + catch (MySqlException) + { + // Restore can fail if the trigger forbids regression; drop current then reinsert. + await using var deleteCurrent = connection.CreateCommand(); + deleteCurrent.CommandText = "DELETE FROM nntppostfiltercurrent WHERE policy_id = 1"; + await deleteCurrent.ExecuteNonQueryAsync(); + await using var insertCurrent = connection.CreateCommand(); + insertCurrent.CommandText = + "INSERT INTO nntppostfiltercurrent (policy_id, revision) VALUES (1, @revision)"; + insertCurrent.Parameters.AddWithValue("@revision", saved); + await insertCurrent.ExecuteNonQueryAsync(); + } + } + + foreach (var revision in _revisions.OrderByDescending(static value => value)) + { + await DeleteRevisionAsync(connection, revision); + } + } + + if (_nntpDb is not null) + { + await _nntpDb.DisposeAsync().ConfigureAwait(false); + } + } + + public long NextRevision() + { + var revision = RevisionBase + _revisions.Count + 1; + _revisions.Add(revision); + return revision; + } + + /// + /// Allocates a revision that remains after dispose so a live validation + /// publish can stay current. Must be higher than . + /// + public long NextPersistentRevision() => + RevisionBase + 90_000 + (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() % 1_000); + + /// + /// Remembers a published revision so dispose restores the pointer here + /// instead of the pre-test value. + /// + public void RememberPublishedRevision(long revision) => _savedCurrentRevision = revision; + + public async Task InsertDisabledRevisionAsync(long revision, Action? customize = null) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + @revision, UTC_TIMESTAMP(3), 'Disabled', + 86400000, 600000, + 0, 0, 0, 0, 0, 0, + 'N', NULL, 131072, 783, + '1.5', 4, 'RoundRobin', + 5000, 30000 + ) + """; + command.Parameters.AddWithValue("@revision", revision); + customize?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + public async Task InsertArtTypeAsync(long revision, string listKind, string artType) + { + await ExecuteAsync( + "INSERT INTO nntppostfilterarttypes (revision, list_kind, art_type) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue("@value", artType); + }); + } + + public async Task InsertAccountAsync(long revision, string listKind, string account) + { + await ExecuteAsync( + "INSERT INTO nntppostfilteraccounts (revision, list_kind, account_name) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue( + "@value", + PostFilterAccountIdentity.FromPolicyEntry(account)); + }); + } + + public async Task InsertCidrAsync(long revision, string listKind, string cidr) + { + await ExecuteAsync( + "INSERT INTO nntppostfiltercidrs (revision, list_kind, cidr) VALUES (@revision, @kind, @value)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@kind", listKind); + command.Parameters.AddWithValue("@value", cidr); + }); + } + + public async Task InsertHostAsync(long revision, int order, string host) + { + await ExecuteAsync( + "INSERT INTO nntppostfiltersahosts (revision, host_order, host) VALUES (@revision, @order, @host)", + command => + { + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@order", order); + command.Parameters.AddWithValue("@host", host); + }); + } + + public async Task PublishOptionsAsync(long revision, PostFilterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + await InsertPolicyScalarsAsync(revision, options); + foreach (var account in options.DeniedAccounts) + { + await InsertAccountAsync(revision, NntpPostFilterQueries.ListKindDeny, account); + } + + foreach (var account in options.AllowlistedAccounts) + { + await InsertAccountAsync(revision, NntpPostFilterQueries.ListKindAllow, account); + } + + foreach (var cidr in options.DeniedCidrs) + { + await InsertCidrAsync(revision, NntpPostFilterQueries.ListKindDeny, cidr); + } + + foreach (var cidr in options.AllowlistedCidrs) + { + await InsertCidrAsync(revision, NntpPostFilterQueries.ListKindAllow, cidr); + } + + foreach (var artType in options.RejectArtTypes) + { + await InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindReject, artType); + } + + foreach (var artType in options.SpamAssassin.ExcludeArtTypes) + { + await InsertArtTypeAsync(revision, NntpPostFilterQueries.ListKindSaExclude, artType); + } + + for (var i = 0; i < options.SpamAssassin.Hosts.Length; i++) + { + await InsertHostAsync(revision, i + 1, options.SpamAssassin.Hosts[i]); + } + + await PublishAsync(revision); + } + + public async Task InsertPolicyScalarsAsync(long revision, PostFilterOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.Quota); + ArgumentNullException.ThrowIfNull(options.SpamAssassin); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = + """ + INSERT INTO nntppostfilterpolicy ( + revision, updated_utc, gate, + long_window_ms, short_window_ms, + max_messages_long, max_bytes_long, max_identical_long, + max_messages_short, max_bytes_short, max_identical_short, + sa_enabled, sa_on_failure, sa_max_article_size, sa_port, + sa_protocol_version, sa_max_connections, sa_host_selection, + sa_connect_timeout_ms, sa_operation_timeout_ms + ) VALUES ( + @revision, UTC_TIMESTAMP(3), @gate, + @long_window_ms, @short_window_ms, + @max_messages_long, @max_bytes_long, @max_identical_long, + @max_messages_short, @max_bytes_short, @max_identical_short, + @sa_enabled, @sa_on_failure, @sa_max_article_size, @sa_port, + @sa_protocol_version, @sa_max_connections, @sa_host_selection, + @sa_connect_timeout_ms, @sa_operation_timeout_ms + ) + """; + command.Parameters.AddWithValue("@revision", revision); + command.Parameters.AddWithValue("@gate", options.Gate.ToString()); + command.Parameters.AddWithValue( + "@long_window_ms", + checked((long)options.Quota.LongWindow.TotalMilliseconds)); + command.Parameters.AddWithValue( + "@short_window_ms", + checked((long)options.Quota.ShortWindow.TotalMilliseconds)); + command.Parameters.AddWithValue("@max_messages_long", options.Quota.MaxMessagesLong); + command.Parameters.AddWithValue("@max_bytes_long", options.Quota.MaxBytesLong); + command.Parameters.AddWithValue("@max_identical_long", options.Quota.MaxIdenticalLong); + command.Parameters.AddWithValue("@max_messages_short", options.Quota.MaxMessagesShort); + command.Parameters.AddWithValue("@max_bytes_short", options.Quota.MaxBytesShort); + command.Parameters.AddWithValue("@max_identical_short", options.Quota.MaxIdenticalShort); + command.Parameters.AddWithValue("@sa_enabled", options.SpamAssassin.Enabled ? "Y" : "N"); + command.Parameters.AddWithValue( + "@sa_on_failure", + options.SpamAssassin.OnFailure is { } onFailure + ? onFailure.ToString() + : DBNull.Value); + command.Parameters.AddWithValue("@sa_max_article_size", options.SpamAssassin.MaxArticleSize); + command.Parameters.AddWithValue("@sa_port", options.SpamAssassin.Port); + command.Parameters.AddWithValue("@sa_protocol_version", options.SpamAssassin.ProtocolVersion); + command.Parameters.AddWithValue("@sa_max_connections", options.SpamAssassin.MaxConnections); + command.Parameters.AddWithValue("@sa_host_selection", options.SpamAssassin.HostSelection.ToString()); + command.Parameters.AddWithValue( + "@sa_connect_timeout_ms", + checked((int)options.SpamAssassin.ConnectTimeout.TotalMilliseconds)); + command.Parameters.AddWithValue( + "@sa_operation_timeout_ms", + checked((int)options.SpamAssassin.OperationTimeout.TotalMilliseconds)); + await command.ExecuteNonQueryAsync(); + } + + public async Task PublishAsync(long revision) + { + await ExecuteAsync( + "UPDATE nntppostfiltercurrent SET revision = @revision WHERE policy_id = 1", + command => command.Parameters.AddWithValue("@revision", revision)); + } + + public async Task LoadAsync() + { + if (Repository is null) + { + throw new InvalidOperationException("PostFilter MySQL fixture is not configured."); + } + + return await Repository.LoadAsync(); + } + + public async Task ExecuteRawAsync(string sql, Action? bind = null) + { + await ExecuteAsync(sql, bind); + } + + public async Task ExecuteRawScalarAsync(string sql, Action? bind = null) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + bind?.Invoke(command); + return await command.ExecuteScalarAsync(); + } + + public async Task ShowCreateTableAsync(string table) + { + ArgumentException.ThrowIfNullOrWhiteSpace(table); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = "SHOW CREATE TABLE `" + table.Replace("`", string.Empty, StringComparison.Ordinal) + "`"; + await using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + throw new InvalidOperationException("SHOW CREATE TABLE returned no rows."); + } + + return reader.GetString(1); + } + + public async Task InsertRejectionAsync(PostFilterRejectionEvidence evidence) + { + ArgumentNullException.ThrowIfNull(evidence); + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var db = new MySqlNntpDbConnection(connection); + await db.InsertPostFilterRejectionAsync(evidence, CancellationToken.None); + } + + public async Task BeginHeldWriteAsync() + { + var connection = MySqlNntpDbConnectionFactory.CreateConnection(RequireConnectionString()); + await connection.OpenAsync(); + var transaction = await connection.BeginTransactionAsync(); + return new HeldPostFilterWrite(connection, transaction); + } + + private async Task ExecuteAsync(string sql, Action? bind) + { + var connectionString = RequireConnectionString(); + await using var connection = MySqlNntpDbConnectionFactory.CreateConnection(connectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + bind?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + private string RequireConnectionString() => + NntpDbIntegration.TryGetConnectionString() + ?? throw new InvalidOperationException(NntpDbIntegration.SkipReason); + + private static string FormatPrepareFailure(Exception ex) + { + if (ex is MySqlException mysql) + { + return "PostFilter MySQL schema could not be prepared: MySqlException " + + mysql.Number.ToString(CultureInfo.InvariantCulture) + + " " + + (mysql.SqlState ?? "-") + + " " + + mysql.Message; + } + + return "PostFilter MySQL schema could not be prepared: " + + ex.GetType().Name + + " " + + ex.Message; + } + + private static async Task HasTableAsync(MySqlConnection connection, string table) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COUNT(*) FROM information_schema.tables " + + "WHERE table_schema = DATABASE() AND table_name = @name"; + command.Parameters.AddWithValue("@name", table); + var count = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + return count > 0; + } + + private static async Task EnsureExtendedSchemaAsync(MySqlConnection connection) + { + if (!await HasTableAsync(connection, "nntppostfilterrejections")) + { + var create = PostFilterSchemaScript.ReadStatements() + .Single(statement => statement.StartsWith( + "CREATE TABLE nntppostfilterrejections", + StringComparison.Ordinal)); + await using var command = connection.CreateCommand(); + command.CommandText = create; + await command.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntppostfilteraccounts") + && await ColumnTypeAsync(connection, "nntppostfilteraccounts", "account_name") is { } accountType + && !accountType.Contains("char(32)", StringComparison.OrdinalIgnoreCase)) + { + await using var hash = connection.CreateCommand(); + hash.CommandText = + """ + UPDATE nntppostfilteraccounts + SET account_name = LOWER(MD5(account_name)) + WHERE CHAR_LENGTH(account_name) <> 32 + """; + await hash.ExecuteNonQueryAsync(); + await using var alter = connection.CreateCommand(); + alter.CommandText = + "ALTER TABLE nntppostfilteraccounts MODIFY account_name CHAR(32) NOT NULL"; + await alter.ExecuteNonQueryAsync(); + try + { + await using var check = connection.CreateCommand(); + check.CommandText = + "ALTER TABLE nntppostfilteraccounts ADD CONSTRAINT chk_nntppostfilteraccounts_name " + + "CHECK (account_name REGEXP '^[0-9a-f]{32}$')"; + await check.ExecuteNonQueryAsync(); + } + catch (MySqlException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Constraint may already exist on a partially migrated schema. + } + } + + if (await HasTableAsync(connection, "nntppostfilterarttypes") + && await ColumnTypeAsync(connection, "nntppostfilterarttypes", "art_type") is { } artType + && !artType.Contains("enum", StringComparison.OrdinalIgnoreCase)) + { + await using var alter = connection.CreateCommand(); + alter.CommandText = + """ + ALTER TABLE nntppostfilterarttypes + MODIFY art_type ENUM( + 'Default','Control','Cancel','Mime','Binary','UuEncode','Base64','YEncoded', + 'BommaNews','UniData','Multipart','Html','PostScript','BinHex','Partial','PgpMessage' + ) NOT NULL + """; + await alter.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntppostfilterrejections") + && await ColumnTypeAsync(connection, "nntppostfilterrejections", "source_ip") is { } sourceIpType + && !sourceIpType.Contains("varbinary", StringComparison.OrdinalIgnoreCase)) + { + await using var convert = connection.CreateCommand(); + convert.CommandText = + """ + ALTER TABLE nntppostfilterrejections + ADD COLUMN source_ip_bin VARBINARY(16) NULL + """; + await convert.ExecuteNonQueryAsync(); + await using var copy = connection.CreateCommand(); + copy.CommandText = + """ + UPDATE nntppostfilterrejections + SET source_ip_bin = INET6_ATON(CAST(source_ip AS CHAR)) + """; + await copy.ExecuteNonQueryAsync(); + await using var drop = connection.CreateCommand(); + drop.CommandText = "ALTER TABLE nntppostfilterrejections DROP COLUMN source_ip"; + await drop.ExecuteNonQueryAsync(); + await using var rename = connection.CreateCommand(); + rename.CommandText = + """ + ALTER TABLE nntppostfilterrejections + CHANGE source_ip_bin source_ip VARBINARY(16) NOT NULL + """; + await rename.ExecuteNonQueryAsync(); + } + + if (await HasTableAsync(connection, "nntpusers") + && !await HasColumnAsync(connection, "nntpusers", "account_art_type")) + { + try + { + await using var alter = connection.CreateCommand(); + alter.CommandText = NntpUserQueries.AddAccountArtTypeColumn; + await alter.ExecuteNonQueryAsync(); + } + catch (MySqlException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Account capability is additive on nntpusers. PostFilter + // schema tests must still run when this ALTER is denied. + } + } + } + + private static async Task ColumnTypeAsync(MySqlConnection connection, string table, string column) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COLUMN_TYPE FROM information_schema.columns " + + "WHERE table_schema = DATABASE() AND table_name = @table AND column_name = @column"; + command.Parameters.AddWithValue("@table", table); + command.Parameters.AddWithValue("@column", column); + var value = await command.ExecuteScalarAsync(); + return value is null or DBNull ? null : Convert.ToString(value, CultureInfo.InvariantCulture); + } + + private static async Task HasColumnAsync(MySqlConnection connection, string table, string column) + { + await using var command = connection.CreateCommand(); + command.CommandText = + "SELECT COUNT(*) FROM information_schema.columns " + + "WHERE table_schema = DATABASE() AND table_name = @table AND column_name = @column"; + command.Parameters.AddWithValue("@table", table); + command.Parameters.AddWithValue("@column", column); + var count = Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + return count > 0; + } + + private static async Task ReadCurrentRevisionAsync(MySqlConnection connection) + { + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT revision FROM nntppostfiltercurrent WHERE policy_id = 1"; + var scalar = await command.ExecuteScalarAsync(); + return scalar is null or DBNull + ? null + : Convert.ToInt64(scalar, CultureInfo.InvariantCulture); + } + + private static async Task DeleteRevisionAsync(MySqlConnection connection, long revision) + { + foreach (var table in new[] + { + "nntppostfilteraccounts", + "nntppostfiltercidrs", + "nntppostfilterarttypes", + "nntppostfiltersahosts", + }) + { + await using var command = connection.CreateCommand(); + command.CommandText = "DELETE FROM " + table + " WHERE revision = @revision"; + command.Parameters.AddWithValue("@revision", revision); + await command.ExecuteNonQueryAsync(); + } + + await using var policy = connection.CreateCommand(); + policy.CommandText = "DELETE FROM nntppostfilterpolicy WHERE revision = @revision"; + policy.Parameters.AddWithValue("@revision", revision); + await policy.ExecuteNonQueryAsync(); + } +} + +/// Open writer transaction used to prove uncommitted policy is invisible. +public sealed class HeldPostFilterWrite : IAsyncDisposable +{ + public HeldPostFilterWrite(MySqlConnection connection, MySqlTransaction transaction) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(transaction); + Connection = connection; + Transaction = transaction; + } + + public MySqlConnection Connection { get; } + + public MySqlTransaction Transaction { get; } + + public async Task ExecuteAsync(string sql, Action? bind = null) + { + await using var command = Connection.CreateCommand(); + command.Transaction = Transaction; + command.CommandText = sql; + bind?.Invoke(command); + await command.ExecuteNonQueryAsync(); + } + + public Task CommitAsync() => Transaction.CommitAsync(); + + public Task RollbackAsync() => Transaction.RollbackAsync(); + + public async ValueTask DisposeAsync() + { + await Transaction.DisposeAsync(); + await Connection.DisposeAsync(); + } +} + +/// Serializes live PostFilter schema tests against one MySQL target. +[CollectionDefinition("NntpDbPostFilter")] +public sealed class NntpDbPostFilterCollection : ICollectionFixture +{ +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegration.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegration.cs new file mode 100644 index 00000000..aa65550d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegration.cs @@ -0,0 +1,48 @@ +using MySqlConnector; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Opt-in live NntpDB seam. Ordinary tests never read ConnectionStrings:NntpDB +/// or the committed production appsettings target. +/// +internal static class NntpDbIntegration +{ + /// + /// Dedicated MySQL connection string for live nntpmoderators tests. + /// + public const string ConnectionStringEnvironmentVariable = "VECTORNNTP_NNTPDB_INTEGRATION"; + + /// Reason used when the live MySQL tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_NNTPDB_INTEGRATION to a dedicated MySQL connection string " + + "that contains nntpmoderators. Ordinary tests do not open MySQL and do not " + + "use ConnectionStrings:NntpDB."; + + /// Returns the opt-in connection string, or when unset. + public static string? TryGetConnectionString() + { + var value = Environment.GetEnvironmentVariable(ConnectionStringEnvironmentVariable); + return string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + } + + /// Describes the target without passwords or the full connection string. + public static string DescribeTarget(string connectionString) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + var builder = new MySqlConnectionStringBuilder(connectionString); + return "Server=" + builder.Server + + ";Database=" + builder.Database + + ";User ID=" + builder.UserID; + } + + /// Builds NntpDB options for the live factory. Never logs the string. + public static NntpDbOptions CreateOptions(string connectionString) => + new() + { + ConnectionString = connectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs new file mode 100644 index 00000000..b9f77ce9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationFactAttribute.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_NNTPDB_INTEGRATION is set. +/// When the variable is set, the test runs and must fail if MySQL is unreachable. +/// +public sealed class NntpDbIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in string is absent. + public NntpDbIntegrationFactAttribute() + { + if (NntpDbIntegration.TryGetConnectionString() is null) + { + Skip = NntpDbIntegration.SkipReason; + } + } +} + +/// +/// xUnit 2 theory skip unless VECTORNNTP_NNTPDB_INTEGRATION is set. +/// +public sealed class NntpDbIntegrationTheoryAttribute : TheoryAttribute +{ + /// Initializes the attribute and skips when the opt-in string is absent. + public NntpDbIntegrationTheoryAttribute() + { + if (NntpDbIntegration.TryGetConnectionString() is null) + { + Skip = NntpDbIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationTests.cs new file mode 100644 index 00000000..84057bfd --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpDbIntegrationTests.cs @@ -0,0 +1,39 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +public sealed class NntpDbIntegrationTests +{ + [Fact] + public void DescribeTarget_OmitsPasswordAndFullConnectionString() + { + const string connectionString = + "Server=198.18.0.70;Port=3306;Database=nntpdb;User ID=nntpd;Password=super-secret;"; + + var description = NntpDbIntegration.DescribeTarget(connectionString); + + Assert.Contains("Server=198.18.0.70", description, StringComparison.Ordinal); + Assert.Contains("Database=nntpdb", description, StringComparison.Ordinal); + Assert.Contains("User ID=nntpd", description, StringComparison.Ordinal); + Assert.DoesNotContain("super-secret", description, StringComparison.Ordinal); + Assert.DoesNotContain("Password", description, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(connectionString, description, StringComparison.Ordinal); + } + + [Fact] + public void TryGetConnectionString_ReadsOnlyOptInVariable() + { + Assert.Equal("VECTORNNTP_NNTPDB_INTEGRATION", NntpDbIntegration.ConnectionStringEnvironmentVariable); + Assert.Contains("do not use ConnectionStrings:NntpDB", NntpDbIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (NntpDbIntegration.TryGetConnectionString() is not null) + { + return; + } + + var attribute = new NntpDbIntegrationFactAttribute(); + Assert.Equal(NntpDbIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs new file mode 100644 index 00000000..340302b6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/NntpUsersArtTypeMigrationScript.cs @@ -0,0 +1,10 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// Loads docs/schema/nntpusers-account-art-type.sql. +internal static class NntpUsersArtTypeMigrationScript +{ + public static string FindPath() => PostFilterSchemaScript.FindDocsSql("nntpusers-account-art-type.sql"); + + public static IReadOnlyList ReadStatements() => + PostFilterSchemaScript.ReadStatements(FindPath()); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs new file mode 100644 index 00000000..0a36cc18 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterQuotaRedisIntegrationFixture.cs @@ -0,0 +1,182 @@ +using System.Collections.Concurrent; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.TransitService.PostFilter.Quota; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Shared live Redis connection for PostFilter quota Lua EVAL tests. Deletes only +/// keys created for uniquely named test accounts. Reuses +/// opt-in. +/// +public sealed class PostFilterQuotaRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _accounts = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + internal RedisPostFilterQuotaStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = SessionStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + Store = new RedisPostFilterQuotaStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var account in _accounts) + { + await DeleteKeysAsync(account); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "PostFilter quota live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateAccountAsync() + { + EnsureConfigured(); + var account = "vnntp.pf.q.lua." + Guid.NewGuid().ToString("N"); + _accounts.Add(account); + await DeleteKeysAsync(account); + return account; + } + + public async Task DeleteKeysAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(accountName)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + foreach (var account in _accounts.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(account))) + { + remaining.Add(Encoding.UTF8.GetString(PostFilterQuotaKeys.CreateQuota(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(account))) + { + remaining.Add(Encoding.UTF8.GetString(PostFilterQuotaKeys.CreateMultipost(account))); + } + } + + return remaining; + } + + public async Task SeedQuotaAsync(string accountName, string field, string value) + { + EnsureConfigured(); + _ = await Database.HashSetAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName), field, value); + } + + public async Task ReadQuotaFieldAsync(string accountName, string field) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName), field); + return value.IsNullOrEmpty ? null : (string)value!; + } + + public async Task ReadMultipostFieldAsync(string accountName, string field) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)PostFilterQuotaKeys.CreateMultipost(accountName), field); + return value.IsNullOrEmpty ? null : (string)value!; + } + + public async Task QuotaFieldCountAsync(string accountName) + { + EnsureConfigured(); + return (int)await Database.HashLengthAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + } + + public async Task QuotaKeyTtlAsync(string accountName) + { + EnsureConfigured(); + return await Database.KeyTimeToLiveAsync((RedisKey)PostFilterQuotaKeys.CreateQuota(accountName)); + } + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(SessionStateRedisIntegration.SkipReason); + } + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterSchemaScript.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterSchemaScript.cs new file mode 100644 index 00000000..70e87319 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/PostFilterSchemaScript.cs @@ -0,0 +1,125 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// Loads the operator DDL at docs/schema/postfilter.sql. +internal static class PostFilterSchemaScript +{ + public static string FindPath() => FindDocsSql("postfilter.sql"); + + /// Finds a file under repository docs/ or docs/schema/. + public static string FindDocsSql(string fileName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fileName); + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var schemaCandidate = Path.Join(dir.FullName, "docs", "schema", fileName); + if (File.Exists(schemaCandidate)) + { + return schemaCandidate; + } + + var docsCandidate = Path.Join(dir.FullName, "docs", fileName); + if (File.Exists(docsCandidate)) + { + return docsCandidate; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("docs/" + fileName + " was not found."); + } + + public static IReadOnlyList ReadStatements() => ReadStatements(FindPath()); + + /// Splits one operator SQL file using the same DELIMITER rules as postfilter.sql. + public static IReadOnlyList ReadStatements(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + var lines = File.ReadAllLines(path); + var statements = new List(); + var buffer = new List(); + var delimiter = ";"; + foreach (var line in lines) + { + var trimmed = line.Trim(); + if (TryReadDelimiter(trimmed, out var nextDelimiter)) + { + Flush(buffer, statements, delimiter); + delimiter = nextDelimiter; + continue; + } + + if (trimmed.StartsWith("-- VECTORNNTP_STMT", StringComparison.Ordinal) + || trimmed.StartsWith("-- BEGIN TRIGGER", StringComparison.Ordinal) + || trimmed.StartsWith("-- END TRIGGER", StringComparison.Ordinal)) + { + Flush(buffer, statements, delimiter); + continue; + } + + if (trimmed.StartsWith("--", StringComparison.Ordinal)) + { + continue; + } + + buffer.Add(line); + if (BufferEndsWith(buffer, delimiter)) + { + Flush(buffer, statements, delimiter); + } + } + + Flush(buffer, statements, delimiter); + return statements; + } + + private static bool TryReadDelimiter(string trimmed, out string delimiter) + { + delimiter = ";"; + const string prefix = "DELIMITER"; + if (!trimmed.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (trimmed.Length != prefix.Length + && !char.IsWhiteSpace(trimmed[prefix.Length])) + { + return false; + } + + var value = trimmed[prefix.Length..].Trim(); + delimiter = value.Length == 0 ? ";" : value; + return true; + } + + private static bool BufferEndsWith(List buffer, string delimiter) + { + var text = string.Join('\n', buffer).TrimEnd(); + return text.EndsWith(delimiter, StringComparison.Ordinal); + } + + private static void Flush(List buffer, List statements, string delimiter) + { + var text = string.Join('\n', buffer).Trim(); + buffer.Clear(); + if (text.Length == 0) + { + return; + } + + if (text.EndsWith(delimiter, StringComparison.Ordinal)) + { + text = text[..^delimiter.Length].TrimEnd(); + } + + text = text.TrimEnd(';').Trim(); + if (text.Length == 0) + { + return; + } + + statements.Add(text); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/SerilogCollection.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SerilogCollection.cs new file mode 100644 index 00000000..5afcc5e0 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SerilogCollection.cs @@ -0,0 +1,10 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Serializes tests that mutate the process-wide Serilog.Log.Logger static. +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class SerilogCollection +{ + public const string Name = "Serilog"; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegration.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegration.cs new file mode 100644 index 00000000..0b94e82e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegration.cs @@ -0,0 +1,373 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Net; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Opt-in live SessionState Redis seam. Ordinary tests never read Redis:Host +/// or open the test Redis. +/// +internal static class SessionStateRedisIntegration +{ + /// Dedicated Redis endpoint for live SessionState Lua tests (host:port). + public const string EndpointEnvironmentVariable = "VECTORNNTP_REDIS_INTEGRATION"; + + /// Authorized SessionState integration-test Redis. + public const string AuthorizedEndpoint = "198.18.0.70:6379"; + + /// Reason used when the live Redis tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_REDIS_INTEGRATION=198.18.0.70:6379 to run SessionState Lua " + + "against the dedicated test Redis. Ordinary tests do not open Redis and do " + + "not use Redis:Host."; + + /// Parses the opt-in endpoint, or when unset. + public static RedisOptions? TryGetOptions() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var endpoint = value.Trim(); + var port = RedisOptions.DefaultPort; + var host = endpoint; + var separator = endpoint.LastIndexOf(':'); + if (separator > 0 + && separator < endpoint.Length - 1 + && int.TryParse(endpoint.AsSpan(separator + 1), NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) + && parsed is > 0 and <= 65535) + { + host = endpoint[..separator]; + port = parsed; + } + + if (string.IsNullOrWhiteSpace(host)) + { + return null; + } + + return new RedisOptions { Host = [host], Port = port }; + } + + /// Describes the target without inventing extra connection material. + public static string DescribeTarget(RedisOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return options.Host[0] + ":" + options.Port.ToString(CultureInfo.InvariantCulture); + } +} + +/// +/// Shared live Redis connection for SessionState Lua EVAL tests. Deletes only +/// keys created for uniquely named test accounts. +/// +public sealed class SessionStateRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _accounts = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + public string Endpoint { get; private set; } = string.Empty; + + public string ServerVersion { get; private set; } = string.Empty; + + internal RedisSessionStateStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = SessionStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + Endpoint = SessionStateRedisIntegration.DescribeTarget(options); + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + var endpoint = live.Multiplexer.GetEndPoints()[0]; + ServerVersion = live.Multiplexer.GetServer(endpoint).Version.ToString(); + Store = new RedisSessionStateStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var account in _accounts) + { + await DeleteKeysAsync(account); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "SessionState live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateAccountAsync() + { + EnsureConfigured(); + var account = "vnntp.sess.lua." + Guid.NewGuid().ToString("N"); + _accounts.Add(account); + await DeleteKeysAsync(account); + return account; + } + + public async Task DeleteKeysAsync(string accountName) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)SessionStateKeys.CreateSession(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)SessionStateKeys.CreateSource(accountName)); + _ = await Database.KeyDeleteAsync((RedisKey)AccountByteKeys.Create(accountName)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + foreach (var account in _accounts.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)SessionStateKeys.CreateSession(account))) + { + remaining.Add(Encoding.UTF8.GetString(SessionStateKeys.CreateSession(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)SessionStateKeys.CreateSource(account))) + { + remaining.Add(Encoding.UTF8.GetString(SessionStateKeys.CreateSource(account))); + } + + if (await Database.KeyExistsAsync((RedisKey)AccountByteKeys.Create(account))) + { + remaining.Add(Encoding.UTF8.GetString(AccountByteKeys.Create(account))); + } + } + + return remaining; + } + + public ValueTask TryAdmitAsync( + string accountName, + string ip, + string ownerId, + int sessionLimit, + int srcIpLimit, + long sessionGeneration, + long sourceGeneration, + long nowUnixMs, + long leaseMs, + bool trackSessions = false) => + Store.TryAdmitAsync( + accountName, + ip, + ownerId, + sessionLimit, + srcIpLimit, + sessionGeneration, + sourceGeneration, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs), + trackSessions: trackSessions); + + public ValueTask ReleaseAsync( + string accountName, + string ip, + string ownerId, + long sessionGeneration, + long sourceGeneration) => + Store.ReleaseAsync(accountName, ip, ownerId, sessionGeneration, sourceGeneration); + + public ValueTask RenewAsync( + string accountName, + string ownerId, + long sessionGeneration, + IReadOnlyList<(string Ip, long Generation)> sources, + long nowUnixMs, + long leaseMs) => + Store.RenewAsync( + accountName, + ownerId, + sessionGeneration, + sources, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseOwnerAsync(string accountName, string ownerId) => + Store.ReleaseOwnerAsync(accountName, ownerId); + + public async Task SeedAsync(string accountName, string field, long expiryUnixMs, long generation, int count, bool session) + { + EnsureConfigured(); + var key = session ? SessionStateKeys.CreateSession(accountName) : SessionStateKeys.CreateSource(accountName); + _ = await Database.HashSetAsync( + (RedisKey)key, + field, + SessionStateKeys.Value(expiryUnixMs, generation, count)); + } + + public Task SeedSessionAsync(string accountName, string ownerId, long expiryUnixMs, long generation, int count) => + SeedAsync(accountName, ownerId, expiryUnixMs, generation, count, session: true); + + public Task SeedSourceAsync(string accountName, string ip, string ownerId, long expiryUnixMs, long generation, int count) => + SeedAsync(accountName, SessionStateKeys.SourceField(ip, ownerId), expiryUnixMs, generation, count, session: false); + + public async Task ReadSessionAsync(string accountName, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)SessionStateKeys.CreateSession(accountName), ownerId); + return Parse(value); + } + + public async Task ReadSourceAsync(string accountName, string ip, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync( + (RedisKey)SessionStateKeys.CreateSource(accountName), + SessionStateKeys.SourceField(ip, ownerId)); + return Parse(value); + } + + public async Task SessionCountAsync(string accountName) + { + var fields = await HashValuesAsync(SessionStateKeys.CreateSession(accountName)); + var total = 0; + // codeql[cs/linq/missed-where] Parse pattern-match; avoid double Parse from Where+body. + foreach (var value in fields) + { + if (Parse(value) is { } ownership) + { + total += ownership.Count; + } + } + + return total; + } + + public async Task DistinctSourceCountAsync(string accountName) + { + var names = await HashFieldNamesAsync(SessionStateKeys.CreateSource(accountName)); + var ips = new HashSet(StringComparer.Ordinal); + // codeql[cs/linq/missed-where] TrySplitField out-params; foreach preserves parse-and-collect. + foreach (var field in names) + { + if (SessionStateKeys.TrySplitField(field, out var ip, out _)) + { + ips.Add(ip); + } + } + + return ips.Count; + } + + public async Task KeyTimeToLiveAsync(string accountName, bool session) + { + EnsureConfigured(); + var key = session ? SessionStateKeys.CreateSession(accountName) : SessionStateKeys.CreateSource(accountName); + return await Database.KeyTimeToLiveAsync((RedisKey)key); + } + + public static string Format(string ip) => SourceAddressIdentity.Format(IPAddress.Parse(ip)); + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(SessionStateRedisIntegration.SkipReason); + } + } + + private async Task HashValuesAsync(byte[] key) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)key); + return [.. entries.Select(static entry => entry.Value)]; + } + + private async Task HashFieldNamesAsync(byte[] key) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)key); + return [.. entries.Select(static entry => (string)entry.Name!)]; + } + + private static Ownership? Parse(RedisValue value) + { + if (value.IsNullOrEmpty) + { + return null; + } + + return SessionStateKeys.TrySplitValue((string)value!, out var expiry, out var generation, out var count) + ? new Ownership(expiry, generation, count) + : null; + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} + +/// Parsed SessionState HASH value. +public readonly record struct Ownership(long ExpiryUnixMs, long Generation, int Count); diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs new file mode 100644 index 00000000..6349450c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationConventionTests.cs @@ -0,0 +1,34 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +public sealed class SessionStateRedisIntegrationConventionTests +{ + [Fact] + public void DescribeTarget_UsesHostAndPortOnly() + { + var description = SessionStateRedisIntegration.DescribeTarget( + new RedisOptions { Host = ["198.18.0.70"], Port = 6379 }); + Assert.Equal(SessionStateRedisIntegration.AuthorizedEndpoint, description); + } + + [Fact] + public void TryGetOptions_ReadsOnlyOptInVariable() + { + Assert.Equal("VECTORNNTP_REDIS_INTEGRATION", SessionStateRedisIntegration.EndpointEnvironmentVariable); + Assert.Contains("do not use Redis:Host", SessionStateRedisIntegration.SkipReason, StringComparison.Ordinal); + Assert.Contains(SessionStateRedisIntegration.AuthorizedEndpoint, SessionStateRedisIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (SessionStateRedisIntegration.TryGetOptions() is not null) + { + return; + } + + var attribute = new SessionStateRedisIntegrationFactAttribute(); + Assert.Equal(SessionStateRedisIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs new file mode 100644 index 00000000..115dd51c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SessionStateRedisIntegrationFactAttribute.cs @@ -0,0 +1,17 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_REDIS_INTEGRATION is set. +/// When the variable is set, the test runs and must fail if Redis is unreachable. +/// +public sealed class SessionStateRedisIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public SessionStateRedisIntegrationFactAttribute() + { + if (SessionStateRedisIntegration.TryGetOptions() is null) + { + Skip = SessionStateRedisIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs new file mode 100644 index 00000000..d509e8d7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/SpamdIntegrationFactAttribute.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Skips at discovery unless VECTORNNTP_SPAMD_INTEGRATION is set to host:port. +/// Ordinary CI does not set the variable, so these tests never dial a default address. +/// When the variable is set, the test runs and fails if that endpoint is unreachable or malformed. +/// +public sealed class SpamdIntegrationFactAttribute : FactAttribute +{ + /// Environment variable that opts in to a real SPAMD endpoint. + public const string EndpointEnvironmentVariable = "VECTORNNTP_SPAMD_INTEGRATION"; + + /// Skip text used when the opt-in variable is absent. + public const string SkipReason = + "Set VECTORNNTP_SPAMD_INTEGRATION=host:port to run opt-in SPAMD compatibility tests. " + + "Ordinary CI does not set this and must not contact an external scanner."; + + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public SpamdIntegrationFactAttribute() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + Skip = SkipReason; + } + } + + /// Parses host:port. An unset variable is not an endpoint. + /// The variable is set but is not host:port. + public static (string Host, int Port) RequireEndpoint() + { + var value = Environment.GetEnvironmentVariable(EndpointEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException(SkipReason); + } + + var separator = value.LastIndexOf(':'); + if (separator <= 0 + || separator == value.Length - 1 + || !int.TryParse(value[(separator + 1)..], out var port) + || port is < 1 or > 65535) + { + throw new InvalidOperationException( + $"{EndpointEnvironmentVariable} must be host:port. The value '{value}' is not usable."); + } + + return (value[..separator].Trim(), port); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/TestHostFactory.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TestHostFactory.cs new file mode 100644 index 00000000..6f76a8d1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TestHostFactory.cs @@ -0,0 +1,216 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +internal static class TestHostFactory +{ + internal static IApplicationService WrapDns(CloudflareDnsReconciliationService service) => + new CloudflareDnsReconciliationApplicationService(service); + + public const string TestCloudFlareApiKey = "unit-test-cloudflare-api-key"; + + /// 64-hex AES-256 test key for POST X-Trace (not a production secret). + public const string TestXTraceKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + /// Offline placeholder for ConnectionStrings:NntpDB (not a live server). + public const string TestNntpDbConnectionString = "Server=127.0.0.1;Database=nntpdb;User ID=test;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=32;ConnectionIdleTimeout=300;"; + + /// + /// Deliberately malformed NntpDB string similar to the observed parse failure + /// (unclosed quote; rejected by MySqlConnectionStringBuilder). + /// + public const string MalformedNntpDbConnectionString = + "Server=127.0.0.1;Port=3306;Database=nntpdb;User ID=nntpd;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=128;ConnectionIdleTimeout=300;Password=\"unclosed"; + + /// Fake password used only in malformed-string secret-redaction tests. + public const string FakeNntpDbPassword = "fake-nntpdb-password-not-real"; + + /// Malformed NntpDB string that contains . + public const string MalformedNntpDbConnectionStringWithPassword = + "Server=127.0.0.1;Database=nntpdb;User ID=nntpd;Password=\"" + FakeNntpDbPassword + ";"; + + /// Shared parent for test file logs so hosts do not write under the repo logs/. + public static readonly string SharedTestLogDir = Path.Join(Path.GetTempPath(), "vectornntp-nntpd-testhost-logs"); + + /// Unique Nntpd:LogDir so parallel hosts do not share a Serilog file lock. + public static string NewTestLogDir() + { + var dir = Path.Join(SharedTestLogDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + public static readonly IPAddress TestIpv4 = IPAddress.Parse("198.18.0.10"); + public static readonly IPAddress TestIpv6 = IPAddress.Parse("2001:db8::10"); + + public static NntpdOptions CreateOptions( + TimeSpan? gracefulShutdownTimeout = null, + TimeSpan? startupTimeout = null) + { + return CreateValidOptions(gracefulShutdownTimeout, startupTimeout); + } + + public static NntpdOptions CreateValidOptions( + TimeSpan? gracefulShutdownTimeout = null, + TimeSpan? startupTimeout = null) + { + return new NntpdOptions + { + ApplicationName = "VectorNNTPD.TransitService.Tests", + GracefulShutdownTimeout = gracefulShutdownTimeout ?? TimeSpan.FromSeconds(5), + StartupTimeout = startupTimeout, + StopHostOnUnexpectedServiceTermination = true, + BindAddress = ["*"], + BindPort = GetFreeTcpPort(), + BindPortTls = 0, + CloudFlareApiKey = TestCloudFlareApiKey, + CloudFlareZoneId = "5811a29d39a0732afb5f160c9b137c3d", + XTraceKey = TestXTraceKey, + DnsSuffix = "usenet.ninja", + ServerId = 1, + }; + } + + /// Reserves an ephemeral TCP port on loopback for offline listener tests. + public static int GetFreeTcpPort() + { + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return ((IPEndPoint)socket.LocalEndPoint!).Port; + } + + internal static ApplicationServiceManager CreateServiceManager( + IEnumerable services, + NntpdOptions? options = null) + { + return new ApplicationServiceManager( + services, + options ?? CreateOptions(), + NullLogger.Instance); + } + + public static ApplicationLifecycle CreateLifecycle( + IEnumerable services, + NntpdOptions? options = null) + { + options ??= CreateOptions(); + var manager = CreateServiceManager(services, options); + return new ApplicationLifecycle( + manager, + options, + NullLogger.Instance); + } + + /// + /// Registers deterministic networking and Cloudflare test configuration before hosting services. + /// + public static void ConfigureNntpdTestHost(HostApplicationBuilder builder, ILocalIpAddressAssignee? assignee = null) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + [NntpdOptions.CloudFlareApiKeyConfigurationKey] = TestCloudFlareApiKey, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ServerId)}"] = "1", + [$"{NntpdOptions.SectionName}:{NntpdOptions.XTraceKeyConfigurationKey}"] = TestXTraceKey, + [NntpdOptions.XTraceKeyConfigurationKey] = TestXTraceKey, + // Replace appsettings BindAddress entirely (in-memory must clear leftover indices). + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "*", + [$"{NntpdOptions.SectionName}:BindAddress:1"] = null, + ["Redis:Host:0"] = "127.0.0.1", + ["Redis:Port"] = "6379", + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "guest", + ["RabbitMQ:Password"] = "guest", + ["RabbitMQ:Management:BaseUrl"] = "http://127.0.0.1:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = TestNntpDbConnectionString, + [$"{NntpdOptions.SectionName}:LogDir"] = NewTestLogDir(), + }); + + var localAssignee = assignee ?? new FakeLocalIpAddressAssignee(TestIpv4, TestIpv6); + builder.Services.AddSingleton(localAssignee); + builder.Services.AddSingleton(new FakeCloudflareDnsClient()); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + PromoteSharedKeysToRoot(builder.Configuration); + IsolateTransit(builder.Services); + + // Ensure machine-specific appsettings bind entries cannot leak into host tests. + // Force TLS off so ACME never contacts Let's Encrypt during offline host tests + // (committed appsettings may set BindPortTls > 0 for local operator use). + // Use an ephemeral plain port so NntpPlainListenerService can bind without conflicts. + builder.Services.PostConfigure(static options => + { + options.BindAddress = ["*"]; + options.BindPort = GetFreeTcpPort(); + options.BindPortTls = 0; + options.AcmeEmail = string.Empty; + }); + } + + /// + /// Copies leftover Nntpd: shared keys to the configuration root so older + /// test fixtures keep working after the Common-owned root contract. + /// + public static void PromoteSharedKeysToRoot(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + if (configuration is not IConfigurationBuilder builder) + { + return; + } + + var extras = new Dictionary(StringComparer.OrdinalIgnoreCase); + string[] keys = + [ + NntpdOptions.CloudFlareApiKeyConfigurationKey, + AcmeCloudflareOptions.AcmeAccountConfigurationKey, + nameof(NntpdOptions.AcmeCertificatePassword), + nameof(NntpdOptions.AcmeStateDir), + nameof(NntpdOptions.AcmeDirectoryUrl), + nameof(NntpdOptions.AcmeRenewalThresholdDays), + ]; + + // codeql[cs/linq/missed-where] Dual configuration lookup; foreach preserves short-circuit clarity. + foreach (var key in keys) + { + if (configuration[key] is null && configuration[$"{NntpdOptions.SectionName}:{key}"] is { } value) + { + extras[key] = value; + } + } + + if (extras.Count > 0) + { + builder.AddInMemoryCollection(extras); + } + } + + /// + /// Drops operator Transit peers inherited from VectorNNTPD.TransitService.json. + /// Host tests use deny-by-default Transit so local operator peers do not leak in. + /// + public static void IsolateTransit(IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.PostConfigure(static options => options.Clear()); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs new file mode 100644 index 00000000..5a1f58de --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegration.cs @@ -0,0 +1,280 @@ +using System.Collections.Concurrent; +using System.Text; +using StackExchange.Redis; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +/// +/// Opt-in live TransitPeerState Redis seam. Reuses +/// VECTORNNTP_REDIS_INTEGRATION and never reads Redis:Host. +/// +internal static class TransitPeerStateRedisIntegration +{ + /// Dedicated Redis endpoint for live Transit Lua tests (host:port). + public const string EndpointEnvironmentVariable = SessionStateRedisIntegration.EndpointEnvironmentVariable; + + /// Authorized TransitPeerState integration-test Redis. + public const string AuthorizedEndpoint = SessionStateRedisIntegration.AuthorizedEndpoint; + + /// Reason used when the live Redis tests are not configured. + public const string SkipReason = + "Set VECTORNNTP_REDIS_INTEGRATION=198.18.0.70:6379 to run TransitPeerState Lua " + + "against the dedicated test Redis. Ordinary tests do not open Redis and do " + + "not use Redis:Host."; + + /// Parses the opt-in endpoint, or when unset. + public static VectorNNTPD.TransitService.Configuration.RedisOptions? TryGetOptions() => + SessionStateRedisIntegration.TryGetOptions(); +} + +/// +/// Shared live Redis connection for TransitPeerState Lua EVAL tests. Deletes only +/// keys created for uniquely named test peer identifiers. +/// +public sealed class TransitPeerStateRedisIntegrationFixture : IAsyncLifetime +{ + private readonly ConcurrentBag _identifiers = []; + private IRedisConnection? _connection; + private IDatabase? _database; + + public bool IsConfigured { get; private set; } + + public string Endpoint { get; private set; } = string.Empty; + + public string ServerVersion { get; private set; } = string.Empty; + + internal RedisTransitPeerStateStore Store { get; private set; } = null!; + + public async Task InitializeAsync() + { + var options = TransitPeerStateRedisIntegration.TryGetOptions(); + if (options is null) + { + return; + } + + IsConfigured = true; + Endpoint = SessionStateRedisIntegration.DescribeTarget(options); + _connection = await new StackExchangeRedisConnectionFactory() + .ConnectAsync(options, CancellationToken.None); + var live = (StackExchangeRedisConnection)_connection; + _database = live.Multiplexer.GetDatabase(); + var endpoint = live.Multiplexer.GetEndPoints()[0]; + ServerVersion = live.Multiplexer.GetServer(endpoint).Version.ToString(); + Store = new RedisTransitPeerStateStore(new LiveRedisService(_connection.GetDatabase())); + } + + public async Task DisposeAsync() + { + if (_database is not null) + { + foreach (var identifier in _identifiers) + { + await DeleteKeyAsync(identifier); + } + + var remaining = await RemainingTestKeysAsync(); + if (remaining.Count > 0) + { + throw new InvalidOperationException( + "TransitPeerState live Redis cleanup left keys: " + string.Join(", ", remaining)); + } + } + + if (_connection is not null) + { + await _connection.DisposeAsync(); + } + } + + public async Task CreateIdentifierAsync() + { + EnsureConfigured(); + var identifier = "vnntp.tconn.lua." + Guid.NewGuid().ToString("N"); + _identifiers.Add(identifier); + await DeleteKeyAsync(identifier); + return identifier; + } + + public async Task DeleteKeyAsync(string identifier) + { + EnsureConfigured(); + _ = await Database.KeyDeleteAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + public async Task> RemainingTestKeysAsync() + { + EnsureConfigured(); + var remaining = new List(); + // codeql[cs/linq/missed-where] Async KeyExists filter; Where cannot await. + foreach (var identifier in _identifiers.Distinct(StringComparer.Ordinal)) + { + if (await Database.KeyExistsAsync((RedisKey)TransitPeerStateKeys.Create(identifier))) + { + remaining.Add(Encoding.UTF8.GetString(TransitPeerStateKeys.Create(identifier))); + } + } + + return remaining; + } + + public ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + long nowUnixMs, + long leaseMs) => + Store.TryAdmitAsync( + identifier, + ownerId, + maxIncoming, + generation, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseAsync(string identifier, string ownerId, long generation) => + Store.ReleaseAsync(identifier, ownerId, generation); + + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + long nowUnixMs, + long leaseMs) => + Store.RenewAsync( + identifier, + ownerId, + generation, + DateTimeOffset.FromUnixTimeMilliseconds(nowUnixMs), + TimeSpan.FromMilliseconds(leaseMs)); + + public ValueTask ReleaseOwnerAsync(string identifier, string ownerId) => + Store.ReleaseOwnerAsync(identifier, ownerId); + + public async Task SeedAsync(string identifier, string ownerId, long expiryUnixMs, long generation, int count) + { + EnsureConfigured(); + _ = await Database.HashSetAsync( + (RedisKey)TransitPeerStateKeys.Create(identifier), + ownerId, + TransitPeerStateKeys.Value(expiryUnixMs, generation, count)); + } + + public async Task ReadAsync(string identifier, string ownerId) + { + EnsureConfigured(); + var value = await Database.HashGetAsync((RedisKey)TransitPeerStateKeys.Create(identifier), ownerId); + return Parse(value); + } + + public async Task CountAsync(string identifier) + { + EnsureConfigured(); + var entries = await Database.HashGetAllAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + var total = 0; + // codeql[cs/linq/missed-where] Parse pattern-match; avoid double Parse from Where+body. + foreach (var entry in entries) + { + if (Parse(entry.Value) is { } ownership) + { + total += ownership.Count; + } + } + + return total; + } + + public async Task KeyTimeToLiveAsync(string identifier) + { + EnsureConfigured(); + return await Database.KeyTimeToLiveAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + public async Task FieldCountAsync(string identifier) + { + EnsureConfigured(); + return (int)await Database.HashLengthAsync((RedisKey)TransitPeerStateKeys.Create(identifier)); + } + + private IDatabase Database => + _database ?? throw new InvalidOperationException("Live Redis fixture is not connected."); + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException(TransitPeerStateRedisIntegration.SkipReason); + } + } + + private static Ownership? Parse(RedisValue value) + { + if (value.IsNullOrEmpty) + { + return null; + } + + return TransitPeerStateKeys.TrySplitValue((string)value!, out var expiry, out var generation, out var count) + ? new Ownership(expiry, generation, count) + : null; + } + + private sealed class LiveRedisService : IRedisService + { + private readonly IRedisDatabase _database; + + public LiveRedisService(IRedisDatabase database) + { + _database = database; + } + + public IRedisDatabase Database => _database; + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + _database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + _database.SetAsync(key, value, expiry, cancellationToken); + } +} + +/// +/// xUnit 2 discovery-time skip unless VECTORNNTP_REDIS_INTEGRATION is set. +/// +public sealed class TransitPeerStateRedisIntegrationFactAttribute : FactAttribute +{ + /// Initializes the attribute and skips when the opt-in endpoint is absent. + public TransitPeerStateRedisIntegrationFactAttribute() + { + if (TransitPeerStateRedisIntegration.TryGetOptions() is null) + { + Skip = TransitPeerStateRedisIntegration.SkipReason; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs new file mode 100644 index 00000000..49d21b12 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/TransitPeerStateRedisIntegrationConventionTests.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.TransitService.Tests.Fixtures; + +public sealed class TransitPeerStateRedisIntegrationConventionTests +{ + [Fact] + public void OptInVariable_MatchesSessionStateDedicatedTestRedis() + { + Assert.Equal( + SessionStateRedisIntegration.EndpointEnvironmentVariable, + TransitPeerStateRedisIntegration.EndpointEnvironmentVariable); + Assert.Equal( + SessionStateRedisIntegration.AuthorizedEndpoint, + TransitPeerStateRedisIntegration.AuthorizedEndpoint); + Assert.Contains("do not use Redis:Host", TransitPeerStateRedisIntegration.SkipReason, StringComparison.Ordinal); + Assert.Contains(TransitPeerStateRedisIntegration.AuthorizedEndpoint, TransitPeerStateRedisIntegration.SkipReason, StringComparison.Ordinal); + } + + [Fact] + public void IntegrationFact_SkipsWhenOptInVariableIsUnset() + { + if (TransitPeerStateRedisIntegration.TryGetOptions() is not null) + { + return; + } + + var attribute = new TransitPeerStateRedisIntegrationFactAttribute(); + Assert.Equal(TransitPeerStateRedisIntegration.SkipReason, attribute.Skip); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json b/tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json new file mode 100644 index 00000000..8ba3df5d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Fixtures/startup-smoke-transit.json @@ -0,0 +1,2987 @@ +{ + "Moderation": { + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/moderators", + "RetrievedFrom": "InnUrl", + "Url": "https://github.com/InterNetNews/inn/blob/main/samples/moderators" + } + }, + "Email": { + "Enabled": false, + "DefaultFrom": "noreply@usenet.ninja", + "EnvelopeSender": "noreply@usenet.ninja", + "Smtp": { + "Host": "", + "Port": 587, + "Security": "StartTls", + "Username": "", + "Password": "", + "RequireTlsForAuthentication": true, + "ConnectTimeout": "00:00:15", + "CommandTimeout": "00:00:30", + "MaxAttempts": 3, + "InitialRetryDelay": "00:00:02", + "MaximumRetryDelay": "00:01:00" + }, + "Spool": { + "Directory": "spool/smtp", + "ShutdownTimeout": "00:00:15", + "ScanInterval": "00:00:01" + } + }, + "Control": { + "PgpAuthorities": { + "Authorities": [ + { + "AdminGroup": "aioe.system", + "Authorizations": [ + { + "From": "usenet@aioe.org", + "Message": "checkgroups", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "newgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + }, + { + "From": "usenet@aioe.org", + "Message": "rmgroup", + "Newsgroups": "aioe.*", + "VerificationIdentity": "usenet@aioe.org" + } + ], + "Contact": "usenet@aioe.org", + "KeyFingerprint": "22031AAC51E7C7FD664F1D8090DF6C712322A7F8", + "KeyUrl": "http://news.aioe.org/hierarchy/aioe.txt", + "Name": "AIOE", + "SyncableServer": "nntp.aioe.org", + "Url": "http://news.aioe.org/hierarchy/" + }, + { + "AdminGroup": "alabama.config", + "Authorizations": [ + { + "From": "jpc@suespammers.org", + "Message": "checkgroups", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "newgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + }, + { + "From": "jpc@suespammers.org", + "Message": "rmgroup", + "Newsgroups": "alabama.*|hsv.*", + "VerificationIdentity": "alabama-group-admin" + } + ], + "Contact": "jpc@suespammers.org", + "Name": "ALABAMA & HSV" + }, + { + "AdminGroup": "at.usenet.gruppen", + "Authorizations": [ + { + "From": "control@usenet.backbone.at", + "Message": "checkgroups", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "newgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + }, + { + "From": "control@usenet.backbone.at", + "Message": "rmgroup", + "Newsgroups": "at.*", + "VerificationIdentity": "control@usenet.backbone.at" + } + ], + "KeyUrl": "http://www.usenet.at/pgpkey.asc", + "Name": "AT", + "Url": "http://www.usenet.at/" + }, + { + "AdminGroup": "aus.net.news", + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "aus.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "AUS", + "Url": "http://aus.news-admin.org/" + }, + { + "AdminGroup": "be.announce", + "Authorizations": [ + { + "From": "group-admin@usenet.be", + "Message": "checkgroups", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "newgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + }, + { + "From": "group-admin@usenet.be", + "Message": "rmgroup", + "Newsgroups": "be.*", + "VerificationIdentity": "be.announce.newgroups" + } + ], + "Contact": "be-hierarchy-admin@usenet.be", + "KeyFingerprint": "302A459470DE1FD5818C5864D2F70871", + "KeyUrl": "http://usenet.be/be.announce.newgroups.asc", + "Name": "BE", + "Url": "http://usenet.be/" + }, + { + "AdminGroup": "belwue.infos", + "Authorizations": [ + { + "From": "news@news.belwue.de", + "Message": "checkgroups", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "newgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + }, + { + "From": "news@news.belwue.de", + "Message": "rmgroup", + "Newsgroups": "belwue.*", + "VerificationIdentity": "belwue-hir-control" + } + ], + "Contact": "news@belwue.de", + "KeyFingerprint": "2EFF2B8A891FE5AA6F89672450B542D9", + "KeyUrl": "http://www.belwue.de/fileadmin/belwue/Dokumente/Newsserver/belwue-hir-control.asc", + "Name": "BELWUE", + "Url": "http://www.belwue.de/produkte/dienste/dienste-usenet1.html" + }, + { + "AdminGroup": "bionet.general", + "Authorizations": [ + { + "From": "Biosci-control-key@net.bio.net", + "Message": "checkgroups", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "newgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + }, + { + "From": "Biosci-control-key@net.bio.net", + "Message": "rmgroup", + "Newsgroups": "bionet.*", + "VerificationIdentity": "Biosci-control-key@net.bio.net" + } + ], + "KeyFingerprint": "EBC0F1BA260BC6D6FB8DEDC4AE5D1054", + "Name": "BIONET", + "Url": "http://www.bio.net/" + }, + { + "AdminGroup": "bit.admin", + "Authorizations": [ + { + "From": "bit@newsadmin.com", + "Message": "checkgroups", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "newgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + }, + { + "From": "bit@newsadmin.com", + "Message": "rmgroup", + "Newsgroups": "bit.*", + "VerificationIdentity": "bit@newsadmin.com" + } + ], + "Name": "BIT", + "Url": "http://www.newsadmin.com/bit/bit.htm" + }, + { + "AdminGroup": "bln.misc", + "Authorizations": [ + { + "From": "news@fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + }, + { + "From": "news@fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "bln.*", + "VerificationIdentity": "news@fu-berlin.de" + } + ], + "Contact": "news@fu-berlin.de", + "KeyFingerprint": "2B24B5C78A4861407BCD5A88281D90763F368EA0", + "KeyUrl": "https://ftp.fu-berlin.de/doc/news/bln/bln-key.txt", + "Name": "BLN", + "Url": "https://ftp.fu-berlin.de/doc/news/bln/bln-info.txt" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "bne.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "BNE", + "Url": "http://bne.news-admin.org/" + }, + { + "Contact": "bofh-control@lists.killfile.org", + "KeyFingerprint": "40B58A561E006152083E38B3CEF569807DC1A266", + "Name": "BOFH" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "canb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "CANB", + "Url": "http://canb.news-admin.org/" + }, + { + "Authorizations": [ + { + "From": "felix.rauch@nice.ch", + "Message": "checkgroups", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "newgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + }, + { + "From": "felix.rauch@nice.ch", + "Message": "rmgroup", + "Newsgroups": "ch.*", + "VerificationIdentity": "ch-news-admin@use-net.ch" + } + ], + "Contact": "ch-news-admin@use-net.ch", + "KeyFingerprint": "7180D68CA7DE2C70624A486ED99602DF", + "KeyUrl": "http://www.use-net.ch/Usenet/adminkey.html", + "Name": "CH", + "Url": "http://www.use-net.ch/Usenet/" + }, + { + "Contact": "koordination@cl-netz.de", + "KeyUrl": "http://www.cl-netz.de/control.txt", + "Name": "CL", + "Url": "http://www.cl-netz.de/" + }, + { + "AdminGroup": "clari.net.admin", + "Authorizations": [ + { + "From": "cl*@clarinet.com", + "Message": "rmgroup", + "Newsgroups": "clari.*", + "VerificationIdentity": "ClariNet.Group" + } + ], + "Contact": "support@clari.net", + "KeyUrl": "http://www.clari.net/tech/clarikey.txt", + "Name": "CLARI" + }, + { + "AdminGroup": "cn.announce", + "Authorizations": [ + { + "From": "control@bentium.com", + "Message": "checkgroups", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "newgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + }, + { + "From": "control@bentium.com", + "Message": "rmgroup", + "Newsgroups": "cn.*", + "VerificationIdentity": "cn.admin.news.announce" + } + ], + "KeyFingerprint": "6297EE33F71625C1A49E47BAC53E5E9E", + "Name": "CN", + "Url": "http://news.yaako.com/" + }, + { + "AdminGroup": "cn.bbs.admin.announce", + "Authorizations": [ + { + "From": "control@cn-bbs.org", + "Message": "checkgroups", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "newgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + }, + { + "From": "control@cn-bbs.org", + "Message": "rmgroup", + "Newsgroups": "cn.bbs.*", + "VerificationIdentity": "cn.bbs.admin.announce" + } + ], + "Name": "CN.BBS", + "Url": "http://bbs.cn.news-admin.org/" + }, + { + "AdminGroup": "news.announce.newgroups", + "Authorizations": [ + { + "From": "group-admin@isc.org", + "Message": "checkgroups", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "newgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + }, + { + "From": "group-admin@isc.org", + "Message": "rmgroup", + "Newsgroups": "comp.*|humanities.*|misc.*|news.*|rec.*|sci.*|soc.*|talk.*", + "VerificationIdentity": "news.announce.newgroups" + } + ], + "Contact": "board@big-8.org", + "KeyFingerprint": "8E181BEF0BE638962C1A5C98FAFE7B550C18C8B7", + "Name": "COMP, HUMANITIES, MISC, NEWS, REC, SCI, SOC, TALK", + "Url": "https://www.big-8.org/" + }, + { + "AdminGroup": "de.admin.news.announce", + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "checkgroups", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "newgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + }, + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Contact": "moderator@dana.de", + "KeyFingerprint": "5BB05288BF55194F667DC2AE16262825", + "KeyUrl": "http://www.dana.de/pgp/dana.txt", + "Name": "DE", + "Url": "http://www.dana.de/" + }, + { + "Authorizations": [ + { + "From": "moderator@dana.de", + "Message": "rmgroup", + "Newsgroups": "de.alt.*", + "VerificationIdentity": "de.admin.news.announce" + } + ], + "Name": "DE.ALT" + }, + { + "AdminGroup": "demon.news", + "Authorizations": [ + { + "From": "newsmaster@demon.net", + "Message": "checkgroups", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "newgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + }, + { + "From": "newsmaster@demon.net", + "Message": "rmgroup", + "Newsgroups": "demon.*", + "VerificationIdentity": "demon.news" + } + ], + "Contact": "newsmaster@demon.net", + "KeyUrl": "ftp://ftp.demon.co.uk/pub/news/doc/demon.news.txt", + "Name": "DEMON" + }, + { + "Authorizations": [ + { + "From": "news@news.dknet.dk", + "Message": "checkgroups", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "newgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + }, + { + "From": "news@news.dknet.dk", + "Message": "rmgroup", + "Newsgroups": "dk.*", + "VerificationIdentity": "news@news.dknet.dk" + } + ], + "KeyFingerprint": "7CB2C750F37D5D738CEE2E3F558072FF", + "KeyUrl": "http://www.usenet.dk/dk-admin/pubkey.html", + "Name": "DK", + "Url": "http://www.usenet.dk/dk-admin/" + }, + { + "AdminGroup": "easynet.support", + "Authorizations": [ + { + "From": "newsmaster@easynet.net", + "Message": "checkgroups", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "newgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + }, + { + "From": "newsmaster@easynet.net", + "Message": "rmgroup", + "Newsgroups": "easynet.*", + "VerificationIdentity": "easynet.news" + } + ], + "Contact": "Christiaan Keet ", + "Name": "EASYNET" + }, + { + "Authorizations": [ + { + "From": "news@news.ut.ee", + "Message": "checkgroups", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "newgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + }, + { + "From": "news@news.ut.ee", + "Message": "rmgroup", + "Newsgroups": "ee.*", + "VerificationIdentity": "ee.news" + } + ], + "Contact": "usenet@news.ut.ee", + "KeyUrl": "http://news.ut.ee/pubkey.asc", + "Name": "EE", + "Url": "http://news.ut.ee/" + }, + { + "AdminGroup": "eug.config", + "Authorizations": [ + { + "From": "newsadmin@efn.org", + "Message": "checkgroups", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "newgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + }, + { + "From": "newsadmin@efn.org", + "Message": "rmgroup", + "Newsgroups": "efn.*|eug.*", + "VerificationIdentity": "eug.config" + } + ], + "Name": "EFN & EUG" + }, + { + "AdminGroup": "england.news.policy", + "Authorizations": [ + { + "From": "admin@england.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "newgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + }, + { + "From": "admin@england.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "england.*", + "VerificationIdentity": "england-usenet" + } + ], + "Contact": "admin@england.news-admin.org", + "KeyFingerprint": "DA3EC20146E561CBA24309CA136D311F", + "Name": "ENGLAND" + }, + { + "AdminGroup": "es.news.anuncios", + "Authorizations": [ + { + "From": "moderador@corus-es.org", + "Message": "checkgroups", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "newgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + }, + { + "From": "moderador@corus-es.org", + "Message": "rmgroup", + "Newsgroups": "es.*", + "VerificationIdentity": "es.news" + } + ], + "Contact": "moderador@corus-es.org", + "KeyUrl": "http://www.corus-es.org/docs/esnews.asc", + "Name": "ES", + "Url": "http://www.corus-es.org/docs/es_newsadmins_faq.txt" + }, + { + "Authorizations": [ + { + "From": "mod-ena@ennui.org", + "Message": "checkgroups", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "newgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + }, + { + "From": "mod-ena@ennui.org", + "Message": "rmgroup", + "Newsgroups": "esp.*", + "VerificationIdentity": "esp.news.administracion" + } + ], + "Contact": "", + "KeyUrl": "http://ennui.org/esp/mod-ena.asc", + "Name": "ESP", + "Url": "http://ennui.org/esp/" + }, + { + "Authorizations": [ + { + "From": "news@eternal-september.org", + "Message": "checkgroups", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "newgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + }, + { + "From": "news@eternal-september.org", + "Message": "rmgroup", + "Newsgroups": "eternal-september.*", + "VerificationIdentity": "news@eternal-september.org" + } + ], + "Contact": "news@eternal-september.org", + "KeyFingerprint": "B79A2BC67D5AFF7918E2AC914BC125F1", + "KeyUrl": "http://www.eternal-september.org/control/pgpkey.txt", + "Name": "ETERNAL-SEPTEMBER", + "Url": "http://www.eternal-september.org/" + }, + { + "AdminGroup": "europa.usenet.admin", + "Authorizations": [ + { + "From": "group-admin@usenet.eu.org", + "Message": "checkgroups", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "newgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + }, + { + "From": "group-admin@usenet.eu.org", + "Message": "rmgroup", + "Newsgroups": "europa.*", + "VerificationIdentity": "group-admin@usenet.eu.org" + } + ], + "KeyFingerprint": "3A05A849FB16292575E3DEBB69E01DB4", + "KeyUrl": "http://www.europa.usenet.eu.org/pgp/index.html", + "Name": "EUROPA", + "Url": "http://www.europa.usenet.eu.org/" + }, + { + "AdminGroup": "ffm.admin", + "Authorizations": [ + { + "From": "ffm.admin@arcor.de", + "Message": "checkgroups", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "newgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + }, + { + "From": "ffm.admin@arcor.de", + "Message": "rmgroup", + "Newsgroups": "ffm.*", + "VerificationIdentity": "ffm.admin" + } + ], + "KeyFingerprint": "53A082626FC781C9CF53AB00A3F8C211", + "KeyUrl": "ftp://ftp.arcor-online.net/pub/news/PGPKEY.FFM", + "Name": "FFM", + "Url": "http://ffm.arcornews.de/" + }, + { + "AdminGroup": "fido.belg.news", + "Authorizations": [ + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "checkgroups", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "newgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + }, + { + "From": "fidobelg@mail.z2.fidonet.org", + "Message": "rmgroup", + "Newsgroups": "fido.belg.*", + "VerificationIdentity": "fido.belg.news" + } + ], + "Name": "FIDO.BELG" + }, + { + "Authorizations": [ + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "checkgroups", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "newgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + }, + { + "From": "fido.ger@news.fu-berlin.de", + "Message": "rmgroup", + "Newsgroups": "fido.ger.*", + "VerificationIdentity": "fido.ger@news.fu-berlin.de" + } + ], + "KeyUrl": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/PGP-Key", + "Name": "FIDO.GER", + "Url": "ftp://ftp.fu-berlin.de/doc/news/fido.ger/fido.ger-info.english" + }, + { + "AdminGroup": "fido7.postmasters", + "Authorizations": [ + { + "From": "newgroups-request@fido7.org", + "Message": "checkgroups", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "newgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + }, + { + "From": "newgroups-request@fido7.org", + "Message": "rmgroup", + "Newsgroups": "fido7.*", + "VerificationIdentity": "fido7.announce.newgroups" + } + ], + "KeyUrl": "http://www.fido7.org/pgpcontrol.html", + "Name": "FIDO7", + "Url": "http://www.fido7.org/" + }, + { + "AdminGroup": "fj.news.announce", + "Authorizations": [ + { + "From": "committee@fj-news.org", + "Message": "checkgroups", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "newgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + }, + { + "From": "committee@fj-news.org", + "Message": "rmgroup", + "Newsgroups": "fj.*", + "VerificationIdentity": "fj.news.announce" + } + ], + "Contact": "committee@fj-news.org", + "KeyUrl": "http://www.is.tsukuba.ac.jp/~yas/fj/fj.asc", + "Name": "FJ", + "Url": "http://www.fj-news.org/index.html.en" + }, + { + "AdminGroup": "flora.general", + "Authorizations": [ + { + "From": "news@flora.ottawa.on.ca", + "Message": "checkgroups", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "newgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + }, + { + "From": "news@flora.ottawa.on.ca", + "Message": "rmgroup", + "Newsgroups": "flora.*", + "VerificationIdentity": "flora-news" + } + ], + "Contact": "russell@flora.org", + "Name": "FLORA" + }, + { + "AdminGroup": "fr.usenet.forums.annonces", + "Authorizations": [ + { + "From": "control@usenet-fr.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "newgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + }, + { + "From": "control@usenet-fr.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "fr.*", + "VerificationIdentity": "control@usenet-fr.news.eu.org" + } + ], + "KeyFingerprint": "D5F369B297573622015354E7FA423E89970EB10B", + "KeyUrl": "https://www.usenet-fr.net/pgp-fr-2020.txt", + "Name": "FR", + "Url": "https://www.usenet-fr.net/" + }, + { + "AdminGroup": "france.admin.evolutions", + "Authorizations": [ + { + "From": "control@usenet-france.news.eu.org", + "Message": "checkgroups", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "newgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + }, + { + "From": "control@usenet-france.news.eu.org", + "Message": "rmgroup", + "Newsgroups": "france.*", + "VerificationIdentity": "control@usenet-france.news.eu.org" + } + ], + "Contact": "control@usenet-france.news.eu.org", + "Name": "FRANCE" + }, + { + "AdminGroup": "gnu.gnusenet.config", + "Authorizations": [ + { + "From": "usenet@gnu.org", + "Message": "checkgroups", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "newgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + }, + { + "From": "usenet@gnu.org", + "Message": "rmgroup", + "Newsgroups": "gnu.*", + "VerificationIdentity": "usenet@gnu.org" + } + ], + "KeyUrl": "http://www.gnu.org/usenet/usenet-pgp-key.txt", + "Name": "GNU", + "Url": "http://www.gnu.org/usenet/usenet.html" + }, + { + "Authorizations": [ + { + "From": "news@gnuu.de", + "Message": "rmgroup", + "Newsgroups": "gnuu.*", + "VerificationIdentity": "news@gnuu.de" + } + ], + "KeyUrl": "http://www.gnuu.de/config/PGPKEY.GNUU", + "Name": "GNUU", + "Url": "http://www.gnuu.de/" + }, + { + "AdminGroup": "grisbi.admin", + "Authorizations": [ + { + "From": "grisbi-control@grisbi.org", + "Message": "checkgroups", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "newgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + }, + { + "From": "grisbi-control@grisbi.org", + "Message": "rmgroup", + "Newsgroups": "grisbi.*", + "VerificationIdentity": "grisbi-control@grisbi.org" + } + ], + "Contact": "newsmaster@grisbi.org", + "KeyFingerprint": "EB350C030080BD2A7E0CA4C9F2C62A3DC86CC6E1", + "KeyUrl": "http://news.grisbi.org/public-key.asc", + "Name": "GRISBI", + "Url": "http://news.grisbi.org/" + }, + { + "Authorizations": [ + { + "From": "news@zhaum.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "newgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + }, + { + "From": "news@zhaum.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "hacktic.*", + "VerificationIdentity": "news@zhaum.xs4all.nl" + } + ], + "Name": "HACKTIC" + }, + { + "AdminGroup": "hamburg.koordination", + "Authorizations": [ + { + "From": "hamburg@steering-group.net", + "Message": "checkgroups", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "newgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + }, + { + "From": "hamburg@steering-group.net", + "Message": "rmgroup", + "Newsgroups": "hamburg.*", + "VerificationIdentity": "hamburg.koordination" + } + ], + "Contact": "hamburg@steering-group.net", + "KeyFingerprint": "3EE70CBB6E0194EE456FC557F4B9548E", + "KeyUrl": "http://www.steering-group.net/hamburg/hamburg.koordination.txt", + "Name": "HAMBURG", + "Url": "http://www.steering-group.net/hamburg/" + }, + { + "AdminGroup": "hamster.de.config", + "Authorizations": [ + { + "From": "hamster-control@snafu.de", + "Message": "checkgroups", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "newgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + }, + { + "From": "hamster-control@snafu.de", + "Message": "rmgroup", + "Newsgroups": "hamster.*", + "VerificationIdentity": "hamster-control@snafu.de" + } + ], + "Contact": "hamster-contact@snafu.de", + "KeyFingerprint": "1275A9428AD61F776ACFB40C79155F93", + "Name": "HAMSTER" + }, + { + "AdminGroup": "han.news.admin", + "Authorizations": [ + { + "From": "newgroups-request@usenet.or.kr", + "Message": "checkgroups", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "newgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + }, + { + "From": "newgroups-request@usenet.or.kr", + "Message": "rmgroup", + "Newsgroups": "han.*", + "VerificationIdentity": "han.news.admin" + } + ], + "Contact": "newgroups-request@usenet.or.kr", + "KeyUrl": "ftp://ftp.usenet.or.kr/pub/korea/usenet/pgp/PGPKEY.han", + "Name": "HAN" + }, + { + "AdminGroup": "hiv.config", + "Authorizations": [ + { + "From": "news@hivnet.org", + "Message": "checkgroups", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "newgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + }, + { + "From": "news@hivnet.org", + "Message": "rmgroup", + "Newsgroups": "hiv.*", + "VerificationIdentity": "news@hivnet.org" + } + ], + "Contact": "news@hivnet.org", + "KeyFingerprint": "5DD60EDC1E2DEA0BB0564DD65253D7A4", + "Name": "HIV" + }, + { + "AdminGroup": "houston.usenet.config", + "Authorizations": [ + { + "From": "news@academ.com", + "Message": "checkgroups", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "newgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + }, + { + "From": "news@academ.com", + "Message": "rmgroup", + "Newsgroups": "houston.*", + "VerificationIdentity": "houston.usenet.config" + } + ], + "Name": "HOUSTON" + }, + { + "AdminGroup": "hr.news.admin", + "Authorizations": [ + { + "From": "newsmaster@carnet.hr", + "Message": "checkgroups", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "newgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + }, + { + "From": "newsmaster@carnet.hr", + "Message": "rmgroup", + "Newsgroups": "hr.*", + "VerificationIdentity": "newsmaster@carnet.hr" + } + ], + "Contact": "newsmaster@carnet.hr", + "KeyFingerprint": "0EE574FB1C407ADB0AACA52F71921BA3ED63AD9A", + "KeyUrl": "http://newsfeed.carnet.hr/control/key.txt", + "Name": "HR", + "SyncableServer": "news.carnet.hr", + "Url": "http://newsfeed.carnet.hr/control/" + }, + { + "Authorizations": [ + { + "From": "news-admin@humanityquest.com", + "Message": "checkgroups", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "newgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + }, + { + "From": "news-admin@humanityquest.com", + "Message": "rmgroup", + "Newsgroups": "humanityquest.*", + "VerificationIdentity": "humanityquest.admin.config" + } + ], + "Contact": "news-admin@humanityquest.com", + "KeyFingerprint": "BA3DB306B6F552AABA8F32F08C4F504016F9C046", + "KeyUrl": "http://www.humanityquest.com/projects/newsgroups/PGP.htm", + "Name": "HUMANITYQUEST", + "Url": "http://www.humanityquest.com/projects/newsgroups/" + }, + { + "AdminGroup": "hun.admin.news", + "Authorizations": [ + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "checkgroups", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "newgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + }, + { + "From": "hun-mnt@news.sztaki.hu", + "Message": "rmgroup", + "Newsgroups": "hun.*", + "VerificationIdentity": "hun.admin.news" + } + ], + "KeyUrl": "http://gatling.ikk.sztaki.hu/~kissg/news/hun.admin.news.asc", + "Name": "HUN", + "Url": "http://gatling.ikk.sztaki.hu/~kissg/news/" + }, + { + "AdminGroup": "ie.news.group", + "Authorizations": [ + { + "From": "control@usenet.ie", + "Message": "checkgroups", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "newgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + }, + { + "From": "control@usenet.ie", + "Message": "rmgroup", + "Newsgroups": "ie.*", + "VerificationIdentity": "control@usenet.ie" + } + ], + "Contact": "control@usenet.ie", + "Name": "IE" + }, + { + "AdminGroup": "is.isnet", + "Authorizations": [ + { + "From": "group-admin@usenet.is", + "Message": "checkgroups", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "newgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + }, + { + "From": "group-admin@usenet.is", + "Message": "rmgroup", + "Newsgroups": "is.*", + "VerificationIdentity": "group-admin@usenet.is" + } + ], + "Contact": "IS Group Admins ", + "KeyFingerprint": "33328D461E5E1C7F48608E72E53ECAEA", + "KeyUrl": "http://www.usenet.is/group-admin.asc", + "Name": "IS", + "Url": "http://www.usenet.is/" + }, + { + "AdminGroup": "isu.newgroups", + "Authorizations": [ + { + "From": "news@news.isu.edu.tw", + "Message": "checkgroups", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "newgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + }, + { + "From": "news@news.isu.edu.tw", + "Message": "rmgroup", + "Newsgroups": "isu.*", + "VerificationIdentity": "news@news.isu.edu.tw" + } + ], + "Contact": "news@news.isu.edu.tw", + "KeyUrl": "http://news.isu.edu.tw/isu.asc", + "Name": "ISU", + "Url": "http://news.isu.edu.tw/" + }, + { + "AdminGroup": "it.news.annunci", + "Authorizations": [ + { + "From": "gcn@news.nic.it", + "Message": "checkgroups", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "newgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + }, + { + "From": "gcn@news.nic.it", + "Message": "rmgroup", + "Newsgroups": "it.*", + "VerificationIdentity": "gcn@news.nic.it" + } + ], + "Contact": "gcn@news.nic.it", + "KeyFingerprint": "94A4F7B54696D6C7A673F298C48CD0E0", + "KeyUrl": "http://www.news.nic.it/pgp.txt", + "Name": "IT", + "Url": "http://www.news.nic.it/" + }, + { + "AdminGroup": "italia.announce.newgroups", + "Authorizations": [ + { + "From": "news@news.cineca.it", + "Message": "checkgroups", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "newgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + }, + { + "From": "news@news.cineca.it", + "Message": "rmgroup", + "Newsgroups": "italia.*", + "VerificationIdentity": "italia.announce.newgroups" + } + ], + "Contact": "news@news.cineca.it", + "KeyFingerprint": "0FBB7162DA5D5DB8D586FC2802671A6B", + "KeyUrl": "http://news.cineca.it/italia/italia-pgp.txt", + "Name": "ITALIA", + "Url": "http://news.cineca.it/italia/" + }, + { + "AdminGroup": "japan.admin.announce", + "Authorizations": [ + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "checkgroups", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "newgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + }, + { + "From": "japan.admin.announce@news.efnet.com", + "Message": "rmgroup", + "Newsgroups": "japan.*", + "VerificationIdentity": "japan.admin.announce@news.efnet.com" + } + ], + "Contact": "Tsuneo Tanaka ", + "KeyFingerprint": "6AFA1947691B107438534B1BD8BA3E85", + "KeyUrl": "http://grex.cyberspace.org/~tt/japan.admin.announce.asc", + "Name": "JAPAN", + "Url": "http://www.asahi-net.or.jp/~AE5T-KSN/japan-e.html" + }, + { + "AdminGroup": "jlug.config", + "Authorizations": [ + { + "From": "news@linux.or.jp", + "Message": "checkgroups", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "newgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + }, + { + "From": "news@linux.or.jp", + "Message": "rmgroup", + "Newsgroups": "jlug.*", + "VerificationIdentity": "news@linux.or.jp" + } + ], + "Contact": "news@linux.or.jp", + "KeyUrl": "http://www.linux.or.jp/pgpkey/news", + "Name": "JLUG", + "Url": "http://www.linux.or.jp/community/news/index.html" + }, + { + "AdminGroup": "ka.admin", + "Contact": "usenet@karlsruhe.org", + "KeyFingerprint": "DE19BB2576198117F067D223E8C87C90", + "KeyUrl": "http://www.karlsruhe.org/pubkey-news.karlsruhe.org.asc", + "Name": "KA", + "Url": "http://www.karlsruhe.org/" + }, + { + "Authorizations": [ + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "checkgroups", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + }, + { + "From": "ty@kamoi.imasy.or.jp", + "Message": "rmgroup", + "Newsgroups": "kanto.*", + "VerificationIdentity": "kanto.news.network" + } + ], + "Name": "KANTO" + }, + { + "AdminGroup": "kassel.admin", + "Authorizations": [ + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + }, + { + "From": "dirk.meyer@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "kassel.*", + "VerificationIdentity": "kassel-admin" + } + ], + "Name": "KASSEL" + }, + { + "AdminGroup": "linux.admin.news", + "Authorizations": [ + { + "From": "linux-admin@bofh.it", + "Message": "checkgroups", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "newgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + }, + { + "From": "linux-admin@bofh.it", + "Message": "rmgroup", + "Newsgroups": "linux.*", + "VerificationIdentity": "linux-admin@bofh.it" + } + ], + "Contact": "Marco d'Itri ", + "KeyFingerprint": "81B327994FCE32D11BC9010DBBB32E41", + "Name": "LINUX" + }, + { + "AdminGroup": "malta.config", + "Authorizations": [ + { + "From": "cmeli@cis.um.edu.mt", + "Message": "checkgroups", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "newgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + }, + { + "From": "cmeli@cis.um.edu.mt", + "Message": "rmgroup", + "Newsgroups": "malta.*", + "VerificationIdentity": "malta.config" + } + ], + "Contact": "cmeli@cis.um.edu.mt", + "KeyFingerprint": "2017015CF0D01A42E41330580B1448A6", + "KeyUrl": "http://www.cis.um.edu.mt/news-malta/PGP.PUBLICKEY", + "Name": "MALTA", + "Url": "http://www.malta.news-admin.org/" + }, + { + "AdminGroup": "maus.info", + "Authorizations": [ + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "checkgroups", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "newgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.hb.provi.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + }, + { + "From": "guenter@gst0hb.north.de", + "Message": "rmgroup", + "Newsgroups": "maus.*", + "VerificationIdentity": "maus-info" + } + ], + "KeyFingerprint": "8252C77026B972A1379855983F26623E", + "Name": "MAUS" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "melb.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "MELB", + "Url": "http://melb.news-admin.org/" + }, + { + "AdminGroup": "mensa.config", + "Authorizations": [ + { + "From": "usenet@newsgate.mensa.org", + "Message": "checkgroups", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "newgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + }, + { + "From": "usenet@newsgate.mensa.org", + "Message": "rmgroup", + "Newsgroups": "mensa.*", + "VerificationIdentity": "mensa.config" + } + ], + "Contact": "usenet@newsgate.mensa.org", + "KeyFingerprint": "52B9396385D908068E197344973C5005DC7DB7A7", + "Name": "MENSA" + }, + { + "AdminGroup": "microsoft.public.news.server", + "Authorizations": [ + { + "From": "control-microsoft@trigofacile.com", + "Message": "checkgroups", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "newgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + }, + { + "From": "control-microsoft@trigofacile.com", + "Message": "rmgroup", + "Newsgroups": "microsoft.*", + "VerificationIdentity": "control-microsoft@trigofacile.com" + } + ], + "Contact": "control-microsoft@trigofacile.com", + "KeyFingerprint": "DF705FC9F615D52E02DBA3CB63A98D13E60E2FAA", + "KeyUrl": "http://www.trigofacile.com/divers/usenet/clefs/pgpkey-microsoft.asc", + "Name": "MICROSOFT", + "SyncableServer": "msnews.microsoft.com", + "Url": "http://www.trigofacile.com/divers/usenet/clefs/index.htm" + }, + { + "AdminGroup": "milw.config", + "Authorizations": [ + { + "From": "milw@usenet.mil.wi.us", + "Message": "checkgroups", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "newgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + }, + { + "From": "milw@usenet.mil.wi.us", + "Message": "rmgroup", + "Newsgroups": "milw.*", + "VerificationIdentity": "milw.config" + } + ], + "Contact": "milw@usenet.mil.wi.us", + "KeyFingerprint": "6E9B9F7098AB9CE5C3C00582215BF49E", + "KeyUrl": "http://usenet.mil.wi.us/pgpkey", + "Name": "MILW", + "Url": "http://usenet.mil.wi.us/" + }, + { + "AdminGroup": "muc.admin", + "Authorizations": [ + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "checkgroups", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "newgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + }, + { + "From": "muc-cmsg@muenchen.pro-bahn.org", + "Message": "rmgroup", + "Newsgroups": "muc.*", + "VerificationIdentity": "muc.admin" + } + ], + "KeyFingerprint": "43C70E7C45C706E0BD6F76CE07395E66", + "KeyUrl": "http://home.arcor.de/andreas-barth/muc-admin.html", + "Name": "MUC" + }, + { + "AdminGroup": "netscape.public.admin", + "Authorizations": [ + { + "From": "news@netscape.com", + "Message": "checkgroups", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "newgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + }, + { + "From": "news@netscape.com", + "Message": "rmgroup", + "Newsgroups": "netscape.*", + "VerificationIdentity": "netscape.public.admin" + } + ], + "Contact": "news@netscape.com", + "KeyFingerprint": "B78055121F9C170B8666AD3BDB6835EC", + "Name": "NETSCAPE", + "Url": "http://www.mozilla.org/community.html" + }, + { + "AdminGroup": "nippon.news.group", + "Authorizations": [ + { + "From": "news@gcd.org", + "Message": "rmgroup", + "Newsgroups": "nippon.*", + "VerificationIdentity": "nippon.news.group" + } + ], + "KeyFingerprint": "BCCF15CDB13CDFB3C3DE356F2FF746DB", + "KeyUrl": "http://www.gcd.org/news/nippon/", + "Name": "NIPPON", + "Url": "http://www.gcd.org/news/nippon/" + }, + { + "AdminGroup": "nl.newsgroups", + "Authorizations": [ + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "checkgroups", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "newgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + }, + { + "From": "nl-admin@nic.surfnet.nl", + "Message": "rmgroup", + "Newsgroups": "nl.*", + "VerificationIdentity": "nl.newsgroups" + } + ], + "Contact": "nl-admin@nic.surfnet.nl", + "KeyFingerprint": "45200BD5A121EA7CEFB2956C25754D27", + "Name": "NL", + "Url": "http://nl.news-admin.org/info/nladmin.html" + }, + { + "Authorizations": [ + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "checkgroups", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "nl-alt-janitor@surfer.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + }, + { + "From": "news@kink.xs4all.nl", + "Message": "rmgroup", + "Newsgroups": "nl-alt.*", + "VerificationIdentity": "nl-alt.config.admin" + } + ], + "KeyFingerprint": "6B62EB534D5D2F9635D9C89CB0650E4C", + "Name": "NL-ALT" + }, + { + "Authorizations": [ + { + "From": "news@nl.linux.org", + "Message": "checkgroups", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "newgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + }, + { + "From": "news@nl.linux.org", + "Message": "rmgroup", + "Newsgroups": "nlo.*", + "VerificationIdentity": "nlo.newsgroups" + } + ], + "KeyFingerprint": "63DCB2510AF3DD72C2BDC6FDC1C544CF", + "Name": "NLO", + "SyncableServer": "news.nl.linux.org", + "Url": "http://mail.nl.linux.org/" + }, + { + "AdminGroup": "nntpworld.config", + "Authorizations": [ + { + "From": "usenet@nntpworld.net", + "Message": "checkgroups", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "newgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + }, + { + "From": "usenet@nntpworld.net", + "Message": "rmgroup", + "Newsgroups": "nntpworld.*", + "VerificationIdentity": "usenet@nntpworld.net" + } + ], + "Contact": "usenet@nntpworld.net", + "KeyFingerprint": "C7BBE0567A8A5A8609B5E329E1BB8CE1", + "KeyUrl": "http://www.nntpworld.net/nntpworld.asc", + "Name": "NNTPWORLD", + "Url": "http://www.nntpworld.net/" + }, + { + "AdminGroup": "no.usenet.admin", + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "checkgroups", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "newgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + }, + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.*", + "VerificationIdentity": "no-hir-control" + } + ], + "KeyUrl": "http://www.usenet.no/pgp-key.txt", + "Name": "NO", + "Url": "http://www.usenet.no/" + }, + { + "Authorizations": [ + { + "From": "control@usenet.no", + "Message": "rmgroup", + "Newsgroups": "no.alt.*", + "VerificationIdentity": "no-hir-control" + } + ], + "Name": "NO.ALT" + }, + { + "AdminGroup": "nrw.admin.announce", + "Authorizations": [ + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "checkgroups", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "newgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + }, + { + "From": "moderator@nrw.usenetverwaltung.de", + "Message": "rmgroup", + "Newsgroups": "nrw.*", + "VerificationIdentity": "moderator@nrw.usenetverwaltung.de" + } + ], + "Contact": "moderator@nrw.usenetverwaltung.de", + "KeyFingerprint": "134A80FED634B464AF32083F620EB1E2", + "KeyUrl": "http://nrw.usenetverwaltung.de/pgp/nrw.asc", + "Name": "NRW", + "Url": "http://nrw.usenetverwaltung.de/" + }, + { + "AdminGroup": "nz.net.announce", + "Authorizations": [ + { + "From": "root@usenet.net.nz", + "Message": "checkgroups", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "newgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + }, + { + "From": "root@usenet.net.nz", + "Message": "rmgroup", + "Newsgroups": "nz.*", + "VerificationIdentity": "nz-hir-control" + } + ], + "Contact": "root@usenet.net.nz", + "KeyFingerprint": "07DF48AAD0EDAA881670C591653D1A28", + "Name": "NZ", + "Url": "http://www.faqs.org/faqs/usenet/nz-news-hierarchy" + }, + { + "Authorizations": [ + { + "From": "news@news2.open-news-network.org", + "Message": "checkgroups", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "newgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + }, + { + "From": "news@news2.open-news-network.org", + "Message": "rmgroup", + "Newsgroups": "opennews.*", + "VerificationIdentity": "news@news2.open-news-network.org" + } + ], + "Name": "OPENNEWS", + "Url": "http://www.open-news-network.org/" + }, + { + "AdminGroup": "openwatcom.contributors", + "Authorizations": [ + { + "From": "admin@openwatcom.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "newgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + }, + { + "From": "admin@openwatcom.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "openwatcom.*", + "VerificationIdentity": "admin@openwatcom.news-admin.org" + } + ], + "Contact": "admin@openwatcom.news-admin.org", + "KeyUrl": "http://cmeerw.org/files/openwatcom/pgp-openwatcom.asc", + "Name": "OPENWATCOM", + "SyncableServer": "news.openwatcom.org", + "Url": "http://www.openwatcom.org/" + }, + { + "Authorizations": [ + { + "From": "*@opera.com", + "Message": "checkgroups", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "newgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + }, + { + "From": "*@opera.com", + "Message": "rmgroup", + "Newsgroups": "opera.*", + "VerificationIdentity": "opera-group-admin" + } + ], + "Contact": "usenet@opera.com", + "Name": "OPERA", + "SyncableServer": "news.opera.com" + }, + { + "Authorizations": [ + { + "From": "postmaster@upb.de", + "Message": "checkgroups", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "newgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + }, + { + "From": "postmaster@upb.de", + "Message": "rmgroup", + "Newsgroups": "pbinfo.*", + "VerificationIdentity": "news@uni-paderborn.de" + } + ], + "Contact": "news@uni-paderborn.de", + "Name": "PBINFO" + }, + { + "Authorizations": [ + { + "From": "newsadmin@perl.org", + "Message": "checkgroups", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "newgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + }, + { + "From": "newsadmin@perl.org", + "Message": "rmgroup", + "Newsgroups": "perl.*", + "VerificationIdentity": "newsadmin@perl.org" + } + ], + "Contact": "newsadmin@perl.org", + "KeyFingerprint": "438FD1BA4DCC3B1ABED82BCC32988A7D8B2ACFBB", + "KeyUrl": "http://www.nntp.perl.org/about/newsadmin@perl.org.pgp", + "Name": "PERL", + "Url": "http://www.nntp.perl.org/about/" + }, + { + "AdminGroup": "pgh.config", + "Authorizations": [ + { + "From": "pgh-config@psc.edu", + "Message": "checkgroups", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "newgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + }, + { + "From": "pgh-config@psc.edu", + "Message": "rmgroup", + "Newsgroups": "pgh.*", + "VerificationIdentity": "pgh.config" + } + ], + "Name": "PGH" + }, + { + "Authorizations": [ + { + "From": "news@postgresql.org", + "Message": "checkgroups", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "newgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + }, + { + "From": "news@postgresql.org", + "Message": "rmgroup", + "Newsgroups": "pgsql.*", + "VerificationIdentity": "news@postgresql.org" + } + ], + "Contact": "news@postgresql.org", + "KeyUrl": "http://news.hub.org/gpg_public_keys.html", + "Name": "PGSQL", + "Url": "http://news.hub.org/gpg_public_keys.html" + }, + { + "AdminGroup": "pl.news.admin", + "Authorizations": [ + { + "From": "michalj@*fuw.edu.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "checkgroups", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "newgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "michalj@*fuw.edu.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + }, + { + "From": "newgroup@usenet.pl", + "Message": "rmgroup", + "Newsgroups": "pl.*", + "VerificationIdentity": "pl.announce.newgroups" + } + ], + "KeyUrl": "http://www.usenet.pl/doc/news-pl-new-site-faq.html#pgp", + "Name": "PL", + "Url": "http://www.usenet.pl/doc/news-pl-new-site-faq.html" + }, + { + "AdminGroup": "pt.internet.usenet", + "Authorizations": [ + { + "From": "pmelo@*.inescc.pt", + "Message": "checkgroups", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "newgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + }, + { + "From": "pmelo@*.inescc.pt", + "Message": "rmgroup", + "Newsgroups": "pt.*", + "VerificationIdentity": "control@usenet-pt.org" + } + ], + "KeyUrl": "http://www.usenet-pt.org/control@usenet-pt.org.asc", + "Name": "PT", + "Url": "http://www.usenet-pt.org/" + }, + { + "AdminGroup": "relcom.netnews", + "Authorizations": [ + { + "From": "coord@*.relcom.ru", + "Message": "checkgroups", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "newgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + }, + { + "From": "coord@*.relcom.ru", + "Message": "rmgroup", + "Newsgroups": "relcom.*", + "VerificationIdentity": "relcom.newsgroups" + } + ], + "KeyUrl": "ftp://ftp.relcom.ru/pub/relcom/netinfo/coordpubkey.txt", + "Name": "RELCOM", + "Url": "ftp://ftp.relcom.ru/pub/relcom/netinfo/" + }, + { + "AdminGroup": "rocksolid.shared.rocksolid", + "Authorizations": [ + { + "From": "usenet@novabbs.com", + "Message": "checkgroups", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "newgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + }, + { + "From": "usenet@novabbs.com", + "Message": "rmgroup", + "Newsgroups": "rocksolid.*", + "VerificationIdentity": "usenet@novabbs.com" + } + ], + "Contact": "usenet@novabbs.com", + "KeyFingerprint": "FF846D730B72CC0CB69B64802A1C655BF6D10B13", + "KeyUrl": "http://news.novabbs.com/hierarchy/rocksolid.txt", + "Name": "ROCKSOLID", + "Url": "http://news.novabbs.com/hierarchy/" + }, + { + "AdminGroup": "saar.admin.news", + "Authorizations": [ + { + "From": "control@saar-admin-news.de", + "Message": "checkgroups", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "newgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + }, + { + "From": "control@saar-admin-news.de", + "Message": "rmgroup", + "Newsgroups": "saar.*", + "VerificationIdentity": "saar-control" + } + ], + "KeyUrl": "http://www.saar-admin-news.de/saar-control.asc", + "Name": "SAAR", + "Url": "http://www.saar-admin-news.de/" + }, + { + "AdminGroup": "sat.usenet.config", + "Authorizations": [ + { + "From": "satgroup@endicor.com", + "Message": "checkgroups", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "newgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + }, + { + "From": "satgroup@endicor.com", + "Message": "rmgroup", + "Newsgroups": "sat.*", + "VerificationIdentity": "satgroup@endicor.com" + } + ], + "Contact": "satgroup@endicor.com", + "Name": "SAT" + }, + { + "AdminGroup": "schule.admin", + "Authorizations": [ + { + "From": "newsctrl@schule.de", + "Message": "checkgroups", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "newgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + }, + { + "From": "newsctrl@schule.de", + "Message": "rmgroup", + "Newsgroups": "schule.*", + "VerificationIdentity": "schule.konfig" + } + ], + "Contact": "schule-admin@roxel.ms.sub.org", + "KeyFingerprint": "6406F0AEE146850CBDCA0E538B1E73D2", + "KeyUrl": "http://www.afaik.de/usenet/admin/schule/control/schule.asc", + "Name": "SCHULE", + "Url": "http://home.pages.de/~schule-admin/" + }, + { + "AdminGroup": "scot.newsgroups.discuss", + "Authorizations": [ + { + "From": "control@scot.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "newgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + }, + { + "From": "control@scot.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "scot.*", + "VerificationIdentity": "control@scot.news-admin.org" + } + ], + "KeyUrl": "http://scot.news-admin.org/signature.html", + "Name": "SCOT", + "Url": "http://scot.news-admin.org/" + }, + { + "AdminGroup": "scout.admin", + "Authorizations": [ + { + "From": "control@news.scoutnet.org", + "Message": "checkgroups", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "newgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + }, + { + "From": "control@news.scoutnet.org", + "Message": "rmgroup", + "Newsgroups": "scout.*", + "VerificationIdentity": "control@news.scoutnet.org" + } + ], + "KeyUrl": "http://news.scoutnet.org/scout-pgpkey.asc", + "Name": "SCOUT", + "Url": "http://news.scoutnet.org/" + }, + { + "AdminGroup": "se.internet.news.meddelanden", + "Authorizations": [ + { + "From": "usenet@usenet-se.net", + "Message": "checkgroups", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "newgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + }, + { + "From": "usenet@usenet-se.net", + "Message": "rmgroup", + "Newsgroups": "se.*", + "VerificationIdentity": "usenet-se" + } + ], + "Contact": "usenet@usenet-se.net", + "KeyFingerprint": "6803F0FD0CC34E696F0D0C603C586396", + "Name": "SE" + }, + { + "AdminGroup": "sfnet.ryhmat+listat", + "Authorizations": [ + { + "From": "sfnet@*cs.tut.fi", + "Message": "checkgroups", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "newgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + }, + { + "From": "sfnet@*cs.tut.fi", + "Message": "rmgroup", + "Newsgroups": "sfnet.*", + "VerificationIdentity": "sfnet@cs.tut.fi" + } + ], + "Contact": "sfnet@cs.tut.fi", + "KeyFingerprint": "DE7933C2D359D12844E56A0CB6E30E536933A636", + "Name": "SFNET", + "Url": "http://www.cs.tut.fi/sfnet/" + }, + { + "AdminGroup": "si.news.announce.newsgroups", + "Authorizations": [ + { + "From": "news-admin@arnes.si", + "Message": "checkgroups", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "newgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + }, + { + "From": "news-admin@arnes.si", + "Message": "rmgroup", + "Newsgroups": "si.*", + "VerificationIdentity": "si.news.announce.newsgroups" + } + ], + "KeyUrl": "http://www.arnes.si/news/config/", + "Name": "SI", + "Url": "http://www.arnes.si/news/config/" + }, + { + "AdminGroup": "staroffice.admin", + "Authorizations": [ + { + "From": "news@stardivision.de", + "Message": "checkgroups", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "newgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + }, + { + "From": "news@stardivision.de", + "Message": "rmgroup", + "Newsgroups": "staroffice.*", + "VerificationIdentity": "staroffice.admin" + } + ], + "Contact": "news@starnews.sun.com", + "KeyFingerprint": "C63E816F2A19D3847251F91BE3B9B2C9", + "Name": "STAROFFICE", + "SyncableServer": "starnews.sun.com" + }, + { + "AdminGroup": "stgt.net", + "Authorizations": [ + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "checkgroups", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "newgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + }, + { + "From": "stgt-control@news.uni-stuttgart.de", + "Message": "rmgroup", + "Newsgroups": "stgt.*", + "VerificationIdentity": "stgt-control" + } + ], + "Contact": "news@news.uni-stuttgart.de", + "KeyFingerprint": "BAA40A548FF5F51E724851AE0951DE44", + "KeyUrl": "http://news.uni-stuttgart.de/hierarchie/stgt/stgt-control.txt", + "Name": "STGT", + "Url": "http://news.uni-stuttgart.de/hierarchie/stgt/" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "syd.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "SYD", + "Url": "http://syd.news-admin.org/" + }, + { + "AdminGroup": "szaf.admin", + "Authorizations": [ + { + "From": "hirtenrat@szaf.org", + "Message": "rmgroup", + "Newsgroups": "szaf.*", + "VerificationIdentity": "Hirtenrat" + } + ], + "Contact": "hirtenrat@szaf.org", + "KeyUrl": "http://news.szaf.org/szaf/szaf-key.txt", + "Name": "SZAF" + }, + { + "AdminGroup": "thur.net.news.groups", + "Authorizations": [ + { + "From": "usenet@thur.de", + "Message": "checkgroups", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "newgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + }, + { + "From": "usenet@thur.de", + "Message": "rmgroup", + "Newsgroups": "thur.*", + "VerificationIdentity": "thur.net.news.groups" + } + ], + "Contact": "usenet@thur.de", + "KeyFingerprint": "7E3D731393D4CA7839DE3CE737EE22F1", + "Name": "THUR", + "SyncableServer": "news.thur.de", + "Url": "http://www.thur.de/thurnet/old/thurnews.html" + }, + { + "Authorizations": [ + { + "From": "usenet@agate.berkeley.edu", + "Message": "checkgroups", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "newgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + }, + { + "From": "usenet@agate.berkeley.edu", + "Message": "rmgroup", + "Newsgroups": "ucb.*", + "VerificationIdentity": "ucb.news" + } + ], + "Contact": "Chris van den Berg ", + "KeyFingerprint": "96B88E9A9809377D0EEC8188DB9029BF", + "KeyUrl": "http://www.net.berkeley.edu/usenet/usenet.asc", + "Name": "UCB", + "Url": "http://www.net.berkeley.edu/usenet/" + }, + { + "AdminGroup": "ufra.admin", + "KeyFingerprint": "F7AD96D87A3F7E84020C839ADB8FEBB8", + "Name": "UFRA" + }, + { + "AdminGroup": "uk.net.news.announce", + "Authorizations": [ + { + "From": "control@usenet.org.uk", + "Message": "checkgroups", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "newgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + }, + { + "From": "control@usenet.org.uk", + "Message": "rmgroup", + "Newsgroups": "uk.*", + "VerificationIdentity": "uk.net.news.announce" + } + ], + "KeyUrl": "http://www.usenet.org.uk/newsadmins.html", + "Name": "UK", + "Url": "http://www.usenet.org.uk/" + }, + { + "AdminGroup": "un.public.usenet.admin", + "Authorizations": [ + { + "From": "news@news.itu.int", + "Message": "checkgroups", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "newgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + }, + { + "From": "news@news.itu.int", + "Message": "rmgroup", + "Newsgroups": "un.*", + "VerificationIdentity": "ungroups@news.itu.int" + } + ], + "Name": "UN" + }, + { + "AdminGroup": "us.config", + "Authorizations": [ + { + "From": "us-control@lists.killfile.org", + "Message": "checkgroups", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "newgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + }, + { + "From": "us-control@lists.killfile.org", + "Message": "rmgroup", + "Newsgroups": "us.*", + "VerificationIdentity": "us-control@lists.killfile.org" + } + ], + "Contact": "us-control@lists.killfile.org", + "KeyFingerprint": "BB96EB2CCFD075C8E9DEC2C21DA29D87B73CAF1B", + "Name": "US" + }, + { + "AdminGroup": "uw.newsgroups", + "Authorizations": [ + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "checkgroups", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "newgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + }, + { + "From": "newsgroups@news.uwaterloo.ca", + "Message": "rmgroup", + "Newsgroups": "uw.*", + "VerificationIdentity": "uw.newsgroups" + } + ], + "Name": "UW", + "SyncableServer": "news.uwaterloo.ca" + }, + { + "Authorizations": [ + { + "From": "ausadmin@aus.news-admin.org", + "Message": "checkgroups", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "newgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + }, + { + "From": "ausadmin@aus.news-admin.org", + "Message": "rmgroup", + "Newsgroups": "wa.*", + "VerificationIdentity": "ausadmin@aus.news-admin.org" + } + ], + "Contact": "ausadmin@aus.news-admin.org", + "KeyUrl": "http://aus.news-admin.org/ausadmin.asc", + "Name": "WA", + "Url": "http://wa.news-admin.org/" + }, + { + "AdminGroup": "wales.usenet.config", + "Authorizations": [ + { + "From": "control@wales-usenet.org", + "Message": "checkgroups", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "newgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + }, + { + "From": "control@wales-usenet.org", + "Message": "rmgroup", + "Newsgroups": "wales.*", + "VerificationIdentity": "wales-usenet" + } + ], + "Contact": "committee@wales-usenet.org", + "KeyFingerprint": "2D9EDEDF12DA345C49E1EE28E3AB0DAD", + "KeyUrl": "http://www.wales-usenet.org/english/newsadmin.txt", + "Name": "WALES", + "Url": "http://www.wales-usenet.org/" + }, + { + "AdminGroup": "x-privat.info", + "Authorizations": [ + { + "From": "dmitry@x-privat.org", + "Message": "checkgroups", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "newgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + }, + { + "From": "dmitry@x-privat.org", + "Message": "rmgroup", + "Newsgroups": "x-privat.*", + "VerificationIdentity": "dmitry@x-privat.org" + } + ], + "Contact": "dmitry@x-privat.org", + "KeyFingerprint": "9B0A7E682780C796476B039051056843", + "KeyUrl": "http://www.x-privat.org/dmitry.asc", + "Name": "X-PRIVAT", + "Url": "http://www.x-privat.org/" + }, + { + "AdminGroup": "z-netz.koordination.user+sysops", + "Authorizations": [ + { + "From": "teko@dinoex.sub.org", + "Message": "checkgroups", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "newgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + }, + { + "From": "teko@dinoex.sub.org", + "Message": "rmgroup", + "Newsgroups": "z-netz.*", + "VerificationIdentity": "z-netz.koordination.user+sysops" + } + ], + "Contact": "teko@dinoex.sub.org", + "KeyUrl": "ftp://ftp.dinoex.de/pub/keys/z-netz.koordination.user+sysops.asc", + "Name": "Z-NETZ" + } + ], + "Source": { + "InnUrl": "https://raw.githubusercontent.com/InterNetNews/inn/main/samples/control.ctl", + "LastModified": "2023-08-05", + "RetrievedFrom": "InnUrl", + "Url": "https://downloads.isc.org/pub/usenet/CONFIG/control.ctl" + } + } + }, + "Nntpd": { + "AcmeStateDir": "certs/", + "AllowCleartextAuth": true, + "BindAddress": [ + "127.0.0.1" + ], + "BindPort": 11991, + "BindPortTls": 15634, + "GracefulShutdownTimeout": "00:01:00", + "HistoryTime": "02:00:00", + "IdleTime": 300, + "LogDir": "logs/", + "MailComplaintsTo": "abuse@usenet.ninja", + "MaxArticleSize": 5242880, + "ProxyHosts": [], + "ServerId": 1, + "XTraceKey": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "StartupTimeout": null, + "StopHostOnUnexpectedServiceTermination": true, + "Top1000": [ + "top1000@anthologeek.net", + "cknipe@opticnetworks.net" + ], + "Transit": { + "StreamOutstandingArticleDepth": 8 + }, + "Systemd": { + "EnableWatchdog": true, + "NotifyReadyOnApplicationRunning": true, + "NotifyStoppingOnApplicationShutdown": true, + "ReportLifecycleStatus": true, + "WatchdogIntervalFraction": 0.5 + }, + "ArticleIngestion": { + "MaxArticleBytes": 4194304, + "MinWorkers": 2, + "MaxWorkers": 32, + "MaxPublishConcurrency": 32, + "OverviewDbWorkQueueMemoryLimit": 1073741824, + "OverviewDbMinPublisherWorkers": 2, + "OverviewDbMaxPublisherWorkers": 32, + "OverviewDbPublisherBatchSize": 100, + "OverviewDbPublisherShutdownSeconds": 2, + "ScaleIntervalSeconds": 2, + "ScaleUpPressureThreshold": 0.4, + "ScaleDownPressureThreshold": 0.1, + "ScaleUpConsecutiveIntervals": 2, + "ScaleDownConsecutiveIntervals": 3 + }, + "TransitQueueMemoryLimit": 4294967296 + }, + "NntpDb": { + "StartupTimeout": "00:00:15" + }, + "Redis": { + "Host": [ "198.18.0.70" ], + "Port": 6379 + }, + "Serilog": { + "Enrich": [ "FromLogContext" ], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information", + "System": "Warning", + "VectorNNTPD.TransitService": "Debug" + } + }, + "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File", "Serilog.Sinks.Async" ], + "WriteTo": [ + { + "Args": { + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "restrictedToMinimumLevel": "Information" + }, + "Name": "Console" + }, + { + "Args": { + "blockWhenFull": true, + "bufferSize": 50000, + "configure": [ + { + "Args": { + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + "path": "logs/pending-.log", + "restrictedToMinimumLevel": "Debug", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Name": "File" + } + ] + }, + "Name": "Async" + } + ], + "News": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/news-.log", + "retainedFileCountLimit": 14, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + }, + "Inpaths": { + "blockWhenFull": true, + "bufferSize": 50000, + "buffered": true, + "flushToDiskInterval": "00:00:01", + "fileSizeLimitBytes": null, + "path": "logs/inpaths-.log", + "retainedFileCountLimit": 1, + "rollOnFileSizeLimit": false, + "rollingInterval": "Day" + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/GlobalUsings.cs b/tests/VectorNNTPD.TransitService.Tests/GlobalUsings.cs new file mode 100644 index 00000000..ce12bb6c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/GlobalUsings.cs @@ -0,0 +1,4 @@ +global using Xunit; +global using VectorNNTPD.Common.NntpDb; +global using VectorNNTPD.TransitService.Tests.Fixtures; +global using VectorNNTPD.TransitService.Tests.TestDoubles; diff --git a/tests/VectorNNTPD.TransitService.Tests/History/HistoryCheckCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/History/HistoryCheckCommandTests.cs new file mode 100644 index 00000000..ea3f5e16 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/History/HistoryCheckCommandTests.cs @@ -0,0 +1,360 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.History; + +public sealed class HistoryCheckCommandTests +{ + private static readonly IPAddress TransitPeer = IPAddress.Parse("198.18.0.70"); + private static readonly byte[] WantedId = ""u8.ToArray(); + + [Fact] + public async Task DoubleMiss_Returns238_ByteIdenticalToStaticCompose() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + var line = await duplex.ReadClientLineAsync(); + Assert.Equal("238 send article to be transferred", line); + + var expected = NntpResponseCompose.Concat( + NntpResponses.CheckPrefix.Span, + WantedId, + NntpResponses.CheckSuffix.Span); + Assert.Equal(Encoding.ASCII.GetString(expected).TrimEnd('\r', '\n'), line); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.Equal(0, redis.Database.SetCount); + Assert.Equal(0, history.Writes.Count); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + } + + [Fact] + public async Task MemoryHit_Returns431_AndDoesNotQueryRedis() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + _ = await history.LookupAsync(WantedId); + redis.Database.KeyExistsCount = 0; + + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task RedisHit_Returns431_Not238_AndDoesNotWriteRedis() + { + var redis = new FakeRedisService(); + redis.Database.Seed(HistoryRedisKeys.Create(HistoryDigest.FromMessageId(WantedId))); + var history = CreateHistory(redis); + + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + Assert.Equal(0, redis.Database.SetCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task RedisUnavailable_Returns431_Not238() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("down"); + var history = CreateHistory(redis); + + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task RedisRecovery_Returns238Then238_WithoutRecording() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("down"); + var history = CreateHistory(redis); + + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + redis.Recover(); + redis.Database.ExistsException = null; + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + Assert.Equal(0, history.Writes.Count); + Assert.Equal(0, redis.Database.SetCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DoubleMiss_DoesNotWaitForRedisPersistence() + { + var redis = new FakeRedisService(); + redis.Database.BlockSet = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var line = await duplex.ReadClientLineAsync(); + Assert.Equal("238 send article to be transferred", line); + Assert.Equal(0, redis.Database.SetCount); + Assert.Equal(0, history.Writes.Count); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + redis.Database.BlockSet.SetResult(); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(cts.Token); + } + + [Fact] + public async Task RepeatedUnknownCheck_Remains238_AndDoesNotMutateHistory() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string wanted = "238 send article to be transferred"; + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(wanted, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(wanted, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(wanted, await duplex.ReadClientLineAsync()); + + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + Assert.Equal(0, history.Writes.Count); + Assert.Equal(0, redis.Database.SetCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task UnknownCheck_ThenTakeThis_AcceptsAndRemembers() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await HistoryCheckDuplex.CreateAsync(); + var session = duplex.CreateSession(history, queue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + Assert.Equal(0, history.Writes.Count); + + await duplex.WriteClientAsync( + "TAKETHIS \r\n" + + CanonicalArticleText.Destuffed("") + + ".\r\n"); + Assert.Equal( + "239 ", + await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(WantedId))); + Assert.Equal(1, history.Writes.Count); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static HistoryDb CreateHistory(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private sealed class HistoryCheckDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new HistoryCheckDuplex()); + + public NntpSession CreateSession(IHistoryDb history, IArticleIngestionQueue? queue = null) + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(TransitPeer, 40000))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers, + historyDb: history); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/History/HistoryDbTests.cs b/tests/VectorNNTPD.TransitService.Tests/History/HistoryDbTests.cs new file mode 100644 index 00000000..0f9b4bd9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/History/HistoryDbTests.cs @@ -0,0 +1,328 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.History; + +public sealed class HistoryDbTests +{ + private static readonly byte[] MessageId = ""u8.ToArray(); + private static readonly byte[] OtherId = ""u8.ToArray(); + + [Fact] + public void HistoryTime_Default_IsTwoHours() + { + Assert.Equal(TimeSpan.FromHours(2), new VectorNNTPD.TransitService.Configuration.NntpdOptions().HistoryTime); + } + + [Fact] + public void IdenticalMessageIds_ProduceIdenticalKeys() + { + var left = HistoryDigest.FromMessageId(MessageId); + var right = HistoryDigest.FromMessageId(MessageId); + Assert.Equal(left, right); + Assert.Equal(HistoryRedisKeys.Create(left), HistoryRedisKeys.Create(right)); + } + + [Fact] + public void HistoryDigest_MatchesCommonArticleIdBytes() + { + var history = HistoryDigest.FromMessageId(MessageId); + var articleId = VectorNNTPD.Common.Articles.ArticleId.FromMessageId(MessageId); + Span historyBytes = stackalloc byte[HistoryDigest.Length]; + Span articleIdBytes = stackalloc byte[VectorNNTPD.Common.Articles.ArticleId.Length]; + history.CopyTo(historyBytes); + articleId.CopyTo(articleIdBytes); + Assert.True(historyBytes.SequenceEqual(articleIdBytes)); + } + + [Fact] + public void DifferentMessageIds_ProduceDifferentKeys() + { + var left = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)); + var right = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(OtherId)); + Assert.False(left.AsSpan().SequenceEqual(right)); + } + + [Fact] + public void RedisKey_IsBinaryNamespacePlusDigest() + { + var digest = HistoryDigest.FromMessageId(MessageId); + var key = HistoryRedisKeys.Create(digest); + Assert.Equal(HistoryRedisKeys.KeyLength, key.Length); + Assert.True(key.AsSpan().StartsWith(HistoryRedisKeys.NamespacePrefix)); + Assert.Equal("nntpd:hist:"u8.Length + HistoryDigest.Length, key.Length); + + var suffix = key.AsSpan(HistoryRedisKeys.NamespacePrefix.Length); + var reconstructed = HistoryDigest.FromSpan(suffix); + Assert.Equal(digest, reconstructed); + Assert.False(suffix.StartsWith("nntpd:hist:"u8)); + } + + [Fact] + public void Namespace_IsolatesHistoryKeys() + { + var key = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)); + Assert.StartsWith("nntpd:hist:", Encoding.ASCII.GetString(key[..HistoryRedisKeys.NamespacePrefix.Length])); + } + + [Fact] + public async Task MemoryMiss_RedisMiss_ReturnsUnseen_AndRecordsLocal() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unseen, result); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(1, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task MemoryHit_DoesNotQueryRedis() + { + var redis = new FakeRedisService(); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + redis.Database.KeyExistsCount = 0; + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Seen, result); + Assert.Equal(0, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task MemoryMiss_RedisHit_ReturnsSeen_AndWarmsMemory() + { + var redis = new FakeRedisService(); + var digest = HistoryDigest.FromMessageId(MessageId); + redis.Database.Seed(HistoryRedisKeys.Create(digest)); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Seen, result); + Assert.True(history.ContainsLocal(digest)); + Assert.Equal(0, redis.Database.SetCount); + } + + [Fact] + public async Task DoubleMiss_EnqueuesAsyncRedisWrite_WithoutWaiting() + { + var redis = new FakeRedisService(); + redis.Database.BlockSet = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = Create(redis); + + var lookup = history.LookupAsync(MessageId); + var completed = await Task.WhenAny(lookup.AsTask(), Task.Delay(TimeSpan.FromSeconds(1))); + Assert.Same(lookup.AsTask(), completed); + Assert.Equal(HistoryLookupResult.Unseen, await lookup); + Assert.Equal(0, redis.Database.SetCount); + + redis.Database.BlockSet.SetResult(); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.True(redis.Database.Contains(HistoryRedisKeys.Create(HistoryDigest.FromMessageId(MessageId)))); + await writer.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task RedisLookupFailure_IsUnavailable_NotUnseen() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + + var result = await history.LookupAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unavailable, result); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + } + + [Fact] + public async Task RedisLookupFailure_DoesNotPopulateLocal_AndDoesNotHammerRedis() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(MessageId)); + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.True(redis.IsUnavailable); + + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(OtherId)); + Assert.Equal(1, redis.Database.KeyExistsCount); + Assert.Equal(1, redis.BeginRejectCount); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(OtherId))); + } + + [Fact] + public async Task RedisRecovery_RestoresMissAndHit() + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("timeout"); + var history = Create(redis); + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(MessageId)); + + redis.Recover(); + redis.Database.ExistsException = null; + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + redis.Database.Seed(HistoryRedisKeys.Create(HistoryDigest.FromMessageId(OtherId))); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(OtherId)); + } + + [Fact] + public async Task Writer_SetFailure_DoesNotChangeAlreadyIssuedUnseen() + { + var redis = new FakeRedisService(); + redis.Database.SetException = new RedisUnavailableException("write-fail"); + var history = Create(redis); + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await writer.StopAsync(CancellationToken.None); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(MessageId)); + } + + [Fact] + public async Task LocalExpiry_MatchesConfiguredRetention() + { + var clock = new ManualTimeProvider(); + var redis = new FakeRedisService(); + var queue = new HistoryWriteQueue(); + var history = new HistoryDb(redis, TimeSpan.FromMinutes(5), NullLogger.Instance, clock, queue); + _ = await history.LookupAsync(MessageId); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + + clock.Advance(TimeSpan.FromMinutes(5).Add(TimeSpan.FromSeconds(1))); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + } + + [Fact] + public void WriteQueue_DropsWhenFull() + { + var queue = new HistoryWriteQueue(capacity: 1); + var first = HistoryDigest.FromMessageId(MessageId); + var second = HistoryDigest.FromMessageId(OtherId); + Assert.True(queue.TryEnqueue(first)); + Assert.False(queue.TryEnqueue(second)); + } + + [Fact] + public async Task Writer_Stop_DrainsOrCancelsWithoutThrowing() + { + var redis = new FakeRedisService(); + redis.Database.BlockSet = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await writer.StopAsync(cts.Token); + redis.Database.BlockSet.TrySetResult(); + } + + [Fact] + public async Task Writer_RedisWriteFailure_IsObserved() + { + var redis = new FakeRedisService(); + redis.Database.SetException = new RedisUnavailableException("write-fail"); + var history = Create(redis); + _ = await history.LookupAsync(MessageId); + var writer = new HistoryWriteService(history, redis, NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.SetCount == 0) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await writer.StopAsync(CancellationToken.None); + Assert.Equal(1, redis.Database.SetCount); + } + + [Fact] + public async Task Peek_Miss_DoesNotRecordLocal() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + var result = await history.PeekAsync(MessageId); + + Assert.Equal(HistoryLookupResult.Unseen, result); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(1, redis.Database.KeyExistsCount); + } + + [Fact] + public async Task Capture_CountsLookupsHitsMissesAndErrors() + { + var redis = new FakeRedisService(); + var history = Create(redis); + + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(MessageId)); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(MessageId)); + redis.IsUnavailable = true; + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(OtherId)); + + var snapshot = history.Capture(); + Assert.Equal(3, snapshot.Lookups); + Assert.Equal(1, snapshot.Hits); + Assert.Equal(1, snapshot.Misses); + Assert.Equal(1, snapshot.Errors); + Assert.True(snapshot.WaitTicks >= 0); + } + + [Fact] + public async Task Remember_RecordsLocalWithoutExists() + { + var redis = new FakeRedisService(); + var history = Create(redis); + redis.Database.KeyExistsCount = 0; + + history.Remember(MessageId); + + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(MessageId))); + Assert.Equal(0, redis.Database.KeyExistsCount); + var peek = await history.PeekAsync(MessageId); + Assert.Equal(HistoryLookupResult.Seen, peek); + Assert.Equal(0, redis.Database.KeyExistsCount); + } + + private static HistoryDb Create(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/History/HistoryStoreCapacityTests.cs b/tests/VectorNNTPD.TransitService.Tests/History/HistoryStoreCapacityTests.cs new file mode 100644 index 00000000..8ac46164 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/History/HistoryStoreCapacityTests.cs @@ -0,0 +1,135 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.History; + +public sealed class HistoryStoreCapacityTests +{ + [Fact] + public void Add_DoesNotScan_AndHonorsHardCap() + { + const int cap = 2048; + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromHours(2), clock, cap); + + for (var i = 0; i < cap + 256; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + Assert.Equal(cap, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.False(store.Contains(HistoryDigest.FromMessageId(Id(cap + 10)))); + } + + [Fact] + public void Maintain_RemovesExpired_AndFreesSlots_OffCheckPath() + { + const int cap = 64; + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromMinutes(5), clock, cap); + for (var i = 0; i < cap; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + clock.Advance(TimeSpan.FromMinutes(5).Add(TimeSpan.FromSeconds(1))); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.False(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.Equal(cap - 1, store.ApproximateCount); + + var removed = store.Maintain(); + Assert.Equal(1, store.MaintenanceScanCount); + Assert.Equal(cap - 1, removed); + Assert.Equal(0, store.ApproximateCount); + + store.Add(HistoryDigest.FromMessageId(Id(10_000))); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(10_000)))); + Assert.Equal(1, store.MaintenanceScanCount); + } + + [Theory] + [InlineData(10_000)] + [InlineData(100_000)] + public void UniqueInserts_StayBounded_WithoutMaintenanceScan(int count) + { + var store = new LocalHistoryStore(TimeSpan.FromHours(2), TimeProvider.System); + for (var i = 0; i < count; i++) + { + store.Add(HistoryDigest.FromMessageId(Id(i))); + } + + Assert.Equal(count, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(0)))); + Assert.True(store.Contains(HistoryDigest.FromMessageId(Id(count - 1)))); + } + + [Fact] + public void Maintain_DoesNotRemoveRefreshedLiveEntry() + { + var clock = new ManualTimeProvider(); + var store = new LocalHistoryStore(TimeSpan.FromMinutes(5), clock); + var digest = HistoryDigest.FromMessageId(Id(1)); + var observedExpiry = clock.GetUtcNow().Add(TimeSpan.FromMinutes(5)).UtcTicks; + store.Add(digest); + + clock.Advance(TimeSpan.FromMinutes(6)); + store.Add(digest); + + Assert.False(store.TryRemoveObservedExpiry(digest, observedExpiry)); + Assert.Equal(0, store.Maintain()); + Assert.True(store.Contains(digest)); + Assert.Equal(1, store.EntryCountForTests); + Assert.Equal(1, store.ApproximateCount); + } + + [Fact] + public void ConcurrentInserts_NeverExceedHardCap() + { + const int cap = 128; + var store = new LocalHistoryStore(TimeSpan.FromHours(2), TimeProvider.System, cap); + Parallel.For(0, 2_000, i => store.Add(HistoryDigest.FromMessageId(Id(i)))); + + Assert.True(store.EntryCountForTests <= cap); + Assert.True(store.ApproximateCount <= cap); + Assert.Equal(store.EntryCountForTests, store.ApproximateCount); + Assert.Equal(0, store.MaintenanceScanCount); + } + + [Fact] + public async Task MaintenanceService_Stop_CompletesWithoutThrowing() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var service = new HistoryMaintenanceService( + history, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await service.StartAsync(CancellationToken.None); + Assert.NotNull(service.Execution); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, history.Local.MaintenanceScanCount); + } + + private static byte[] Id(int i) => Encoding.ASCII.GetBytes($""); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/HostIntegrationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/HostIntegrationTests.cs new file mode 100644 index 00000000..a1f789f3 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/HostIntegrationTests.cs @@ -0,0 +1,185 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Hosting.Systemd; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting; + +[Collection(SerilogCollection.Name)] +public sealed class HostIntegrationTests +{ + [Fact] + public async Task Host_StartsAndStopsCleanly() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Running, lifecycle.State); + + await host.StopAsync(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_RepeatedStop_IsIdempotent() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + await host.StopAsync(); + + Assert.Equal(ApplicationState.Stopped, host.Services.GetRequiredService().State); + } + + [Fact] + public async Task Host_WithSystemdNotifierFakes_ReportsReadyOnlyAfterRunning() + { + var notify = new FakeSystemdNotifyBridge(); + var runtime = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(); + + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + services.AddSingleton(notify); + services.AddSingleton(runtime); + }); + + Assert.Equal(0, notify.ReadyCount); + await host.StartAsync(); + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + Assert.Equal(1, notify.ReadyCount); + + await host.StopAsync(); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Host_StartupFailure_Propagates_AndLeavesApplicationStopped() + { + using var host = CreateTestHost(services => + { + services.AddSingleton( + new FakeApplicationService("bad", onStart: _ => throw new InvalidOperationException("startup-fail"))); + }); + + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Equal("startup-fail", ex.Message); + + var lifecycle = host.Services.GetRequiredService(); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_StopApplication_TriggersGracefulShutdown() + { + using var host = CreateTestHost(services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + + var lifetime = host.Services.GetRequiredService(); + var lifecycle = host.Services.GetRequiredService(); + + lifetime.StopApplication(); + await host.WaitForShutdownAsync(); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task Host_RespectsGracefulShutdownTimeoutConfiguration() + { + var service = new FakeApplicationService("slow") + { + HoldStopUntilReleased = true, + }; + + using var host = CreateTestHost( + services => + { + services.AddSingleton(service); + }, + configure: options => + { + options.GracefulShutdownTimeout = TimeSpan.FromSeconds(1); + }, + hostShutdownTimeout: TimeSpan.FromSeconds(5)); + + await host.StartAsync(); + + // Host StopAsync should surface the application shutdown timeout from the service manager. + await Assert.ThrowsAsync(() => host.StopAsync()); + Assert.Equal(ApplicationState.Stopped, host.Services.GetRequiredService().State); + } + + [Fact] + public async Task Host_UnexpectedServiceTermination_StopsHost() + { + var service = new FakeApplicationService("fragile", withExecution: true); + + using var host = CreateTestHost(services => + { + services.AddSingleton(service); + }); + + await host.StartAsync(); + + var lifecycle = host.Services.GetRequiredService(); + var shutdown = host.WaitForShutdownAsync(); + + service.CompleteExecution(new InvalidOperationException("boom")); + + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(5)); + var shutdownError = await Assert.ThrowsAsync( + () => shutdown.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.Equal( + "An application service terminated unexpectedly while the application was Running.", + shutdownError.Message); + + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + private static IHost CreateTestHost( + Action? configureServices = null, + Action? configure = null, + TimeSpan? hostShutdownTimeout = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => + { + // Tests avoid noisy console output; exclusivity is asserted separately. + lc.MinimumLevel.Fatal(); + }); + + builder.Services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + if (hostShutdownTimeout is { } timeout) + { + options.ShutdownTimeout = timeout; + } + }); + + builder.Services.AddNntpTransitHosting(configure, includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + + return builder.Build(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/NntpServerRoleIsolationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/NntpServerRoleIsolationTests.cs new file mode 100644 index 00000000..46fd8a61 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/NntpServerRoleIsolationTests.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.DependencyInjection; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Hosting +{ + public sealed class NntpServerRoleIsolationTests + { + [Fact] + public void Reader_hosting_on_transit_executable_throws() + { + var services = new ServiceCollection(); + services.AddLogging(); + var ex = Assert.Throws(() => services.AddNntpReaderHosting()); + Assert.Contains("Reader", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Transit_host_registers_transit_markers_without_reader_post_filter() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddNntpTransitHosting(); + + Assert.Contains( + services, + static d => d.ServiceType == typeof(NntpServerRole) + && Equals(d.ImplementationInstance, NntpServerRole.Transit)); + Assert.Contains(services, static d => d.ImplementationType == typeof(TransitCatalogueService)); + Assert.Contains(services, static d => d.ImplementationType == typeof(TransitPeerAuthorization)); + Assert.DoesNotContain(services, static d => d.ImplementationType == typeof(PostFilterPolicyService)); + Assert.DoesNotContain(services, static d => d.ImplementationType == typeof(SessionStateService)); + + Assert.Contains( + services, + static d => d.ServiceType == typeof(INntpAuthenticationProvider) + && d.ImplementationFactory is not null); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs new file mode 100644 index 00000000..c7726db5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/ObsoleteConsoleFormatterCleanupTests.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Hosting; + +[Collection(SerilogCollection.Name)] +public sealed class ObsoleteConsoleFormatterCleanupTests +{ + [Fact] + public void RemoveObsoleteMicrosoftConsoleFormatterConfiguration_StripsConsoleLoggerOptionsBindings() + { + var services = new ServiceCollection(); + services.Configure(options => + { + options.FormatterName = ConsoleFormatterNames.Systemd; + }); + + Assert.Contains(services, NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + + NntpdServiceCollectionExtensions.RemoveObsoleteMicrosoftConsoleFormatterConfiguration(services); + + Assert.DoesNotContain(services, NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + } + + [Fact] + public void PlatformHosting_AfterAddSystemd_DoesNotRetainMicrosoftConsoleFormatterConfiguration() + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Nntpd:LogDir"] = TestHostFactory.NewTestLogDir(), + }); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + + Assert.DoesNotContain( + builder.Services, + NntpdServiceCollectionExtensions.IsMicrosoftConsoleLoggerOptionsConfiguration); + + using var host = builder.Build(); + + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.Empty(host.Services.GetLoggerProviders()); + + // ConsoleLoggerOptions may still resolve via Options infrastructure defaults, but must not + // carry the systemd MEL formatter name that AddSystemd would otherwise leave behind. + var consoleOptions = host.Services.GetService>(); + Assert.True( + consoleOptions is null + || string.IsNullOrEmpty(consoleOptions.Value.FormatterName) + || !string.Equals( + consoleOptions.Value.FormatterName, + ConsoleFormatterNames.Systemd, + StringComparison.Ordinal)); + } + + [Fact] + public void PlatformHosting_PreservesSerilogConsoleSinkConfiguration() + { + var builder = Host.CreateApplicationBuilder(); + builder.Configuration.AddInMemoryCollection( + new Dictionary + { + ["Nntpd:LogDir"] = TestHostFactory.NewTestLogDir(), + }); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + + using var host = builder.Build(); + var logger = host.Services.GetRequiredService>(); + + // Behavioral check: logging remains functional through Serilog after cleanup. + logger.LogInformation("Serilog console pipeline remains active after formatter cleanup"); + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs new file mode 100644 index 00000000..c9e15d78 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdLifecycleNotifierTests.cs @@ -0,0 +1,134 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +public sealed class SystemdLifecycleNotifierTests +{ + [Fact] + public async Task Readiness_FollowsSuccessfulInitialization_AndIsNotDuplicated() + { + var notify = new FakeSystemdNotifyBridge(); + var service = new FakeApplicationService("svc"); + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + + await notifier.StartAsync(CancellationToken.None); + Assert.Equal(0, notify.ReadyCount); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(1, notify.ReadyCount); + Assert.Contains(notify.Notifications, n => n.StartsWith("STATUS=", StringComparison.Ordinal)); + + // Force another Running notification attempt via duplicate event is not possible; + // stopping and ensuring ready count stays at 1 after shutdown start. + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Readiness_IsNotSentDuringStartup() + { + var notify = new FakeSystemdNotifyBridge(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var service = new FakeApplicationService( + "slow", + onStart: async ct => + { + entered.TrySetResult(); + await release.Task.WaitAsync(ct); + }); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + var start = lifecycle.StartAsync(CancellationToken.None); + await entered.Task; + + Assert.Equal(ApplicationState.Starting, lifecycle.State); + Assert.Equal(0, notify.ReadyCount); + Assert.Contains(notify.Notifications, n => n.Contains("Starting", StringComparison.Ordinal)); + + release.TrySetResult(); + await start; + + Assert.Equal(1, notify.ReadyCount); + } + + [Fact] + public async Task Readiness_IsNotSentAfterFailedStartup() + { + var notify = new FakeSystemdNotifyBridge(); + var service = new FakeApplicationService( + "bad", + onStart: _ => throw new InvalidOperationException("boom")); + + await using var lifecycle = TestHostFactory.CreateLifecycle([service]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => lifecycle.StartAsync(CancellationToken.None)); + + Assert.Equal(0, notify.ReadyCount); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Shutdown_PreventsSubsequentReadinessReporting() + { + var notify = new FakeSystemdNotifyBridge(); + await using var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.StoppingCount); + Assert.True(lifecycle.ShutdownRequested); + + // A second stop is idempotent and must not emit another readiness notification. + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(1, notify.ReadyCount); + Assert.Equal(1, notify.StoppingCount); + } + + [Fact] + public async Task Notifications_AreNoOpsWhenBridgeDisabled() + { + var notify = new FakeSystemdNotifyBridge { IsEnabled = false }; + await using var lifecycle = TestHostFactory.CreateLifecycle([new FakeApplicationService("svc")]); + using var notifier = CreateNotifier(lifecycle, notify); + await notifier.StartAsync(CancellationToken.None); + + await lifecycle.StartAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + + Assert.Empty(notify.Notifications); + } + + private static SystemdLifecycleNotifier CreateNotifier( + ApplicationLifecycle lifecycle, + FakeSystemdNotifyBridge notify, + Action? configure = null) + { + var options = TestHostFactory.CreateOptions(); + configure?.Invoke(options); + + return new SystemdLifecycleNotifier( + lifecycle, + notify, + RecordingSystemdRuntimeFactory.LinuxUnderSystemd(), + Options.Create(options), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs new file mode 100644 index 00000000..fa2fea2d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdRuntimeDetectionTests.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +public sealed class SystemdRuntimeDetectionTests +{ + [Fact] + public void NonLinux_DisablesWatchdogEvenIfEnvironmentLooksConfigured() + { + using var scope = new EnvironmentVariableScope( + (SystemdRuntime.WatchdogUsecVariable, "5000000"), + (SystemdRuntime.WatchdogPidVariable, Environment.ProcessId.ToString())); + + var runtime = new SystemdRuntime([], NullLogger.Instance); + + if (runtime.IsLinux) + { + // On a real Linux CI agent this assertion is skipped; detection is covered by fakes. + return; + } + + Assert.False(runtime.IsLinux); + Assert.False(runtime.IsSystemdService); + Assert.False(runtime.IsWatchdogConfigured); + Assert.Null(runtime.WatchdogTimeout); + } + + [Fact] + public void MissingWatchdogConfiguration_IsSafe() + { + using var scope = new EnvironmentVariableScope( + (SystemdRuntime.WatchdogUsecVariable, null), + (SystemdRuntime.WatchdogPidVariable, null)); + + var fake = RecordingSystemdRuntimeFactory.LinuxConsole(); + Assert.False(fake.IsWatchdogConfigured); + Assert.Null(fake.WatchdogTimeout); + } + + [Fact] + public void InvalidWatchdogUsec_IsIgnored_OnLinuxPath() + { + // Parse behavior is exercised via interval helper + fake runtimes offline. + // Malformed WATCHDOG_USEC must never invent an interval. + var fake = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(watchdog: null); + Assert.False(fake.IsWatchdogConfigured); + } + + [Fact] + public void FakeDetection_DistinguishesSystemdFromConsoleAndNonLinux() + { + var underSystemd = RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromSeconds(5)); + var console = RecordingSystemdRuntimeFactory.LinuxConsole(); + var windows = RecordingSystemdRuntimeFactory.NonLinux(); + + Assert.True(underSystemd.IsLinux && underSystemd.IsSystemdService && underSystemd.IsWatchdogConfigured); + Assert.True(console.IsLinux && !console.IsSystemdService && !console.IsWatchdogConfigured); + Assert.True(!windows.IsLinux && !windows.IsSystemdService && !windows.IsWatchdogConfigured); + } +} + +internal sealed class EnvironmentVariableScope : IDisposable +{ + private readonly (string Name, string? Previous)[] _previous; + + public EnvironmentVariableScope(params (string Name, string? Value)[] values) + { + _previous = values + .Select(v => (v.Name, Environment.GetEnvironmentVariable(v.Name))) + .ToArray(); + + foreach (var (name, value) in values) + { + Environment.SetEnvironmentVariable(name, value); + } + } + + public void Dispose() + { + foreach (var (name, previous) in _previous) + { + Environment.SetEnvironmentVariable(name, previous); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdTestDoubles.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdTestDoubles.cs new file mode 100644 index 00000000..87d9bc2c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdTestDoubles.cs @@ -0,0 +1,140 @@ +using VectorNNTPD.TransitService.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +internal sealed class FakeSystemdNotifyBridge : ISystemdNotifyBridge +{ + private readonly object _sync = new(); + private readonly List _notifications = []; + private int _readyCount; + private int _stoppingCount; + private int _watchdogCount; + + public bool IsEnabled { get; set; } = true; + + public Exception? ThrowOnNotify { get; set; } + + public IReadOnlyList Notifications + { + get + { + lock (_sync) + { + return _notifications.ToArray(); + } + } + } + + public int ReadyCount => Volatile.Read(ref _readyCount); + + public int StoppingCount => Volatile.Read(ref _stoppingCount); + + public int WatchdogCount => Volatile.Read(ref _watchdogCount); + + public TaskCompletionSource ReadySent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StoppingSent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource WatchdogSent { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void NotifyReady() + { + Record("READY=1", ref _readyCount); + ReadySent.TrySetResult(); + } + + public void NotifyStopping() + { + Record("STOPPING=1", ref _stoppingCount); + StoppingSent.TrySetResult(); + } + + public void NotifyStatus(string status) + { + if (ThrowOnNotify is not null) + { + throw ThrowOnNotify; + } + + if (!IsEnabled) + { + return; + } + + lock (_sync) + { + _notifications.Add("STATUS=" + status); + } + } + + public void NotifyWatchdog() + { + Record("WATCHDOG=1", ref _watchdogCount); + WatchdogSent.TrySetResult(); + } + + private void Record(string value, ref int counter) + { + if (ThrowOnNotify is not null) + { + throw ThrowOnNotify; + } + + if (!IsEnabled) + { + return; + } + + lock (_sync) + { + _notifications.Add(value); + } + + Interlocked.Increment(ref counter); + } +} + +internal sealed class FakeSystemdRuntime : ISystemdRuntime +{ + public bool IsLinux { get; set; } + + public bool IsSystemdService { get; set; } + + public bool IsNotifyEnabled { get; set; } + + public TimeSpan? WatchdogTimeout { get; set; } + + public bool IsWatchdogConfigured => WatchdogTimeout is not null; +} + +internal sealed class FakeApplicationHealth : IApplicationHealth +{ + public bool IsHealthyForWatchdog { get; set; } +} + +internal sealed class RecordingSystemdRuntimeFactory +{ + public static FakeSystemdRuntime LinuxUnderSystemd(TimeSpan? watchdog = null) => new() + { + IsLinux = true, + IsSystemdService = true, + IsNotifyEnabled = true, + WatchdogTimeout = watchdog, + }; + + public static FakeSystemdRuntime LinuxConsole() => new() + { + IsLinux = true, + IsSystemdService = false, + IsNotifyEnabled = false, + WatchdogTimeout = null, + }; + + public static FakeSystemdRuntime NonLinux() => new() + { + IsLinux = false, + IsSystemdService = false, + IsNotifyEnabled = false, + WatchdogTimeout = null, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs new file mode 100644 index 00000000..00883912 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogIntervalTests.cs @@ -0,0 +1,34 @@ +using VectorNNTPD.TransitService.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +public sealed class SystemdWatchdogIntervalTests +{ + [Fact] + public void Calculate_UsesConfiguredFraction_AndLeavesMargin() + { + var interval = SystemdWatchdogInterval.Calculate(TimeSpan.FromSeconds(10), fraction: 0.5); + + Assert.Equal(TimeSpan.FromSeconds(5), interval); + Assert.True(interval < TimeSpan.FromSeconds(10)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1)] + [InlineData(1.5)] + [InlineData(double.NaN)] + public void Calculate_RejectsInvalidFraction(double fraction) + { + Assert.Throws( + () => SystemdWatchdogInterval.Calculate(TimeSpan.FromSeconds(10), fraction)); + } + + [Fact] + public void Calculate_RejectsNonPositiveTimeout() + { + Assert.Throws( + () => SystemdWatchdogInterval.Calculate(TimeSpan.Zero)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs new file mode 100644 index 00000000..8b7f6692 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Hosting/Systemd/SystemdWatchdogServiceTests.cs @@ -0,0 +1,240 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting.Systemd; + +namespace VectorNNTPD.TransitService.Tests.Hosting.Systemd; + +public sealed class SystemdWatchdogServiceTests +{ + [Fact] + public async Task Watchdog_DisabledWhenNotConfigured() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(watchdog: null), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + Assert.Equal(0, notify.WatchdogCount); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_DisabledWhenOptionFalse() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromSeconds(1)), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }, + configure: o => o.Systemd.EnableWatchdog = false); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_DisabledOnNonLinux() + { + var notify = new FakeSystemdNotifyBridge(); + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.NonLinux(), + health: new FakeApplicationHealth { IsHealthyForWatchdog = true }); + + await service.StartAsync(CancellationToken.None); + Assert.False(service.IsActive); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_SendsHeartbeatsWhileHealthy_AndStopsOnShutdown() + { + var notify = new FakeSystemdNotifyBridge(); + var health = new FakeApplicationHealth { IsHealthyForWatchdog = true }; + var lifetime = new TestHostApplicationLifetime(); + + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(100)), + health: health, + hostLifetime: lifetime, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsActive); + Assert.True(service.HeartbeatInterval > TimeSpan.Zero); + Assert.True(service.HeartbeatInterval < TimeSpan.FromMilliseconds(100)); + + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(notify.WatchdogCount >= 1); + + await service.StopAsync(CancellationToken.None); + var countAfterStop = notify.WatchdogCount; + Assert.Equal(countAfterStop, notify.WatchdogCount); + Assert.False(service.IsActive); + } + + [Fact] + public async Task Watchdog_DoesNotReportHealthyDuringStartupOrUnhealthyPeriods() + { + var notify = new FakeSystemdNotifyBridge(); + var health = new FakeApplicationHealth { IsHealthyForWatchdog = false }; + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(80)), + health: health, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsActive); + + var unhealthyGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = Task.Run(async () => + { + await Task.Delay(service.HeartbeatInterval + TimeSpan.FromMilliseconds(30)); + unhealthyGate.TrySetResult(); + }); + await unhealthyGate.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, notify.WatchdogCount); + + health.IsHealthyForWatchdog = true; + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(notify.WatchdogCount >= 1); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_StopsAfterFatalSupervisedServiceFailure() + { + var appService = new FakeApplicationService("fragile", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([appService]); + await lifecycle.StartAsync(CancellationToken.None); + + var health = new ApplicationHealth(lifecycle); + Assert.True(health.IsHealthyForWatchdog); + + var notify = new FakeSystemdNotifyBridge(); + using var watchdog = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(80)), + health: health, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await watchdog.StartAsync(CancellationToken.None); + await notify.WatchdogSent.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + appService.CompleteExecution(new InvalidOperationException("fatal")); + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(health.IsHealthyForWatchdog); + + var count = notify.WatchdogCount; + var settle = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _ = Task.Run(async () => + { + await Task.Delay(watchdog.HeartbeatInterval + TimeSpan.FromMilliseconds(40)); + settle.TrySetResult(); + }); + await settle.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(count, notify.WatchdogCount); + + await watchdog.StopAsync(CancellationToken.None); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Watchdog_NotificationFailure_IsSurfacedAndStopsHost() + { + var notify = new FakeSystemdNotifyBridge + { + ThrowOnNotify = new InvalidOperationException("notify-failed"), + }; + var health = new FakeApplicationHealth { IsHealthyForWatchdog = true }; + var lifetime = new TestHostApplicationLifetime(); + + using var service = CreateWatchdog( + notify, + runtime: RecordingSystemdRuntimeFactory.LinuxUnderSystemd(TimeSpan.FromMilliseconds(60)), + health: health, + hostLifetime: lifetime, + configure: o => o.Systemd.WatchdogIntervalFraction = 0.5); + + await service.StartAsync(CancellationToken.None); + await lifetime.StoppingRequested.Task.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.True(lifetime.StopRequested); + + await service.StopAsync(CancellationToken.None); + Assert.False(service.IsActive); + } + + [Fact] + public async Task ApplicationHealth_TracksLifecycleAndUnexpectedTermination() + { + var appService = new FakeApplicationService("svc", withExecution: true); + await using var lifecycle = TestHostFactory.CreateLifecycle([appService]); + var health = new ApplicationHealth(lifecycle); + + Assert.False(health.IsHealthyForWatchdog); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.True(health.IsHealthyForWatchdog); + + appService.CompleteExecution(new InvalidOperationException("fatal")); + await lifecycle.UnexpectedTermination.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(health.IsHealthyForWatchdog); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.False(health.IsHealthyForWatchdog); + Assert.True(lifecycle.ShutdownRequested); + } + + private static SystemdWatchdogService CreateWatchdog( + FakeSystemdNotifyBridge notify, + ISystemdRuntime runtime, + IApplicationHealth health, + IHostApplicationLifetime? hostLifetime = null, + Action? configure = null) + { + var options = TestHostFactory.CreateOptions(); + configure?.Invoke(options); + + return new SystemdWatchdogService( + runtime, + notify, + health, + hostLifetime ?? new TestHostApplicationLifetime(), + Options.Create(options), + NullLogger.Instance); + } +} + +internal sealed class TestHostApplicationLifetime : IHostApplicationLifetime +{ + private readonly CancellationTokenSource _started = new(); + private readonly CancellationTokenSource _stopping = new(); + private readonly CancellationTokenSource _stopped = new(); + + public CancellationToken ApplicationStarted => _started.Token; + + public CancellationToken ApplicationStopping => _stopping.Token; + + public CancellationToken ApplicationStopped => _stopped.Token; + + public bool StopRequested { get; private set; } + + public TaskCompletionSource StoppingRequested { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void StopApplication() + { + StopRequested = true; + StoppingRequested.TrySetResult(); + _stopping.Cancel(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/FileLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/FileLoggingTests.cs new file mode 100644 index 00000000..baf26d6e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/FileLoggingTests.cs @@ -0,0 +1,587 @@ +using System.Collections; +using System.Globalization; +using System.IO.Compression; +using System.Reflection; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Serilog; +using Serilog.Core; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class FileLoggingTests +{ + [Fact] + public void ProductionAppsettings_DeclaresAsyncFileSinkContract() + { + using var doc = JsonDocument.Parse(File.ReadAllText(FindProductionAppsettings())); + var async = doc.RootElement.GetProperty("Serilog").GetProperty("WriteTo")[1]; + Assert.Equal("Async", async.GetProperty("Name").GetString()); + Assert.Equal(50000, async.GetProperty("Args").GetProperty("bufferSize").GetInt32()); + Assert.True(async.GetProperty("Args").GetProperty("blockWhenFull").GetBoolean()); + var file = async.GetProperty("Args").GetProperty("configure")[0]; + Assert.Equal("File", file.GetProperty("Name").GetString()); + var args = file.GetProperty("Args"); + Assert.Equal("Debug", args.GetProperty("restrictedToMinimumLevel").GetString()); + Assert.Equal("Day", args.GetProperty("rollingInterval").GetString()); + // 14 daily files including the active file. .log and .log.gz for one day count once. + Assert.Equal(14, args.GetProperty("retainedFileCountLimit").GetInt32()); + Assert.True(args.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", args.GetProperty("flushToDiskInterval").GetString()); + Assert.False(args.GetProperty("rollOnFileSizeLimit").GetBoolean()); + Assert.Equal(JsonValueKind.Null, args.GetProperty("fileSizeLimitBytes").ValueKind); + Assert.False(args.TryGetProperty("hooks", out _)); + Assert.Equal( + "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}", + args.GetProperty("outputTemplate").GetString()); + Assert.Equal("logs/pending-.log", args.GetProperty("path").GetString()); + var news = doc.RootElement.GetProperty("Serilog").GetProperty("News"); + Assert.True(news.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", news.GetProperty("flushToDiskInterval").GetString()); + Assert.False(news.TryGetProperty("hooks", out _)); + var inpaths = doc.RootElement.GetProperty("Serilog").GetProperty("Inpaths"); + Assert.True(inpaths.GetProperty("buffered").GetBoolean()); + Assert.Equal("00:00:01", inpaths.GetProperty("flushToDiskInterval").GetString()); + Assert.False(inpaths.TryGetProperty("hooks", out _)); + var usingNames = doc.RootElement.GetProperty("Serilog").GetProperty("Using") + .EnumerateArray().Select(static e => e.GetString()).ToArray(); + Assert.Contains("Serilog.Sinks.File", usingNames); + Assert.Contains("Serilog.Sinks.Async", usingNames); + Assert.DoesNotContain("Serilog.Sinks.File.Archive", usingNames); + Assert.Equal("-.log", NntpdFileLogging.RollingPathSuffix); + Assert.Equal(".gz", NntpdFileLogging.GzipArchiveSuffix); + } + + [Fact] + public void BindResolvedFilePath_OverwritesProductionJsonPlaceholderFromLogDir() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.TransitService", + }); + + NntpdFileLogging.BindResolvedFilePath(configuration); + + var expected = NntpdFileLogging.RollingFilePath( + logDir, + ApplicationJsonConfiguration.EntryAssemblyName); + Assert.Equal(expected, configuration["Serilog:WriteTo:1:Args:configure:0:Args:path"]); + Assert.DoesNotContain("VectorNNTPD.TransitService-", configuration["Serilog:WriteTo:1:Args:configure:0:Args:path"], StringComparison.Ordinal); + Assert.Equal(NntpdNewsLogging.NewsRollingFilePath(logDir), configuration["Serilog:News:path"]); + Assert.Equal(NntpdPathSurveyLogging.PathSurveyRollingFilePath(logDir), configuration["Serilog:Inpaths:path"]); + Assert.True(Directory.Exists(logDir)); + } + finally + { + TryDelete(logDir); + } + } + + [Fact] + public void ProductionJson_BindsUnlimitedDailyAsyncGzipFileSink() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.TransitService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + var loggerConfiguration = new LoggerConfiguration() + .ReadFrom.Configuration(NntpdConfiguredSinks.WithoutWriteTo(configuration)); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + + var sinks = WalkLogEventSinks(logger).ToArray(); + var fileSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.Null(ReadInstanceField(fileSink, "_fileSizeLimitBytes")); + Assert.False(Assert.IsType(ReadInstanceField(fileSink, "_rollOnFileSizeLimit")!)); + Assert.Equal(14, ReadInstanceField(fileSink, "_retainedFileCountLimit")); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_buffered")!)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + + var asyncSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); + var queue = ReadInstanceField(asyncSink, "_queue") + ?? throw new InvalidOperationException("BackgroundWorkerSink._queue was not found."); + var boundedCapacity = queue.GetType().GetProperty("BoundedCapacity")?.GetValue(queue) + ?? throw new InvalidOperationException("BoundedCapacity was not found."); + Assert.Equal(50000, Convert.ToInt32(boundedCapacity, CultureInfo.InvariantCulture)); + AssertFlushesOncePerSecond(sinks); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void DirectRegistration_MatchesProductionAsyncGzipFileSink() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.TransitService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + var levels = NntpdConfiguredSinks.WithoutWriteTo(configuration); + Assert.Empty(levels.GetSection("Serilog:WriteTo").GetChildren()); + Assert.Equal("Information", levels["Serilog:MinimumLevel:Default"]); + Assert.Equal("Debug", levels["Serilog:MinimumLevel:Override:VectorNNTPD.TransitService"]); + + var loggerConfiguration = new LoggerConfiguration() + .ReadFrom.Configuration(levels); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + + var sinks = WalkLogEventSinks(logger).ToArray(); + var fileSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.Null(ReadInstanceField(fileSink, "_fileSizeLimitBytes")); + Assert.False(Assert.IsType(ReadInstanceField(fileSink, "_rollOnFileSizeLimit")!)); + Assert.Equal(14, ReadInstanceField(fileSink, "_retainedFileCountLimit")); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_buffered")!)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + + var asyncSink = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); + var queue = ReadInstanceField(asyncSink, "_queue") + ?? throw new InvalidOperationException("BackgroundWorkerSink._queue was not found."); + var boundedCapacity = queue.GetType().GetProperty("BoundedCapacity")?.GetValue(queue) + ?? throw new InvalidOperationException("BoundedCapacity was not found."); + Assert.Equal(50000, Convert.ToInt32(boundedCapacity, CultureInfo.InvariantCulture)); + AssertFlushesOncePerSecond(sinks); + Assert.Single(sinks, static n => n.GetType().Name.Equals("ConsoleSink", StringComparison.Ordinal)); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void DirectRegistration_OmitsFileWhenOnlyConsoleIsConfigured() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Warning", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + }) + .Build(); + + var loggerConfiguration = new LoggerConfiguration(); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + var sinks = WalkLogEventSinks(logger).ToArray(); + Assert.Single(sinks, static n => n.GetType().Name.Equals("ConsoleSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)); + Assert.DoesNotContain(sinks, static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); + } + + [Fact] + public void DirectRegistration_LeavesUnsupportedFileArgumentsOnConfigurationBinding() + { + var logDir = CreateTempLogDir(); + try + { + var path = NntpdFileLogging.RollingFilePath(logDir, "VectorNNTPD.TransitService"); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Serilog:Using:0"] = "Serilog.Sinks.File", + ["Serilog:WriteTo:0:Name"] = "File", + ["Serilog:WriteTo:0:Args:path"] = path, + ["Serilog:WriteTo:0:Args:shared"] = "true", + ["Serilog:WriteTo:0:Args:rollingInterval"] = "Day", + }) + .Build(); + + var loggerConfiguration = new LoggerConfiguration(); + NntpdConfiguredSinks.Apply(loggerConfiguration, configuration); + using var logger = loggerConfiguration.CreateLogger(); + var fileSink = Assert.Single( + WalkLogEventSinks(logger), + static n => n.GetType().Name.Equals("RollingFileSink", StringComparison.Ordinal)); + Assert.True(Assert.IsType(ReadInstanceField(fileSink, "_shared")!)); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void ProductionNewsAndInpaths_FlushCurrentFileOncePerSecond() + { + var logDir = CreateTempLogDir(); + try + { + using var configuration = new ConfigurationManager(); + configuration.AddJsonFile(FindProductionAppsettings(), optional: false, reloadOnChange: false); + configuration.AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.LogDir)}"] = logDir, + [$"{NntpdOptions.SectionName}:{nameof(NntpdOptions.ApplicationName)}"] = "VectorNNTPD.TransitService", + }); + NntpdFileLogging.BindResolvedFilePath(configuration); + + Assert.Equal(TimeSpan.FromSeconds(1), NntpdNewsLogging.ReadSettings(configuration).FlushToDiskInterval); + Assert.Equal(TimeSpan.FromSeconds(1), NntpdPathSurveyLogging.ReadSettings(configuration).FlushToDiskInterval); + + using var news = NntpdNewsLogging.CreateLogger(configuration); + using var inpaths = NntpdPathSurveyLogging.CreateLogger( + configuration, + new IgnoringCompletedPathSurveyFileHandler(), + NullLogger.Instance); + var newsSinks = WalkLogEventSinks(news).ToArray(); + var inpathsSinks = WalkLogEventSinks(inpaths).ToArray(); + Assert.Equal(14, ReadInstanceField(Assert.Single(newsSinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)), "_retainedFileCountLimit")); + Assert.Equal(1, ReadInstanceField(Assert.Single(inpathsSinks, static n => n.GetType().Name.Equals("GzipRollingFileSink", StringComparison.Ordinal)), "_retainedFileCountLimit")); + AssertFlushesOncePerSecond(newsSinks); + AssertFlushesOncePerSecond(inpathsSinks); + } + finally + { + Log.CloseAndFlush(); + TryDelete(logDir); + } + } + + [Fact] + public void ResolveDirectory_UsesCommonHelperAgainstApplicationBase() + { + var relative = NntpdFileLogging.ResolveDirectory(NntpdOptions.DefaultLogDir); + Assert.Equal( + ApplicationLocalPath.ResolveApplicationLocalPath(NntpdOptions.DefaultLogDir, AppContext.BaseDirectory), + relative); + + var absolute = Path.Join(Path.GetTempPath(), "vectornntp-logdir-abs"); + Assert.Equal(Path.GetFullPath(absolute), NntpdFileLogging.ResolveDirectory(absolute)); + Assert.DoesNotContain("logs/", NntpdFileLogging.RollingFilePath(absolute, "VectorNNTPD.TransitService"), StringComparison.Ordinal); + } + + [Fact] + public void RollingFilePath_UsesApplicationName() + { + var path = NntpdFileLogging.RollingFilePath("C:\\logs", "VectorNNTPD.TransitService"); + Assert.Equal(Path.Join("C:\\logs", "VectorNNTPD.TransitService-.log"), path); + } + + [Fact] + public void CompletedApplicationLog_IsReplacedByGzip_WithTheOriginalBytes() + { + var dir = CreateTempLogDir(); + try + { + var rolled = Path.Join(dir, "VectorNNTPD.TransitService-20260101.log"); + var original = "completed-day"u8.ToArray(); + File.WriteAllBytes(rolled, original); + var expectedName = NntpdFileLogging.GzipArchiveFileName(rolled); + Assert.Equal("VectorNNTPD.TransitService-20260101.log.gz", expectedName); + + Assert.True(new GzipLogFileCompressor().TryCompressAndReplace(rolled)); + + var archive = Path.Join(dir, expectedName); + Assert.True(File.Exists(archive)); + Assert.False(File.Exists(rolled)); + using var gz = new GZipStream(File.OpenRead(archive), CompressionMode.Decompress); + using var reader = new MemoryStream(); + gz.CopyTo(reader); + Assert.Equal(original, reader.ToArray()); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void FailedCompression_DoesNotDeleteTheOriginalLog() + { + var dir = CreateTempLogDir(); + try + { + var rolled = Path.Join(dir, "VectorNNTPD.TransitService-20260102.log"); + File.WriteAllText(rolled, "newer-day"); + using (new FileStream(rolled, FileMode.Open, FileAccess.Read, FileShare.None)) + { + Assert.False(new GzipLogFileCompressor().TryCompressAndReplace(rolled)); + } + + Assert.True(File.Exists(rolled)); + Assert.False(File.Exists(rolled + ".gz")); + Assert.Equal("newer-day", File.ReadAllText(rolled)); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void ConsoleReceivesInformation_ButNotDebug_FileReceivesDebugAndTakeThis() + { + var dir = CreateTempLogDir(); + using var captured = new StringWriter(); + var previous = Console.Out; + Console.SetOut(captured); + try + { + using var host = CreateFileLoggingHost(dir); + var factory = host.Services.GetRequiredService(); + var logger = factory.CreateLogger("VectorNNTPD.TransitService.Session.NntpSession"); + + Assert.True(logger.IsEnabled(LogLevel.Information)); + Assert.True(logger.IsEnabled(LogLevel.Debug)); + Assert.True(logger.IsEnabled(LogLevel.Trace)); + + logger.LogInformation("file-logging-info-marker"); + logger.LogDebug("file-logging-debug-marker"); + CommandLogMessages.CommandRx(logger, "198.51.100.20:9", "TAKETHIS "); + + Log.CloseAndFlush(); + } + finally + { + Console.SetOut(previous); + } + + var console = captured.ToString(); + Assert.Contains("file-logging-info-marker", console, StringComparison.Ordinal); + Assert.DoesNotContain("file-logging-debug-marker", console, StringComparison.Ordinal); + Assert.DoesNotContain("TAKETHIS", console, StringComparison.Ordinal); + + var applicationName = ApplicationJsonConfiguration.EntryAssemblyName; + var daily = Assert.Single( + Directory.GetFiles(dir, applicationName + "-*.log"), + path => Path.GetFileName(path).StartsWith(applicationName + "-", StringComparison.Ordinal) + && path.EndsWith(".log", StringComparison.Ordinal) + && !path.EndsWith(".log.gz", StringComparison.Ordinal)); + Assert.Matches("^" + System.Text.RegularExpressions.Regex.Escape(applicationName) + @"-\d{8}\.log$", Path.GetFileName(daily)); + Assert.DoesNotContain("VectorNNTPD.TransitService-", Path.GetFileName(daily), StringComparison.Ordinal); + Assert.Empty(Directory.GetFiles(dir, "*.gz")); + + var fileText = File.ReadAllText(daily); + Assert.Contains("file-logging-info-marker", fileText, StringComparison.Ordinal); + Assert.Contains("file-logging-debug-marker", fileText, StringComparison.Ordinal); + Assert.Contains("TAKETHIS ", fileText, StringComparison.Ordinal); + + TryDelete(dir); + } + + [Fact] + public void ConfiguredLogDir_IsHonoured_AndCreatedWhenMissing() + { + var parent = Path.Join(Path.GetTempPath(), "vectornntp-logdir-" + Guid.NewGuid().ToString("N")); + var dir = Path.Join(parent, "nested-logs"); + Assert.False(Directory.Exists(dir)); + try + { + using var host = CreateFileLoggingHost(dir); + var logger = host.Services.GetRequiredService() + .CreateLogger("VectorNNTPD.TransitService"); + logger.LogInformation("created-dir-marker"); + Log.CloseAndFlush(); + + Assert.True(Directory.Exists(dir)); + Assert.Contains( + Directory.GetFiles(dir, "*.log"), + path => File.ReadAllText(path).Contains("created-dir-marker", StringComparison.Ordinal)); + } + finally + { + TryDelete(parent); + } + } + + private static IHost CreateFileLoggingHost(string logDir) + { + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + ContentRootPath = AppContext.BaseDirectory, + }); + TestHostFactory.ConfigureNntpdTestHost(builder); + var settings = new Dictionary + { + [$"{NntpdOptions.SectionName}:LogDir"] = logDir, + [$"{NntpdOptions.SectionName}:ApplicationName"] = "VectorNNTPD.TransitService", + ["Serilog:Using:0"] = "Serilog.Sinks.Console", + ["Serilog:Using:1"] = "Serilog.Sinks.File", + ["Serilog:Using:2"] = "Serilog.Sinks.Async", + ["Serilog:MinimumLevel:Default"] = "Information", + ["Serilog:MinimumLevel:Override:Microsoft"] = "Warning", + ["Serilog:MinimumLevel:Override:Microsoft.Hosting.Lifetime"] = "Information", + ["Serilog:MinimumLevel:Override:System"] = "Warning", + ["Serilog:MinimumLevel:Override:VectorNNTPD.TransitService"] = "Verbose", + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Information", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + }; + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + settings[pair.Key] = pair.Value; + } + + builder.Configuration.AddInMemoryCollection(settings); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static string CreateTempLogDir() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-filelog-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } + + private static void TryDelete(string path) + { + try + { + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private static object? ReadInstanceField(object instance, string name) + { + for (var type = instance.GetType(); type is not null; type = type.BaseType) + { + var field = type.GetField(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (field is not null) + { + return field.GetValue(instance); + } + } + + return null; + } + + private static void AssertFlushesOncePerSecond(IEnumerable sinks) + { + var flush = Assert.Single( + sinks, + static n => n.GetType().Name.Equals("PeriodicFileFlushSink", StringComparison.Ordinal)); + var timer = ReadInstanceField(flush, "_timer") + ?? throw new InvalidOperationException("PeriodicFileFlushSink._timer was not found."); + var holder = timer.GetType().GetField("_timer", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(timer) + ?? throw new InvalidOperationException("Timer._timer was not found."); + var queueTimer = holder.GetType().GetField("_timer", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(holder) + ?? throw new InvalidOperationException("TimerHolder._timer was not found."); + var period = queueTimer.GetType().GetField("_period", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(queueTimer) + ?? throw new InvalidOperationException($"{queueTimer.GetType().FullName}._period was not found."); + Assert.Equal(1000u, Assert.IsType(period)); + } + + private sealed class IgnoringCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler + { + public void OnCompletedFile(string completedFilePath) + { + } + } + + private static IEnumerable WalkLogEventSinks(object root) + { + var seen = new HashSet(ReferenceEqualityComparer.Instance); + var pending = new Stack(); + pending.Push(root); + while (pending.Count > 0) + { + var current = pending.Pop(); + if (!seen.Add(current)) + { + continue; + } + + yield return current; + foreach (var value in current.GetType() + .GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Select(field => field.GetValue(current))) + { + if (value is ILogEventSink sink) + { + pending.Push(sink); + continue; + } + + if (value is IEnumerable sinks) + { + foreach (var inner in sinks) + { + pending.Push(inner); + } + } + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/InnNewsTextFormatterTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/InnNewsTextFormatterTests.cs new file mode 100644 index 00000000..8c73b1f3 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/InnNewsTextFormatterTests.cs @@ -0,0 +1,172 @@ +using System.Text; +using Serilog.Events; +using Serilog.Parsing; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +public sealed class InnNewsTextFormatterTests +{ + [Fact] + public void Format_IgnoresMessageTemplate_AndEmitsInnLine() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {NewsDisposition} {Message}"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.SitesProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Aug 25 13:37:41.839 + ? 0 ?\n", writer.ToString()); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_Junk_UsesJ_SpacePaddedDay_AndOmitsSite() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('j')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Jan 5 00:00:00.000 j ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain("newsgroup not carried", line, StringComparison.Ordinal); + Assert.DoesNotContain(" SITE", line, StringComparison.Ordinal); + } + + [Fact] + public void Format_Junk_UsesSuppliedReason_NotDerived() + { + var timestamp = new DateTimeOffset(2024, 1, 5, 0, 0, 0, 0, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('j')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.ReasonProperty, new ScalarValue("peer-only: junk.local")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Jan 5 00:00:00.000 j ? 0 peer-only: junk.local\n", writer.ToString()); + } + + [Fact] + public void Format_Rejected_UsesSuppliedCodeAndReason() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {Message}"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('-')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("<23k82@bar.net>")), + new LogEventProperty(NntpdNewsLogging.ResponseCodeProperty, new ScalarValue(437)), + new LogEventProperty(NntpdNewsLogging.ReasonProperty, new ScalarValue("Poison newsgroup")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal("Aug 25 13:37:54.638 - ? <23k82@bar.net> 0 Poison newsgroup\n", writer.ToString()); + Assert.DoesNotContain("437", writer.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_Moderated_UsesM_NotPlusOrQuestion() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('m')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Aug 25 13:37:41.839 m ? 0\n", line); + Assert.DoesNotContain(" 0 ?", line, StringComparison.Ordinal); + Assert.DoesNotContain(" + ", line, StringComparison.Ordinal); + Assert.NotEqual('?', (char)NewsLogDisposition.Moderated); + } + + [Fact] + public void Format_PreservesMessageIdBytesAsAscii() + { + var timestamp = new DateTimeOffset(2024, 8, 25, 13, 37, 41, 839, TimeSpan.Zero); + var id = ""; + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("?")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue(id)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Contains(id, writer.ToString(), StringComparison.Ordinal); + Assert.EndsWith(id + " 0 ?\n", writer.ToString(), StringComparison.Ordinal); + Assert.Equal(id, Encoding.ASCII.GetString(Encoding.ASCII.GetBytes(id))); + } + + [Fact] + public void Format_NamedPeer_RoundTripsSizeAndOutboundPlaceholder_WithoutSession() + { + var timestamp = new DateTimeOffset(2026, 9, 27, 0, 4, 4, 293, TimeSpan.Zero); + var evt = new LogEvent( + timestamp, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("news"), + [ + new LogEventProperty(NntpdNewsLogging.DispositionProperty, new ScalarValue('+')), + new LogEventProperty(NntpdNewsLogging.FeedProperty, new ScalarValue("BlueWorldHosting")), + new LogEventProperty(NntpdNewsLogging.MessageIdProperty, new ScalarValue("")), + new LogEventProperty(NntpdNewsLogging.SizeProperty, new ScalarValue(1584)), + new LogEventProperty(NntpdNewsLogging.SitesProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnNewsTextFormatter().Format(evt, writer); + Assert.Equal( + "Sep 27 00:04:04.293 + BlueWorldHosting 1584 ?\n", + writer.ToString()); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/InnPathSurveyTextFormatterTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/InnPathSurveyTextFormatterTests.cs new file mode 100644 index 00000000..1233feb6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/InnPathSurveyTextFormatterTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using Serilog.Events; +using Serilog.Parsing; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +public sealed class InnPathSurveyTextFormatterTests +{ + [Fact] + public void Format_IgnoresMessageTemplate_AndEmitsPathLine() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("HACK {Message} {PathSurveyPath}"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue("peer.example!not-for-mail")), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: peer.example!not-for-mail\n", writer.ToString()); + Assert.DoesNotContain("HACK", writer.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Format_EmptyPath_EmitsPrefixAndNewline() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue(string.Empty)), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: \n", writer.ToString()); + } + + [Fact] + public void Format_PreservesExactPathBytesAsLatin1() + { + var path = "AbC!Example.COM!not-for-mail"; + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue(path)), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + Assert.Equal("Path: " + path + "\n", writer.ToString()); + Assert.Equal(path, Encoding.Latin1.GetString(Encoding.Latin1.GetBytes(path))); + } + + [Fact] + public void Format_DoesNotEmitMessageIdNewsgroupsOrDisposition() + { + var evt = new LogEvent( + DateTimeOffset.UtcNow, + LogEventLevel.Information, + exception: null, + new MessageTemplateParser().Parse("inpaths"), + [ + new LogEventProperty(NntpdPathSurveyLogging.PathProperty, new ScalarValue("only.path")), + new LogEventProperty("NewsMessageId", new ScalarValue("")), + new LogEventProperty("NewsDisposition", new ScalarValue('+')), + ]); + + using var writer = new StringWriter(); + new InnPathSurveyTextFormatter().Format(evt, writer); + var line = writer.ToString(); + Assert.Equal("Path: only.path\n", line); + Assert.DoesNotContain("", line, StringComparison.Ordinal); + Assert.DoesNotContain("+", line, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/LoggingPipelineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/LoggingPipelineTests.cs new file mode 100644 index 00000000..a7be9027 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/LoggingPipelineTests.cs @@ -0,0 +1,424 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Logging.Debug; +using Microsoft.Extensions.Options; +using Serilog; +using Serilog.Extensions.Logging; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class LoggingPipelineTests +{ + [Fact] + public void ApplicationInformationAndDebug_AreEnabled_ForVectorNntpCategory() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var root = factory.CreateLogger("VectorNNTPD.TransitService"); + var listener = factory.CreateLogger("VectorNNTPD.TransitService.Networking.Listeners.NntpPlainListenerService"); + + Assert.True(root.IsEnabled(LogLevel.Information)); + Assert.True(root.IsEnabled(LogLevel.Debug)); + Assert.True(listener.IsEnabled(LogLevel.Information)); + Assert.True(listener.IsEnabled(LogLevel.Debug)); + } + + [Fact] + public void MicrosoftAndSystem_RemainWarning_WhileHostingLifetimeIsInformation() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var microsoft = factory.CreateLogger("Microsoft.Extensions.Options"); + var system = factory.CreateLogger("System.Net.Http"); + var lifetime = factory.CreateLogger("Microsoft.Hosting.Lifetime"); + + Assert.False(microsoft.IsEnabled(LogLevel.Information)); + Assert.True(microsoft.IsEnabled(LogLevel.Warning)); + Assert.False(system.IsEnabled(LogLevel.Information)); + Assert.True(system.IsEnabled(LogLevel.Warning)); + Assert.True(lifetime.IsEnabled(LogLevel.Information)); + } + + [Fact] + public void SerilogFactory_IsExclusive_NoMicrosoftProviders() + { + using var host = CreateProductionConfiguredHost(); + var factories = host.Services.GetServices().ToArray(); + Assert.Single(factories); + Assert.Equal("SerilogLoggerFactory", factories[0].GetType().Name); + var providers = host.Services.GetLoggerProviders(); + Assert.Empty(providers); + Assert.DoesNotContain(providers, static p => p is ConsoleLoggerProvider); + Assert.DoesNotContain(providers, static p => p is DebugLoggerProvider); + } + + [Fact] + public void ConnectionAcceptance_ReachesSerilogSink() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + var logger = host.Services.GetRequiredService>(); + + Assert.True(logger.IsEnabled(LogLevel.Information)); + NetworkingLogMessages.PlainConnectionAccepted(logger, "192.0.2.10:40000"); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("Plain connection accepted", StringComparison.Ordinal)); + Assert.Equal(Serilog.Events.LogEventLevel.Information, evt.Level); + Assert.Equal( + "VectorNNTPD.TransitService.Networking.Listeners.NntpPlainListenerService", + evt.Properties["SourceContext"].ToString().Trim('"')); + Assert.DoesNotContain("password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void FeedDiagnosticsSnapshot_ReachesSerilogSink() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + var logger = host.Services.GetRequiredService>(); + var text = FeedDiagnosticsFormatter.Format( + new FeedDiagnosticsSnapshot + { + CapturedAt = DateTimeOffset.UtcNow, + Interval = TimeSpan.FromSeconds(10), + Peers = [], + Sessions = [], + IntervalArticles = 12, + IntervalBytes = 9_000_000, + }, + includeSessions: false); + + FeedDiagnosticsLogMessages.Snapshot(logger, text); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("FEED interval=", StringComparison.Ordinal)); + Assert.Contains("rate=", evt.RenderMessage(), StringComparison.Ordinal); + Assert.Equal( + "VectorNNTPD.TransitService.Diagnostics.FeedDiagnosticsService", + evt.Properties["SourceContext"].ToString().Trim('"')); + Assert.DoesNotContain("@", evt.RenderMessage(), StringComparison.Ordinal); + Assert.DoesNotContain("password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void LoggingInitialized_ReachesSerilogSink_ThroughILogger() + { + var sink = new CollectingSink(); + using var host = CreateProductionConfiguredHost(sink); + + NntpdLoggingExtensions.WriteLoggingInitialized( + host.Services, + "Production", + AppContext.BaseDirectory); + + var evt = Assert.Single( + sink.Events, + e => e.RenderMessage().Contains("Application logging initialized", StringComparison.Ordinal)); + Assert.Equal(Serilog.Events.LogEventLevel.Information, evt.Level); + Assert.Contains("SerilogLoggerFactory", evt.RenderMessage(), StringComparison.Ordinal); + Assert.Contains(NntpdLogCategories.Hosting, evt.RenderMessage(), StringComparison.Ordinal); + Assert.True(evt.Properties.TryGetValue("Application", out var application)); + Assert.Equal( + "\"" + ApplicationJsonConfiguration.EntryAssemblyName + "\"", + application.ToString()); + Assert.DoesNotContain("not-the-entry-assembly", application.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("CloudFlare", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Password", evt.RenderMessage(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ConfiguredConsoleSink_ReceivesInformationEvent() + { + using var captured = new StringWriter(); + var previous = Console.Out; + Console.SetOut(captured); + try + { + using var host = CreateProductionConfiguredHost(); + var logger = host.Services.GetRequiredService>(); + NetworkingLogMessages.PlainConnectionAccepted(logger, "198.51.100.10:119"); + } + finally + { + Console.SetOut(previous); + } + + var output = captured.ToString(); + Assert.Contains("Plain connection accepted", output, StringComparison.Ordinal); + Assert.Contains("198.51.100.10:119", output, StringComparison.Ordinal); + Assert.DoesNotContain("password", output, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void DateAndTakeThis_AreNotHotPathSuppressed() + { + Assert.False(NntpCommandLogFormat.SuppressHotPathCommand(NntpVerb.Date)); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommandLog("DATE")); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommand(NntpVerb.TakeThis)); + Assert.False(NntpCommandLogFormat.SuppressHotPathCommandLog("TAKETHIS ")); + } + + [Fact] + public async Task LivePlainAccept_AndDate_EmitThroughSerilog() + { + var sink = new CollectingSink(); + using var serilog = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.FromLogContext() + .WriteTo.Sink(sink) + .CreateLogger(); + using var factory = new SerilogLoggerFactory(serilog); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + options.ProxyHosts = []; + + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + factory, + factory.CreateLogger()); + + await service.StartAsync(CancellationToken.None); + var endpoint = service.LocalEndPoints[0]; + using var client = new TcpClient(); + await client.ConnectAsync(endpoint); + await using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + await using var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true) + { + NewLine = "\r\n", + AutoFlush = true, + }; + + var greeting = await reader.ReadLineAsync(); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + + await writer.WriteLineAsync("DATE"); + var date = await reader.ReadLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + await writer.WriteLineAsync("QUIT"); + _ = await reader.ReadLineAsync(); + await service.StopAsync(CancellationToken.None); + + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("Plain connection accepted from", StringComparison.Ordinal) + && e.Properties.ContainsKey("TcpPeer")); + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("RX:", StringComparison.Ordinal) + && e.Properties.TryGetValue("Command", out var rx) + && rx.ToString().Contains("DATE", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("TX:", StringComparison.Ordinal) + && e.Properties.TryGetValue("Command", out var tx) + && tx.ToString().Contains("DATE", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.Properties.TryGetValue("SourceContext", out var sessionCtx) + && sessionCtx.ToString().Contains("VectorNNTPD.TransitService.Session.NntpSession", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.Properties.TryGetValue("SourceContext", out var dateCtx) + && dateCtx.ToString().Contains("VectorNNTPD.TransitService.Session.Commands.Date", StringComparison.Ordinal)); + } + + [Fact] + public void ProductionAssembly_ReferencesFileSinksAndCommon() + { + var names = typeof(NntpdLoggingExtensions).Assembly + .GetReferencedAssemblies() + .Select(static a => a.Name) + .ToHashSet(StringComparer.Ordinal); + Assert.Contains("Serilog.Sinks.Console", names); + Assert.Contains("Serilog.Sinks.File", names); + Assert.Contains("VectorNNTPD.Common", names); + Assert.DoesNotContain("Serilog.Sinks.File.Archive", names); + } + + [Fact] + public void ProductionSerilogSection_ConsoleIsInformationOnly() + { + var section = ProductionSerilogSection(); + Assert.Equal("Serilog.Sinks.Console", section["Serilog:Using:0"]); + Assert.Equal("Console", section["Serilog:WriteTo:0:Name"]); + Assert.Equal("Information", section["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"]); + Assert.Equal("Debug", section["Serilog:MinimumLevel:Override:VectorNNTPD.TransitService"]); + Assert.Equal("Async", section["Serilog:WriteTo:1:Name"]); + Assert.Equal("50000", section["Serilog:WriteTo:1:Args:bufferSize"]); + Assert.Equal("true", section["Serilog:WriteTo:1:Args:blockWhenFull"]); + Assert.Equal("File", section["Serilog:WriteTo:1:Args:configure:0:Name"]); + Assert.Equal("Debug", section["Serilog:WriteTo:1:Args:configure:0:Args:restrictedToMinimumLevel"]); + Assert.Equal("Day", section["Serilog:WriteTo:1:Args:configure:0:Args:rollingInterval"]); + Assert.Equal("14", section["Serilog:WriteTo:1:Args:configure:0:Args:retainedFileCountLimit"]); + Assert.Null(section["Serilog:WriteTo:1:Args:configure:0:Args:fileSizeLimitBytes"]); + } + + [Fact] + public void CommandLoggers_UseHostFactory_NotNullLogger() + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + _ = new NntpCommandDispatcher(factory); + var dateLogger = NntpCommandLoggers.For(typeof(Date)); + Assert.True(dateLogger.IsEnabled(LogLevel.Information)); + Assert.NotEqual(NullLogger.Instance.GetType(), dateLogger.GetType()); + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + } + + [Fact] + public async Task LivePlainAccept_AndDate_AppearOnConfiguredConsole() + { + using var captured = new StringWriter(); + var previous = Console.Out; + NntpPlainListenerService? service = null; + Console.SetOut(captured); + try + { + using var host = CreateProductionConfiguredHost(); + var factory = host.Services.GetRequiredService(); + var logger = host.Services.GetRequiredService>(); + var sessionLogger = factory.CreateLogger(); + var dateLogger = factory.CreateLogger("VectorNNTPD.TransitService.Session.Commands.Date"); + + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + Assert.True(logger.IsEnabled(LogLevel.Information)); + Assert.True(sessionLogger.IsEnabled(LogLevel.Information)); + Assert.True(dateLogger.IsEnabled(LogLevel.Information)); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + options.ProxyHosts = []; + + service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + factory, + logger); + + await service.StartAsync(CancellationToken.None); + var endpoint = service.LocalEndPoints[0]; + using var client = new TcpClient(); + await client.ConnectAsync(endpoint); + await using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, leaveOpen: true); + await using var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true) + { + NewLine = "\r\n", + AutoFlush = true, + }; + + var greeting = await reader.ReadLineAsync(); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + await writer.WriteLineAsync("DATE"); + var date = await reader.ReadLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + await writer.WriteLineAsync("QUIT"); + _ = await reader.ReadLineAsync(); + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + service = null; + } + finally + { + Console.SetOut(previous); + if (service is not null) + { + await service.DisposeAsync(); + } + } + + var output = captured.ToString(); + Assert.Contains("Plain connection accepted", output, StringComparison.Ordinal); + Assert.DoesNotContain("RX: DATE", output, StringComparison.Ordinal); + Assert.DoesNotContain("TX: DATE executed in", output, StringComparison.Ordinal); + Assert.DoesNotContain("TX: DATE [", output, StringComparison.Ordinal); + Assert.DoesNotContain("TAKETHIS", output, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("password", output, StringComparison.OrdinalIgnoreCase); + } + + private static IHost CreateProductionConfiguredHost(CollectingSink? sink = null) + { + var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings + { + ContentRootPath = AppContext.BaseDirectory, + }); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.Configuration.AddInMemoryCollection(ProductionSerilogSection()); + builder.ConfigureNntpdLogging(sink is null + ? null + : lc => + { + lc.WriteTo.Sink(sink); + }); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + return builder.Build(); + } + + private static Dictionary ProductionSerilogSection() + { + var section = new Dictionary + { + ["Serilog:Using:0"] = "Serilog.Sinks.Console", + ["Serilog:Using:1"] = "Serilog.Sinks.File", + ["Serilog:Using:2"] = "Serilog.Sinks.Async", + ["Serilog:MinimumLevel:Default"] = "Information", + ["Serilog:MinimumLevel:Override:Microsoft"] = "Warning", + ["Serilog:MinimumLevel:Override:Microsoft.Hosting.Lifetime"] = "Information", + ["Serilog:MinimumLevel:Override:System"] = "Warning", + ["Serilog:MinimumLevel:Override:VectorNNTPD.TransitService"] = "Debug", + ["Serilog:WriteTo:0:Name"] = "Console", + ["Serilog:WriteTo:0:Args:restrictedToMinimumLevel"] = "Information", + ["Serilog:WriteTo:0:Args:outputTemplate"] = NntpdLoggingExtensions.ConsoleOutputTemplate, + ["Serilog:Enrich:0"] = "FromLogContext", + ["Serilog:Properties:Application"] = "not-the-entry-assembly", + }; + foreach (var pair in NntpdFileLogging.AsyncFileWriteToKeys()) + { + section[pair.Key] = pair.Value; + } + + return section; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/PathSurveyCompletedFileTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/PathSurveyCompletedFileTests.cs new file mode 100644 index 00000000..45004e96 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/PathSurveyCompletedFileTests.cs @@ -0,0 +1,148 @@ +using System.IO.Compression; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +public sealed class PathSurveyCompletedFileTests +{ + [Fact] + public void CompletedFile_IsExposedToHandler_BeforeGzip_AndOriginalBytesAreInTheArchive() + { + var dir = CreateTempLogDir(); + try + { + var handler = new RecordingCompletedPathSurveyFileHandler(); + using (var writer = new SerilogPathSurveyWriter( + Configuration(dir, retention: 5, sizeLimit: 80), + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 8; i++) + { + writer.Write("aaaaaaaaaaaaaaaaaaaaaaaa"u8); + } + } + + Assert.NotEmpty(handler.Completed); + Assert.All(handler.GzipExistedAtHandoff, static existed => Assert.False(existed)); + Assert.All(handler.Completed, static path => Assert.False(File.Exists(path))); + var archives = Directory.GetFiles(dir, "inpaths*.log.gz"); + Assert.NotEmpty(archives); + Assert.NotEmpty(Directory.GetFiles(dir, "inpaths*.log")); + using var gz = new GZipStream(File.OpenRead(archives[0]), CompressionMode.Decompress); + using var reader = new StreamReader(gz); + Assert.Contains("Path: ", reader.ReadToEnd(), StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void HandlerFailure_DoesNotSkipGzip_AndDoesNotDropTheCompletedBytes() + { + var dir = CreateTempLogDir(); + try + { + var handler = new ThrowingCompletedPathSurveyFileHandler(); + using (var writer = new SerilogPathSurveyWriter( + Configuration(dir, retention: 5, sizeLimit: 80), + handler, + NullLogger.Instance)) + { + for (var i = 0; i < 8; i++) + { + writer.Write("bbbbbbbbbbbbbbbbbbbbbbbb"u8); + } + } + + Assert.False(string.IsNullOrEmpty(handler.Path)); + Assert.False(File.Exists(handler.Path)); + var archive = handler.Path + ".gz"; + Assert.True(File.Exists(archive)); + using var gz = new GZipStream(File.OpenRead(archive), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + Assert.Contains("Path: ", reader.ReadToEnd(), StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + [Fact] + public void NewsRoll_CompressesTheCompletedFile_WithoutAPathSurveyHandler() + { + var dir = CreateTempLogDir(); + try + { + using (var writer = new SerilogNewsLogWriter(NewsConfiguration(dir))) + { + writer.Write(new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray())); + writer.Write(new NewsLogEvent(NewsLogDisposition.Accepted, ""u8.ToArray())); + } + + var archives = Directory.GetFiles(dir, "news*.log.gz"); + Assert.NotEmpty(archives); + Assert.All(archives, static archive => Assert.False(File.Exists(archive[..^3]))); + Assert.Contains(Directory.GetFiles(dir), static path => path.EndsWith(".log", StringComparison.OrdinalIgnoreCase) + && !path.EndsWith(".log.gz", StringComparison.OrdinalIgnoreCase)); + using var gz = new GZipStream(File.OpenRead(archives[0]), CompressionMode.Decompress); + using var reader = new StreamReader(gz); + var text = reader.ReadToEnd(); + Assert.Contains("", text, StringComparison.Ordinal); + Assert.DoesNotContain("", text, StringComparison.Ordinal); + } + finally + { + TryDelete(dir); + } + } + + private static IConfiguration Configuration(string dir, int retention, long sizeLimit) => + new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Serilog:Inpaths:path"] = Path.Join(dir, "inpaths-.log"), + ["Serilog:Inpaths:rollingInterval"] = "Infinite", + ["Serilog:Inpaths:retainedFileCountLimit"] = retention.ToString(), + ["Serilog:Inpaths:buffered"] = "false", + ["Serilog:Inpaths:rollOnFileSizeLimit"] = "true", + ["Serilog:Inpaths:fileSizeLimitBytes"] = sizeLimit.ToString(), + }).Build(); + + private static IConfiguration NewsConfiguration(string dir) => + new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Serilog:News:path"] = Path.Join(dir, "news-.log"), + ["Serilog:News:rollingInterval"] = "Infinite", + ["Serilog:News:retainedFileCountLimit"] = "5", + ["Serilog:News:buffered"] = "false", + ["Serilog:News:rollOnFileSizeLimit"] = "true", + ["Serilog:News:fileSizeLimitBytes"] = "1", + }).Build(); + + private static string CreateTempLogDir() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-inpaths-completed-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + return dir; + } + + private static void TryDelete(string dir) + { + try + { + Directory.Delete(dir, recursive: true); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/SerilogLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/SerilogLoggingTests.cs new file mode 100644 index 00000000..851e99ab --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/SerilogLoggingTests.cs @@ -0,0 +1,277 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Logging.Debug; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +[Collection(SerilogCollection.Name)] +public sealed class SerilogLoggingTests +{ + [Fact] + public void ConfigureNntpdLogging_RegistersSerilog_AndRemovesDefaultProviders() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + builder.Services.AddSingleton(new FakeApplicationService("svc")); + + using var host = builder.Build(); + + var factory = host.Services.GetRequiredService(); + Assert.Equal("SerilogLoggerFactory", factory.GetType().Name); + + var providers = host.Services.GetLoggerProviders(); + Assert.DoesNotContain(providers, static p => p is ConsoleLoggerProvider); + Assert.DoesNotContain(providers, static p => p is DebugLoggerProvider); + Assert.DoesNotContain( + providers, + static p => p.GetType().Name.Contains("EventLog", StringComparison.Ordinal)); + + // AddSerilog replaces ILoggerFactory rather than stacking MEL providers. + Assert.Empty(providers); + } + + [Fact] + public void ConfigureNntpdLogging_DoesNotRegisterDuplicateSerilogFactories() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var factories = host.Services.GetServices().ToArray(); + Assert.Single(factories); + Assert.Equal("SerilogLoggerFactory", factories[0].GetType().Name); + } + + [Fact] + public void ApplicationILogger_ResolvesThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink); + + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Structured probe {ProbeId} for {Component}", 42, "lifecycle"); + + var evt = Assert.Single( + sink.Events, + e => e.MessageTemplate.Text.Contains("Structured probe", StringComparison.Ordinal)); + Assert.Equal(LogEventLevel.Information, evt.Level); + Assert.True(evt.Properties.TryGetValue("ProbeId", out var probeId)); + Assert.True(evt.Properties.ContainsKey("Component")); + Assert.Equal("42", probeId.ToString().Trim('"')); + } + + [Fact] + public void StructuredLogging_PreservesExceptions() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink); + var logger = host.Services.GetRequiredService>(); + + var ex = new InvalidOperationException("boom"); + logger.LogError(ex, "Failure in {Phase}", "test"); + + var evt = Assert.Single(sink.Events, e => e.Exception is not null); + Assert.Same(ex, evt.Exception); + Assert.True(evt.Properties.ContainsKey("Phase")); + } + + [Fact] + public async Task LifecycleEvents_FlowThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + + Assert.Contains( + sink.Events, + e => e.MessageTemplate.Text.Contains("lifecycle state transition", StringComparison.OrdinalIgnoreCase) + || e.RenderMessage().Contains("Running", StringComparison.Ordinal)); + Assert.Contains( + sink.Events, + e => e.RenderMessage().Contains("Stopped", StringComparison.Ordinal) + || e.MessageTemplate.Text.Contains("shutdown", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Logger_RemainsAvailableDuringShutdown_AndHostDisposesCleanly() + { + var sink = new CollectingSink(); + var stopLogged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var service = new FakeApplicationService( + "svc", + onStop: ct => + { + // Resolve logger during stop to prove it is still usable. + return Task.CompletedTask; + }); + + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(service); + }); + + await host.StartAsync(); + var logger = host.Services.GetRequiredService>(); + logger.LogInformation("Pre-stop marker"); + + await host.StopAsync(); + logger.LogInformation("Post-stop marker still reachable from resolved instance"); + + Assert.Contains(sink.Events, e => e.RenderMessage().Contains("Pre-stop marker", StringComparison.Ordinal)); + Assert.True(service.StopCount >= 1); + stopLogged.TrySetResult(); + } + + [Fact] + public async Task StartupFailure_IsLogged_AndDoesNotLeakHost() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton( + new FakeApplicationService("bad", onStart: _ => throw new InvalidOperationException("startup-fail"))); + }); + + var ex = await Assert.ThrowsAsync(() => host.StartAsync()); + Assert.Equal("startup-fail", ex.Message); + + Assert.Contains( + sink.Events, + e => e.Exception is InvalidOperationException + || e.RenderMessage().Contains("startup", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void BootstrapLogger_CapturesFatalBeforeHostBuild() + { + var previous = Log.Logger; + try + { + var sink = new CollectingSink(); + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(sink) + .CreateLogger(); + + var boom = new InvalidOperationException("bootstrap-fail"); + Log.Fatal(boom, "Host construction failed for {ApplicationName}", "VectorNNTPD.TransitService"); + + var evt = Assert.Single(sink.Events); + Assert.Equal(LogEventLevel.Fatal, evt.Level); + Assert.Same(boom, evt.Exception); + Assert.True(evt.Properties.ContainsKey("ApplicationName")); + } + finally + { + Log.CloseAndFlush(); + Log.Logger = previous; + } + } + + [Fact] + public void CloseAndFlush_CanBeCalledExactlyOnceWithoutThrowing() + { + var previous = Log.Logger; + try + { + Log.Logger = new LoggerConfiguration() + .WriteTo.Sink(new CollectingSink()) + .CreateLogger(); + + Log.Information("flush-probe"); + Log.CloseAndFlush(); + Log.CloseAndFlush(); + } + finally + { + Log.Logger = previous; + } + } + + [Fact] + public void PlatformHosting_StillRegistersWithSerilogExclusiveLogging() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(); + builder.ConfigureNntpdPlatformHosting(); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + Assert.Equal("SerilogLoggerFactory", host.Services.GetRequiredService().GetType().Name); + Assert.Empty(host.Services.GetLoggerProviders()); + Assert.NotNull(host.Services.GetService()); + } + + [Fact] + public async Task HostedServiceStop_EmitsShutdownLogsThroughSerilog() + { + var sink = new CollectingSink(); + using var host = CreateSerilogHost(sink, services => + { + services.AddSingleton(new FakeApplicationService("svc")); + }); + + await host.StartAsync(); + await host.StopAsync(); + + Assert.Contains( + sink.Events, + e => e.RenderMessage().Contains("stopping", StringComparison.OrdinalIgnoreCase) + || e.RenderMessage().Contains("shutdown", StringComparison.OrdinalIgnoreCase) + || e.MessageTemplate.Text.Contains("Host stopping", StringComparison.OrdinalIgnoreCase)); + } + + private static IHost CreateSerilogHost( + CollectingSink sink, + Action? configureServices = null) + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.Configuration[$"{NntpdOptions.SectionName}:ServerId"] = "1"; + builder.Configuration[$"{NntpdOptions.SectionName}:{NntpdOptions.CloudFlareZoneIdConfigurationKey}"] = + "5811a29d39a0732afb5f160c9b137c3d"; + builder.Configuration[$"{NntpdOptions.SectionName}:DnsSuffix"] = "usenet.ninja"; + builder.ConfigureNntpdLogging(lc => + { + lc.MinimumLevel.Verbose(); + lc.WriteTo.Sink(sink); + }); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + configureServices?.Invoke(builder.Services); + return builder.Build(); + } +} + +internal sealed class CollectingSink : ILogEventSink +{ + private readonly ConcurrentQueue _events = new(); + + public IReadOnlyCollection Events => _events.ToArray(); + + public void Emit(LogEvent logEvent) + { + ArgumentNullException.ThrowIfNull(logEvent); + _events.Enqueue(logEvent); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Logging/SourceGeneratedLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Logging/SourceGeneratedLoggingTests.cs new file mode 100644 index 00000000..52e74a47 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Logging/SourceGeneratedLoggingTests.cs @@ -0,0 +1,317 @@ +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Serilog.Events; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.Logging; + +namespace VectorNNTPD.TransitService.Tests.Logging; + +/// +/// Observable contracts for source-generated logging: templates, structured properties, +/// EventIds, levels, and exception preservation. +/// +[Collection(SerilogCollection.Name)] +public sealed class SourceGeneratedLoggingTests +{ + [Fact] + public void CommandRx_PreservesStructuredTemplateAndEventId() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandRx(logger, "198.18.0.70:49860", "HELP"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1600, logger.EventId.Id); + Assert.Equal("[198.18.0.70:49860] RX: HELP", logger.Formatted); + Assert.Equal("198.18.0.70:49860", logger.GetString("Client")); + Assert.Equal("HELP", logger.GetString("Command")); + Assert.DoesNotContain("{Command}", logger.Formatted, StringComparison.Ordinal); + Assert.DoesNotContain("$\"", logger.Formatted, StringComparison.Ordinal); + } + + [Fact] + public void CommandRejected_PreservesStructuredProtocolFields() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandRejected( + logger, + "198.18.0.70:49860", + NntpVerb.Unknown, + NntpVerb.None, + NntpParseStatus.UnknownVerb, + "empty command"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1601, logger.EventId.Id); + Assert.Contains("RX rejected:", logger.Formatted, StringComparison.Ordinal); + Assert.Equal("198.18.0.70:49860", logger.GetString("Client")); + Assert.Equal(NntpVerb.Unknown, logger.GetValue("Verb")); + Assert.Equal(NntpVerb.None, logger.GetValue("Qualifier")); + Assert.Equal(NntpParseStatus.UnknownVerb, logger.GetValue("Status")); + Assert.Equal("empty command", logger.GetString("Detail")); + } + + [Fact] + public void CommandFailed_PreservesException() + { + var logger = new CapturingLogger(); + var boom = new InvalidOperationException("handler-fail"); + + CommandLogMessages.CommandFailed(logger, boom, "198.18.0.70:49860", "DATE"); + + Assert.Equal(LogLevel.Error, logger.Level); + Assert.Equal(1604, logger.EventId.Id); + Assert.Same(boom, logger.Exception); + Assert.Equal("[198.18.0.70:49860] DATE failed", logger.Formatted); + Assert.Equal("DATE", logger.GetString("Command")); + } + + [Fact] + public void CommandTx_PreservesTimingShape() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTx(logger, "198.18.0.70:49860", "HELP", 0.012); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1602, logger.EventId.Id); + Assert.Equal("[198.18.0.70:49860] TX: HELP executed in 0.012s", logger.Formatted); + Assert.Equal(0.012, logger.GetValue("ElapsedSeconds")); + } + + [Fact] + public void CommandTxWithStatus_PreservesStatusLineAndTimingShape() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTxWithStatus( + logger, + "198.18.0.70:49860", + "CHECK", + "238 send article to be transferred", + 0.012); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1615, logger.EventId.Id); + Assert.Equal( + "[198.18.0.70:49860] TX: CHECK [238 send article to be transferred] executed in 0.012s", + logger.Formatted); + Assert.Equal("CHECK", logger.GetString("Command")); + Assert.Equal( + "238 send article to be transferred", + logger.GetString("StatusLine")); + Assert.Equal(0.012, logger.GetValue("ElapsedSeconds")); + } + + [Fact] + public void CommandTxWithStatusAndDetail_PreservesBothStatusAndDetail() + { + var logger = new CapturingLogger(); + + CommandLogMessages.CommandTxWithStatusAndDetail( + logger, + "198.18.0.70:49860", + "STARTTLS", + "382 Continue with TLS negotiation", + 0.001, + "TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384"); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1616, logger.EventId.Id); + Assert.Equal( + "[198.18.0.70:49860] TX: STARTTLS [382 Continue with TLS negotiation] executed in 0.001s [TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384]", + logger.Formatted); + Assert.Equal("382 Continue with TLS negotiation", logger.GetString("StatusLine")); + Assert.Equal("TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384", logger.GetString("Detail")); + } + + [Fact] + public void RedactRxCommand_IsExplicitLoggingStringBoundary() + { + var command = NntpCommandParser.Parse("HELP"u8); + var text = NntpCommandLogFormat.RedactRxCommand(command, "HELP"u8); + + Assert.Equal("HELP", text); + + var pass = NntpCommandParser.Parse("AUTHINFO PASS secret"u8); + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxCommand(pass, "AUTHINFO PASS secret"u8)); + } + + [Fact] + public void DisabledInformation_DoesNotInvokeFormatter() + { + var logger = new CapturingLogger(enabled: false); + + CommandLogMessages.CommandRx(logger, "198.18.0.70:49860", "HELP"); + + Assert.Null(logger.Formatted); + Assert.Empty(logger.Properties); + } + + [Fact] + public void LifecycleTransition_UsesReservedEventId1000() + { + var logger = new CapturingLogger(); + + LifecycleLogMessages.LifecycleTransition( + logger, + ApplicationStateLog.Starting, + ApplicationStateLog.Running); + + Assert.Equal(LogLevel.Information, logger.Level); + Assert.Equal(1000, logger.EventId.Id); + Assert.Equal( + "Application lifecycle state transition: Starting -> Running", + logger.Formatted); + } + + [Fact] + public void SessionEndedWithError_PreservesClientAndException() + { + var logger = new CapturingLogger(); + var boom = new IOException("peer reset"); + var client = IPAddress.Parse("198.18.0.70"); + + SessionLogMessages.SessionEndedWithError(logger, boom, client); + + Assert.Equal(LogLevel.Debug, logger.Level); + Assert.Equal(1611, logger.EventId.Id); + Assert.Same(boom, logger.Exception); + Assert.Equal(client, logger.GetValue("Client")); + } + + [Fact] + public void PlainConnectionAccepted_PreservesEndpointProperty() + { + var logger = new CapturingLogger(); + + NetworkingLogMessages.PlainConnectionAccepted(logger, "198.18.0.70:42122"); + + Assert.Equal(LogLevel.Information, logger.Level); + Assert.Equal(1405, logger.EventId.Id); + Assert.Equal("Plain connection accepted from 198.18.0.70:42122", logger.Formatted); + Assert.Equal("198.18.0.70:42122", logger.GetString("TcpPeer")); + } + + [Fact] + public void GeneratedMessages_FlowThroughSerilogStructuredProperties() + { + var sink = new CollectingSink(); + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(lc => + { + lc.MinimumLevel.Verbose(); + lc.WriteTo.Sink(sink); + }); + + using var host = builder.Build(); + var logger = host.Services.GetRequiredService() + .CreateLogger("VectorNNTPD.TransitService.Tests.SourceGenerated"); + + var boom = new SocketException((int)SocketError.ConnectionReset); + CommandLogMessages.QuitUncaught(logger, boom, "198.18.0.70:49860", boom.GetType().Name, boom.SocketErrorCode); + + var evt = Assert.Single(sink.Events, e => e.Exception is SocketException); + Assert.Equal(LogEventLevel.Error, evt.Level); + Assert.Same(boom, evt.Exception); + Assert.Equal( + "[{Client}] QUIT uncaught during termination: {Type} socket={Socket}", + evt.MessageTemplate.Text); + Assert.True(evt.Properties.ContainsKey("Client")); + Assert.True(evt.Properties.ContainsKey("Type")); + Assert.True(evt.Properties.ContainsKey("Socket")); + Assert.Equal("198.18.0.70:49860", Scalar(evt, "Client")); + } + + [Fact] + public void EventIds_AreUniqueAcrossGeneratedMessageFiles() + { + var ids = new[] + { + 1000, 1001, 1002, 1003, 1004, 1005, + 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, + 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, + 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, + 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, + 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, + 1314, 1315, 1316, 1317, 1318, + 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, + 1500, 1501, 1502, 1503, + 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, + 1615, 1616, + 1700, 1701, 1702, 1703, 1704, 1705, 1706, + 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, + 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, + 1828, 1829, + 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1913, + 1914, 1915, 1916, 1917, 1918, + 2000, 2001, 2002, 2003, 2004, + }; + + Assert.Equal(ids.Length, ids.Distinct().Count()); + } + + private static string Scalar(LogEvent evt, string name) => + evt.Properties[name].ToString().Trim('"'); + + private sealed class CapturingLogger : ILogger + { + private readonly bool _enabled; + + public CapturingLogger(bool enabled = true) => _enabled = enabled; + + public LogLevel? Level { get; private set; } + + public EventId EventId { get; private set; } + + public Exception? Exception { get; private set; } + + public string? Formatted { get; private set; } + + public Dictionary Properties { get; } = new(StringComparer.Ordinal); + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => _enabled; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Level = logLevel; + EventId = eventId; + Exception = exception; + Formatted = formatter(state, exception); + if (state is IEnumerable> pairs) + { + foreach (var pair in pairs) + { + Properties[pair.Key] = pair.Value; + } + } + } + + public string GetString(string name) => + Convert.ToString(Properties[name], CultureInfo.InvariantCulture) ?? string.Empty; + + public T GetValue(string name) => + Assert.IsType(Properties[name]); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs new file mode 100644 index 00000000..ac0cf6a1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Moderation/EmailModerationSubmissionServiceTests.cs @@ -0,0 +1,125 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Tests.Email; + +namespace VectorNNTPD.TransitService.Tests.Moderation; + +public sealed class EmailModerationSubmissionServiceTests +{ + [Fact] + public async Task DisabledEmail_IsUnavailableWithoutSpoolFile() + { + using var harness = new EmailSpoolHarness(enabled: false); + var service = Create(harness, enabled: false); + var result = await service.SubmitAsync(Submission()); + Assert.Equal(ModerationSubmissionStatus.Unavailable, result.Status); + Assert.False(Directory.Exists(harness.Directory)); + Assert.Empty(harness.EmlFiles()); + } + + [Fact] + public async Task EnabledEmail_SpoolIsAccepted() + { + using var harness = new EmailSpoolHarness(); + var service = Create(harness, enabled: true); + var result = await service.SubmitAsync(Submission()); + Assert.Equal(ModerationSubmissionStatus.Accepted, result.Status); + Assert.Contains("spool", result.Detail, StringComparison.Ordinal); + Assert.Single(harness.EmlFiles()); + var bytes = await File.ReadAllBytesAsync(harness.EmlFiles()[0]); + Assert.True(EmailSpoolRecord.TryParse(bytes, out var item)); + Assert.Equal("comp-example@moderators.isc.org", item!.Recipients[0].Address); + Assert.Equal("noreply@example.com", item.EnvelopeSender.Address); + var encoded = System.Text.Encoding.ASCII.GetString(item.EncodedMessage.Span); + Assert.Contains("application/news-transmission; usage=moderate", encoded, StringComparison.Ordinal); + Assert.Contains("X-Moderated-Newsgroup: comp.example", encoded, StringComparison.Ordinal); + Assert.Contains("Newsgroups: comp.example", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Info:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Injection-Date:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("X-Trace:", encoded, StringComparison.Ordinal); + Assert.DoesNotContain("Path:", encoded, StringComparison.Ordinal); + } + + [Fact] + public async Task SpoolWriteFailure_IsFailed() + { + var blocker = Path.Join(Path.GetTempPath(), "vectornntp-moderation-blocker-" + Guid.NewGuid().ToString("N")); + await File.WriteAllBytesAsync(blocker, "x"u8.ToArray()); + try + { + var options = EnabledOptions(); + options.Spool.Directory = blocker; + var spool = new FilesystemEmailSpool(Options.Create(options), NullLogger.Instance); + var service = new EmailModerationSubmissionService( + new EmailService(spool, new Rfc5322MessageEncoder(), Options.Create(options), NullLogger.Instance), + new ModerationEmailComposer(Options.Create(options)), + Options.Create(options)); + Assert.Equal(ModerationSubmissionStatus.Failed, (await service.SubmitAsync(Submission())).Status); + } + finally + { + File.Delete(blocker); + } + } + + [Fact] + public void PercentS_Address_ExpandsInComposerDestination() + { + var composer = new ModerationEmailComposer(Options.Create(EnabledOptions())); + var message = composer.Compose(Submission(moderator: "comp-example@moderators.isc.org")); + Assert.Equal("comp-example@moderators.isc.org", message.To[0].Address); + Assert.Equal("comp.example ", message.Subject); + } + + [Fact] + public void Destuff_RemovesStuffingDots() + { + var destuffed = ModerationEmailComposer.DestuffProtoArticle("..hidden\r\nbody\r\n"u8); + Assert.Equal(".hidden\r\nbody\r\n", System.Text.Encoding.ASCII.GetString(destuffed)); + } + + internal static EmailModerationSubmissionService Create(EmailSpoolHarness harness, bool enabled) + { + var options = enabled ? harness.Options : new EmailOptions(); + if (enabled) + { + options.DefaultFrom = "noreply@example.com"; + options.EnvelopeSender = "noreply@example.com"; + } + + var boxed = Options.Create(options); + return new EmailModerationSubmissionService( + new EmailService(harness.Spool, new Rfc5322MessageEncoder(), boxed, NullLogger.Instance), + new ModerationEmailComposer(boxed), + boxed); + } + + internal static EmailOptions EnabledOptions() => + new() + { + Enabled = true, + DefaultFrom = "noreply@example.com", + EnvelopeSender = "noreply@example.com", + Smtp = new SmtpOptions { Host = "127.0.0.1", Port = 25 }, + Spool = new EmailSpoolOptions + { + Directory = Path.Join(Path.GetTempPath(), "vectornntp-email-unused", Guid.NewGuid().ToString("N")), + }, + }; + + internal static ModerationSubmission Submission(string moderator = "comp-example@moderators.isc.org") => + new() + { + ProtoArticle = "From: poster@example.com\r\nNewsgroups: comp.example\r\nSubject: hi\r\nMessage-ID: \r\nDate: Fri, 25 Sep 2026 00:00:00 +0000\r\n\r\nbody\r\n"u8.ToArray(), + MessageId = "", + Newsgroups = ["comp.example"], + TargetModeratedGroup = "comp.example", + ModeratorAddress = moderator, + AuthenticatedUsername = "alice", + Sender = ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorCatalogueServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorCatalogueServiceTests.cs new file mode 100644 index 00000000..2a625bf4 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorCatalogueServiceTests.cs @@ -0,0 +1,184 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Moderation; + +public sealed class ModeratorCatalogueServiceTests +{ + [Fact] + public async Task StartAsync_LoadsEnabledRows_InModeratorIdOrder() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 2, + GroupPattern = "later.*", + ModeratorAddress = "later@example.com", + AccountName = "later", + }); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "first.*", + ModeratorAddress = "first@example.com", + AccountName = "first", + }); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 3, + GroupPattern = "disabled.*", + ModeratorAddress = "disabled@example.com", + AccountName = "disabled", + Enabled = false, + }); + + await using var catalogue = Create(repository); + await catalogue.StartAsync(CancellationToken.None); + + Assert.Equal(1, repository.QueryCount); + Assert.True(catalogue.HasSnapshot); + Assert.Equal(2, catalogue.Current.Count); + Assert.True(catalogue.Current.TryResolve("first.group"u8, out var first)); + Assert.Equal("first@example.com", first.Address); + Assert.False(catalogue.Current.TryResolve("disabled.group"u8, out _)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenRepositoryFails() + { + var repository = new MemoryNntpModeratorRepository + { + Exception = new NntpDbUnavailableException("down"), + }; + await using var catalogue = Create(repository); + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task Refresh_Failure_RetainsLastKnownGood() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "group.a", + ModeratorAddress = "a@example.com", + AccountName = "mod-a", + }); + var clock = new ControllableTimeProvider(); + await using var catalogue = Create(repository, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var original = catalogue.Current; + + repository.Exception = new NntpDbUnavailableException("down"); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => repository.QueryCount >= 2); + + Assert.Same(original, catalogue.Current); + Assert.True(catalogue.Current.TryResolve("group.a"u8, out _)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_Success_AtomicallyReplacesCurrent() + { + var repository = new MemoryNntpModeratorRepository(); + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 1, + GroupPattern = "old.*", + ModeratorAddress = "old@example.com", + AccountName = "old", + }); + var clock = new ControllableTimeProvider(); + await using var catalogue = Create(repository, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var original = catalogue.Current; + + repository.Add(new MemoryModeratorRecord + { + ModeratorId = 2, + GroupPattern = "new.*", + ModeratorAddress = "new@example.com", + AccountName = "new", + }); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => !ReferenceEquals(original, catalogue.Current)); + + Assert.True(catalogue.Current.TryResolve("new.group"u8, out var identity)); + Assert.Equal("new@example.com", identity.Address); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task ConcurrentPublish_IsSafe() + { + await using var catalogue = Create(new MemoryNntpModeratorRepository()); + var first = ModeratorTestSnapshot.Create(("a.*", "a@example.com", "a")); + var second = ModeratorTestSnapshot.Create(("b.*", "b@example.com", "b")); + Parallel.For(0, 32, i => catalogue.PublishForTests(i % 2 == 0 ? first : second)); + Assert.True(catalogue.HasSnapshot); + Assert.True( + catalogue.Current.TryResolve("a.group"u8, out _) + || catalogue.Current.TryResolve("b.group"u8, out _)); + } + + [Fact] + public void QueryConstant_SelectsEnabledRowsByModeratorId() + { + Assert.Contains("FROM nntpmoderators", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("is_enabled = 'Y'", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("ORDER BY moderator_id", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("group_pattern", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("moderator_address", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + Assert.Contains("account_name", NntpModeratorQueries.SelectEnabledModerators, StringComparison.Ordinal); + } + + [Fact] + public async Task FakeNntpDb_MapsEnabledRowsAndBackendFailure() + { + var factory = new FakeNntpDbConnectionFactory + { + Moderators = + [ + new NntpModeratorRow(1, "comp.foo.*", "moderator@example.org", "MODERATOR01"), + ], + }; + await using var connection = (FakeNntpDbConnection)await factory.OpenAsync("Server=test;", CancellationToken.None); + var rows = await connection.QueryEnabledModeratorsAsync(CancellationToken.None); + var row = Assert.Single(rows); + Assert.Equal(1, row.ModeratorId); + Assert.Equal("comp.foo.*", row.GroupPattern); + Assert.Equal("moderator@example.org", row.ModeratorAddress); + Assert.Equal("MODERATOR01", row.AccountName); + + connection.QueryModeratorsException = new NntpDbUnavailableException("down"); + await Assert.ThrowsAsync( + async () => await connection.QueryEnabledModeratorsAsync(CancellationToken.None)); + } + + private static ModeratorCatalogueService Create( + INntpModeratorRepository repository, + TimeProvider? clock = null, + TimeSpan? interval = null) => + new( + repository, + NullLogger.Instance, + clock ?? TimeProvider.System, + interval ?? ModeratorCatalogueService.RefreshInterval); + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorSnapshotTests.cs b/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorSnapshotTests.cs new file mode 100644 index 00000000..5f8ba6a8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Moderation/ModeratorSnapshotTests.cs @@ -0,0 +1,206 @@ +using System.Text; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Moderation; + +public sealed class ModeratorSnapshotTests +{ + [Fact] + public void FirstMatch_WinsOverLaterGeneralPattern() + { + var auth = ModeratorTestSnapshot.Create( + ("comp.example.moderated", "specific@example.com", "mod-specific"), + ("comp.example.*", "general@example.com", "mod-general")); + + Assert.True(auth.TryResolve(Bytes("comp.example.moderated"), out var identity)); + Assert.Equal("specific@example.com", identity.Address); + Assert.Equal("mod-specific", identity.Username); + } + + [Fact] + public void Resolve_UnknownGroup_Fails() + { + var auth = ModeratorTestSnapshot.Create(("group.a", "a@example.com", "moderator-a")); + Assert.False(auth.TryResolve(Bytes("group.b"), out _)); + } + + [Fact] + public void CanApprove_RequiresMatchingUsernameAndAddress() + { + var auth = Standard(); + Assert.True(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("moderator-b@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorB, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(MapNntpAuthenticationProvider.NormalUser, Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.False(auth.CanApprove(null, Bytes("moderator-a@example.com"), Bytes("group.a"))); + } + + [Fact] + public void CanApprove_AddressComparison_IsCaseInsensitive() + { + var auth = Standard(); + Assert.True(auth.CanApprove(MapNntpAuthenticationProvider.ModeratorA, Bytes("Moderator-A@Example.COM"), Bytes("group.a"))); + } + + [Fact] + public void UsernameComparison_IsOrdinal() + { + var auth = Standard(); + Assert.False(auth.CanApprove("Moderator-A", Bytes("moderator-a@example.com"), Bytes("group.a"))); + Assert.True(auth.IsAuthenticatedModerator(MapNntpAuthenticationProvider.ModeratorA)); + Assert.False(auth.IsAuthenticatedModerator("Moderator-A")); + Assert.False(auth.IsAuthenticatedModerator(MapNntpAuthenticationProvider.NormalUser)); + } + + [Fact] + public void TryAuthorizeApproval_AllModeratedTargetsMustBeCovered() + { + var auth = Standard(); + Assert.True(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com"], + ["group.a"], + out _)); + Assert.False(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com"], + ["group.a", "group.b"], + out var detail)); + Assert.Equal("unauthorized moderator principal", detail); + } + + [Fact] + public void TryAuthorizeApproval_RejectsExtraUnrelatedIdentity() + { + var auth = Standard(); + Assert.False(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["moderator-a@example.com", "other@example.com"], + ["group.a"], + out var detail)); + Assert.Equal("conflicting Approved", detail); + } + + [Fact] + public void TryAuthorizeApproval_SameModeratorOnTwoPatterns_Succeeds() + { + var auth = ModeratorTestSnapshot.Create( + ("group.a", "shared@example.com", "moderator-a"), + ("group.b", "shared@example.com", "moderator-a")); + Assert.True(auth.TryAuthorizeApproval( + MapNntpAuthenticationProvider.ModeratorA, + ["shared@example.com"], + ["group.a", "group.b"], + out _)); + } + + [Fact] + public void EmptyCatalogue_ResolvesNothing() + { + var auth = ModeratorSnapshot.Empty; + Assert.False(auth.TryResolve(Bytes("group.a"), out _)); + Assert.False(auth.TryAuthorizeApproval("moderator-a", ["a@example.com"], ["group.a"], out var detail)); + Assert.Equal("unresolved moderator", detail); + } + + [Theory] + [InlineData("fido7.some.group", "fido7-some-group@fido7.org")] + [InlineData("perl.foo.bar", "news-moderator-perl-foo-bar@perl.org")] + [InlineData("comp.test", "comp-test@moderators.isc.org")] + public void InnTemplate_ExpandsPercentSWithDotsToDashes(string group, string expected) + { + Assert.Equal(expected, ModeratorAddressTemplate.Expand(AddressFor(group), Bytes(group))); + } + + [Fact] + public void InnRoutingCatalogue_FirstMatchAndCatchAll() + { + var auth = InnRouting(); + Assert.True(auth.TryResolve(Bytes("fido7.some.group"), out var fido7)); + Assert.Equal("fido7-some-group@fido7.org", fido7.Address); + Assert.Equal(string.Empty, fido7.Username); + Assert.True(auth.TryResolve(Bytes("perl.foo.bar"), out var perl)); + Assert.Equal("news-moderator-perl-foo-bar@perl.org", perl.Address); + Assert.True(auth.TryResolve(Bytes("comp.test"), out var catchAll)); + Assert.Equal("comp-test@moderators.isc.org", catchAll.Address); + Assert.Equal("*", catchAll.Pattern); + } + + [Fact] + public void InnRoutingCatalogue_DoesNotAuthorizeInjection() + { + var auth = InnRouting(); + Assert.False(auth.IsAuthenticatedModerator("moderator-a")); + Assert.False(auth.CanApprove("moderator-a", Bytes("fido7-some-group@fido7.org"), Bytes("fido7.some.group"))); + Assert.False(auth.TryAuthorizeApproval( + "moderator-a", + ["fido7-some-group@fido7.org"], + ["fido7.some.group"], + out var detail)); + Assert.Equal("unauthorized moderator principal", detail); + } + + [Fact] + public void RoutingOnlyEntry_CompileDoesNotDropMissingUsername() + { + var auth = ModeratorTestSnapshot.Create(("fido7.*", "%s@fido7.org", string.Empty)); + Assert.True(auth.TryResolve(Bytes("fido7.announce"), out var identity)); + Assert.Equal("fido7-announce@fido7.org", identity.Address); + Assert.Equal(string.Empty, identity.Username); + } + + [Fact] + public void StaticMailbox_IsUsedLiterally() + { + var auth = ModeratorTestSnapshot.Create(("comp.foo.*", "moderator@example.org", "MODERATOR01")); + Assert.True(auth.TryResolve(Bytes("comp.foo.test"), out var identity)); + Assert.Equal("moderator@example.org", identity.Address); + Assert.True(auth.CanApprove("MODERATOR01", Bytes("moderator@example.org"), Bytes("comp.foo.test"))); + } + + [Fact] + public void Create_PreservesModeratorIdOrder_NotAlphabetical() + { + var auth = ModeratorSnapshot.Create( + [ + new NntpModeratorRow(2, "zzz.*", "later@example.com", "later"), + new NntpModeratorRow(1, "aaa.*", "first@example.com", "first"), + ]); + Assert.True(auth.TryResolve(Bytes("zzz.group"), out var identity)); + Assert.Equal("later@example.com", identity.Address); + } + + [Fact] + public void DatabaseFailureDoesNotAuthorize_EmptySnapshot() + { + Assert.False(ModeratorSnapshot.Empty.TryAuthorizeApproval( + "moderator-a", + ["a@example.com"], + ["group.a"], + out _)); + } + + private static ModeratorSnapshot InnRouting() => + ModeratorTestSnapshot.Create( + ("fido7.*", "%s@fido7.org", string.Empty), + ("fj.*", "%s@moderators.fj-news.org", string.Empty), + ("medlux.*", "%s@news.medlux.ru", string.Empty), + ("nl.*", "%s@nl.news-admin.org", string.Empty), + ("perl.*", "news-moderator-%s@perl.org", string.Empty), + ("relcom.*", "%s@moderators.relcom.ru", string.Empty), + ("si.*", "%s@arnes.si", string.Empty), + ("*", "%s@moderators.isc.org", string.Empty)); + + private static string AddressFor(string group) => + group.StartsWith("fido7.", StringComparison.Ordinal) ? "%s@fido7.org" + : group.StartsWith("perl.", StringComparison.Ordinal) ? "news-moderator-%s@perl.org" + : "%s@moderators.isc.org"; + + private static ModeratorSnapshot Standard() => + ModeratorTestSnapshot.Create( + ("group.a", "moderator-a@example.com", MapNntpAuthenticationProvider.ModeratorA), + ("group.b", "moderator-b@example.com", MapNntpAuthenticationProvider.ModeratorB)); + + private static byte[] Bytes(string value) => Encoding.ASCII.GetBytes(value); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs new file mode 100644 index 00000000..87e087cd --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Moderation/MySqlNntpModeratorRepositoryIntegrationTests.cs @@ -0,0 +1,205 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Moderation; + +/// +/// Live MySQL coverage for . +/// Skips unless VECTORNNTP_NNTPDB_INTEGRATION is set. Fails when that +/// variable is set and MySQL is unreachable. +/// +public sealed class MySqlNntpModeratorRepositoryIntegrationTests + : IClassFixture +{ + private readonly MySqlModeratorIntegrationFixture _fixture; + + public MySqlNntpModeratorRepositoryIntegrationTests(MySqlModeratorIntegrationFixture fixture) + { + ArgumentNullException.ThrowIfNull(fixture); + _fixture = fixture; + } + + [NntpDbIntegrationFact] + public async Task LoadsEnabledModeratorsFromRealMySql() + { + var ours = await LoadOursAsync(); + + AssertMapped(ours, _fixture.Static); + AssertMapped(ours, _fixture.Template); + AssertMapped(ours, _fixture.SharedFirst); + AssertMapped(ours, _fixture.SharedSecond); + AssertMapped(ours, _fixture.Boundary); + Assert.Equal(5, ours.Count); + } + + [NntpDbIntegrationFact] + public async Task ExcludesDisabledModeratorsFromRealMySql() + { + var ours = await LoadOursAsync(); + + Assert.DoesNotContain(ours, row => row.ModeratorId == _fixture.Disabled.ModeratorId); + Assert.DoesNotContain( + ours, + row => string.Equals(row.AccountName, _fixture.Disabled.AccountName, StringComparison.Ordinal) + && string.Equals(row.GroupPattern, _fixture.Disabled.GroupPattern, StringComparison.Ordinal)); + } + + [NntpDbIntegrationFact] + public async Task PreservesModeratorIdOrdering() + { + var ours = await LoadOursAsync(); + + for (var i = 1; i < ours.Count; i++) + { + Assert.True( + ours[i - 1].ModeratorId < ours[i].ModeratorId, + "Enabled nntpmoderators rows must be returned in moderator_id ASC."); + } + + var shared = ours + .Where(row => string.Equals(row.GroupPattern, _fixture.SharedPattern, StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(2, shared.Length); + Assert.Equal(Math.Min(_fixture.SharedFirst.ModeratorId, _fixture.SharedSecond.ModeratorId), shared[0].ModeratorId); + Assert.Equal(Math.Max(_fixture.SharedFirst.ModeratorId, _fixture.SharedSecond.ModeratorId), shared[1].ModeratorId); + Assert.NotEqual(shared[0].ModeratorAddress, shared[1].ModeratorAddress); + } + + [NntpDbIntegrationFact] + public async Task MapsCharAccountNameCorrectly() + { + var ours = await LoadOursAsync(); + + var shortName = Find(ours, _fixture.Static); + Assert.Equal("TESTMOD01", shortName.AccountName); + Assert.Equal(9, shortName.AccountName.Length); + Assert.False(shortName.AccountName.EndsWith(' ')); + + var fullWidth = Find(ours, _fixture.Boundary); + Assert.Equal(_fixture.LongAccountName, fullWidth.AccountName); + Assert.Equal(32, fullWidth.AccountName.Length); + Assert.False(fullWidth.AccountName.EndsWith(' ')); + } + + [NntpDbIntegrationFact] + public async Task MapsTemplateModeratorAddressCorrectly() + { + var ours = await LoadOursAsync(); + var template = Find(ours, _fixture.Template); + + Assert.Equal("moderator-%s@example.org", template.ModeratorAddress); + Assert.Contains("%s", template.ModeratorAddress, StringComparison.Ordinal); + Assert.Equal("TESTMOD02", template.AccountName); + } + + [NntpDbIntegrationFact] + public async Task MapsBoundaryLengthsFromRealMySql() + { + var ours = await LoadOursAsync(); + var boundary = Find(ours, _fixture.Boundary); + + Assert.Equal(255, boundary.GroupPattern.Length); + Assert.Equal(_fixture.Boundary.GroupPattern, boundary.GroupPattern); + Assert.Equal(320, boundary.ModeratorAddress.Length); + Assert.Equal(_fixture.LongModeratorAddress, boundary.ModeratorAddress); + Assert.Equal(_fixture.LongAccountName, boundary.AccountName); + } + + [NntpDbIntegrationFact] + public async Task CataloguePublishesMappedRowsFromRealMySql() + { + RequireConfigured(); + var repository = _fixture.Repository!; + var rows = await repository.GetEnabledAsync(CancellationToken.None); + var ours = _fixture.Ours(rows); + var snapshot = ModeratorSnapshot.Create(ours); + + Assert.Equal(5, snapshot.Count); + Assert.True(snapshot.IsAuthenticatedModerator("TESTMOD01")); + Assert.True(snapshot.IsAuthenticatedModerator("TESTMOD02")); + Assert.True(snapshot.IsAuthenticatedModerator(_fixture.LongAccountName)); + Assert.False(snapshot.IsAuthenticatedModerator("TESTMOD03")); + + Assert.True(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".static.child"), out var staticIdentity)); + Assert.Equal("moderator@example.org", staticIdentity.Address); + Assert.Equal("TESTMOD01", staticIdentity.Username); + + Assert.True(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".template.child"), out var templateIdentity)); + var expectedTemplate = "moderator-" + + (_fixture.PatternPrefix + ".template.child").Replace('.', '-') + + "@example.org"; + Assert.Equal(expectedTemplate, templateIdentity.Address); + Assert.Equal("TESTMOD02", templateIdentity.Username); + + var sharedGroup = _fixture.PatternPrefix + ".shared.child"; + Assert.True(snapshot.TryResolve(GroupBytes(sharedGroup), out var sharedIdentity)); + var firstShared = _fixture.SharedFirst.ModeratorId < _fixture.SharedSecond.ModeratorId + ? _fixture.SharedFirst + : _fixture.SharedSecond; + Assert.Equal(firstShared.ModeratorAddress, sharedIdentity.Address); + Assert.Equal(firstShared.AccountName, sharedIdentity.Username); + + Assert.False(snapshot.TryResolve(GroupBytes(_fixture.PatternPrefix + ".disabled.child"), out _)); + + await using var catalogue = new ModeratorCatalogueService( + repository, + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + await catalogue.StartAsync(CancellationToken.None); + try + { + Assert.True(catalogue.Current.Count >= 5); + Assert.True(catalogue.IsAuthenticatedModerator("TESTMOD01")); + Assert.True(catalogue.IsAuthenticatedModerator(_fixture.LongAccountName)); + Assert.False(catalogue.IsAuthenticatedModerator("TESTMOD03")); + } + finally + { + await catalogue.StopAsync(CancellationToken.None); + } + } + + [NntpDbIntegrationFact] + public async Task GetEnabledAsync_HonorsCancellation() + { + RequireConfigured(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync( + () => _fixture.Repository!.GetEnabledAsync(cts.Token).AsTask()); + } + + private async Task> LoadOursAsync() + { + RequireConfigured(); + var rows = await _fixture.Repository!.GetEnabledAsync(CancellationToken.None); + return _fixture.Ours(rows); + } + + private void RequireConfigured() + { + Assert.True(_fixture.IsConfigured, NntpDbIntegration.SkipReason); + Assert.NotNull(_fixture.Repository); + } + + private static void AssertMapped(IReadOnlyList rows, SeededModeratorRow expected) + { + var actual = Find(rows, expected); + Assert.Equal(expected.ModeratorId, actual.ModeratorId); + Assert.Equal(expected.GroupPattern, actual.GroupPattern); + Assert.Equal(expected.ModeratorAddress, actual.ModeratorAddress); + Assert.Equal(expected.AccountName, actual.AccountName); + } + + private static NntpModeratorRow Find(IReadOnlyList rows, SeededModeratorRow expected) + { + var match = rows.SingleOrDefault(row => row.ModeratorId == expected.ModeratorId); + Assert.NotNull(match); + return match; + } + + private static byte[] GroupBytes(string group) => Encoding.ASCII.GetBytes(group); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/BindAddressResolverTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/BindAddressResolverTests.cs new file mode 100644 index 00000000..693d1560 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/BindAddressResolverTests.cs @@ -0,0 +1,149 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; + +namespace VectorNNTPD.TransitService.Tests.Networking; + +public sealed class BindAddressResolverTests +{ + [Fact] + public void Resolve_ExplicitIpv4AndIpv6_ReturnsBoth() + { + var v4 = IPAddress.Parse("198.18.0.66"); + var v6 = IPAddress.Parse("2001:db8::66"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [v4.ToString(), v6.ToString()]; + + var resolved = resolver.Resolve(options); + + Assert.Equal(2, resolved.All.Count); + Assert.Single(resolved.IPv4); + Assert.Single(resolved.IPv6); + Assert.Contains(v4, resolved.IPv4); + Assert.Contains(v6, resolved.IPv6); + } + + [Fact] + public void Resolve_WildcardStar_ExpandsEligibleInterfaceAddresses() + { + var publicV4 = IPAddress.Parse("203.0.113.10"); + var privateV4 = IPAddress.Parse("10.0.0.5"); + var v6 = IPAddress.Parse("2001:db8::5"); + var loopback = IPAddress.Loopback; + var linkLocal = IPAddress.Parse("169.254.10.10"); + var multicast = IPAddress.Parse("224.0.0.1"); + + var resolver = CreateResolver(publicV4, privateV4, v6, loopback, linkLocal, multicast); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var resolved = resolver.Resolve(options); + + Assert.Equal(3, resolved.All.Count); + Assert.Contains(publicV4, resolved.IPv4); + Assert.Contains(privateV4, resolved.IPv4); + Assert.Contains(v6, resolved.IPv6); + Assert.DoesNotContain(loopback, resolved.All); + Assert.DoesNotContain(linkLocal, resolved.All); + Assert.DoesNotContain(multicast, resolved.All); + } + + [Fact] + public void Resolve_Ipv4AnyWildcard_ExpandsOnlyIpv4() + { + var v4 = IPAddress.Parse("198.18.0.20"); + var v6 = IPAddress.Parse("2001:db8::20"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["0.0.0.0"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v4, resolved.IPv4[0]); + Assert.Empty(resolved.IPv6); + } + + [Fact] + public void Resolve_Ipv6AnyWildcard_ExpandsOnlyIpv6() + { + var v4 = IPAddress.Parse("198.18.0.21"); + var v6 = IPAddress.Parse("2001:db8::21"); + var resolver = CreateResolver(v4, v6); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["::"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v6, resolved.IPv6[0]); + Assert.Empty(resolved.IPv4); + } + + [Fact] + public void Resolve_Duplicates_AreEliminated() + { + var v4 = IPAddress.Parse("198.18.0.30"); + var resolver = CreateResolver(v4); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [v4.ToString(), v4.ToString(), "*"]; + + var resolved = resolver.Resolve(options); + + Assert.Single(resolved.All); + Assert.Equal(v4, resolved.IPv4[0]); + } + + [Fact] + public void Resolve_ExplicitLoopbackOnly_YieldsEmptySet() + { + var resolver = CreateResolver(IPAddress.Loopback); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + + var resolved = resolver.Resolve(options); + + Assert.False(resolved.HasAny); + Assert.Empty(resolved.All); + } + + [Fact] + public void Resolve_NoEligibleAddressesOnWildcard_YieldsEmptySet() + { + var resolver = CreateResolver(IPAddress.Loopback, IPAddress.IPv6Loopback, IPAddress.Parse("169.254.1.1")); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["*"]; + + var resolved = resolver.Resolve(options); + + Assert.False(resolved.HasAny); + } + + [Theory] + [InlineData("127.0.0.1", false)] + [InlineData("::1", false)] + [InlineData("0.0.0.0", false)] + [InlineData("::", false)] + [InlineData("169.254.1.1", false)] + [InlineData("fe80::1", false)] + [InlineData("224.0.0.1", false)] + [InlineData("ff02::1", false)] + [InlineData("10.0.0.1", true)] + [InlineData("192.168.1.1", true)] + [InlineData("203.0.113.1", true)] + [InlineData("2001:db8::1", true)] + public void IsEligibleForDns_ClassifiesAddresses(string text, bool expected) + { + var address = IPAddress.Parse(text); + Assert.Equal(expected, IpAddressEligibility.IsEligibleForDns(address)); + } + + private static BindAddressResolver CreateResolver(params IPAddress[] local) + { + return new BindAddressResolver( + new FakeLocalIpAddressAssignee(local), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs new file mode 100644 index 00000000..fec79daa --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ConnectionAcceptanceLoggingTests.cs @@ -0,0 +1,163 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Listeners; + +/// Connection-acceptance log format contract (TCP peer vs PROXY effective client; TLS params). +public sealed class ConnectionAcceptanceLoggingTests +{ + [Fact] + public void LogPlainAccepted_Direct_OmitsClientSuffix() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.18.0.70"), 42122)); + ConnectionAcceptanceLogging.LogPlainAccepted(logger, identity); + + Assert.Single(logger.Messages); + var message = logger.Messages.Single(); + Assert.Equal("Plain connection accepted from 198.18.0.70:42122", message); + Assert.DoesNotContain("client ", message, StringComparison.Ordinal); + Assert.DoesNotContain("proxy ", message, StringComparison.Ordinal); + } + + [Fact] + public void LogPlainAccepted_Proxy_DistinguishesTcpPeerAndEffectiveClient() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Parse("198.18.0.10"), 40000), + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 42122), + proxyProtocolVersion: 2); + ConnectionAcceptanceLogging.LogPlainAccepted(logger, identity); + + Assert.Single(logger.Messages); + Assert.Equal( + "Proxy connection accepted from 198.18.0.10:40000; proxy 198.18.0.70:42122", + logger.Messages.Single()); + } + + [Fact] + public void LogTlsAccepted_Direct_IncludesNegotiatedParameters() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.18.0.70"), 40106)); + ConnectionAcceptanceLogging.LogTlsAccepted(logger, identity, "TLSv1.3", "TLS_AES_256_GCM_SHA384"); + + Assert.Single(logger.Messages); + Assert.Equal( + "TLS connection accepted from 198.18.0.70:40106 (TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384)", + logger.Messages.Single()); + } + + [Fact] + public void LogTlsAccepted_Proxy_IncludesNegotiatedParameters() + { + var logger = new RecordingLogger(); + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Parse("198.18.0.10"), 40000), + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 40106), + proxyProtocolVersion: 2); + ConnectionAcceptanceLogging.LogTlsAccepted(logger, identity, "TLSv1.3", "TLS_AES_256_GCM_SHA384"); + + Assert.Single(logger.Messages); + Assert.Equal( + "TLS/Proxy connection accepted from 198.18.0.10:40000; proxy 198.18.0.70:40106 (TlsVersion=TLSv1.3, Cipher=TLS_AES_256_GCM_SHA384)", + logger.Messages.Single()); + } + + [Fact] + public void FormatSslProtocol_UsesOperationalLabels() + { + Assert.Equal("TLSv1.3", TlsNegotiationLogging.FormatSslProtocol(SslProtocols.Tls13)); + Assert.Equal("TLSv1.2", TlsNegotiationLogging.FormatSslProtocol(SslProtocols.Tls12)); + } + + [Fact] + public async Task PlainListener_DirectAccept_LogsRequiredFormat() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.ProxyHosts = []; + + var recording = new RecordingLogger(); + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + recording); + + await service.StartAsync(CancellationToken.None); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(service.LocalEndPoints[0]); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!recording.Messages.Any(m => m.StartsWith("Plain connection accepted from ", StringComparison.Ordinal)) + && !cts.IsCancellationRequested) + { + await Task.Delay(10, cts.Token); + } + + var accept = Assert.Single( + recording.Messages, + m => m.StartsWith("Plain connection accepted from ", StringComparison.Ordinal)); + Assert.DoesNotContain("; client ", accept, StringComparison.Ordinal); + Assert.DoesNotContain("proxy ", accept, StringComparison.Ordinal); + await service.StopAsync(CancellationToken.None); + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) => + Messages.Add(formatter(state, exception)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ListenerBindStartupTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ListenerBindStartupTests.cs new file mode 100644 index 00000000..abf76e6c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Listeners/ListenerBindStartupTests.cs @@ -0,0 +1,554 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Hosting.Systemd; +using VectorNNTPD.Common.Networking.Listeners; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Networking.Listeners; + +/// +/// Bind-failure startup contract: all-or-nothing listener start, cleanup, and lifecycle propagation. +/// +public sealed class ListenerBindStartupTests +{ + [Fact] + public async Task PlainStartAsync_FirstBindSucceedsSecondFails_DisposesFirstAndFails() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var logger = new CapturingLogger(); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder, logger); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(2, binder.BindAttempts); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + Assert.Contains( + logger.Records, + r => r.Level == LogLevel.Error + && r.Exception == bindError + && r.Message.Contains("Failed to bind Plain NNTP listener", StringComparison.Ordinal) + && r.Message.Contains("listener startup cannot continue", StringComparison.Ordinal) + && r.Message.Contains(options.BindPort.ToString(), StringComparison.Ordinal)); + } + + [Fact] + public async Task PlainStartAsync_FirstBindFailsImmediately_FailsWithoutAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var service = CreatePlain(MultiEndpointPlainOptions(), binder); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(1, binder.BindAttempts); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + } + + [Fact] + public async Task TlsStartAsync_FirstBindSucceedsSecondFails_DisposesFirstAndFails() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var logger = new CapturingLogger(); + var options = MultiEndpointTlsOptions(); + await using var service = CreateTls(options, binder, logger); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(2, binder.BindAttempts); + Assert.False(service.ListenersBound); + Assert.Empty(service.LocalEndPoints); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + Assert.Contains( + logger.Records, + r => r.Level == LogLevel.Error + && r.Exception == bindError + && r.Message.Contains("Failed to bind TLS NNTP listener", StringComparison.Ordinal) + && r.Message.Contains("listener startup cannot continue", StringComparison.Ordinal) + && r.Message.Contains(options.BindPortTls.ToString(), StringComparison.Ordinal)); + } + + [Fact] + public async Task TlsStartAsync_FirstBindFailsImmediately_FailsWithoutAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var service = CreateTls(MultiEndpointTlsOptions(), binder); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Equal(1, binder.BindAttempts); + Assert.False(service.ListenersBound); + Assert.False(service.HasActiveAcceptLoops); + Assert.Null(service.Execution); + Assert.All(binder.Sockets, static socket => Assert.True(socket.SafeHandle.IsClosed)); + } + + [Fact] + public async Task PlainBindFailure_PropagatesToApplicationServiceManager() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder); + var managerLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + + var ex = await Assert.ThrowsAsync(() => manager.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Empty(manager.StartedServices); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("Application service NntpPlainListener started", StringComparison.Ordinal)); + Assert.Contains( + managerLogger.Records, + r => r.Level == LogLevel.Error + && r.Message.Contains("NntpPlainListener failed during startup", StringComparison.Ordinal)); + } + + [Fact] + public async Task TlsBindFailure_PropagatesToApplicationServiceManager() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + var options = MultiEndpointTlsOptions(); + await using var service = CreateTls(options, binder); + var managerLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + + var ex = await Assert.ThrowsAsync(() => manager.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.Empty(manager.StartedServices); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.Contains( + managerLogger.Records, + r => r.Level == LogLevel.Error + && r.Message.Contains("NntpTlsListener failed during startup", StringComparison.Ordinal)); + } + + [Fact] + public async Task ListenerBindFailure_DoesNotReachRunning_AndOmitsSuccessLifecycleLogs() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(null, bindError); + var options = MultiEndpointPlainOptions(); + await using var service = CreatePlain(options, binder); + var managerLogger = new CapturingLogger(); + var lifecycleLogger = new CapturingLogger(); + var hostedLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + await using var lifecycle = new ApplicationLifecycle( + manager, + options, + lifecycleLogger); + var hosted = new NntpdHostedService( + lifecycle, + new NntpdHostLifetime( + lifecycle, + new TestHostApplicationLifetime(), + Options.Create(options), + NullLogger.Instance), + Options.Create(options), + hostedLogger); + + var ex = await Assert.ThrowsAsync(() => hosted.StartAsync(CancellationToken.None)); + + Assert.Same(bindError, ex); + Assert.NotEqual(ApplicationState.Running, lifecycle.State); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + Assert.Empty(manager.StartedServices); + Assert.False(service.HasActiveAcceptLoops); + Assert.DoesNotContain( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.DoesNotContain( + lifecycleLogger.Records, + r => r.Message.Contains("Starting -> Running", StringComparison.Ordinal)); + Assert.DoesNotContain( + lifecycleLogger.Records, + r => r.Message.Contains("Application initialization completed", StringComparison.Ordinal)); + Assert.DoesNotContain( + hostedLogger.Records, + r => r.Message.Contains("Application entered Running", StringComparison.Ordinal) + || r.Message.Contains("Application started", StringComparison.Ordinal)); + } + + [Fact] + public async Task PlainStartAsync_SuccessfulBind_StartsAllEndpoints() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + await using var service = CreatePlain(options, listenBinder: null); + + await service.StartAsync(CancellationToken.None); + + Assert.NotEmpty(service.LocalEndPoints); + Assert.Equal(options.BindPort, service.LocalEndPoints[0].Port); + Assert.NotNull(service.Execution); + Assert.False(service.Execution!.IsCompleted); + Assert.True(service.HasActiveAcceptLoops); + await service.StopAsync(CancellationToken.None); + Assert.False(service.HasActiveAcceptLoops); + } + + [Fact] + public async Task PlainStartAsync_SuccessfulBind_UsesNntpdSectionBindSettings() + { + var port = TestHostFactory.GetFreeTcpPort(); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"{NntpdOptions.SectionName}:BindAddress:0"] = "127.0.0.1", + [$"{NntpdOptions.SectionName}:BindPort"] = port.ToString(), + [$"{NntpdOptions.SectionName}:BindPortTls"] = "0", + ["BindAddress:0"] = "203.0.113.10", + ["BindPort"] = "2119", + ["BindPortTls"] = "2563", + }) + .Build(); + + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = []; + configuration.GetSection(NntpdOptions.SectionName).Bind(options); + NntpdAcmeCloudflareOptionsOverlay.OverlaySharedFromRootPreservingApplicationAcme( + options, + configuration); + + await using var service = CreatePlain(options, listenBinder: null); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(["127.0.0.1"], options.BindAddress); + Assert.Equal(port, options.BindPort); + Assert.Equal(0, options.BindPortTls); + Assert.NotEmpty(service.LocalEndPoints); + Assert.Equal(port, service.LocalEndPoints[0].Port); + Assert.Equal(IPAddress.Loopback, service.LocalEndPoints[0].Address); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task SuccessfulStartup_ReachesRunning_AndKeepsSuccessLogs() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 0; + await using var service = CreatePlain(options, listenBinder: null); + var managerLogger = new CapturingLogger(); + var lifecycleLogger = new CapturingLogger(); + var manager = new ApplicationServiceManager( + [service], + options, + managerLogger); + await using var lifecycle = new ApplicationLifecycle( + manager, + options, + lifecycleLogger); + + await lifecycle.StartAsync(CancellationToken.None); + + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.Contains(manager.StartedServices, s => ReferenceEquals(s, service)); + Assert.Contains( + managerLogger.Records, + r => r.Message.Contains("All application services started successfully", StringComparison.Ordinal)); + Assert.Contains( + lifecycleLogger.Records, + r => r.Message.Contains("Starting -> Running", StringComparison.Ordinal)); + await lifecycle.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_DoesNotFail_WhenAcceptFailsAfterSuccessfulBind() + { + var binder = new ScriptedListenSocketBinder([null]); + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = 23456; + await using var service = CreatePlain(options, binder); + + await service.StartAsync(CancellationToken.None); + + Assert.NotNull(service.Execution); + Assert.False(service.Execution!.IsFaulted); + Assert.False(service.Execution.IsCompleted); + Assert.NotEmpty(service.LocalEndPoints); + Assert.True(service.HasActiveAcceptLoops); + + foreach (var socket in binder.Sockets) + { + socket.Dispose(); + } + + Assert.False(service.Execution.IsFaulted); + Assert.False(service.Execution.IsCompleted); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task SocketAcceptListener_Start_PropagatesBindFailure_BeforeAcceptLoop() + { + var bindError = AddressInUse(); + var binder = new ScriptedListenSocketBinder(bindError); + await using var listener = new SocketAcceptListener( + new ListenBinding(IPAddress.Loopback, 23456, DualMode: false), + static (_, _) => ValueTask.CompletedTask, + NullLogger.Instance, + binder); + + var ex = Assert.Throws(() => listener.Start()); + + Assert.Same(bindError, ex); + Assert.False(listener.AcceptLoopActive); + } + + private static SocketException AddressInUse() => + new((int)SocketError.AddressAlreadyInUse); + + private static NntpdOptions MultiEndpointPlainOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1", "::1"]; + options.BindPort = 23456; + options.BindPortTls = 0; + return options; + } + + private static NntpdOptions MultiEndpointTlsOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1", "::1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + options.BindPortTls = 23457; + return options; + } + + [Fact] + public async Task Plain_bind_failure_with_tls_disabled_does_not_publish() + { + var options = MultiEndpointPlainOptions(); + options.BindPortTls = 0; + options.BindAddress = [TestHostFactory.TestIpv4.ToString(), TestHostFactory.TestIpv6.ToString()]; + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "healthy-a", + Type = "A", + Name = options.Fqdn, + Content = "198.18.0.10", + Ttl = 300, + Proxied = false, + }); + var publication = CreatePublication(options, client); + await publication.StartAsync(CancellationToken.None); + await using var service = CreatePlain(options, new ScriptedListenSocketBinder(AddressInUse()), publication: publication); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await publication.StopAsync(CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Contains(client.Snapshot(), static record => record.Id == "healthy-a"); + } + + [Fact] + public async Task Tls_bind_failure_does_not_publish() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = [TestHostFactory.TestIpv4.ToString()]; + options.BindPortTls = 563; + var client = new FakeCloudflareDnsClient(); + client.Seed(new CloudflareDnsRecord + { + Id = "healthy-a", + Type = "A", + Name = options.Fqdn, + Content = "198.18.0.10", + Ttl = 300, + Proxied = false, + }); + var publication = CreatePublication(options, client); + await publication.StartAsync(CancellationToken.None); + await using var service = CreateTls(options, new ScriptedListenSocketBinder(AddressInUse()), publication: publication); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + await publication.StopAsync(CancellationToken.None); + + Assert.Equal(0, client.CreateCallCount); + Assert.Equal(0, client.DeleteCallCount); + Assert.Contains(client.Snapshot(), static record => record.Id == "healthy-a"); + } + + private static NntpPlainListenerService CreatePlain( + NntpdOptions options, + IListenSocketBinder? listenBinder, + ILogger? logger = null, + CloudflareDnsReconciliationService? publication = null) + { + return new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + logger ?? NullLogger.Instance, + listenBinder: listenBinder, + publication: publication); + } + + private static NntpTlsListenerService CreateTls( + NntpdOptions options, + IListenSocketBinder listenBinder, + ILogger? logger = null, + CloudflareDnsReconciliationService? publication = null) + { + return new NntpTlsListenerService( + Options.Create(options), + new AvailableTlsCertificateProvider(), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + logger ?? NullLogger.Instance, + listenBinder: listenBinder, + publication: publication); + } + + private static CloudflareDnsReconciliationService CreatePublication(NntpdOptions options, FakeCloudflareDnsClient client) + { + var resolver = new BindAddressResolver( + new FakeLocalIpAddressAssignee(TestHostFactory.TestIpv4, TestHostFactory.TestIpv6), + NullLogger.Instance); + var reconciler = new CloudflareDnsReconciler( + client, + Options.Create(options), + NullLogger.Instance); + return new CloudflareDnsReconciliationService( + Options.Create(options), + resolver, + reconciler, + NullLogger.Instance, + new ListenerBoundDnsPublicationLease()); + } + + private sealed class ScriptedListenSocketBinder : IListenSocketBinder + { + private readonly Exception?[] _outcomes; + + public ScriptedListenSocketBinder(params Exception?[] outcomes) + { + _outcomes = outcomes; + } + + public List Sockets { get; } = []; + + public int BindAttempts { get; private set; } + + public void BindAndListen(Socket socket, IPEndPoint endpoint, int backlog) + { + Sockets.Add(socket); + var index = BindAttempts++; + var outcome = index < _outcomes.Length ? _outcomes[index] : null; + if (outcome is not null) + { + throw outcome; + } + + // Bind an ephemeral loopback port so AcceptAsync parks instead of + // throwing synchronously (which would spin Start() on the caller). + var address = endpoint.AddressFamily == AddressFamily.InterNetwork + ? IPAddress.Loopback + : IPAddress.IPv6Loopback; + socket.Bind(new IPEndPoint(address, 0)); + socket.Listen(backlog); + } + } + + private sealed class AvailableTlsCertificateProvider : ITlsCertificateContextProvider + { + public bool IsAvailable => true; + + public TlsCertificateLease Acquire() => + throw new InvalidOperationException("Bind-startup tests do not perform TLS handshakes."); + + public void PublishFromPfx(ReadOnlySpan pfxBytes, string password) + { + } + } + + private sealed record CapturedLog(LogLevel Level, EventId EventId, string Message, Exception? Exception); + + private sealed class CapturingLogger : ILogger + { + public ConcurrentBag Records { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Records.Add(new CapturedLog(logLevel, eventId, formatter(state, exception), exception)); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Proxy/ProxyProtocolTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Proxy/ProxyProtocolTests.cs new file mode 100644 index 00000000..cbbb8a49 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Proxy/ProxyProtocolTests.cs @@ -0,0 +1,1049 @@ +using System.Buffers.Binary; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Networking.Transport; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Networking.Proxy; + +public sealed class TrustedProxyHostsTests +{ + [Fact] + public void EmptyProxyHosts_IsDisabled() + { + var hosts = new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + Assert.False(hosts.IsEnabled); + Assert.False(hosts.IsTrusted(IPAddress.Loopback)); + } + + [Fact] + public void MissingProxyHosts_IsDisabled() + { + var hosts = new TrustedProxyHosts(Options.Create(new NntpdOptions())); + Assert.False(hosts.IsEnabled); + } + + [Fact] + public void SingleIpv4_IsTrusted() + { + var hosts = Create(["198.51.100.10"]); + Assert.True(hosts.IsEnabled); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("198.51.100.11"))); + } + + [Fact] + public void MultipleIpv4_MatchExactly() + { + var hosts = Create(["198.51.100.10", "198.51.100.11"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.11"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("198.51.100.12"))); + } + + [Fact] + public void Ipv6_IsTrusted() + { + var hosts = Create(["2001:db8::10"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("2001:db8::10"))); + Assert.True(hosts.IsTrusted(IPAddress.Parse("2001:db8:0:0:0:0:0:10"))); + Assert.False(hosts.IsTrusted(IPAddress.Parse("2001:db8::11"))); + } + + [Fact] + public void Ipv4MappedPeer_MatchesConfiguredIpv4() + { + var hosts = Create(["198.51.100.10"]); + var mapped = IPAddress.Parse("::ffff:198.51.100.10"); + Assert.True(hosts.IsTrusted(mapped)); + } + + [Fact] + public void DuplicateAddresses_AreDeduped() + { + var hosts = Create(["198.51.100.10", "198.51.100.10", "::ffff:198.51.100.10"]); + Assert.True(hosts.IsTrusted(IPAddress.Parse("198.51.100.10"))); + } + + private static TrustedProxyHosts Create(string[] entries) => + new(Options.Create(new NntpdOptions { ProxyHosts = entries })); +} + +public sealed class ProxyHostsConfigurationTests +{ + [Fact] + public void Validate_RejectsInvalidProxyHost() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["not-an-ip"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("ProxyHosts", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsCidr() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["198.51.100.0/24"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_RejectsAnyAddressWildcard() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["0.0.0.0"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_AcceptsIpv4AndIpv6() + { + var options = TestHostFactory.CreateValidOptions(); + options.ProxyHosts = ["198.51.100.10", "2001:db8::1"]; + var result = new NntpdOptionsValidator(new FakeLocalIpAddressAssignee(assignAll: true)) + .Validate(null, options); + Assert.True(result.Succeeded); + } +} + +public sealed class ProxyProtocolParserTests +{ + private static readonly IPEndPoint TcpPeer = new(IPAddress.Parse("198.51.100.1"), 45000); + + [Fact] + public void V1_Tcp4_ParsesClientEndpoint() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 41234 119\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out var error); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(string.Empty, error); + Assert.Equal(IPAddress.Parse("203.0.113.10"), result.Identity.ClientAddress); + Assert.Equal(41234, result.Identity.ClientPort); + Assert.Equal(TcpPeer, result.Identity.TcpPeer); + Assert.Equal(header.Length, result.HeaderLength); + } + + [Fact] + public void V1_Tcp6_ParsesClientEndpoint() + { + var header = Encoding.ASCII.GetBytes( + "PROXY TCP6 2001:db8::2 2001:db8::1 41234 119\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(IPAddress.Parse("2001:db8::2"), result.Identity.ClientAddress); + Assert.Equal(41234, result.Identity.ClientPort); + } + + [Fact] + public void V1_Unknown_UsesTcpPeer() + { + var header = Encoding.ASCII.GetBytes("PROXY UNKNOWN\r\n"); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(TcpPeer, result.Identity.Client); + } + + [Fact] + public void V1_PartialDelivery_NeedsMore() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 41234 119\r\n"); + for (var i = 1; i < header.Length; i++) + { + var status = ProxyProtocolParser.TryParse(header.AsSpan(0, i), TcpPeer, out _, out _); + Assert.Equal(ProxyParseStatus.NeedMore, status); + } + + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V1_Malformed_Fails() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 not-an-ip 198.51.100.20 41234 119\r\n"); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("invalid address", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V1_ExceedsMaxLine_Fails() + { + var line = "PROXY UNKNOWN " + new string('x', 200) + "\r\n"; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(Encoding.ASCII.GetBytes(line), TcpPeer, out _, out _)); + } + + [Fact] + public void V1_LeadingZeroPort_Fails() + { + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.10 198.51.100.20 01234 119\r\n"); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_Tcp4_ParsesClientEndpoint() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + srcPort: 40000, + dstPort: 119); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out var error); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(string.Empty, error); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + Assert.Equal(40000, result.Identity.ClientPort); + Assert.Equal(2, result.Version); + } + + [Fact] + public void V2_Tcp6_ParsesClientEndpoint() + { + var header = BuildV2Tcp6( + IPAddress.Parse("2001:db8::aa"), + IPAddress.Parse("2001:db8::bb"), + srcPort: 40001, + dstPort: 119); + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(IPAddress.Parse("2001:db8::aa"), result.Identity.ClientAddress); + Assert.Equal(40001, result.Identity.ClientPort); + } + + [Fact] + public void V2_LocalCommand_UsesTcpPeer() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; // version 2, LOCAL + header[13] = 0x00; + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.True(result.LocalCommand); + Assert.Equal(TcpPeer, result.Identity.Client); + } + + [Fact] + public void V2_PartialDelivery_NeedsMore() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + for (var i = 1; i < header.Length; i++) + { + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, i), TcpPeer, out _, out _)); + } + } + + [Fact] + public void V2_InvalidSignature_Fails() + { + var header = new byte[16]; + header.AsSpan().Fill(0x41); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void UnrecognizedFirstByte_Fails() + { + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse([(byte)'G'], TcpPeer, out _, out var error)); + Assert.Contains("PROXY header required", error, StringComparison.Ordinal); + } + + [Fact] + public void V2_UnexpectedCommand_Fails() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + header[12] = 0x22; // version 2, unsupported command 2 + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("unsupported command", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_UnlistedFamilyProtocol_FallsBackToTcpPeer() + { + // Valid nibbles (family=1, protocol=0) but not a listed combination → UNSPEC fallback. + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x10; + header[14] = 0x00; + header[15] = 0x00; + var status = ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _); + Assert.Equal(ProxyParseStatus.Ok, status); + Assert.Equal(TcpPeer.Address, result.Identity.ClientAddress); + Assert.Equal(TcpPeer.Port, result.Identity.ClientPort); + } + + [Fact] + public void V2_InvalidProtocolNibble_Fails() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x13; // AF_INET + invalid protocol 3 + header[14] = 0x00; + header[15] = 0x00; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_AbsentCrc_IsAccepted() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + } + + [Fact] + public void V2_ValidCrc32C_IsAccepted() + { + var header = BuildV2Tcp4WithCrc( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119, + corruptCrc: false); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.Equal(IPAddress.Parse("203.0.113.50"), result.Identity.ClientAddress); + Assert.Equal(40000, result.Identity.ClientPort); + } + + [Fact] + public void V2_InvalidCrc32C_IsRejected() + { + var header = BuildV2Tcp4WithCrc( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119, + corruptCrc: true); + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("CRC32C", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_UnsupportedFamilyNibble_Fails() + { + var header = new byte[16]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x41; // family 4 (unspecified) + STREAM + header[14] = 0x00; + header[15] = 0x00; + Assert.Equal( + ProxyParseStatus.Error, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out var error)); + Assert.Contains("unsupported address family/protocol", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void V2_MaxAddressLength_LocalCommand_Parses() + { + // Length field is uint16; MaxV2AddressOctets (65535) is the wire maximum — no larger value exists. + var header = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; // LOCAL + header[13] = 0x00; + BinaryPrimitives.WriteUInt16BigEndian(header.AsSpan(14, 2), ProxyProtocolParser.MaxV2AddressOctets); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out var result, out _)); + Assert.True(result.LocalCommand); + Assert.Equal(ProxyProtocolParser.MaxV2HeaderOctets, result.HeaderLength); + } + + [Fact] + public void V2_OneByteBelowMaxAddressLength_NeedsMore() + { + var header = new byte[ProxyProtocolParser.MaxV2HeaderOctets]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x20; + header[13] = 0x00; + BinaryPrimitives.WriteUInt16BigEndian(header.AsSpan(14, 2), ProxyProtocolParser.MaxV2AddressOctets); + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, header.Length - 1), TcpPeer, out _, out _)); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + [Fact] + public void V2_OneByteBelowComplete_NeedsMore() + { + var header = BuildV2Tcp4( + IPAddress.Parse("203.0.113.50"), + IPAddress.Parse("198.51.100.20"), + 40000, + 119); + Assert.Equal( + ProxyParseStatus.NeedMore, + ProxyProtocolParser.TryParse(header.AsSpan(0, header.Length - 1), TcpPeer, out _, out _)); + Assert.Equal( + ProxyParseStatus.Ok, + ProxyProtocolParser.TryParse(header, TcpPeer, out _, out _)); + } + + internal static byte[] BuildV2Tcp4(IPAddress src, IPAddress dst, ushort srcPort, ushort dstPort) + { + var header = new byte[28]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; // ver 2 PROXY + header[13] = 0x11; // TCP4 + header[14] = 0x00; + header[15] = 12; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 20); + header[24] = (byte)(srcPort >> 8); + header[25] = (byte)srcPort; + header[26] = (byte)(dstPort >> 8); + header[27] = (byte)dstPort; + return header; + } + + internal static byte[] BuildV2Tcp6(IPAddress src, IPAddress dst, ushort srcPort, ushort dstPort) + { + var header = new byte[52]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x21; // TCP6 + header[14] = 0x00; + header[15] = 36; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 32); + header[48] = (byte)(srcPort >> 8); + header[49] = (byte)srcPort; + header[50] = (byte)(dstPort >> 8); + header[51] = (byte)dstPort; + return header; + } + + internal static byte[] BuildV2Tcp4WithCrc( + IPAddress src, + IPAddress dst, + ushort srcPort, + ushort dstPort, + bool corruptCrc) + { + // Address block (12) + CRC TLV (3 + 4) = 19. + var header = new byte[16 + 19]; + "\r\n\r\n\0\r\nQUIT\n"u8.CopyTo(header); + header[12] = 0x21; + header[13] = 0x11; + header[14] = 0x00; + header[15] = 19; + src.GetAddressBytes().CopyTo(header, 16); + dst.GetAddressBytes().CopyTo(header, 20); + header[24] = (byte)(srcPort >> 8); + header[25] = (byte)srcPort; + header[26] = (byte)(dstPort >> 8); + header[27] = (byte)dstPort; + header[28] = 0x03; // PP2_TYPE_CRC32C + header[29] = 0x00; + header[30] = 0x04; + header[31] = 0x00; + header[32] = 0x00; + header[33] = 0x00; + header[34] = 0x00; + var crc = Crc32C.Compute(header); + if (corruptCrc) + { + crc ^= 0xFFFFFFFFu; + } + + BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(31, 4), crc); + return header; + } +} + +public sealed class ConnectionClientIdentityTests +{ + [Fact] + public void Direct_DoesNotAliasMutableEndpoints() + { + var original = new IPEndPoint(IPAddress.Parse("198.51.100.8"), 3333); + var identity = ConnectionClientIdentity.Direct(original); + + Assert.False(ReferenceEquals(identity.TcpPeer, identity.Client)); + Assert.False(ReferenceEquals(identity.TcpPeer, original)); + Assert.False(ReferenceEquals(identity.Client, original)); + + original.Port = 9999; + Assert.Equal(3333, identity.ClientPort); + Assert.Equal(3333, identity.TcpPeer.Port); + + identity.Client.Port = 1; + Assert.Equal(3333, identity.TcpPeer.Port); + Assert.Equal(IPAddress.Parse("198.51.100.8"), identity.TcpPeer.Address); + } + + [Fact] + public void FromTrustedProxy_ClonesBothEndpoints() + { + var peer = new IPEndPoint(IPAddress.Loopback, 40000); + var client = new IPEndPoint(IPAddress.Parse("203.0.113.9"), 51515); + var identity = ConnectionClientIdentity.FromTrustedProxy(peer, client, proxyProtocolVersion: 1); + + peer.Port = 1; + client.Port = 2; + Assert.Equal(40000, identity.TcpPeer.Port); + Assert.Equal(51515, identity.ClientPort); + Assert.False(ReferenceEquals(identity.TcpPeer, peer)); + Assert.False(ReferenceEquals(identity.Client, client)); + } +} + +public sealed class ProxyPreambleLiveTests +{ + [Fact] + public async Task TrustedPeer_FragmentedV1Header_EstablishesClientIdentity() + { + var proxyIp = IPAddress.Loopback; + var trusted = new TrustedProxyHosts([proxyIp]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + Assert.True(trusted.IsTrusted(tcpPeer.Address)); + + var header = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.77 127.0.0.1 51515 119\r\nHELLO"); + await client.SendAsync(header.AsMemory(0, 10)); + await client.SendAsync(header.AsMemory(10)); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(IPAddress.Parse("203.0.113.77"), resolution.Identity.ClientAddress); + Assert.Equal(51515, resolution.Identity.ClientPort); + Assert.True(resolution.Identity.IsTrustedProxy); + Assert.Equal("HELLO"u8.ToArray(), resolution.Leftover.ToArray()); + } + + [Fact] + public async Task TrustedPeer_SendsNothing_TimesOutWithoutFallbackIdentity() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var ex = await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + Assert.Contains("timed out", ex.Message, StringComparison.OrdinalIgnoreCase); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_IncompleteV1ThenStall_TimesOut() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("PROXY TCP4 203.0.113.1 "u8.ToArray()); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_IncompleteV2ThenStall_TimesOut() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("\r\n\r\n\0\r\n"u8.ToArray()); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromMilliseconds(200)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_NonProxyBytes_FailsWithoutTcpFallback() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + await client.SendAsync("CAPABILITIES\r\n"u8.ToArray()); + + var ex = await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromSeconds(2)).AsTask()); + Assert.Contains("PROXY header required", ex.Message, StringComparison.Ordinal); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_ClosesWithoutHeader_Fails() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + client.Dispose(); + + await Assert.ThrowsAsync(() => + ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None, + preambleTimeout: TimeSpan.FromSeconds(2)).AsTask()); + accepted.Dispose(); + } + + [Fact] + public async Task TrustedPeer_MissingProxyOnTlsListener_DoesNotPublishSession() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + // First peer: trusted TCP but ordinary application bytes — must not become a session. + using (var bad = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await bad.ConnectAsync(host.EndPoint); + await bad.SendAsync("CAPABILITIES\r\n"u8.ToArray()); + await WaitForRemoteCloseAsync(bad); + } + + // Second peer: valid PROXY then TLS — must be the only published connection. + using var good = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await good.ConnectAsync(host.EndPoint); + await good.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.66 127.0.0.1 45000 563\r\n")); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(good, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.Equal(IPAddress.Parse("203.0.113.66"), server.ClientIdentity.ClientAddress); + Assert.Equal(45000, server.ClientIdentity.ClientPort); + Assert.True(server.ClientIdentity.IsTrustedProxy); + } + + [Fact] + public async Task UntrustedPeer_CannotReplaceIdentityWithProxyHeader() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + Assert.False(trusted.IsTrusted(tcpPeer.Address)); + + var forged = Encoding.ASCII.GetBytes("PROXY TCP4 203.0.113.1 127.0.0.1 1 119\r\n"); + await client.SendAsync(forged); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.False(resolution.Identity.IsTrustedProxy); + Assert.False(resolution.Identity.UsedProxyHeader); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.Equal(tcpPeer.Port, resolution.Identity.ClientPort); + Assert.True(resolution.Leftover.IsEmpty); + + var leftover = new byte[forged.Length]; + var read = await accepted.ReceiveAsync(leftover.AsMemory(), SocketFlags.None); + Assert.Equal(forged.Length, read); + Assert.Equal(forged, leftover); + } + + [Fact] + public async Task UntrustedPeer_V2SignatureBytesRemainOnSocket() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var forged = ProxyProtocolParserTests.BuildV2Tcp4( + IPAddress.Parse("203.0.113.9"), + IPAddress.Loopback, + 1, + 119); + await client.SendAsync(forged); + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.False(resolution.Identity.UsedProxyHeader); + + var leftover = new byte[forged.Length]; + var read = await accepted.ReceiveAsync(leftover.AsMemory(), SocketFlags.None); + Assert.Equal(forged.Length, read); + Assert.Equal(forged, leftover); + } + + [Fact] + public async Task UntrustedPeer_EstablishesDirectIdentityWithoutWaitingForProxy() + { + var trusted = new TrustedProxyHosts([IPAddress.Parse("198.51.100.9")]); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(250)); + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + cts.Token); + Assert.False(resolution.Identity.UsedProxyHeader); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.False(cts.IsCancellationRequested); + } + + [Fact] + public async Task EmptyTrustedSet_UsesTcpPeerWithoutReading() + { + var trusted = new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + await using var listener = await StartAcceptAsync(); + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(listener.EndPoint); + var accepted = await listener.Accepted; + var tcpPeer = (IPEndPoint)accepted.RemoteEndPoint!; + + var resolution = await ProxyPreambleResolver.ResolveAsync( + accepted, + tcpPeer, + trusted, + CancellationToken.None); + Assert.Equal(tcpPeer.Address, resolution.Identity.ClientAddress); + Assert.Equal(tcpPeer.Port, resolution.Identity.ClientPort); + Assert.False(resolution.Identity.UsedProxyHeader); + } + + [Fact] + public async Task TlsWithTrustedProxy_HeaderThenHandshake_UsesProxyClientIdentity() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(host.EndPoint); + + var header = Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.44 127.0.0.1 42424 563\r\n"); + await socket.SendAsync(header); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(socket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.True(server.IsTls); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.44"), server.ClientIdentity.ClientAddress); + Assert.Equal(42424, server.ClientIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, server.ClientIdentity.TcpPeer.Address); + + var payload = new byte[] { 0x10, 0x20, 0x30 }; + await ssl.WriteAsync(payload); + await ssl.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task TlsWithTrustedProxy_HeaderAndHandshakePipelined_Succeeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartTlsWithProxyAsync(pfx, trusted); + + using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(host.EndPoint); + + // PROXY header first; TLS ClientHello follows immediately on the same connection. + var header = ProxyProtocolParserTests.BuildV2Tcp4( + IPAddress.Parse("203.0.113.55"), + IPAddress.Loopback, + 43000, + 563); + await socket.SendAsync(header); + + var acceptTask = host.AcceptAsync(); + await using var network = new NetworkStream(socket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + + await using var server = await acceptTask; + Assert.Equal(IPAddress.Parse("203.0.113.55"), server.ClientIdentity.ClientAddress); + Assert.Equal(43000, server.ClientIdentity.ClientPort); + Assert.Equal(2, server.ClientIdentity.ProxyProtocolVersion); + + var payload = new byte[] { 0xAA, 0xBB }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public void Session_ExposesEffectiveClientWithoutInspectingTransport() + { + var identity = ConnectionClientIdentity.FromTrustedProxy( + new IPEndPoint(IPAddress.Loopback, 40000), + new IPEndPoint(IPAddress.Parse("203.0.113.9"), 51515), + proxyProtocolVersion: 1); + + var sessionFromIdentity = new NntpSession( + new IdentityOnlyConnection(identity), + NullLogger.Instance); + Assert.Equal(IPAddress.Parse("203.0.113.9"), sessionFromIdentity.ClientAddress); + Assert.Equal(51515, sessionFromIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, sessionFromIdentity.TcpPeer.Address); + Assert.Same(identity, sessionFromIdentity.ClientIdentity); + + var direct = ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("198.51.100.8"), 3333)); + var directSession = new NntpSession( + new IdentityOnlyConnection(direct), + NullLogger.Instance); + Assert.Equal(IPAddress.Parse("198.51.100.8"), directSession.ClientAddress); + Assert.Equal(3333, directSession.ClientPort); + Assert.Equal(direct.TcpPeer.Address, directSession.TcpPeer.Address); + Assert.Equal(direct.TcpPeer.Port, directSession.TcpPeer.Port); + } + + private static async Task WaitForRemoteCloseAsync(Socket socket) + { + var buffer = new byte[64]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + var read = await socket.ReceiveAsync(buffer.AsMemory(), SocketFlags.None, cts.Token); + if (read == 0) + { + return; + } + } + } + + private static async Task StartAcceptAsync() + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + listener.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + listener.Listen(1); + var acceptTask = AcceptOneAsync(listener, tcs); + return new AcceptFixture(listener, tcs.Task, acceptTask); + } + + private static async Task AcceptOneAsync(Socket listener, TaskCompletionSource tcs) + { + try + { + var accepted = await listener.AcceptAsync().ConfigureAwait(false); + tcs.TrySetResult(accepted); + } + // codeql[cs/catch-of-all-exceptions] Stopping the test listener can fault the accept loop with any transport exception. + catch (Exception ex) + { + tcs.TrySetException(ex); + } + } + + private sealed class AcceptFixture(Socket listener, Task accepted, Task acceptLoop) : IAsyncDisposable + { + public IPEndPoint EndPoint => (IPEndPoint)listener.LocalEndPoint!; + + public Task Accepted { get; } = accepted; + + public async ValueTask DisposeAsync() + { + listener.Dispose(); + try + { + await acceptLoop.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Listener disposed. + } + } + } + + private sealed class IdentityOnlyConnection : INntpConnection + { + public IdentityOnlyConnection(ConnectionClientIdentity identity) + { + ClientIdentity = identity; + } + + public System.IO.Pipelines.PipeReader Input => throw new NotSupportedException(); + + public System.IO.Pipelines.PipeWriter Output => throw new NotSupportedException(); + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => CancellationToken.None; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs new file mode 100644 index 00000000..f9959577 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/ConnectionByteTransportQuiescenceTests.cs @@ -0,0 +1,323 @@ +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Deterministic regression coverage for ConnectionByteTransport quiescence admission. +/// +public sealed class ConnectionByteTransportQuiescenceTests +{ + [Fact] + public async Task AfterPauseReads_TryCommitReadToApplication_StashesClientHello() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + await transport.PauseReadsAsync(CancellationToken.None); + + var clientHello = new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00 }; + Assert.False(transport.TryCommitReadToApplication(clientHello)); + + var prefix = transport.TakePendingUpgradePrefix(); + Assert.Equal(clientHello, prefix.ToArray()); + } + + [Fact] + public async Task Quiesce_DuringHistoricalAdmitWindow_DoesNotStartStreamIoAfterIdle() + { + // Reproduces the pre-fix window: WaitUntilActive observed Active, then Quiesce ran and + // declared idle before outstanding++. With atomic admission, stream I/O must not start. + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesceCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesceCompleted.TrySetResult(); + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[16], readCts.Token).AsTask(); + + await quiesceCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.ConcurrentOps); + + Assert.False(readTask.IsCompleted); + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => readTask); + Assert.Equal(0, stream.ReadStarts); + } + + [Fact] + public async Task Quiesce_WaitsForAdmittedRead_UntilStreamReadReturns() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var readEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancelObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowReadReturn = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + stream.OnReadAsync = async ct => + { + readEntered.TrySetResult(); + try + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Cancel was requested, but the operation still owns the stream until we return. + cancelObserved.TrySetResult(); + await allowReadReturn.Task.ConfigureAwait(false); + throw; + } + + return 0; + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[8], readCts.Token).AsTask(); + await readEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + using var quiesceCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var quiesceTask = transport.QuiesceAsync(quiesceCts.Token); + + await cancelObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(quiesceTask.IsCompleted); + Assert.Equal(1, stream.ConcurrentOps); + + allowReadReturn.TrySetResult(); + await quiesceTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ConcurrentOps); + await Assert.ThrowsAnyAsync(() => readTask); + } + + [Fact] + public async Task Quiesce_WaitsForAdmittedWrite_UntilStreamWriteReturns() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var writeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + stream.OnWriteAsync = async _ => + { + writeEntered.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var writeTask = transport.WriteAsync(new byte[] { 1, 2, 3 }, writeCts.Token).AsTask(); + await writeEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + using var quiesceCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var quiesceTask = transport.QuiesceAsync(quiesceCts.Token); + await Task.Yield(); + Assert.False(quiesceTask.IsCompleted); + Assert.Equal(1, stream.ConcurrentOps); + + releaseWrite.TrySetResult(); + await quiesceTask.WaitAsync(TimeSpan.FromSeconds(5)); + await writeTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ConcurrentOps); + } + + [Fact] + public async Task QuiescedIdle_SimulatedHandshake_CannotOverlapPlaintextStreamIo() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + await transport.QuiesceAsync(CancellationToken.None); + Assert.Same(transport.RateLimiter, transport.TakeQuiescedStreamForTlsWrap()); + Assert.Equal(0, stream.ConcurrentOps); + + stream.ArmPostQuiesceTracking(); + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var blockedRead = transport.ReadAsync(new byte[16], readCts.Token).AsTask(); + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var blockedWrite = transport.WriteAsync(new byte[] { 1 }, writeCts.Token).AsTask(); + + // Ownership window equivalent to AuthenticateAsServerAsync: stream must stay untouched. + for (var i = 0; i < 64; i++) + { + Assert.Equal(0, stream.PostQuiesceIoStarts); + Assert.Equal(0, stream.ConcurrentOps); + Assert.False(blockedRead.IsCompleted); + Assert.False(blockedWrite.IsCompleted); + await Task.Yield(); + } + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => blockedRead); + await Assert.ThrowsAnyAsync(() => blockedWrite); + Assert.Equal(0, stream.PostQuiesceIoStarts); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.WriteStarts); + } + + [Fact] + public async Task Quiesce_RepeatedAdmitRace_NeverStartsIoAfterIdle() + { + for (var iteration = 0; iteration < 64; iteration++) + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesced = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesced.TrySetResult(); + }; + + using var readCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readTask = transport.ReadAsync(new byte[8], readCts.Token).AsTask(); + await quiesced.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.ReadStarts); + Assert.Equal(0, stream.ConcurrentOps); + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => readTask); + Assert.Equal(0, stream.ReadStarts); + } + } + + [Fact] + public async Task QuiescedIdle_WriteAdmitRace_NeverStartsWriteAfterIdle() + { + await using var stream = new ControllableStream(); + await using var transport = new VectorNNTPD.TransitService.Networking.Transport.ConnectionByteTransport( + stream, + isTls: false); + + var quiesced = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.AfterActiveBeforeAdmitProbe = async () => + { + await transport.QuiesceAsync(CancellationToken.None).ConfigureAwait(false); + quiesced.TrySetResult(); + }; + + using var writeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var writeTask = transport.WriteAsync(new byte[] { 9 }, writeCts.Token).AsTask(); + await quiesced.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, stream.WriteStarts); + Assert.Equal(0, stream.ConcurrentOps); + + transport.AbortQuiesceForConnectionTeardown(); + await Assert.ThrowsAnyAsync(() => writeTask); + Assert.Equal(0, stream.WriteStarts); + } + + private sealed class ControllableStream : Stream + { + private int _concurrentOps; + private int _readStarts; + private int _writeStarts; + private int _postQuiesceIoStarts; + private bool _trackPostQuiesce; + + public Func>? OnReadAsync { get; set; } + public Func? OnWriteAsync { get; set; } + + public int ConcurrentOps => Volatile.Read(ref _concurrentOps); + public int ReadStarts => Volatile.Read(ref _readStarts); + public int WriteStarts => Volatile.Read(ref _writeStarts); + public int PostQuiesceIoStarts => Volatile.Read(ref _postQuiesceIoStarts); + + public void ArmPostQuiesceTracking() => _trackPostQuiesce = true; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _readStarts); + if (_trackPostQuiesce) + { + Interlocked.Increment(ref _postQuiesceIoStarts); + } + + Interlocked.Increment(ref _concurrentOps); + try + { + if (OnReadAsync is { } onRead) + { + return await onRead(cancellationToken).ConfigureAwait(false); + } + + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + return 0; + } + finally + { + Interlocked.Decrement(ref _concurrentOps); + } + } + + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _writeStarts); + if (_trackPostQuiesce) + { + Interlocked.Increment(ref _postQuiesceIoStarts); + } + + Interlocked.Increment(ref _concurrentOps); + try + { + if (OnWriteAsync is { } onWrite) + { + await onWrite(cancellationToken).ConfigureAwait(false); + return; + } + + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + finally + { + Interlocked.Decrement(ref _concurrentOps); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs new file mode 100644 index 00000000..7eb39090 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/DeflateUpgradeTransportTests.cs @@ -0,0 +1,291 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +[Collection(nameof(TransportTestHostCollection))] +public sealed class DeflateUpgradeTransportTests +{ + [Fact] + public async Task UpgradeToDeflate_Plain_RoundTripsApplicationBytes() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.False(server.IsCompressed); + + var before = "PLAIN-BEFORE"u8.ToArray(); + await clientSocket.SendAsync(before); + Assert.Equal(before, await TransportTestShared.ReadExactAsync(server.Input, before.Length)); + + var identity = server.ClientIdentity; + var local = server.LocalEndPoint; + var remote = server.RemoteEndPoint; + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + Assert.False(server.IsTls); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(local, server.LocalEndPoint); + Assert.Equal(remote, server.RemoteEndPoint); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + + var payload = "COMPRESSED-PAYLOAD-12345"u8.ToArray(); + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + + var outbound = "SERVER-TO-CLIENT"u8.ToArray(); + await server.Output.WriteAsync(outbound); + await server.Output.FlushAsync(); + var received = new byte[outbound.Length]; + Assert.Equal(outbound.Length, await ReadExactAsync(clientDeflate, received)); + Assert.Equal(outbound, received); + } + + [Fact] + public async Task UpgradeToDeflate_AfterTls_LayersDeflateAboveTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var tlsUpgrade = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await tlsUpgrade; + Assert.True(server.IsTls); + Assert.False(server.IsCompressed); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsTls); + Assert.True(server.IsCompressed); + + // NntpDeflateStream takes ownership of the SslStream. + await using var clientDeflate = new NntpDeflateStream(ssl); + var payload = new byte[] { 0x10, 0x20, 0x30, 0x40 }; + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_AfterDeflate_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("DEFLATE", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.True(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_AlreadyCompressed_Throws() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + await server.UpgradeToDeflateAsync(); + + await Assert.ThrowsAsync(() => server.UpgradeToDeflateAsync()); + } + + [Fact] + public async Task UpgradeToDeflate_UnconsumedInput_RefusesAndKeepsUncompressed() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await clientSocket.SendAsync("UNCONSUMED"u8.ToArray()); + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek)) + { + var hasData = !peek.Buffer.IsEmpty; + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + if (hasData) + { + break; + } + } + + await Task.Yield(); + } + + var ex = await Assert.ThrowsAsync(() => server.UpgradeToDeflateAsync()); + Assert.Contains("unconsumed", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsCompressed); + Assert.False(server.ConnectionClosed.IsCancellationRequested); + Assert.Equal("UNCONSUMED"u8.ToArray(), await TransportTestShared.ReadExactAsync(server.Input, 10)); + } + + [Fact] + public async Task UpgradeToDeflate_PreservesTrustedProxyClientIdentity() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartPlainWithProxyAsync(trusted); + + using var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await clientSocket.ConnectAsync(host.EndPoint); + await clientSocket.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.88 127.0.0.1 4242 119\r\n")); + + await using var server = await host.AcceptAsync(); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.88"), server.ClientIdentity.ClientAddress); + Assert.Equal(4242, server.ClientIdentity.ClientPort); + var identity = server.ClientIdentity; + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(IPAddress.Parse("203.0.113.88"), server.ClientIdentity.ClientAddress); + Assert.Equal(4242, server.ClientIdentity.ClientPort); + } + + [Fact] + public async Task UpgradeToDeflate_MalformedPeerData_FailsTerminally() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.UpgradeToDeflateAsync(); + await clientSocket.SendAsync(new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01 }); + clientSocket.Shutdown(SocketShutdown.Send); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await Assert.ThrowsAnyAsync(async () => + { + while (!cts.Token.IsCancellationRequested) + { + var result = await server.Input.ReadAsync(cts.Token); + if (result.IsCompleted) + { + throw new InvalidDataException("Input completed after malformed DEFLATE."); + } + + server.Input.AdvanceTo(result.Buffer.End); + } + }); + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_CancelledDuringQuiesce_FailsTerminally() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => + server.UpgradeToDeflateAsync(cts.Token)); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToDeflate_ConcurrentWithTlsUpgrade_SecondRejected() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var tls = server.UpgradeToTlsAsync(host.CertificateProvider!); + var deflate = server.UpgradeToDeflateAsync(); + + var secondEx = await Assert.ThrowsAsync(() => deflate); + Assert.Contains("already in progress", secondEx.Message, StringComparison.OrdinalIgnoreCase); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await tls; + Assert.True(server.IsTls); + Assert.False(server.IsCompressed); + } + + [Fact] + public async Task UpgradeToDeflate_FlushedUncompressedOutbound_IsDeliveredBeforeCompression() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "206 Compression active\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var receivedNotice = new byte[notice.Length]; + var noticeTotal = 0; + using (var noticeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (noticeTotal < notice.Length) + { + var n = await clientSocket.ReceiveAsync(receivedNotice.AsMemory(noticeTotal), noticeCts.Token); + Assert.True(n > 0); + noticeTotal += n; + } + } + + Assert.Equal(notice, receivedNotice); + + await server.UpgradeToDeflateAsync(); + Assert.True(server.IsCompressed); + + await using var clientStream = new NetworkStream(clientSocket, ownsSocket: true); + await using var clientDeflate = new NntpDeflateStream(clientStream); + var payload = "after"u8.ToArray(); + await clientDeflate.WriteAsync(payload); + await clientDeflate.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + private static async Task ReadExactAsync(Stream stream, Memory buffer) + { + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await stream.ReadAsync(buffer[total..], cts.Token); + Assert.True(n > 0); + total += n; + } + + return total; + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs new file mode 100644 index 00000000..f6dfc7b5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputReaderOwnershipTests.cs @@ -0,0 +1,144 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Deterministic Input.Reader ownership: the application consumer examines/advances; +/// CompleteAsync (including ObservePump after receive EOF) must not complete the reader +/// while ReadAsync can still run. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpConnectionInputReaderOwnershipTests +{ + [Fact] + public async Task CompleteAsync_WhileInputReadAsyncPending_DoesNotThrowReaderCompleted() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inboundRead = server.Input.ReadAsync().AsTask(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + var complete = server.CompleteAsync(); + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "CompleteAsync completed Input.Reader while ReadAsync was outstanding: " + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + await complete.WaitAsync(timeout.Token); + + Assert.False(server.InputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ReceiveEof_WhileInputReadAsyncPending_DoesNotCompleteReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inboundRead = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Receive EOF completed Input.Reader while ReadAsync was outstanding: " + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + Assert.False(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task CompleteInputReader_AfterConsumerStops_IsOnceAndAllowsNoFurtherRead() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + server.AttachInputReaderConsumer(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + Assert.False(server.InputReaderCompletedForTests); + + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + + var completed = false; + try + { + _ = await server.Input.ReadAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task ConcurrentCompleteAsync_DoesNotCompleteInputReaderWhileConsumerAttached() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + server.AttachInputReaderConsumer(); + var inboundRead = server.Input.ReadAsync().AsTask(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + var first = server.CompleteAsync(); + var second = server.CompleteAsync(); + await second.WaitAsync(timeout.Token); + await first.WaitAsync(timeout.Token); + + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Concurrent CompleteAsync completed Input.Reader while a consumer was attached: " + + ex.Message); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + Assert.False(server.InputReaderCompletedForTests); + + await server.CompleteInputReaderAsync(); + Assert.True(server.InputReaderCompletedForTests); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs new file mode 100644 index 00000000..ae1d1993 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionInputWriterOwnershipTests.cs @@ -0,0 +1,260 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Deterministic Input.Writer ownership: ReceiveAsync GetMemory/Advance/completes; +/// CompleteAsync must not complete the writer while a receive reservation is outstanding. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpConnectionInputWriterOwnershipTests +{ + [Fact] + public async Task CompleteAsync_AfterReadReturns_DoesNotInvalidateAdvance() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.AfterStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync(new byte[] { 0x41 }); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await entered.Task.WaitAsync(timeout.Token); + + await server.CompleteInputReaderAsync(); + var complete = server.CompleteAsync(); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.InputWriterCompletedForTests); + Assert.False(complete.IsCompleted); + Assert.False(server.ReceivePumpTaskForTests!.IsCompleted); + + release.TrySetResult(); + await complete.WaitAsync(timeout.Token); + + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task IdleTimeout_WhileReadOutstanding_DoesNotLogReceivePumpError() + { + var logger = new CapturingConnectionLogger(); + var clock = new ControllableTimeProvider(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var session = new NntpSession( + server, + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(2), + timeProvider: clock); + var run = session.RunAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + transport.AfterStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync(new byte[] { (byte)'D' }); + await entered.Task.WaitAsync(timeout.Token); + + clock.Advance(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + Assert.False(server.InputWriterCompletedForTests); + + release.TrySetResult(); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(TcpDisconnectReason.IdleTimeout, server.DisconnectReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task Quit_WhileNextReadOutstanding_DoesNotLogReceivePumpError() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var session = new NntpSession(server, NullLogger.Instance); + var run = session.RunAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = async () => + { + entered.TrySetResult(); + await release.Task.ConfigureAwait(false); + }; + + await client.SendAsync("QUIT\r\n"u8.ToArray()); + await entered.Task.WaitAsync(timeout.Token); + await WaitUntilAsync(static s => s.ConnectionClosed.IsCancellationRequested, server, timeout.Token); + Assert.False(server.InputWriterCompletedForTests); + + release.TrySetResult(); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + AssertNoAdvanceRangeFault(server.ReceivePumpTaskForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task RemoteEof_CompletesWriterOnce_AndKeepsRemoteClosed() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inbound = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await inbound.WaitAsync(timeout.Token); + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + await WaitUntilAsync(static s => s.InputWriterCompletedForTests, server, timeout.Token); + Assert.Equal(TcpDisconnectReason.RemoteClosed, server.DisconnectReasonForTests); + Assert.True(server.InputWriterCompletedForTests); + Assert.False(server.InputReaderCompletedForTests); + Assert.Empty(logger.PumpErrors); + } + + [Fact] + public async Task ReceiveException_StillCompletesWriter() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = static () => + throw new IOException("forced-receive-failure"); + + await client.SendAsync(new byte[] { 0x01 }); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync( + static s => s.InputWriterCompletedForTests && s.IsCompleted, + server, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + Assert.True(server.InputWriterCompletedForTests); + Assert.Equal(TcpDisconnectReason.ReceiveError, server.DisconnectReasonForTests); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + var receive = server.ReceivePumpTaskForTests; + Assert.NotNull(receive); + Assert.True(receive.IsFaulted); + Assert.Contains( + receive.Exception!.Flatten().InnerExceptions, + static ex => ex is IOException); + } + + private static void AssertNoAdvanceRangeFault(Task? receive) + { + Assert.NotNull(receive); + if (!receive.IsFaulted) + { + return; + } + + foreach (var ex in receive.Exception!.Flatten().InnerExceptions) + { + Assert.False( + ex is ArgumentOutOfRangeException aor + && string.Equals(aor.ParamName, "bytes", StringComparison.Ordinal), + "Receive pump must not Advance after Input.Writer was completed."); + } + } + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class CapturingConnectionLogger : ILogger + { + public List PumpErrors { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var formatted = formatter(state, exception); + if (formatted.Contains("pump ended with an error", StringComparison.Ordinal)) + { + PumpErrors.Add(formatted); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs new file mode 100644 index 00000000..94f0a427 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpConnectionOutputReaderOwnershipTests.cs @@ -0,0 +1,133 @@ +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Deterministic Output.Reader ownership: SendAsync examines/advances/completes; +/// CompleteAsync must not complete the reader while the send pump holds a ReadAsync result. +/// +public sealed class NntpConnectionOutputReaderOwnershipTests +{ + [Fact] + public async Task CompleteAsync_WhileSendBlockedInWrite_DoesNotCompleteReaderBeforeAdvanceTo() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var enteredWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = async () => + { + enteredWrite.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + await server.Output.WriteAsync(new byte[] { 0x01, 0x02, 0x03 }); + await server.Output.FlushAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await enteredWrite.Task.WaitAsync(timeout.Token); + + var complete = server.CompleteAsync(); + await WaitUntilAsync(static s => s.OutputWriterCompletedForTests, server, timeout.Token); + + Assert.True(server.OutputWriterCompletedForTests); + Assert.False(complete.IsCompleted); + Assert.False(server.SendPumpTaskForTests!.IsCompleted); + Assert.False(server.OutputReaderCompletedForTests); + + releaseWrite.TrySetResult(); + await complete.WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task CompleteAsync_WhileSendIdleInReadAsync_CompletesCleanly() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(static s => s.SendPumpAwaitingOutputForTests == 1, server, timeout.Token); + + await server.CompleteAsync().WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ConcurrentCompleteAsync_IsSingleFlight_AndDoesNotRaceOutputReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var enteredWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWrite = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = async () => + { + enteredWrite.TrySetResult(); + await releaseWrite.Task.ConfigureAwait(false); + }; + + await server.Output.WriteAsync(new byte[] { 0x11 }); + await server.Output.FlushAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await enteredWrite.Task.WaitAsync(timeout.Token); + + var first = server.CompleteAsync(); + await WaitUntilAsync(static s => s.IsCompleted, server, timeout.Token); + + var second = server.CompleteAsync(); + await second.WaitAsync(timeout.Token); + Assert.False(first.IsCompleted); + + releaseWrite.TrySetResult(); + await first.WaitAsync(timeout.Token); + + AssertNoAdvanceToAfterReaderCompleted(server.SendPumpTaskForTests); + Assert.True(server.OutputReaderCompletedForTests); + } + + private static void AssertNoAdvanceToAfterReaderCompleted(Task? send) + { + Assert.NotNull(send); + if (!send.IsFaulted) + { + return; + } + + foreach (var ex in send.Exception!.Flatten().InnerExceptions) + { + Assert.False( + ex is InvalidOperationException + && ex.Message.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal), + "Send pump must not fault because Output.Reader was completed before AdvanceTo."); + } + } + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpDeflateStreamTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpDeflateStreamTests.cs new file mode 100644 index 00000000..c9ff7498 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpDeflateStreamTests.cs @@ -0,0 +1,250 @@ +using System.IO.Compression; +using System.Text; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Codec-level tests for raw DEFLATE used by NNTP COMPRESS (RFC 8054 §4 / RFC 1951). +/// +public sealed class NntpDeflateStreamTests +{ + [Fact] + public void DeflateStream_ProducesRawDeflate_NotZlibOrGzip() + { + var plain = "hello hello hello hello NNTP raw deflate"u8.ToArray(); + using var ms = new MemoryStream(); + using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) + { + ds.Write(plain); + } + + var compressed = ms.ToArray(); + Assert.NotEmpty(compressed); + // zlib wrapper starts with CMF/FLG typically 0x78; gzip with 0x1F 0x8B. + Assert.NotEqual(0x78, compressed[0]); + Assert.False(compressed.Length >= 2 && compressed[0] == 0x1F && compressed[1] == 0x8B); + + using var zlibMs = new MemoryStream(); + using (var zs = new ZLibStream(zlibMs, CompressionLevel.Optimal, leaveOpen: true)) + { + zs.Write(plain); + } + + Assert.Equal(0x78, zlibMs.ToArray()[0]); + Assert.ThrowsAny(() => + { + using var reader = new ZLibStream(new MemoryStream(compressed), CompressionMode.Decompress); + reader.CopyTo(Stream.Null); + }); + } + + [Theory] + [InlineData("")] + [InlineData("a")] + [InlineData("hello world")] + [InlineData("AAAAAAABBBBBBCCCCCC")] + public async Task RoundTrip_VariousPayloads(string text) + { + var plain = Encoding.UTF8.GetBytes(text); + await AssertRoundTripAsync(plain); + } + + [Fact] + public async Task RoundTrip_BinaryAndHighEntropy() + { + var plain = new byte[4096]; + Random.Shared.NextBytes(plain); + await AssertRoundTripAsync(plain); + } + + [Fact] + public async Task RoundTrip_LargeStreaming() + { + var plain = new byte[256 * 1024]; + for (var i = 0; i < plain.Length; i++) + { + plain[i] = (byte)('A' + (i % 26)); + } + + await AssertRoundTripAsync(plain, writeChunk: 17, readChunk: 31); + } + + [Fact] + public async Task Flush_MakesCompressedBytesVisibleWithoutDispose() + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(peer.Server); + await using var client = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(peer.Client); + + var payload = "flush-boundary-test-payload"u8.ToArray(); + await server.WriteAsync(payload); + await server.FlushAsync(); + + var buffer = new byte[payload.Length]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await client.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + + Assert.Equal(payload, buffer); + } + + [Fact] + public async Task MalformedCompressedInput_ThrowsWithoutSilentCorruption() + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(peer.Server); + // Write non-DEFLATE junk on the wire (peer.Client is the raw NetworkStream). + await peer.Client.WriteAsync(new byte[] { 0x00, 0x01, 0x02, 0xFF, 0xFE }); + await peer.Client.FlushAsync(); + + var buffer = new byte[64]; + await Assert.ThrowsAnyAsync(async () => + { + _ = await server.ReadAsync(buffer); + }); + } + + [Fact] + public async Task IndependentStreams_DoNotShareDictionaryState() + { + await using var a = await ConnectedStreamPair.CreateAsync(); + await using var b = await ConnectedStreamPair.CreateAsync(); + await using var aServer = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(a.Server); + await using var aClient = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(a.Client); + await using var bServer = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(b.Server); + await using var bClient = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(b.Client); + + await aServer.WriteAsync("connection-A-payload"u8.ToArray()); + await aServer.FlushAsync(); + await bServer.WriteAsync("connection-B-different"u8.ToArray()); + await bServer.FlushAsync(); + + var bufA = new byte[20]; + var bufB = new byte[22]; + Assert.Equal(20, await ReadExactAsync(aClient, bufA)); + Assert.Equal(22, await ReadExactAsync(bClient, bufB)); + Assert.Equal("connection-A-payload"u8.ToArray(), bufA); + Assert.Equal("connection-B-different"u8.ToArray(), bufB); + } + + private static async Task AssertRoundTripAsync(byte[] plain, int writeChunk = 0, int readChunk = 0) + { + await using var peer = await ConnectedStreamPair.CreateAsync(); + await using var server = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(peer.Server); + await using var client = new VectorNNTPD.TransitService.Networking.Transport.NntpDeflateStream(peer.Client); + + if (writeChunk <= 0) + { + await server.WriteAsync(plain); + } + else + { + for (var i = 0; i < plain.Length; i += writeChunk) + { + var len = Math.Min(writeChunk, plain.Length - i); + await server.WriteAsync(plain.AsMemory(i, len)); + } + } + + await server.FlushAsync(); + + var received = new byte[plain.Length]; + if (readChunk <= 0) + { + Assert.Equal(plain.Length, await ReadExactAsync(client, received)); + } + else + { + var total = 0; + while (total < received.Length) + { + var n = await client.ReadAsync(received.AsMemory(total, Math.Min(readChunk, received.Length - total))); + Assert.True(n > 0); + total += n; + } + } + + Assert.Equal(plain, received); + } + + private static async Task ReadExactAsync(Stream stream, Memory buffer) + { + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + while (total < buffer.Length) + { + var n = await stream.ReadAsync(buffer[total..], cts.Token); + if (n == 0) + { + break; + } + + total += n; + } + + return total; + } + + private sealed class ConnectedStreamPair : IAsyncDisposable + { + private readonly System.Net.Sockets.Socket _listener; + private readonly System.Net.Sockets.Socket _serverSocket; + private readonly System.Net.Sockets.Socket _clientSocket; + + private ConnectedStreamPair( + System.Net.Sockets.Socket listener, + System.Net.Sockets.Socket serverSocket, + System.Net.Sockets.Socket clientSocket, + System.Net.Sockets.NetworkStream server, + System.Net.Sockets.NetworkStream client) + { + _listener = listener; + _serverSocket = serverSocket; + _clientSocket = clientSocket; + Server = server; + Client = client; + } + + public System.Net.Sockets.NetworkStream Server { get; } + public System.Net.Sockets.NetworkStream Client { get; } + + public static async Task CreateAsync() + { + var listener = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + listener.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + listener.Listen(1); + + var clientSocket = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + var connect = clientSocket.ConnectAsync(listener.LocalEndPoint!); + var serverSocket = await listener.AcceptAsync(); + await connect; + + return new ConnectedStreamPair( + listener, + serverSocket, + clientSocket, + new System.Net.Sockets.NetworkStream(serverSocket, ownsSocket: true), + new System.Net.Sockets.NetworkStream(clientSocket, ownsSocket: true)); + } + + public async ValueTask DisposeAsync() + { + await Server.DisposeAsync(); + await Client.DisposeAsync(); + _listener.Dispose(); + _serverSocket.Dispose(); + _clientSocket.Dispose(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpTransportTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpTransportTests.cs new file mode 100644 index 00000000..9d2fd584 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/NntpTransportTests.cs @@ -0,0 +1,742 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Tests.Acme; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.Networking.Listeners; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +public sealed class ListenEndpointPlannerTests +{ + [Fact] + public void Star_ProducesSingleDualStackIpv6Any() + { + var bindings = ListenEndpointPlanner.Plan(["*"], 1199); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Star_IgnoresAdditionalExplicits() + { + var bindings = ListenEndpointPlanner.Plan(["*", "127.0.0.1", "::1"], 1199); + Assert.Single(bindings); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv4Any_Alone_ProducesIpv4Wildcard() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0"], 1200); + Assert.Single(bindings); + Assert.Equal(IPAddress.Any, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void Ipv6Any_Alone_ProducesDualStack() + { + var bindings = ListenEndpointPlanner.Plan(["::"], 1200); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void ExplicitIpv4_Alone() + { + var address = IPAddress.Parse("192.0.2.10"); + var bindings = ListenEndpointPlanner.Plan(["192.0.2.10"], 1200); + Assert.Single(bindings); + Assert.Equal(address, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void ExplicitIpv6_Alone() + { + var address = IPAddress.Parse("2001:db8::10"); + var bindings = ListenEndpointPlanner.Plan(["2001:db8::10"], 1200); + Assert.Single(bindings); + Assert.Equal(address, bindings[0].Address); + Assert.False(bindings[0].DualMode); + } + + [Fact] + public void ExplicitAddresses_Deduplicate() + { + var bindings = ListenEndpointPlanner.Plan(["127.0.0.1", "127.0.0.1", "::1"], 1200); + Assert.Equal(2, bindings.Count); + } + + [Fact] + public void MultipleIndependentExplicits_AreRetained() + { + var bindings = ListenEndpointPlanner.Plan(["192.0.2.10", "2001:db8::10"], 1200); + Assert.Equal(2, bindings.Count); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Parse("192.0.2.10"))); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Parse("2001:db8::10"))); + } + + [Fact] + public void Ipv4AnyAndIpv6Any_AreSeparateSingleFamilySockets() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0", "::"], 1201); + Assert.Equal(2, bindings.Count); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.Any) && !b.DualMode); + Assert.Contains(bindings, static b => b.Address.Equals(IPAddress.IPv6Any) && !b.DualMode); + } + + [Fact] + public void Ipv6AnyDualMode_PlusExplicitIpv4_DoesNotOverlap() + { + var bindings = ListenEndpointPlanner.Plan(["::", "192.0.2.10"], 1202); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv6AnyDualMode_PlusIpv4MappedExplicit_DoesNotOverlap() + { + var bindings = ListenEndpointPlanner.Plan(["::", "::ffff:192.0.2.10"], 1203); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } + + [Fact] + public void Ipv4Any_PlusExplicitIpv4_SkipsExplicit() + { + var bindings = ListenEndpointPlanner.Plan(["0.0.0.0", "192.0.2.10"], 1204); + Assert.Single(bindings); + Assert.Equal(IPAddress.Any, bindings[0].Address); + } + + [Fact] + public void Ipv6Any_PlusExplicitIpv6_SkipsExplicit() + { + var bindings = ListenEndpointPlanner.Plan(["::", "2001:db8::10"], 1205); + Assert.Single(bindings); + Assert.Equal(IPAddress.IPv6Any, bindings[0].Address); + Assert.True(bindings[0].DualMode); + } +} + +/// +/// Binds planned dual-stack endpoints on loopback/ephemeral ports (Windows dual-stack semantics). +/// +public sealed class ListenEndpointBindTests +{ + [Fact] + public async Task DualStackIpv6Any_AcceptsIpv4AndIpv6Loopback() + { + if (!Socket.OSSupportsIPv6) + { + return; + } + + var plan = ListenEndpointPlanner.Plan(["::", "127.0.0.1"], port: 1199); + Assert.Single(plan); + Assert.True(plan[0].DualMode); + + // Ephemeral port: one DualMode IPv6Any listener (no separate IPv4 socket from the planner). + var binding = new ListenBinding(IPAddress.IPv6Any, 0, DualMode: true); + await using var listener = new SocketAcceptListener( + binding, + static (_, _) => ValueTask.CompletedTask, + NullLogger.Instance); + listener.Start(); + var port = listener.LocalEndPoint.Port; + + using (var v4 = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await v4.ConnectAsync(new IPEndPoint(IPAddress.Loopback, port)); + } + + using (var v6 = new Socket(AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp)) + { + await v6.ConnectAsync(new IPEndPoint(IPAddress.IPv6Loopback, port)); + } + } +} + +public sealed class NntpPlainTransportTests +{ + [Fact] + public async Task Accept_ExchangeArbitraryBytes_Bidirectional() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var payload = new byte[] { 0x00, 0x01, 0x7F, 0x80, 0xFE, 0xFF, 0x0D, 0x0A, 0x2E, 0x2E }; + await client.SendAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + + var reply = new byte[] { 0xAA, 0x55, 0x00, 0xFF }; + await server.Output.WriteAsync(reply); + await server.Output.FlushAsync(); + var buffer = new byte[reply.Length]; + Assert.Equal(reply.Length, await client.ReceiveAsync(buffer)); + Assert.Equal(reply, buffer); + } + + [Fact] + public async Task FragmentedReceives_PreserveOrder() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await client.SendAsync(new byte[] { 0x01, 0x02 }); + await client.SendAsync(new byte[] { 0x03, 0x04, 0x05 }); + Assert.Equal( + new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }, + await TransportTestShared.ReadExactAsync(server.Input, 5)); + } + + [Fact] + public async Task MultipleConnections_AreIndependent() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var c1 = await host.ConnectPlainClientAsync(); + using var c2 = await host.ConnectPlainClientAsync(); + await using var s1 = await host.AcceptAsync(); + await using var s2 = await host.AcceptAsync(); + + await c1.SendAsync(new byte[] { 0x11 }); + await c2.SendAsync(new byte[] { 0x22 }); + Assert.Equal(new byte[] { 0x11 }, await TransportTestShared.ReadExactAsync(s1.Input, 1)); + Assert.Equal(new byte[] { 0x22 }, await TransportTestShared.ReadExactAsync(s2.Input, 1)); + } + + [Fact] + public async Task RemoteGracefulClose_CompletesInput() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Arm inbound read before FIN so Ordering A/B are both covered without sleeps. + var inboundRead = server.Input.ReadAsync().AsTask(); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + + // Writer completion (receive EOF) must unblock ReadAsync as IsCompleted. + // CompleteAsync must not complete Input.Reader while this consumer is still reading. + ReadResult result; + try + { + result = await inboundRead.WaitAsync(timeout.Token); + } + catch (InvalidOperationException ex) + { + Assert.Fail( + "Input.Reader was completed while ReadAsync was outstanding: " + ex.Message); + return; + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + Assert.Fail("Timed out waiting for inbound completion after remote FIN."); + return; + } + + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + // 2) Connection lifecycle must reach closed (may already be cancelled). + if (!server.ConnectionClosed.IsCancellationRequested) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + server.ConnectionClosed, + timeout.Token); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + Assert.Fail("Expected ConnectionClosed after remote FIN."); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(timeout.IsCancellationRequested); + } + + [Fact] + public async Task ListenerContinues_AfterPriorConnectionCloses() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var c1 = await host.ConnectPlainClientAsync(); + await using var s1 = await host.AcceptAsync(); + await s1.CompleteAsync(); + await s1.DisposeAsync(); + c1.Dispose(); + + using var c2 = await host.ConnectPlainClientAsync(); + await using var s2 = await host.AcceptAsync(); + await c2.SendAsync(new byte[] { 0x42 }); + Assert.Equal(new byte[] { 0x42 }, await TransportTestShared.ReadExactAsync(s2.Input, 1)); + } + + [Fact] + public async Task PlainListenerService_BindsAndAccepts() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPort = TestHostFactory.GetFreeTcpPort(); + await using var service = new NntpPlainListenerService( + Options.Create(options), + new TrustedProxyHosts(Options.Create(options)), + new TlsCertificateContextProvider(NullLogger.Instance), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.NotEmpty(service.LocalEndPoints); + + using var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(service.LocalEndPoints[0]); + await client.SendAsync(new byte[] { 0x01 }); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (service.ActiveConnectionCount < 1 && !cts.IsCancellationRequested) + { + await Task.Delay(10, cts.Token); + } + + Assert.True(service.ActiveConnectionCount >= 1); + await service.StopAsync(CancellationToken.None); + } +} + +public sealed class NntpTlsTransportTests +{ + [Fact] + public async Task TlsDisabled_DoesNotBind() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindPortTls = 0; + await using var service = new NntpTlsListenerService( + Options.Create(options), + new TlsCertificateContextProvider(NullLogger.Instance), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + Assert.False(service.ListenersBound); + } + + [Fact] + public async Task TlsEnabledWithoutCertificate_FailsStart() + { + var options = TestHostFactory.CreateValidOptions(); + options.BindAddress = ["127.0.0.1"]; + options.BindPortTls = TestHostFactory.GetFreeTcpPort(); + await using var service = new NntpTlsListenerService( + Options.Create(options), + new TlsCertificateContextProvider(NullLogger.Instance), + new TrustedProxyHosts(Options.Create(options)), + DenyAllNntpAuthenticationProvider.Instance, + DisabledArticleIngestionQueue.Instance, + TransitPeerAuthorization.Disabled, + NullLoggerFactory.Instance, + NullLogger.Instance); + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + } + + [Fact] + public async Task SuccessfulHandshake_ExchangesBinary_AndPresentsIdentity() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + await using var client = await host.ConnectTlsClientAsync(); + Assert.NotNull(client.RemoteCertificate); + Assert.Contains( + "nntpd01.usenet.ninja", + client.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + + await using var server = await host.AcceptAsync(); + Assert.True(server.IsTls); + var payload = new byte[] { 0x00, 0x80, 0xFF, 0x0D, 0x0A, 0x2E }; + await client.Stream.WriteAsync(payload); + await client.Stream.FlushAsync(); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task TlsOutbound_MultiplePipeBatches_PreserveOrderWithoutSslFlush() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + + // Distinct PipeWriter flushes → distinct send-pump ReadAsync batches. + // Delivery must not depend on SslStream.FlushAsync (NetworkStream is unbuffered). + var batch1 = new byte[] { 0x01, 0x02, 0x03 }; + var batch2 = new byte[] { 0xFE, 0xFF, 0x00 }; + var batch3 = new byte[] { 0x0D, 0x0A, 0x2E, 0x2E }; + + await server.Output.WriteAsync(batch1); + await server.Output.FlushAsync(); + await server.Output.WriteAsync(batch2); + await server.Output.FlushAsync(); + await server.Output.WriteAsync(batch3); + await server.Output.FlushAsync(); + + var expected = batch1.Concat(batch2).Concat(batch3).ToArray(); + var received = new byte[expected.Length]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < expected.Length) + { + var n = await client.Stream.ReadAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0, "TLS stream closed before all outbound batches arrived."); + total += n; + } + + Assert.Equal(expected, received); + } + + [Fact] + public async Task FailedHandshake_DoesNotKillListener() + { + await using var host = await TransportTestHost.StartTlsAsync( + TransportTestShared.CreatePfx("nntpd01.usenet.ninja")); + using (var raw = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp)) + { + await raw.ConnectAsync(host.EndPoint); + await raw.SendAsync(new byte[] { 0x15, 0x00, 0x00, 0x00 }); + } + + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + await client.Stream.WriteAsync(new byte[] { 0x01 }); + await client.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0x01 }, await TransportTestShared.ReadExactAsync(server.Input, 1)); + } + + [Fact] + public async Task CertificateRotation_KeepsOldSession_NewHandshakeUsesNewCert() + { + var pfxA = TransportTestShared.CreatePfx("cert-a.usenet.ninja"); + var pfxB = TransportTestShared.CreatePfx("cert-b.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfxA); + + await using var client1 = await host.ConnectTlsClientAsync(); + Assert.Contains( + "cert-a.usenet.ninja", + client1.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + await using var server1 = await host.AcceptAsync(); + + host.PublishCertificate(pfxB); + + await using var client2 = await host.ConnectTlsClientAsync(); + Assert.Contains( + "cert-b.usenet.ninja", + client2.RemoteCertificate!.Subject, + StringComparison.OrdinalIgnoreCase); + await using var server2 = await host.AcceptAsync(); + + await client1.Stream.WriteAsync(new byte[] { 0xA1 }); + await client1.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0xA1 }, await TransportTestShared.ReadExactAsync(server1.Input, 1)); + + await client2.Stream.WriteAsync(new byte[] { 0xB2 }); + await client2.Stream.FlushAsync(); + Assert.Equal(new byte[] { 0xB2 }, await TransportTestShared.ReadExactAsync(server2.Input, 1)); + } +} + +public sealed class TlsCertificateContextProviderTests +{ + [Fact] + public void Publish_AtomicSwap_HeldLeaseRemainsUsable() + { + var provider = new TlsCertificateContextProvider(NullLogger.Instance); + provider.PublishFromPfx( + TransportTestShared.CreatePfx("a.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + using var leaseA = provider.Acquire(); + provider.PublishFromPfx( + TransportTestShared.CreatePfx("b.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + using var leaseB = provider.Acquire(); + Assert.Contains( + "a.usenet.ninja", + leaseA.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.Contains( + "b.usenet.ninja", + leaseB.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + } +} + +internal static class TransportTestShared +{ + public static byte[] CreatePfx(string cn) => + TestCertificateFactory.CreateMaterial( + [cn, "news.usenet.ninja"], + AcmeConfigurationTests.TestPfxPassword, + DateTimeOffset.UtcNow.AddDays(30)) + .PfxBytes; + + public static async Task ReadExactAsync(PipeReader reader, int count) + { + while (true) + { + var result = await reader.ReadAsync(); + if (result.Buffer.Length >= count) + { + var slice = result.Buffer.Slice(0, count); + var copy = new byte[count]; + slice.CopyTo(copy.AsSpan()); + reader.AdvanceTo(slice.End); + return copy; + } + + if (result.IsCompleted) + { + throw new InvalidOperationException($"Completed before {count} bytes arrived."); + } + + reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + } + } +} + +internal sealed class TransportTestHost : IAsyncDisposable +{ + private readonly SocketAcceptListener _listener; + private readonly TlsCertificateContextProvider? _certs; + private readonly ConcurrentQueue> _waiters = new(); + private readonly ConcurrentQueue _ready = new(); + private readonly object _gate = new(); + + private TransportTestHost(SocketAcceptListener listener, TlsCertificateContextProvider? certs) + { + _listener = listener; + _certs = certs; + } + + public IPEndPoint EndPoint => _listener.LocalEndPoint; + + public TlsCertificateContextProvider? CertificateProvider => _certs; + + public static Task StartPlainAsync(ILogger? connectionLogger = null) => + StartAsync(tls: false, pfx: null, connectionLogger: connectionLogger); + + /// Plain accept path with a certificate provider available for . + public static Task StartPlainWithCertificateAsync(byte[] pfx) => + StartAsync(tls: false, pfx); + + public static Task StartTlsAsync(byte[] pfx) => StartAsync(tls: true, pfx); + + /// + /// Starts a TLS test host that resolves PROXY identity via + /// before the TLS handshake (production listener ordering). + /// + public static Task StartTlsWithProxyAsync(byte[] pfx, ITrustedProxyHosts trustedProxyHosts) => + StartAsync(tls: true, pfx, trustedProxyHosts); + + /// Plain accept with PROXY trust evaluation and optional upgrade certificate provider. + public static Task StartPlainWithProxyAsync( + ITrustedProxyHosts trustedProxyHosts, + byte[]? upgradePfx = null) => + StartAsync(tls: false, upgradePfx, trustedProxyHosts); + + private static Task StartAsync( + bool tls, + byte[]? pfx, + ITrustedProxyHosts? trustedProxyHosts = null, + ILogger? connectionLogger = null) + { + TlsCertificateContextProvider? certs = null; + if (pfx is not null) + { + certs = new TlsCertificateContextProvider(NullLogger.Instance); + certs.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + + if (tls) + { + ArgumentNullException.ThrowIfNull(certs); + } + + trustedProxyHosts ??= new TrustedProxyHosts(Options.Create(new NntpdOptions { ProxyHosts = [] })); + var logger = connectionLogger ?? NullLogger.Instance; + + TransportTestHost? host = null; + var binding = new ListenBinding(IPAddress.Loopback, 0, DualMode: false); + var listener = new SocketAcceptListener( + binding, + async (socket, ct) => + { + if (!ProxyPreambleResolver.TryGetTcpPeer(socket, out var tcpPeer)) + { + socket.Dispose(); + return; + } + + ProxyPreambleResolution preamble; + try + { + preamble = await ProxyPreambleResolver + .ResolveAsync(socket, tcpPeer, trustedProxyHosts, ct) + .ConfigureAwait(false); + } + catch (ProxyProtocolException) + { + socket.Dispose(); + return; + } + + INntpConnection connection = tls + ? await NntpConnection.StartTlsAsync( + socket, + certs!, + preamble.Identity, + logger, + ct, + preamble.Leftover) + .ConfigureAwait(false) + : NntpConnection.StartPlain( + socket, + preamble.Identity, + logger, + preamble.Leftover); + + lock (host!._gate) + { + if (host._waiters.TryDequeue(out var waiter)) + { + waiter.TrySetResult(connection); + } + else + { + host._ready.Enqueue(connection); + } + } + }, + NullLogger.Instance); + + host = new TransportTestHost(listener, certs); + listener.Start(); + return Task.FromResult(host); + } + + public void PublishCertificate(byte[] pfx) + { + ArgumentNullException.ThrowIfNull(_certs); + _certs.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + + public async Task ConnectPlainClientAsync() + { + var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await client.ConnectAsync(EndPoint); + return client; + } + + public async Task ConnectTlsClientAsync() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await socket.ConnectAsync(EndPoint); + var network = new NetworkStream(socket, ownsSocket: true); + var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }); + var remoteCertificate = ssl.RemoteCertificate is null + ? null + : new X509Certificate2(ssl.RemoteCertificate); + return new TlsClient(ssl, remoteCertificate); + } + + public async Task AcceptAsync() + { + TaskCompletionSource tcs; + lock (_gate) + { + if (_ready.TryDequeue(out var ready)) + { + return ready; + } + + tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Enqueue(tcs); + } + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await using var reg = cts.Token.Register( + static s => ((TaskCompletionSource)s!).TrySetCanceled(), + tcs); + return await tcs.Task; + } + + public async ValueTask DisposeAsync() + { + await _listener.DisposeAsync(); + if (_certs is not null) + { + await _certs.DisposeAsync(); + } + } + + internal sealed class TlsClient : IAsyncDisposable + { + public TlsClient(SslStream stream, X509Certificate2? remoteCertificate) + { + Stream = stream; + RemoteCertificate = remoteCertificate; + } + + public SslStream Stream { get; } + + public X509Certificate2? RemoteCertificate { get; } + + public ValueTask DisposeAsync() + { + Stream.Dispose(); + RemoteCertificate?.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SendPipeAdvancementTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SendPipeAdvancementTests.cs new file mode 100644 index 00000000..9fec0d82 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SendPipeAdvancementTests.cs @@ -0,0 +1,169 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Documents outbound pipe advancement: unsent bytes must not be marked consumed while a send +/// remains logically in progress; on terminal abort the connection is completed (no retry). +/// +public sealed class SendPipeAdvancementTests +{ + [Fact] + public async Task PlainSendPattern_OnThrowAfterPartialSegments_DoesNotConsumeUnsentBytes() + { + var pipe = new Pipe(); + // Two segments in one ReadOnlySequence: force non-contiguous via two writes without flush coalesce... + // A single contiguous buffer is fine: treat first half as "sent", then throw before marking End. + var payload = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06 }; + await pipe.Writer.WriteAsync(payload); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + + var result = await pipe.Reader.ReadAsync(); + var buffer = result.Buffer; + Assert.Equal(6, buffer.Length); + + // Mirror SendSocketAsync: only advance consumed on full success. + var consumed = buffer.Start; + var transmitted = 0; + var failed = false; + try + { + foreach (var segment in buffer) + { + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + // Transmit first three bytes across possibly multiple calls, then fail. + if (transmitted >= 3) + { + throw new IOException("simulated send failure after partial transmit"); + } + + var take = Math.Min(2, Math.Min(memory.Length, 3 - transmitted)); + transmitted += take; + return ValueTask.FromResult(take); + }, + segment, + state: null, + CancellationToken.None); + } + + consumed = buffer.End; + } + catch (IOException) + { + failed = true; + } + finally + { + pipe.Reader.AdvanceTo(consumed); + } + + Assert.True(failed); + Assert.Equal(3, transmitted); + Assert.True(consumed.Equals(buffer.Start)); + + // Unsent octets remain readable (would be discarded only when the connection/pipe is completed). + var again = await pipe.Reader.ReadAsync(); + Assert.Equal(6, again.Buffer.Length); + pipe.Reader.AdvanceTo(again.Buffer.Start, again.Buffer.End); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task PlainSendPattern_OnFullSuccess_ConsumesEntireBuffer() + { + var pipe = new Pipe(); + var payload = new byte[] { 0xAA, 0xBB, 0xCC }; + await pipe.Writer.WriteAsync(payload); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + + var result = await pipe.Reader.ReadAsync(); + var buffer = result.Buffer; + var consumed = buffer.Start; + try + { + foreach (var segment in buffer) + { + await SocketPayloadSender.SendAllAsync( + static (memory, _, _) => ValueTask.FromResult(memory.Length), + segment, + state: null, + CancellationToken.None); + } + + consumed = buffer.End; + } + finally + { + pipe.Reader.AdvanceTo(consumed); + } + + Assert.True(consumed.Equals(buffer.End)); + var drained = await pipe.Reader.ReadAsync(); + Assert.True(drained.Buffer.IsEmpty); + Assert.True(drained.IsCompleted); + pipe.Reader.AdvanceTo(drained.Buffer.End); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task LiveConnection_SendAfterPeerAbort_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Abortive close so subsequent server sends fail rather than blocking forever. + client.LingerState = new LingerOption(enable: true, seconds: 0); + client.Dispose(); + + var payload = new byte[64 * 1024]; + Random.Shared.NextBytes(payload); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + server.ConnectionClosed, + timeout.Token); + + try + { + while (!linked.IsCancellationRequested) + { + await server.Output.WriteAsync(payload, linked.Token); + await server.Output.FlushAsync(linked.Token); + } + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Send pump faulted → ObservePump → CompleteAsync cancelled ConnectionClosed. + } + catch (Exception ex) when ( + server.ConnectionClosed.IsCancellationRequested + && ex is InvalidOperationException or IOException or SocketException) + { + // Peer abort completes the connection; Write/Flush may surface several transport exceptions. + } + + if (!server.ConnectionClosed.IsCancellationRequested) + { + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True( + server.ConnectionClosed.IsCancellationRequested, + "Send failure must permanently abort the connection (ConnectionClosed)."); + } +} \ No newline at end of file diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SocketPayloadSenderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SocketPayloadSenderTests.cs new file mode 100644 index 00000000..5a0ac3bf --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/SocketPayloadSenderTests.cs @@ -0,0 +1,124 @@ +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +public sealed class SocketPayloadSenderTests +{ + [Fact] + public async Task SendAllAsync_LoopsUntilEntireBufferSent() + { + var payload = new byte[100]; + for (var i = 0; i < payload.Length; i++) + { + payload[i] = (byte)i; + } + + var scripted = new Queue([37, 41, 22]); + var transmitted = new List(payload.Length); + + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + Assert.False(memory.IsEmpty); + var take = scripted.Dequeue(); + Assert.True(take <= memory.Length); + transmitted.AddRange(memory.Span[..take].ToArray()); + return ValueTask.FromResult(take); + }, + payload, + state: null, + CancellationToken.None); + + Assert.Empty(scripted); + Assert.Equal(payload, transmitted); + } + + [Fact] + public async Task SendAllAsync_ZeroByteSend_ThrowsWithoutSpinning() + { + var calls = 0; + var ex = await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (_, _, _) => + { + calls++; + return ValueTask.FromResult(0); + }, + new byte[16], + state: null, + CancellationToken.None)); + + Assert.Equal(1, calls); + Assert.Contains("zero bytes", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task SendAllAsync_HonorsCancellation() + { + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(async () => + await SocketPayloadSender.SendAllAsync( + static (_, _, _) => ValueTask.FromResult(1), + new byte[8], + state: null, + cts.Token)); + } + + [Fact] + public async Task SendAllAsync_DoesNotAdvancePastFailure_WhenUsedWithPipeSemantics() + { + // Simulate the connection send contract: only mark consumed after full success. + var payload = new byte[10]; + Random.Shared.NextBytes(payload); + var sent = new List(); + var calls = 0; + + await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + calls++; + if (calls == 1) + { + sent.AddRange(memory.Span[..4].ToArray()); + return ValueTask.FromResult(4); + } + + return ValueTask.FromResult(0); + }, + payload, + state: null, + CancellationToken.None)); + + Assert.Equal(4, sent.Count); + Assert.Equal(payload.AsSpan(0, 4).ToArray(), sent.ToArray()); + } + + [Fact] + public async Task SendAllAsync_ThrowAfterPartial_PropagatesWithoutClaimingSuccess() + { + var payload = new byte[] { 1, 2, 3, 4, 5, 6 }; + var sent = 0; + + var ex = await Assert.ThrowsAsync(async () => + await SocketPayloadSender.SendAllAsync( + (memory, _, _) => + { + if (sent > 0) + { + throw new IOException("peer reset after partial send"); + } + + sent += memory.Length >= 3 ? 3 : memory.Length; + return ValueTask.FromResult(3); + }, + payload, + state: null, + CancellationToken.None)); + + Assert.Equal(3, sent); + Assert.Contains("peer reset", ex.Message, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs new file mode 100644 index 00000000..8298bc0d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TcpConnectionDisconnectLoggingTests.cs @@ -0,0 +1,320 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Networking.Listeners; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Exactly one Information TCP disconnect summary per established connection, with a +/// known-cause reason and no exception text on that line. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class TcpConnectionDisconnectLoggingTests +{ + [Fact] + public async Task RemoteClose_EmitsExactlyOneDisconnect_WithRemoteClosed() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var inbound = server.Input.ReadAsync().AsTask(); + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await inbound.WaitAsync(timeout.Token); + Assert.True(result.IsCompleted); + server.Input.AdvanceTo(result.Buffer.End); + + // IsCompleted flips before the summary is logged and a second CompleteAsync returns immediately, + // so wait on the observable (the single disconnect line) rather than on the completion flag. + await WaitUntilAsync( + () => server.IsCompleted && logger.Disconnects.Count > 0, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "RemoteClosed"); + Assert.Equal(TcpDisconnectReason.RemoteClosed, server.DisconnectReasonForTests); + Assert.Empty(logger.DisconnectExceptions); + } + + [Fact] + public async Task LocalCompleteAsync_EmitsExactlyOneDisconnect_WithLocalClose() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "LocalClose"); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + Assert.Empty(logger.DisconnectExceptions); + } + + [Fact] + public async Task ReceivePumpError_EmitsExactlyOneDisconnect_WithReceiveError_AndKeepsDebugException() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamReadProbe = static () => + throw new IOException("forced-receive-failure"); + + // The in-flight receive already passed the probe; one octet starts the next read. + await client.SendAsync(new byte[] { 0x01 }); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync( + () => server.IsCompleted && logger.Disconnects.Count == 1 && logger.PumpErrors.Count > 0, + timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "ReceiveError"); + Assert.Equal(TcpDisconnectReason.ReceiveError, server.DisconnectReasonForTests); + Assert.DoesNotContain("forced-receive-failure", line, StringComparison.Ordinal); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + Assert.Contains( + logger.PumpExceptions, + static e => e.Contains("forced-receive-failure", StringComparison.Ordinal)); + } + + [Fact] + public async Task SendPumpError_EmitsExactlyOneDisconnect_WithSendError_AndKeepsDebugException() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + transport.BeforeStreamWriteProbe = static () => + throw new IOException("forced-send-failure"); + + // The send pump can fail and complete the output pipe before WriteAsync/FlushAsync observe it + // (timing depends on JIT warm-up); the pipe then surfaces the pump's exception to the producer. + try + { + await server.Output.WriteAsync(new byte[] { 0x01 }); + await server.Output.FlushAsync(); + } + catch (IOException ex) when (ex.Message == "forced-send-failure") + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitUntilAsync(() => server.IsCompleted, timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "SendError"); + Assert.Equal(TcpDisconnectReason.SendError, server.DisconnectReasonForTests); + Assert.DoesNotContain("forced-send-failure", line, StringComparison.Ordinal); + Assert.Contains( + logger.PumpErrors, + static e => e.Contains("pump ended with an error", StringComparison.Ordinal)); + Assert.Contains( + logger.PumpExceptions, + static e => e.Contains("forced-send-failure", StringComparison.Ordinal)); + } + + [Fact] + public async Task RequestCloseThenComplete_EmitsExactlyOneDisconnect_WithProtocolClose() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession(server, NullLogger.Instance); + + session.RequestClose(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "ProtocolClose"); + Assert.Equal(TcpDisconnectReason.ProtocolClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task ConcurrentCompleteAndDispose_EmitsExactlyOneDisconnect() + { + var logger = new CapturingConnectionLogger(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + var server = Assert.IsType(await host.AcceptAsync()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + await server.DisposeAsync().AsTask().WaitAsync(timeout.Token); + + Assert.Single(logger.Disconnects); + Assert.Equal(TcpDisconnectReason.LocalClose, server.DisconnectReasonForTests); + } + + [Fact] + public async Task IdleTimeout_EmitsDisconnect_WithIdleTimeoutReason() + { + var logger = new CapturingConnectionLogger(); + var clock = new ControllableTimeProvider(); + await using var host = await TransportTestHost.StartPlainAsync(logger); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession( + server, + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(2), + timeProvider: clock); + var run = session.RunAsync(); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await session.IdleWatchArmed.WaitAsync(timeout.Token); + clock.Advance(TimeSpan.FromSeconds(2)); + await run.WaitAsync(timeout.Token); + + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(TcpDisconnectReason.IdleTimeout, server.DisconnectReasonForTests); + var line = Assert.Single(logger.Disconnects); + AssertSingleLineDisconnect(line, server, "IdleTimeout"); + Assert.Empty(logger.DisconnectExceptions); + Assert.Null(logger.LastException); + Assert.Equal(LogLevel.Information, logger.LastLevel); + } + + [Fact] + public void GeneratedDisconnectMessage_IsSingleLine_WithStructuredReason() + { + var logger = new CapturingConnectionLogger(); + NetworkingLogMessages.TcpConnectionDisconnected( + logger, + "69.80.99.16:49136", + "0.0.0.0:119", + "RemoteClosed"); + + var line = Assert.Single(logger.Disconnects); + Assert.Equal(LogLevel.Information, logger.LastLevel); + Assert.Equal(1425, logger.LastEventId); + Assert.Equal( + "TCP connection disconnected: remote=69.80.99.16:49136 local=0.0.0.0:119 reason=RemoteClosed", + line); + Assert.DoesNotContain('\n', line); + Assert.Null(logger.LastException); + Assert.Equal("69.80.99.16:49136", logger.LastRemote); + Assert.Equal("RemoteClosed", logger.LastReason); + } + + private static void AssertSingleLineDisconnect(string line, NntpConnection server, string reason) + { + Assert.DoesNotContain('\n', line); + Assert.StartsWith("TCP connection disconnected:", line, StringComparison.Ordinal); + Assert.Contains("remote=", line, StringComparison.Ordinal); + Assert.Contains("reason=" + reason, line, StringComparison.Ordinal); + var remote = ConnectionAcceptanceLogging.FormatEndpoint(server.RemoteEndPoint); + Assert.Contains(remote, line, StringComparison.Ordinal); + Assert.DoesNotContain("at VectorNNTP", line, StringComparison.Ordinal); + Assert.DoesNotContain("StackTrace", line, StringComparison.Ordinal); + } + + private static async Task WaitUntilAsync(Func predicate, CancellationToken cancellationToken) + { + while (!predicate()) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class CapturingConnectionLogger : ILogger + { + public List Disconnects { get; } = []; + + public List DisconnectExceptions { get; } = []; + + public List PumpErrors { get; } = []; + + public List PumpExceptions { get; } = []; + + public LogLevel LastLevel { get; private set; } + + public int LastEventId { get; private set; } + + public Exception? LastException { get; private set; } + + public string? LastRemote { get; private set; } + + public string? LastReason { get; private set; } + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var formatted = formatter(state, exception); + LastLevel = logLevel; + LastEventId = eventId.Id; + LastException = exception; + if (state is IReadOnlyList> properties) + { + LastRemote = properties.FirstOrDefault(static p => p.Key == "Remote").Value as string; + LastReason = properties.FirstOrDefault(static p => p.Key == "Reason").Value as string; + } + + if (formatted.Contains("TCP connection disconnected:", StringComparison.Ordinal)) + { + Disconnects.Add(formatted); + if (exception is not null) + { + DisconnectExceptions.Add(exception); + } + } + + if (formatted.Contains("pump ended with an error", StringComparison.Ordinal)) + { + PumpErrors.Add(formatted); + if (exception is not null) + { + PumpExceptions.Add(exception.ToString()); + } + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs new file mode 100644 index 00000000..73fe58c3 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsCertificateLifetimeTests.cs @@ -0,0 +1,195 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Tests.Acme; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +public sealed class TlsCertificateLifetimeTests +{ + [Fact] + public async Task ExistingLease_SurvivesPublication_UntilReleased() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + TlsCertificateHolder holderA; + using (var leaseA = provider.Acquire()) + { + holderA = leaseA.Holder; + Assert.Equal(0, holderA.DisposeCount); + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + Assert.Contains( + "a.usenet.ninja", + leaseA.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.False(holderA.IsDisposed); + Assert.Equal(0, holderA.DisposeCount); + } + + Assert.Equal(1, holderA.DisposeCount); + Assert.True(holderA.IsDisposed); + } + + [Fact] + public async Task NewAcquire_AfterPublish_ReceivesNewGeneration() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + int generationA; + using (var leaseA = provider.Acquire()) + { + generationA = leaseA.Generation; + } + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + using var leaseB = provider.Acquire(); + Assert.NotEqual(generationA, leaseB.Generation); + Assert.Contains( + "b.usenet.ninja", + leaseB.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task FailedPublication_LeavesPriorContextActive() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + using var leaseBefore = provider.Acquire(); + var generationA = leaseBefore.Generation; + + Assert.Throws(() => + provider.PublishFromPfx([0x01, 0x02, 0x03, 0x04], AcmeConfigurationTests.TestPfxPassword)); + + using var leaseAfter = provider.Acquire(); + Assert.Equal(generationA, leaseAfter.Generation); + Assert.Contains( + "a.usenet.ninja", + leaseAfter.Context.TargetCertificate.Subject, + StringComparison.OrdinalIgnoreCase); + Assert.False(leaseBefore.HolderIsDisposed); + } + + [Fact] + public async Task LeaseDoubleDispose_DoesNotDoubleDisposeHolder() + { + await using var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + // codeql[cs/missed-using-statement] Lease must be disposed twice explicitly before publication retires the holder. + var lease = provider.Acquire(); + var holder = lease.Holder; + + lease.Dispose(); + // codeql[cs/dispose-not-called-on-throw] Intentional double-dispose contract exercise for TlsCertificateLease. + lease.Dispose(); + Assert.Equal(0, holder.DisposeCount); + + provider.PublishFromPfx(CreatePfx("b.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public async Task ProviderDispose_BlocksAcquire_AndDisposesAfterLeasesReleased() + { + var provider = CreateProvider(); + provider.PublishFromPfx(CreatePfx("a.usenet.ninja"), AcmeConfigurationTests.TestPfxPassword); + TlsCertificateHolder holder; + using (var lease = provider.Acquire()) + { + holder = lease.Holder; + + await provider.DisposeAsync(); + Assert.Throws(() => provider.Acquire()); + Assert.False(holder.IsDisposed); + } + + Assert.True(holder.IsDisposed); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public void RetiredHolder_RejectsAddRef() + { + var holder = TlsCertificateHolder.CreateFromPfx( + CreatePfx("a.usenet.ninja"), + AcmeConfigurationTests.TestPfxPassword); + holder.Retire(); + Assert.Throws(holder.AddRef); + holder.Release(); + Assert.Equal(1, holder.DisposeCount); + } + + [Fact] + public async Task ConcurrentAcquirePublishRelease_NeverUsesDisposedHolder() + { + await using var provider = CreateProvider(); + var pfxA = CreatePfx("a.usenet.ninja"); + var pfxB = CreatePfx("b.usenet.ninja"); + provider.PublishFromPfx(pfxA, AcmeConfigurationTests.TestPfxPassword); + + var errors = new ConcurrentBag(); + var observedGenerations = new ConcurrentBag(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + var tasks = new Task[8]; + + for (var t = 0; t < tasks.Length; t++) + { + var publish = t % 2 == 0; + tasks[t] = Task.Run(() => + { + try + { + var flip = 0; + while (!cts.IsCancellationRequested) + { + if (publish) + { + var pfx = Interlocked.Increment(ref flip) % 2 == 0 ? pfxA : pfxB; + provider.PublishFromPfx(pfx, AcmeConfigurationTests.TestPfxPassword); + } + else + { + using var lease = provider.Acquire(); + observedGenerations.Add(lease.Generation); + if (lease.HolderIsDisposed) + { + throw new InvalidOperationException("Acquired a disposed holder."); + } + + _ = lease.Context.TargetCertificate.Thumbprint; + } + } + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Provider disposal at test end can race the last iterations. + } + // codeql[cs/catch-of-all-exceptions] Concurrent stress worker must capture any provider disposal race exception. + catch (Exception ex) + { + errors.Add(ex); + } + }, CancellationToken.None); + } + + await Task.Delay(200); + await cts.CancelAsync(); + await Task.WhenAll(tasks); + + Assert.Empty(errors); + Assert.NotEmpty(observedGenerations); + } + + private static TlsCertificateContextProvider CreateProvider() => + new(NullLogger.Instance); + + private static byte[] CreatePfx(string cn) => + TestCertificateFactory.CreateMaterial( + [cn, "news.usenet.ninja"], + AcmeConfigurationTests.TestPfxPassword, + DateTimeOffset.UtcNow.AddDays(30)) + .PfxBytes; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs new file mode 100644 index 00000000..7555bcf2 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TlsUpgradeTransportTests.cs @@ -0,0 +1,475 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +public sealed class TlsUpgradeTransportTests +{ + [Fact] + public async Task UpgradeToTls_SameSocket_PlainThenEncryptedTraffic() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.False(server.IsTls); + + var localBefore = server.LocalEndPoint; + var remoteBefore = server.RemoteEndPoint; + var identityBefore = server.ClientIdentity; + + var plain = "PLAIN-BEFORE"u8.ToArray(); + await clientSocket.SendAsync(plain); + Assert.Equal(plain, await TransportTestShared.ReadExactAsync(server.Input, plain.Length)); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Equal(localBefore, server.LocalEndPoint); + Assert.Equal(remoteBefore, server.RemoteEndPoint); + Assert.Same(identityBefore, server.ClientIdentity); + Assert.False(server.ClientIdentity.UsedProxyHeader); + + var encrypted = new byte[] { 0x10, 0x20, 0x30, 0x0D, 0x0A }; + await ssl.WriteAsync(encrypted); + await ssl.FlushAsync(); + Assert.Equal(encrypted, await TransportTestShared.ReadExactAsync(server.Input, encrypted.Length)); + + var outbound = new byte[] { 0xAA, 0xBB, 0xCC }; + await server.Output.WriteAsync(outbound); + await server.Output.FlushAsync(); + var received = new byte[outbound.Length]; + var total = 0; + while (total < received.Length) + { + var n = await ssl.ReadAsync(received.AsMemory(total)); + Assert.True(n > 0); + total += n; + } + + Assert.Equal(outbound, received); + } + + [Fact] + public async Task ImplicitTls_ClientWithoutCertificate_HandshakeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfx); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + + Assert.True(server.IsTls); + Assert.Null(client.Stream.LocalCertificate); + Assert.NotNull(client.RemoteCertificate); + + var payload = new byte[] { 0x01, 0x02, 0x03 }; + await client.Stream.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_ClientWithoutCertificate_HandshakeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + // Explicitly no client certificate. + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Null(ssl.LocalCertificate); + } + + [Fact] + public async Task UpgradeToTls_PreservesTrustedProxyClientIdentity() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + await using var host = await TransportTestHost.StartPlainWithProxyAsync(trusted, pfx); + + using var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await clientSocket.ConnectAsync(host.EndPoint); + await clientSocket.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 203.0.113.77 127.0.0.1 51515 119\r\n")); + + await using var server = await host.AcceptAsync(); + Assert.False(server.IsTls); + Assert.True(server.ClientIdentity.IsTrustedProxy); + Assert.Equal(IPAddress.Parse("203.0.113.77"), server.ClientIdentity.ClientAddress); + Assert.Equal(51515, server.ClientIdentity.ClientPort); + var identity = server.ClientIdentity; + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.Same(identity, server.ClientIdentity); + Assert.Equal(IPAddress.Parse("203.0.113.77"), server.ClientIdentity.ClientAddress); + Assert.Equal(51515, server.ClientIdentity.ClientPort); + Assert.Equal(IPAddress.Loopback, server.ClientIdentity.TcpPeer.Address); + + var payload = new byte[] { 0x01, 0x02 }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task After382_WithReadsStillActive_ClientHelloEntersApplicationInput() + { + // Documents the historical failure mode: writing 382 while reads remain Active lets + // ClientHello commit into application Input and fail EnsureApplicationInputDrainedForUpgrade. + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + // Minimal TLS record header (Handshake / TLS 1.2) — enough to prove content-type 0x16. + var clientHelloPrefix = new byte[] { 0x16, 0x03, 0x03, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00 }; + await clientSocket.SendAsync(clientHelloPrefix); + + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + byte first = 0; + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek) && !peek.Buffer.IsEmpty) + { + first = peek.Buffer.FirstSpan[0]; + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + break; + } + + await Task.Yield(); + } + + Assert.Equal(0x16, first); + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("unconsumed plaintext", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsTls); + Assert.False(server.IsCompleted); + } + + [Fact] + public async Task PauseReads_Before382_ClientHelloDoesNotEnterApplicationInput() + { + // Required STARTTLS order: PauseReads → write 382 → ClientHello must not hit Input. + // With reads paused, the receive pump does not commit socket octets to Input; SslStream + // reads ClientHello from the network stream (or PrefixedStream if an in-flight read raced). + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await server.PauseReadsAsync(); + + var idleBefore = server.OutboundIdleVersion; + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + await server.WaitForOutboundDeliveryAsync(idleBefore); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + // Start the client handshake immediately after 382 — the historical race window. + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + var clientAuth = ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + + // While handshake is in flight, application Input must not observe ClientHello. + for (var i = 0; i < 20; i++) + { + if (server.Input.TryRead(out var peek)) + { + Assert.True( + peek.Buffer.IsEmpty, + "ClientHello must not enter application Input after PauseReads."); + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + } + + if (upgradeTask.IsCompleted && clientAuth.IsCompleted) + { + break; + } + + await Task.Yield(); + } + + await clientAuth; + await upgradeTask; + + Assert.True(server.IsTls); + Assert.False(server.Input.TryRead(out var leftover) && !leftover.Buffer.IsEmpty); + } + + [Fact] + public async Task PauseReadsBefore382_ThenClientHello_UpgradeSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + // Correct STARTTLS handoff: pause reads BEFORE 382 can reach the peer. + await server.PauseReadsAsync(); + var idleBefore = server.OutboundIdleVersion; + var notice = "382 Continue with TLS negotiation\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + await server.WaitForOutboundDeliveryAsync(idleBefore); + + var received = new byte[notice.Length]; + var total = 0; + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (total < received.Length) + { + var n = await clientSocket.ReceiveAsync(received.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + } + } + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + + Assert.True(server.IsTls); + Assert.False(server.Input.TryRead(out var leftover) && !leftover.Buffer.IsEmpty); + + var payload = new byte[] { 0x0A, 0x0B }; + await ssl.WriteAsync(payload); + Assert.Equal(payload, await TransportTestShared.ReadExactAsync(server.Input, payload.Length)); + } + + [Fact] + public async Task UpgradeToTls_UnconsumedInput_RefusesUpgradeAndKeepsPlaintext() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + await clientSocket.SendAsync("UNCONSUMED"u8.ToArray()); + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + waitCts.Token.ThrowIfCancellationRequested(); + if (server.Input.TryRead(out var peek)) + { + var hasData = !peek.Buffer.IsEmpty; + // Leave bytes unconsumed and unexamined so UpgradeToTlsAsync can still observe them. + server.Input.AdvanceTo(peek.Buffer.Start, peek.Buffer.Start); + if (hasData) + { + break; + } + } + + await Task.Yield(); + } + + var ex = await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + Assert.Contains("unconsumed plaintext", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(server.IsTls); + // Precondition failure refuses the upgrade without tearing down plaintext transport. + Assert.False(server.ConnectionClosed.IsCancellationRequested); + + // Session can still consume the buffered plaintext. + Assert.Equal("UNCONSUMED"u8.ToArray(), await TransportTestShared.ReadExactAsync(server.Input, 10)); + } + + [Fact] + public async Task UpgradeToTls_InvalidClientHello_FailsWithoutPlaintextFallback() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_ClientClosesDuringHandshake_FailsTerminally() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + await Task.Yield(); + clientSocket.Dispose(); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_Cancelled_FailsTerminally() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + using var cts = new CancellationTokenSource(); + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!, cts.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => upgradeTask); + Assert.False(server.IsTls); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + + [Fact] + public async Task UpgradeToTls_AlreadyTls_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartTlsAsync(pfx); + await using var client = await host.ConnectTlsClientAsync(); + await using var server = await host.AcceptAsync(); + Assert.True(server.IsTls); + + await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + } + + [Fact] + public async Task UpgradeToTls_AfterComplete_Throws() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + await server.CompleteAsync(); + + await Assert.ThrowsAsync(() => + server.UpgradeToTlsAsync(host.CertificateProvider!)); + } + + [Fact] + public async Task UpgradeToTls_ConcurrentCalls_SecondRejected() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var first = server.UpgradeToTlsAsync(host.CertificateProvider!); + var second = server.UpgradeToTlsAsync(host.CertificateProvider!); + + var secondEx = await Assert.ThrowsAsync(() => second); + Assert.Contains("already in progress", secondEx.Message, StringComparison.OrdinalIgnoreCase); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await first; + Assert.True(server.IsTls); + } + + [Fact] + public async Task UpgradeToTls_FlushedPlaintextOutbound_IsDeliveredBeforeHandshake() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var notice = "382 Go ahead\r\n"u8.ToArray(); + await server.Output.WriteAsync(notice); + await server.Output.FlushAsync(); + + var receivedNotice = new byte[notice.Length]; + var noticeTotal = 0; + using (var noticeCts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + while (noticeTotal < notice.Length) + { + var n = await clientSocket.ReceiveAsync( + receivedNotice.AsMemory(noticeTotal), + noticeCts.Token); + Assert.True(n > 0); + noticeTotal += n; + } + } + + Assert.Equal(notice, receivedNotice); + + var upgradeTask = server.UpgradeToTlsAsync(host.CertificateProvider!); + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions(clientCertificate: null)); + await upgradeTask; + Assert.True(server.IsTls); + } + + private static SslClientAuthenticationOptions CreateClientSslOptions( + System.Security.Cryptography.X509Certificates.X509Certificate2? clientCertificate) => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + ClientCertificates = clientCertificate is null + ? [] + : [clientCertificate], + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportIoSessionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportIoSessionTests.cs new file mode 100644 index 00000000..d9d5feaf --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportIoSessionTests.cs @@ -0,0 +1,235 @@ +using System.Buffers; +using System.IO.Pipelines; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// Serializes static enable/disable so tests do not race. +[Collection(nameof(TransportIoSessionTests))] +public sealed class TransportIoSessionTests +{ + [Fact] + public void CreateSession_WhenDisabled_ReturnsNull() + { + TransportIoProbe.Disable(); + Assert.False(TransportIoProbe.IsEnabled); + Assert.Null(TransportIoProbe.CreateSession()); + } + + [Fact] + public void RecordReceive_AggregatesBytesAndSyncCounts() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordReceive(4096, 4096, ts, ts, sync: true); + session.RecordReceive(4096, 2048, ts, ts, sync: false); + + var snap = session.Snapshot(); + Assert.Equal(2, snap.Rx.Ops); + Assert.Equal(6144, snap.Rx.Bytes); + Assert.Equal(8192, snap.Rx.Requested); + Assert.Equal(2048, snap.Rx.MinBytes); + Assert.Equal(4096, snap.Rx.MaxBytes); + Assert.Equal(1, snap.Rx.Sync); + Assert.Equal(1, snap.Rx.Async); + Assert.Equal(2, snap.Rx.OpUs.Length); + } + + [Fact] + public void RecordSend_AggregatesAndReportsPercentiles() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + for (var i = 0; i < 10; i++) + { + session.RecordSend(65536, 65536, ts, ts, sync: true); + } + + var snap = session.Snapshot(); + Assert.Equal(10, snap.Tx.Ops); + Assert.Equal(655360, snap.Tx.Bytes); + Assert.Equal(65536, snap.Tx.MinBytes); + Assert.Equal(65536, snap.Tx.MaxBytes); + Assert.Equal(10, snap.Tx.Sync); + Assert.Equal(0, snap.Tx.Async); + var stats = TransportIoReport.Stats(snap.Tx.OpUs); + Assert.True(stats.P50 >= 0); + Assert.True(stats.P99 >= stats.P50); + Assert.True(stats.Max >= stats.P99); + } + + [Fact] + public void Percentile_KnownSeries() + { + var samples = new[] { 10, 20, 30, 40, 50 }; + Assert.Equal(10, TransportIoReport.Percentile(samples, 0)); + Assert.Equal(30, TransportIoReport.Percentile(samples, 0.5)); + Assert.Equal(50, TransportIoReport.Percentile(samples, 1)); + Assert.Equal(0, TransportIoReport.Percentile([], 0.5)); + } + + [Fact] + public void ExcessOperations_UpdateCounters_ButCapSamples() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + for (var i = 0; i < TransportIoSession.SampleCapacity + 25; i++) + { + session.RecordReceive(100, 100, ts, ts, sync: true); + } + + var snap = session.Snapshot(); + Assert.Equal(TransportIoSession.SampleCapacity + 25, snap.Rx.Ops); + Assert.Equal(TransportIoSession.SampleCapacity, snap.Rx.OpUs.Length); + Assert.Equal((TransportIoSession.SampleCapacity + 25) * 100L, snap.Rx.Bytes); + } + + [Fact] + public void RecordTxReadShape_SingleSegment() + { + var session = new TransportIoSession(); + session.RecordTxReadShape(new ReadOnlySequence(new byte[65536])); + session.RecordTxReadShape(ReadOnlySequence.Empty); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.TxRead.Reads); + Assert.Equal(1, snap.TxRead.EmptyReads); + Assert.Equal(65536, snap.TxRead.Bytes); + Assert.Equal(1, snap.TxRead.Segments); + Assert.Equal(65536, snap.TxRead.MinBytes); + Assert.Equal(65536, snap.TxRead.MaxBytes); + Assert.Equal(1, snap.TxRead.MinSegments); + Assert.Equal(1, snap.TxRead.MaxSegments); + Assert.Equal(65536, snap.TxRead.MinSegmentBytes); + Assert.Equal(65536, snap.TxRead.MaxSegmentBytes); + } + + [Fact] + public async Task RecordTxReadShape_MultiSegmentFromPipe() + { + var options = new PipeOptions( + pauseWriterThreshold: 4 * 1024 * 1024, + resumeWriterThreshold: 2 * 1024 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false); + var pipe = new Pipe(options); + var chunk = new byte[65536]; + for (var i = 0; i < 4; i++) + { + var memory = pipe.Writer.GetMemory(chunk.Length); + chunk.CopyTo(memory); + pipe.Writer.Advance(chunk.Length); + _ = await pipe.Writer.FlushAsync(); + } + + var result = await pipe.Reader.ReadAsync(); + var session = new TransportIoSession(); + session.RecordTxReadShape(result.Buffer); + pipe.Reader.AdvanceTo(result.Buffer.End); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.TxRead.Reads); + Assert.Equal(4 * 65536, snap.TxRead.Bytes); + Assert.True(snap.TxRead.Segments >= 4, "expected at least one segment per 64 KiB flush"); + Assert.Equal(snap.TxRead.Segments, snap.TxRead.MaxSegments); + Assert.True(snap.TxRead.MaxSegmentBytes >= 65536 || snap.TxRead.Segments > 4); + } + + [Fact] + public void Write_ProducesReportFile() + { + var session = new TransportIoSession(); + var ts = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordSend(1024, 1024, ts, ts, sync: true); + var dir = Path.Join(Path.GetTempPath(), "vectornntp-transport-io-" + Guid.NewGuid().ToString("N")); + try + { + var path = session.Write(dir, "198.18.0.66:1234"); + Assert.True(File.Exists(path)); + var text = File.ReadAllText(path); + Assert.Contains("TX (stream WriteAsync)", text, StringComparison.Ordinal); + Assert.Contains("operations:", text, StringComparison.Ordinal); + Assert.Contains("TX SendAsync loop stages", text, StringComparison.Ordinal); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + } + + [Fact] + public void EnableDisable_TogglesSessionCreation() + { + var dir = Path.Join(Path.GetTempPath(), "vectornntp-transport-io-" + Guid.NewGuid().ToString("N")); + try + { + TransportIoProbe.Enable(dir); + Assert.True(TransportIoProbe.IsEnabled); + Assert.NotNull(TransportIoProbe.CreateSession()); + } + finally + { + TransportIoProbe.Disable(); + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + + Assert.False(TransportIoProbe.IsEnabled); + Assert.Null(TransportIoProbe.CreateSession()); + } + + [Fact] + public void RecordSendLoopStages_AccumulateWithoutPerOpObjects() + { + var session = new TransportIoSession(); + var t0 = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordSendLoopStart(); + session.RecordReadToFirstWrite(t0, t0 + 1000); + session.RecordBetweenWrites(t0, t0 + 500); + session.RecordTransportWrite(t0, t0 + 2000); + session.RecordLastWriteToAdvance(t0, t0 + 300); + session.RecordAdvanceTo(t0, t0 + 100); + session.RecordAdvanceToNextRead(t0, t0 + 400); + session.RecordSendLoopEnd(); + + var snap = session.Snapshot(); + Assert.Equal(1, snap.Loop.AdvanceCount); + Assert.Equal(1, snap.Loop.BetweenWriteCount); + Assert.Equal(1, snap.Loop.TransportWriteCount); + Assert.True(snap.Loop.SendAsyncElapsed >= TimeSpan.Zero); + Assert.True(snap.Loop.AdvanceTo >= TimeSpan.Zero); + Assert.True(snap.Loop.TransportWrite >= TimeSpan.Zero); + _ = Assert.Single(snap.Loop.AdvanceUs); + var text = TransportIoReport.Format(snap, "loop-test"); + Assert.Contains("AdvanceTo calls:", text, StringComparison.Ordinal); + Assert.Contains("ReadAsync waiting:", text, StringComparison.Ordinal); + Assert.Contains("TX write coalesce:", text, StringComparison.Ordinal); + Assert.Contains("TX write granularity", text, StringComparison.Ordinal); + } + + [Fact] + public void RecordCopy_AggregatesBytesAndDuration() + { + var session = new TransportIoSession { WriteGranularityTargetBytes = 256 * 1024 }; + var t0 = System.Diagnostics.Stopwatch.GetTimestamp(); + session.RecordCopy(65536, t0, t0 + 1000); + session.RecordCopy(65536, t0, t0 + 2000); + + var snap = session.Snapshot(); + Assert.Equal(256 * 1024, snap.Coalesce.TargetBytes); + Assert.Equal(2, snap.Coalesce.CopyCount); + Assert.Equal(131072, snap.Coalesce.CopyBytes); + Assert.True(snap.Coalesce.CopyTime >= TimeSpan.Zero); + Assert.Equal(2, snap.Coalesce.CopyUs.Length); + var text = TransportIoReport.Format(snap, "copy-test"); + Assert.Contains("copy bytes:", text, StringComparison.Ordinal); + Assert.Contains("target:", text, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportTestHostCollection.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportTestHostCollection.cs new file mode 100644 index 00000000..10677e3c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TransportTestHostCollection.cs @@ -0,0 +1,7 @@ +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Serializes tests that bind ephemeral loopback sockets via . +/// +[CollectionDefinition(nameof(TransportTestHostCollection), DisableParallelization = true)] +public sealed class TransportTestHostCollection; diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs new file mode 100644 index 00000000..8d3b8a42 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxPipePauseExperimentTests.cs @@ -0,0 +1,169 @@ +using System.IO.Pipelines; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Serializes the static TX-pipe pause experiment so tests do not leak into production defaults. +/// +[Collection(nameof(TxPipePauseExperimentTests))] +public sealed class TxPipePauseExperimentTests +{ + public TxPipePauseExperimentTests() => TxPipePauseExperiment.Clear(); + + [Fact] + public void Create_RemainsProduction64KiB_WhenExperimentIsSet() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + var input = NntpPipeOptions.Create(); + Assert.Equal(NntpPipeOptions.PauseWriterThreshold, input.PauseWriterThreshold); + Assert.Equal(NntpPipeOptions.ResumeWriterThreshold, input.ResumeWriterThreshold); + Assert.Equal(64 * 1024, input.PauseWriterThreshold); + Assert.Equal(32 * 1024, input.ResumeWriterThreshold); + Assert.Same(PipeScheduler.ThreadPool, input.ReaderScheduler); + Assert.Same(PipeScheduler.ThreadPool, input.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void CreateOutput_Default_IsProduction64KiB() + { + TxPipePauseExperiment.Clear(); + Assert.False(TxPipePauseExperiment.IsConfigured); + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(64 * 1024, output.PauseWriterThreshold); + Assert.Equal(32 * 1024, output.ResumeWriterThreshold); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + + [Fact] + public void CreateOutput_AppliesAllowlistedPause_AndHalfResume() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + Assert.True(TxPipePauseExperiment.TryGetEffective(out var pause, out var resume)); + Assert.Equal(4 * 1024 * 1024, pause); + Assert.Equal(2 * 1024 * 1024, resume); + + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(4 * 1024 * 1024, output.PauseWriterThreshold); + Assert.Equal(2 * 1024 * 1024, output.ResumeWriterThreshold); + Assert.Equal(NntpPipeOptions.MinimumSegmentSize, output.MinimumSegmentSize); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void Set_64KiB_MatchesProductionThresholds() + { + try + { + TxPipePauseExperiment.Set(64 * 1024); + Assert.True(TxPipePauseExperiment.IsConfigured); + var output = NntpPipeOptions.CreateOutput(); + Assert.Equal(NntpPipeOptions.PauseWriterThreshold, output.PauseWriterThreshold); + Assert.Equal(NntpPipeOptions.ResumeWriterThreshold, output.ResumeWriterThreshold); + Assert.Same(PipeScheduler.Inline, output.ReaderScheduler); + Assert.Same(PipeScheduler.Inline, output.WriterScheduler); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public void Set_RejectsValuesOutsideAllowlist() + { + Assert.Throws(() => TxPipePauseExperiment.Set(1)); + Assert.Throws(() => TxPipePauseExperiment.Set(16 * 1024 * 1024)); + Assert.Throws(() => TxPipePauseExperiment.Set(96 * 1024)); + Assert.False(TxPipePauseExperiment.IsConfigured); + Assert.Equal(64 * 1024, NntpPipeOptions.CreateOutput().PauseWriterThreshold); + } + + [Theory] + [InlineData("64KiB", 64 * 1024)] + [InlineData("128k", 128 * 1024)] + [InlineData("256KiB", 256 * 1024)] + [InlineData("512KiB", 512 * 1024)] + [InlineData("1MiB", 1024 * 1024)] + [InlineData("2m", 2 * 1024 * 1024)] + [InlineData("4MiB", 4 * 1024 * 1024)] + [InlineData("8MiB", 8 * 1024 * 1024)] + [InlineData("4194304", 4 * 1024 * 1024)] + public void TryParseAllowed_AcceptsSweepTokens(string token, long expected) + { + Assert.True(TxPipePauseExperiment.TryParseAllowed(token, out var parsed)); + Assert.Equal(expected, parsed); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("16MiB")] + [InlineData("96KiB")] + [InlineData("")] + public void TryParseAllowed_RejectsAccidentalValues(string token) + { + Assert.False(TxPipePauseExperiment.TryParseAllowed(token, out _)); + } + + [Fact] + public async Task CreateOutput_4MiB_DoesNotBackpressureAfterOne64KiBChunk() + { + try + { + TxPipePauseExperiment.Set(4 * 1024 * 1024); + var pipe = new Pipe(NntpPipeOptions.CreateOutput()); + WriteZeros(pipe.Writer, 64 * 1024); + var flush = pipe.Writer.FlushAsync(); + Assert.True(flush.IsCompletedSuccessfully); + pipe.Reader.AdvanceTo((await pipe.Reader.ReadAsync()).Buffer.End); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + finally + { + TxPipePauseExperiment.Clear(); + } + } + + [Fact] + public async Task Create_Production64KiB_BackpressuresAfterOne64KiBChunk() + { + TxPipePauseExperiment.Clear(); + var pipe = new Pipe(NntpPipeOptions.Create()); + WriteZeros(pipe.Writer, 64 * 1024); + var flush = pipe.Writer.FlushAsync(); + Assert.False(flush.IsCompleted); + + var read = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(read.Buffer.End); + var result = await flush; + Assert.False(result.IsCompleted); + + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + private static void WriteZeros(PipeWriter writer, int bytes) + { + var span = writer.GetSpan(bytes); + span[..bytes].Clear(); + writer.Advance(bytes); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteCoalescerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteCoalescerTests.cs new file mode 100644 index 00000000..45571d2d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteCoalescerTests.cs @@ -0,0 +1,318 @@ +using System.Buffers; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Exact-byte and flush-shape contracts for the diagnostic TX write coalescer. +/// +public sealed class TxWriteCoalescerTests +{ + [Theory] + [InlineData(64 * 1024)] + [InlineData(128 * 1024)] + [InlineData(256 * 1024)] + [InlineData(512 * 1024)] + [InlineData(1024 * 1024)] + [InlineData(4 * 1024 * 1024)] + public void Copy_ExactTarget_FillsOnce_PreservesBytes(int target) + { + var coalescer = new TxWriteCoalescer(target); + var source = Pattern(target, seed: 7); + Assert.Equal(target, coalescer.Copy(source, io: null)); + Assert.True(coalescer.IsFull); + Assert.Equal(0, coalescer.RemainingCapacity); + Assert.True(source.AsSpan().SequenceEqual(coalescer.PendingMemory.Span)); + coalescer.Clear(); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public void Copy_MultipleSegments_PreservesOrder_NoDuplicationOrLoss() + { + const int target = 256 * 1024; + var coalescer = new TxWriteCoalescer(target); + var expected = Pattern(target, seed: 11); + var copied = 0; + foreach (var chunk in expected.Chunk(64 * 1024)) + { + copied += coalescer.Copy(chunk, io: null); + } + + Assert.Equal(target, copied); + Assert.True(coalescer.IsFull); + Assert.True(expected.AsSpan().SequenceEqual(coalescer.PendingMemory.Span)); + } + + [Fact] + public void Copy_PartialFinalAggregate_LeavesRemainder() + { + var coalescer = new TxWriteCoalescer(128 * 1024); + var first = Pattern(64 * 1024, seed: 3); + var tail = Pattern(100, seed: 9); + Assert.Equal(first.Length, coalescer.Copy(first, io: null)); + Assert.False(coalescer.IsFull); + Assert.Equal(tail.Length, coalescer.Copy(tail, io: null)); + Assert.Equal(first.Length + tail.Length, coalescer.PendingCount); + Assert.True(first.AsSpan().SequenceEqual(coalescer.PendingMemory.Span[..first.Length])); + Assert.True(tail.AsSpan().SequenceEqual(coalescer.PendingMemory.Span[first.Length..])); + } + + [Fact] + public void Copy_DoesNotOverflow_WhenSegmentExceedsRemaining() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var oversized = Pattern((64 * 1024) + 16, seed: 5); + Assert.Equal(64 * 1024, coalescer.Copy(oversized, io: null)); + Assert.True(coalescer.IsFull); + Assert.Equal(0, coalescer.Copy(oversized.AsSpan(64 * 1024), io: null)); + Assert.True(oversized.AsSpan(0, 64 * 1024).SequenceEqual(coalescer.PendingMemory.Span)); + } + + [Fact] + public async Task Drain_FlushesFullAggregates_ThenPartialRemainder() + { + const int target = 128 * 1024; + var coalescer = new TxWriteCoalescer(target); + var payload = Pattern((128 * 1024) + 64, seed: 13); + var sequence = ToSequence(payload.AsMemory(0, 64 * 1024), payload.AsMemory(64 * 1024, 64 * 1024), payload.AsMemory(128 * 1024)); + var writes = new List(); + + await DrainAsync(sequence, coalescer, writes, flushRemaining: true, CancellationToken.None); + + Assert.Equal(2, writes.Count); + Assert.Equal(target, writes[0].Length); + Assert.Equal(64, writes[1].Length); + Assert.Equal(payload, writes.SelectMany(static w => w).ToArray()); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_WithoutFlushRemaining_HoldsPartialAcrossCalls() + { + var coalescer = new TxWriteCoalescer(256 * 1024); + var first = Pattern(64 * 1024, seed: 17); + var second = Pattern(64 * 1024, seed: 19); + var writes = new List(); + + await DrainAsync(new ReadOnlySequence(first), coalescer, writes, flushRemaining: false, CancellationToken.None); + Assert.Empty(writes); + Assert.Equal(first.Length, coalescer.PendingCount); + + await DrainAsync(new ReadOnlySequence(second), coalescer, writes, flushRemaining: false, CancellationToken.None); + Assert.Empty(writes); + Assert.Equal(first.Length + second.Length, coalescer.PendingCount); + + await DrainAsync(ReadOnlySequence.Empty, coalescer, writes, flushRemaining: true, CancellationToken.None); + Assert.Single(writes); + Assert.Equal(first.Concat(second).ToArray(), writes[0]); + } + + [Fact] + public async Task Drain_TransportFailure_DoesNotClear_AndDoesNotWriteTwice() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var payload = Pattern(64 * 1024, seed: 23); + var writes = 0; + await Assert.ThrowsAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(payload), + coalescer, + writeAsync: (_, _) => + { + writes++; + throw new IOException("simulated transport failure"); + }, + flushRemaining: true, + io: null, + CancellationToken.None); + }); + + Assert.Equal(1, writes); + Assert.Equal(payload.Length, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_Cancellation_DoesNotWrite() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var writes = 0; + await Assert.ThrowsAnyAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(Pattern(64 * 1024, seed: 29)), + coalescer, + writeAsync: (_, token) => + { + writes++; + token.ThrowIfCancellationRequested(); + return ValueTask.CompletedTask; + }, + flushRemaining: true, + io: null, + cts.Token); + }); + + Assert.Equal(0, writes); + Assert.Equal(0, coalescer.PendingCount); + } + + [Fact] + public async Task Drain_WriteCanceled_LeavesPendingUnwritten() + { + var coalescer = new TxWriteCoalescer(64 * 1024); + var writes = 0; + await Assert.ThrowsAnyAsync(async () => + { + await TxWriteCoalesceTestDrain.RunAsync( + new ReadOnlySequence(Pattern(64 * 1024, seed: 29)), + coalescer, + writeAsync: (_, _) => + { + writes++; + throw new OperationCanceledException(); + }, + flushRemaining: true, + io: null, + CancellationToken.None); + }); + + Assert.Equal(1, writes); + Assert.Equal(64 * 1024, coalescer.PendingCount); + } + + [Fact] + public void RecordCopy_IsMeasuredSeparately() + { + var io = new TransportIoSession(); + var coalescer = new TxWriteCoalescer(64 * 1024); + var source = Pattern(64 * 1024, seed: 31); + Assert.Equal(source.Length, coalescer.Copy(source, io)); + var snap = io.Snapshot(); + Assert.Equal(1, snap.Coalesce.CopyCount); + Assert.Equal(source.Length, snap.Coalesce.CopyBytes); + Assert.True(snap.Coalesce.CopyTime >= TimeSpan.Zero); + Assert.Single(snap.Coalesce.CopyUs); + } + + [Fact] + public void Constructor_RejectsNonAllowlistedTarget() + { + Assert.Throws(() => new TxWriteCoalescer(96 * 1024)); + } + + private static async Task DrainAsync( + ReadOnlySequence buffer, + TxWriteCoalescer coalescer, + List writes, + bool flushRemaining, + CancellationToken cancellationToken) + { + await TxWriteCoalesceTestDrain.RunAsync( + buffer, + coalescer, + writeAsync: (memory, _) => + { + writes.Add(memory.ToArray()); + return ValueTask.CompletedTask; + }, + flushRemaining, + io: null, + cancellationToken); + } + + private static byte[] Pattern(int length, byte seed) + { + var bytes = new byte[length]; + for (var i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)(seed + (i * 31)); + } + + return bytes; + } + + private static ReadOnlySequence ToSequence(params ReadOnlyMemory[] parts) + { + if (parts.Length == 0) + { + return ReadOnlySequence.Empty; + } + + if (parts.Length == 1) + { + return new ReadOnlySequence(parts[0]); + } + + var first = new BufferSegment(parts[0]); + var last = first; + for (var i = 1; i < parts.Length; i++) + { + last = last.Append(parts[i]); + } + + return new ReadOnlySequence(first, 0, last, last.Memory.Length); + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(ReadOnlyMemory memory) => Memory = memory; + + public BufferSegment Append(ReadOnlyMemory memory) + { + var next = new BufferSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length, + }; + Next = next; + return next; + } + } +} + +/// +/// Test-only drain that mirrors NntpConnection aggregate copy/write without a live pipe. +/// +internal static class TxWriteCoalesceTestDrain +{ + public static async ValueTask RunAsync( + ReadOnlySequence buffer, + TxWriteCoalescer coalescer, + Func, CancellationToken, ValueTask> writeAsync, + bool flushRemaining, + TransportIoSession? io, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var segment in buffer) + { + if (segment.Length == 0) + { + continue; + } + + var offset = 0; + while (offset < segment.Length) + { + cancellationToken.ThrowIfCancellationRequested(); + if (coalescer.RemainingCapacity == 0) + { + await writeAsync(coalescer.PendingMemory, cancellationToken).ConfigureAwait(false); + coalescer.Clear(); + } + + offset += coalescer.Copy(segment.Span[offset..], io); + } + } + + if (coalescer.IsFull || (flushRemaining && coalescer.PendingCount > 0)) + { + cancellationToken.ThrowIfCancellationRequested(); + await writeAsync(coalescer.PendingMemory, cancellationToken).ConfigureAwait(false); + coalescer.Clear(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs new file mode 100644 index 00000000..554da27f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityConnectionTests.cs @@ -0,0 +1,323 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Live contracts for diagnostic write coalescing. +/// Uses per-transport so other tests keep +/// the production one-segment-one-write path. +/// +public sealed class TxWriteGranularityConnectionTests +{ + [Fact] + public async Task Disabled_WritesEachSegmentImmediately() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + Assert.False(TxWriteGranularityExperiment.TryResolveTarget(transport, out _)); + + var writes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + + var first = Pattern(64 * 1024, 1); + var second = Pattern(64 * 1024, 2); + await WriteSegmentAsync(server.Output, first, timeout.Token); + await WriteSegmentAsync(server.Output, second, timeout.Token); + + var received = await ReadExactAsync(client, first.Length + second.Length, timeout.Token); + Assert.Equal(first.Concat(second).ToArray(), received); + Assert.Equal(2, Volatile.Read(ref writes)); + } + + [Fact] + public async Task Disabled_SmallFlush_HitsTransportImmediately() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.BeforeStreamWriteProbe = () => + { + entered.TrySetResult(); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + await server.Output.WriteAsync(new byte[] { 0x01, 0x02, 0x03 }, timeout.Token); + await server.Output.FlushAsync(timeout.Token); + await entered.Task.WaitAsync(timeout.Token); + Assert.Equal(new byte[] { 0x01, 0x02, 0x03 }, await ReadExactAsync(client, 3, timeout.Token)); + } + + [Theory] + [InlineData(64 * 1024, 1)] + [InlineData(128 * 1024, 2)] + [InlineData(256 * 1024, 4)] + [InlineData(512 * 1024, 8)] + [InlineData(1024 * 1024, 16)] + [InlineData(4 * 1024 * 1024, 64)] + public async Task Coalesce_ExactTarget_OneWrite_PreservesBytes(int target, int chunks) + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var writes = 0; + var writeBytes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, target); + try + { + var expected = new byte[target]; + for (var i = 0; i < chunks; i++) + { + var chunk = Pattern(64 * 1024, (byte)(i + 1)); + chunk.CopyTo(expected.AsSpan(i * 64 * 1024)); + var span = server.Output.GetSpan(chunk.Length); + chunk.CopyTo(span); + server.Output.Advance(chunk.Length); + writeBytes += chunk.Length; + } + + var read = ReadExactAsync(client, target, timeout.Token); + await server.Output.FlushAsync(timeout.Token); + + var received = await read; + Assert.Equal(expected, received); + Assert.Equal(target, writeBytes); + Assert.Equal(1, Volatile.Read(ref writes)); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_PartialFinal_FlushesWithoutWaitingForMore() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + var writes = 0; + transport.BeforeStreamWriteProbe = () => + { + Interlocked.Increment(ref writes); + return ValueTask.CompletedTask; + }; + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 256 * 1024); + try + { + var chunk = Pattern(64 * 1024, 41); + await WriteSegmentAsync(server.Output, chunk, timeout.Token); + var received = await ReadExactAsync(client, chunk.Length, timeout.Token); + Assert.Equal(chunk, received); + Assert.Equal(1, Volatile.Read(ref writes)); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_TransportFailure_CompletesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + transport.BeforeStreamWriteProbe = () => throw new IOException("simulated coalesce write failure"); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 64 * 1024); + try + { + try + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 8), timeout.Token); + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or OperationCanceledException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + await WaitUntilAsync( + static s => s.ConnectionClosed.IsCancellationRequested, + server, + timeout.Token); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_Cancellation_DoesNotHang() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 256 * 1024); + try + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 12), timeout.Token); + await server.CompleteAsync().WaitAsync(timeout.Token); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.True(server.OutputReaderCompletedForTests); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + [Fact] + public async Task Coalesce_PeerDisconnect_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var transport = server.ByteTransportForTests; + Assert.NotNull(transport); + + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await WaitIdleAsync(server, timeout.Token); + TxWriteGranularityExperiment.SetFor(transport, 64 * 1024); + try + { + client.LingerState = new LingerOption(enable: true, seconds: 0); + client.Dispose(); + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(server.ConnectionClosed, timeout.Token); + try + { + while (!linked.IsCancellationRequested) + { + await WriteSegmentAsync(server.Output, Pattern(64 * 1024, 15), linked.Token); + } + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (Exception ex) when ( + server.ConnectionClosed.IsCancellationRequested + && ex is InvalidOperationException or IOException or SocketException) + { + _ = ex; // observed; expected during test teardown/cleanup + } + + if (!server.ConnectionClosed.IsCancellationRequested) + { + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, linked.Token); + } + catch (OperationCanceledException ex) when (server.ConnectionClosed.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + Assert.True(server.ConnectionClosed.IsCancellationRequested); + } + finally + { + TxWriteGranularityExperiment.ClearFor(transport); + } + } + + private static async Task WriteSegmentAsync(PipeWriter writer, ReadOnlyMemory chunk, CancellationToken cancellationToken) + { + var span = writer.GetSpan(chunk.Length); + chunk.Span.CopyTo(span); + writer.Advance(chunk.Length); + await writer.FlushAsync(cancellationToken); + } + + private static byte[] Pattern(int length, byte seed) + { + var bytes = new byte[length]; + for (var i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)(seed + (i * 31)); + } + + return bytes; + } + + private static async Task ReadExactAsync(Socket client, int count, CancellationToken cancellationToken) + { + var buffer = new byte[count]; + var read = 0; + while (read < count) + { + var n = await client.ReceiveAsync(buffer.AsMemory(read), cancellationToken); + if (n == 0) + { + throw new InvalidOperationException($"Peer closed after {read} of {count} bytes."); + } + + read += n; + } + + return buffer; + } + + private static async Task WaitIdleAsync(NntpConnection server, CancellationToken cancellationToken) => + await WaitUntilAsync(static s => s.SendPumpAwaitingOutputForTests == 1, server, cancellationToken); + + private static async Task WaitUntilAsync( + Func predicate, + NntpConnection server, + CancellationToken cancellationToken) + { + while (!predicate(server)) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs new file mode 100644 index 00000000..d6df7ed9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Networking/Transport/TxWriteGranularityExperimentTests.cs @@ -0,0 +1,83 @@ +using VectorNNTPD.TransitService.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Networking.Transport; + +/// +/// Serializes process-wide write-granularity experiment state. These tests do not create +/// instances while +/// is active. +/// +[Collection(nameof(TxWriteGranularityExperimentTests))] +public sealed class TxWriteGranularityExperimentTests +{ + public TxWriteGranularityExperimentTests() => TxWriteGranularityExperiment.Clear(); + + [Fact] + public void Default_IsInactive() + { + TxWriteGranularityExperiment.Clear(); + Assert.False(TxWriteGranularityExperiment.IsConfigured); + Assert.Null(TxWriteGranularityExperiment.ConfiguredTargetBytes); + Assert.False(TxWriteGranularityExperiment.TryGetTarget(out var target)); + Assert.Equal(0, target); + Assert.False(TxWriteGranularityExperiment.TryResolveTarget(transport: null, out _)); + } + + [Theory] + [InlineData(64 * 1024)] + [InlineData(128 * 1024)] + [InlineData(256 * 1024)] + [InlineData(512 * 1024)] + [InlineData(1024 * 1024)] + [InlineData(4 * 1024 * 1024)] + public void Set_AllowlistedTargets_AreResolved(int targetBytes) + { + try + { + TxWriteGranularityExperiment.Set(targetBytes); + Assert.True(TxWriteGranularityExperiment.IsConfigured); + Assert.True(TxWriteGranularityExperiment.TryGetTarget(out var resolved)); + Assert.Equal(targetBytes, resolved); + Assert.Equal(targetBytes, TxWriteGranularityExperiment.ConfiguredTargetBytes); + } + finally + { + TxWriteGranularityExperiment.Clear(); + } + } + + [Fact] + public void Set_RejectsValuesOutsideAllowlist() + { + Assert.Throws(() => TxWriteGranularityExperiment.Set(1)); + Assert.Throws(() => TxWriteGranularityExperiment.Set(2 * 1024 * 1024)); + Assert.Throws(() => TxWriteGranularityExperiment.Set(8 * 1024 * 1024)); + Assert.False(TxWriteGranularityExperiment.IsConfigured); + } + + [Theory] + [InlineData("64KiB", 64 * 1024)] + [InlineData("128k", 128 * 1024)] + [InlineData("256KiB", 256 * 1024)] + [InlineData("512KiB", 512 * 1024)] + [InlineData("1MiB", 1024 * 1024)] + [InlineData("4MiB", 4 * 1024 * 1024)] + [InlineData("4194304", 4 * 1024 * 1024)] + public void TryParseAllowed_AcceptsSweepTokens(string token, int expected) + { + Assert.True(TxWriteGranularityExperiment.TryParseAllowed(token, out var parsed)); + Assert.Equal(expected, parsed); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("2MiB")] + [InlineData("8MiB")] + [InlineData("96KiB")] + [InlineData("")] + public void TryParseAllowed_RejectsAccidentalValues(string token) + { + Assert.False(TxWriteGranularityExperiment.TryParseAllowed(token, out _)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs new file mode 100644 index 00000000..48952d8f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupCatalogueServiceTests.cs @@ -0,0 +1,289 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Newsgroups; + +public sealed class NewsgroupCatalogueServiceTests +{ + [Fact] + public async Task StartAsync_LoadsCompleteSnapshot_BeforeReturning() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = + [ + new NntpGroupRow("comp.risks", "Risks", 10, 2, (byte)'m'), + new NntpGroupRow("misc.test", "Testing", 5, 1, (byte)'y'), + ], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await catalogue.StartAsync(CancellationToken.None); + + Assert.True(catalogue.HasSnapshot); + var snapshot = catalogue.Current; + Assert.Equal(2, snapshot.Groups.Count); + Assert.Equal("comp.risks", snapshot.Groups[0].GroupName); + Assert.Equal("misc.test", snapshot.Groups[1].GroupName); + Assert.Equal(1, factory.QueryNewsgroupsCount); + Assert.Equal(1, Assert.Single(factory.Connections.Skip(1)).DisposeCount); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenQueryFails() + { + var factory = new FakeNntpDbConnectionFactory + { + QueryNewsgroupsException = new NntpDbUnavailableException("down"), + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenRowIsMalformed() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("", "desc", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task StartAsync_PreservesExtendedPostingStatuses() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = + [ + new NntpGroupRow("group.closed", "", 4, 1, (byte)'x'), + new NntpGroupRow("group.junk", "", 5, 2, (byte)'j'), + ], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await catalogue.StartAsync(CancellationToken.None); + + var snapshot = catalogue.Current; + Assert.Equal(NewsgroupPostingStatus.NoPostingOrPeerArticles, snapshot.Groups[0].PostingStatus); + Assert.Equal(NewsgroupPostingStatus.PeerOnly, snapshot.Groups[1].PostingStatus); + Assert.True(snapshot.Groups[0].ListActiveLine.Span.SequenceEqual("group.closed 4 1 x\r\n"u8)); + Assert.True(snapshot.Groups[1].ListActiveLine.Span.SequenceEqual("group.junk 5 2 j\r\n"u8)); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StartAsync_FailsHard_WhenEqualsAliasStatus() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("alias.group", "", 1, 1, (byte)'=')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + await using var catalogue = CreateCatalogue(db); + + await Assert.ThrowsAsync(() => catalogue.StartAsync(CancellationToken.None)); + Assert.False(catalogue.HasSnapshot); + } + + [Fact] + public async Task Refresh_Success_AtomicallyReplacesCurrent() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("old.group", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var first = catalogue.Current; + + factory.Newsgroups = [new NntpGroupRow("new.group", "", 8, 3, (byte)'n')]; + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => + { + var current = catalogue.Current; + return !ReferenceEquals(current, first) + && current.Groups.Count == 1 + && current.Groups[0].GroupName == "new.group"; + }); + + var second = catalogue.Current; + Assert.NotSame(first, second); + Assert.Equal("old.group", Assert.Single(first.Groups).GroupName); + Assert.Equal("new.group", Assert.Single(second.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_Failure_RetainsPreviousSnapshot() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("keep.group", "", 2, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + var first = catalogue.Current; + + factory.QueryNewsgroupsException = new NntpDbUnavailableException("down"); + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => factory.QueryNewsgroupsCount >= 2); + + Assert.Same(first, catalogue.Current); + Assert.Equal("keep.group", Assert.Single(catalogue.Current.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Refresh_DoesNotOverlap() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("g", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + Assert.Equal(1, factory.QueryNewsgroupsCount); + + factory.BlockQueryNewsgroups = new TaskCompletionSource(); + factory.QueryNewsgroupsStarted = new TaskCompletionSource(); + clock.Advance(TimeSpan.FromMinutes(5)); + await factory.QueryNewsgroupsStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(catalogue.IsRefreshing); + clock.Advance(TimeSpan.FromMinutes(5)); + Assert.Equal(1, factory.QueryNewsgroupsCount); + + factory.BlockQueryNewsgroups.SetResult(); + await WaitUntilAsync(() => !catalogue.IsRefreshing); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Current_CanBeReadConcurrently_WhileRefreshPublishes() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("first.group", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var readers = Enumerable.Range(0, 4).Select(_ => Task.Run(() => + { + while (!cts.IsCancellationRequested) + { + var snapshot = catalogue.Current; + _ = snapshot.ListActiveComplete.Length; + _ = snapshot.ListNewsgroupsComplete.Length; + } + }, cts.Token)).ToArray(); + + factory.Newsgroups = [new NntpGroupRow("second.group", "", 3, 1, (byte)'n')]; + clock.Advance(TimeSpan.FromMinutes(5)); + await WaitUntilAsync(() => factory.QueryNewsgroupsCount >= 2); + cts.Cancel(); + try + { + await Task.WhenAll(readers); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Readers observe the test timeout token. + } + + Assert.Equal("second.group", Assert.Single(catalogue.Current.Groups).GroupName); + await catalogue.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopAsync_CancelsInFlightRefresh() + { + var factory = new FakeNntpDbConnectionFactory + { + Newsgroups = [new NntpGroupRow("g", "", 1, 1, (byte)'y')], + }; + var db = CreateDb(factory); + await db.StartAsync(CancellationToken.None); + var clock = new ControllableTimeProvider(); + await using var catalogue = CreateCatalogue(db, clock, TimeSpan.FromMinutes(5)); + await catalogue.StartAsync(CancellationToken.None); + + factory.BlockQueryNewsgroups = new TaskCompletionSource(); + factory.QueryNewsgroupsStarted = new TaskCompletionSource(); + clock.Advance(TimeSpan.FromMinutes(5)); + await factory.QueryNewsgroupsStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var stop = catalogue.StopAsync(CancellationToken.None); + factory.BlockQueryNewsgroups.SetCanceled(); + await stop.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static NntpDbService CreateDb(FakeNntpDbConnectionFactory factory) + { + return new NntpDbService( + factory, + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.FromSeconds(15), + }), + NullLogger.Instance); + } + + private static NewsgroupCatalogueService CreateCatalogue( + NntpDbService db, + TimeProvider? clock = null, + TimeSpan? interval = null) + { + return new NewsgroupCatalogueService( + db, + NullLogger.Instance, + clock ?? TimeProvider.System, + interval ?? NewsgroupCatalogueService.RefreshInterval); + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Delay(10, cts.Token); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupSnapshotTests.cs b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupSnapshotTests.cs new file mode 100644 index 00000000..c6d3f662 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NewsgroupSnapshotTests.cs @@ -0,0 +1,260 @@ +using System.Runtime.InteropServices; +using System.Text; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Newsgroups; + +public sealed class NewsgroupSnapshotTests +{ + [Fact] + public void Create_MapsFields_PreservesUnsignedCountsAndPostingStatus() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition( + "misc.test", + "General testing", + CountHigh: 4_294_967_295UL, + CountLow: 1, + NewsgroupPostingStatus.Moderated), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal("misc.test", group.GroupName); + Assert.Equal("General testing", group.GroupDescription); + Assert.Equal(4_294_967_295UL, group.CountHigh); + Assert.Equal(1UL, group.CountLow); + Assert.Equal(NewsgroupPostingStatus.Moderated, group.PostingStatus); + Assert.Equal(4_294_967_295UL, group.EstimatedCount); + } + + [Fact] + public void Create_SortsDeterministically_ByOrdinalIgnoreCaseThenOrdinal() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("comp.risks", "", 2, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("alt.test", "", 2, 1, NewsgroupPostingStatus.Prohibited), + new NewsgroupDefinition("Misc.Test", "", 2, 1, NewsgroupPostingStatus.Moderated), + ]); + + Assert.Equal(["alt.test", "comp.risks", "Misc.Test"], snapshot.Groups.Select(g => g.GroupName)); + } + + [Fact] + public void TryGet_IsOrdinalCaseInsensitive_AndPreservesOriginalName() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.TEST", "desc", 10, 2, NewsgroupPostingStatus.Allowed), + ]); + + Assert.True(snapshot.TryGet("MISC.test"u8, out var group)); + Assert.Equal("misc.TEST", group.GroupName); + Assert.True(group.GroupNameBytes.Span.SequenceEqual("misc.TEST"u8)); + } + + [Theory] + [InlineData(NewsgroupPostingStatus.Allowed, (byte)'y', "group.example 100 1 y\r\n")] + [InlineData(NewsgroupPostingStatus.Prohibited, (byte)'n', "group.example 100 1 n\r\n")] + [InlineData(NewsgroupPostingStatus.Moderated, (byte)'m', "group.example 100 1 m\r\n")] + [InlineData(NewsgroupPostingStatus.NoPostingOrPeerArticles, (byte)'x', "group.example 100 1 x\r\n")] + [InlineData(NewsgroupPostingStatus.PeerOnly, (byte)'j', "group.example 100 1 j\r\n")] + public void ListActiveLine_EmitsEachSupportedStatusOctet( + NewsgroupPostingStatus status, + byte expectedOctet, + string expectedLine) + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("group.example", "", 100, 1, status), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(status, group.PostingStatus); + Assert.Equal(expectedOctet, (byte)group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual(Encoding.ASCII.GetBytes(expectedLine))); + } + + [Fact] + public void ListActiveLine_IsExactUtf8WithCrlf() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "ignored", 3002322, 3000234, NewsgroupPostingStatus.Allowed), + ]); + + var line = Assert.Single(snapshot.Groups).ListActiveLine; + Assert.True(line.Span.SequenceEqual("misc.test 3002322 3000234 y\r\n"u8)); + Assert.True(snapshot.ListActiveComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test 3002322 3000234 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + Assert.True(Assert.Single(snapshot.Groups).ListCountsLine.Span.SequenceEqual( + "misc.test 3002322 3000234 2089 y\r\n"u8)); + Assert.True(snapshot.ListCountsComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "misc.test 3002322 3000234 2089 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + } + + [Fact] + public void ListNewsgroupsLine_EncodesUtf8Description() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "Café résumé £", 1, 1, NewsgroupPostingStatus.Allowed), + ]); + + var expected = Encode("misc.test Café résumé £\r\n"); + Assert.True(Assert.Single(snapshot.Groups).ListNewsgroupsLine.Span.SequenceEqual(expected)); + Assert.True(snapshot.ListNewsgroupsComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, expected, NntpResponses.MultilineTerminator))); + } + + [Fact] + public void PrecomputedLines_AreStableReferences() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("a.test", "d", 2, 1, NewsgroupPostingStatus.Allowed), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.True(group.ListActiveLine.Equals(snapshot.Groups[0].ListActiveLine)); + Assert.True(snapshot.ListActiveComplete.Equals(snapshot.ListActiveComplete)); + Assert.True(MemoryMarshal.TryGetArray(group.ListActiveLine, out var first)); + Assert.True(MemoryMarshal.TryGetArray(snapshot.Groups[0].ListActiveLine, out var second)); + Assert.Same(first.Array, second.Array); + } + + [Fact] + public void Create_FromDatabaseRows_UsesExactQueryFields() + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NntpGroupRow("tx.natives.recovery", "Texas", 89, 56, (byte)'n'), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(NewsgroupPostingStatus.Prohibited, group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual("tx.natives.recovery 89 56 n\r\n"u8)); + } + + [Theory] + [InlineData((byte)'x', NewsgroupPostingStatus.NoPostingOrPeerArticles, "group.example 100 1 x\r\n")] + [InlineData((byte)'j', NewsgroupPostingStatus.PeerOnly, "group.example 100 1 j\r\n")] + public void Create_PreservesExtendedStatus_AndListActiveBytes( + byte statusOctet, + NewsgroupPostingStatus expected, + string expectedLine) + { + var snapshot = NewsgroupSnapshot.Create( + [ + new NntpGroupRow("group.example", "", 100, 1, statusOctet), + ]); + + var group = Assert.Single(snapshot.Groups); + Assert.Equal(expected, group.PostingStatus); + Assert.Equal(statusOctet, (byte)group.PostingStatus); + Assert.True(group.ListActiveLine.Span.SequenceEqual(Encoding.ASCII.GetBytes(expectedLine))); + } + + [Fact] + public void Create_RejectsInvalidPostingStatus_AndDoesNotPublishPartial() + { + var ex = Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NntpGroupRow("good.group", "", 1, 1, (byte)'y'), + new NntpGroupRow("bad.group", "", 1, 1, (byte)'z'), + ])); + Assert.Contains("posting_status", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Create_RejectsEqualsAliasForm() + { + var ex = Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NntpGroupRow("alias.group", "", 1, 1, (byte)'='), + ])); + Assert.Contains("posting_status", ex.Message, StringComparison.Ordinal); + Assert.False(NewsgroupPostingStatusOctets.IsSupported((byte)'=')); + } + + [Fact] + public void Create_RejectsDuplicateCaseInsensitiveNames() + { + Assert.Throws(() => NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("misc.test", "", 1, 1, NewsgroupPostingStatus.Allowed), + new NewsgroupDefinition("MISC.TEST", "", 2, 1, NewsgroupPostingStatus.Allowed), + ])); + } + + [Fact] + public void PublishedSnapshot_RemainsUnchanged_WhenAnotherIsConstructed() + { + var first = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("old.group", "old", 5, 1, NewsgroupPostingStatus.Allowed), + ]); + var captured = first; + var second = NewsgroupSnapshot.Create( + [ + new NewsgroupDefinition("new.group", "new", 9, 2, NewsgroupPostingStatus.Moderated), + ]); + + Assert.Equal("old.group", Assert.Single(captured.Groups).GroupName); + Assert.Equal("new.group", Assert.Single(second.Groups).GroupName); + Assert.True(captured.ListActiveComplete.Span.SequenceEqual( + Concat( + NntpResponses.ListOfNewsgroupsFollows, + "old.group 5 1 y\r\n"u8.ToArray(), + NntpResponses.MultilineTerminator))); + } + + [Theory] + [InlineData(0UL, 0UL, 0UL)] + [InlineData(5UL, 5UL, 1UL)] + [InlineData(4000UL, 3999UL, 0UL)] + [InlineData(234UL, 567UL, 334UL)] + [InlineData(0UL, ulong.MaxValue, ulong.MaxValue)] + public void EstimateCount_FollowsRfcEmptyAndSpanRules(ulong low, ulong high, ulong expected) + { + Assert.Equal(expected, NewsgroupSnapshot.EstimateCount(low, high)); + } + + [Fact] + public void Empty_HasTerminatorOnlyListBodies() + { + Assert.Empty(NewsgroupSnapshot.Empty.Groups); + Assert.True(NewsgroupSnapshot.Empty.ListActiveComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator))); + Assert.True(NewsgroupSnapshot.Empty.ListCountsComplete.Span.SequenceEqual( + Concat(NntpResponses.ListOfNewsgroupsFollows, NntpResponses.MultilineTerminator))); + } + + private static byte[] Encode(string text) => Encoding.UTF8.GetBytes(text); + + private static byte[] Concat(ReadOnlyMemory first, ReadOnlySpan second, ReadOnlyMemory third) + { + var result = new byte[first.Length + second.Length + third.Length]; + first.Span.CopyTo(result); + second.CopyTo(result.AsSpan(first.Length)); + third.Span.CopyTo(result.AsSpan(first.Length + second.Length)); + return result; + } + + private static byte[] Concat(ReadOnlyMemory first, ReadOnlyMemory second) + { + var result = new byte[first.Length + second.Length]; + first.Span.CopyTo(result); + second.Span.CopyTo(result.AsSpan(first.Length)); + return result; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NntpWildmatTests.cs b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NntpWildmatTests.cs new file mode 100644 index 00000000..cc690bde --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Newsgroups/NntpWildmatTests.cs @@ -0,0 +1,52 @@ +using System.Text; +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.Tests.Newsgroups; + +public sealed class NntpWildmatTests +{ + [Theory] + [InlineData("abc")] + [InlineData("a*")] + [InlineData("a*,!*b")] + [InlineData("*.recovery")] + [InlineData("*")] + public void TryValidate_AcceptsRfcPatterns(string wildmat) + { + Assert.True(NntpWildmat.TryValidate(Encoding.ASCII.GetBytes(wildmat))); + } + + [Theory] + [InlineData("u[ks].*")] + [InlineData(@"a\b")] + [InlineData("a]b")] + [InlineData("")] + [InlineData("a*,")] + [InlineData("!*b")] + public void TryValidate_RejectsForbiddenOrEmptyPatterns(string wildmat) + { + Assert.False(NntpWildmat.TryValidate(Encoding.ASCII.GetBytes(wildmat))); + } + + [Theory] + [InlineData("misc.test", "misc.test", true)] + [InlineData("MISC.TEST", "misc.test", true)] + [InlineData("misc.test", "MISC.TEST", true)] + [InlineData("alt.rfc-writers.recovery", "*.recovery", true)] + [InlineData("tx.natives.recovery", "*.recovery", true)] + [InlineData("misc.test", "*.recovery", false)] + [InlineData("aaa", "a*,!*b", true)] + [InlineData("abb", "a*,!*b", false)] + [InlineData("ccb", "a*,!*b,*c*", true)] + [InlineData("xxx", "a*,!*b,*c*", false)] + [InlineData("misc.test", "misc.?est", true)] + [InlineData("misc.test", "misc.??st", true)] + [InlineData("misc.test", "misc.?", false)] + public void IsMatch_FollowsRfcExamples_CaseInsensitive(string text, string wildmat, bool expected) + { + var textBytes = Encoding.ASCII.GetBytes(text); + var pattern = Encoding.ASCII.GetBytes(wildmat); + Assert.Equal(expected, NntpWildmat.IsMatch(textBytes, pattern)); + Assert.Equal(expected, NntpWildmat.IsMatchValidated(textBytes, pattern)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsLogReaderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsLogReaderTests.cs new file mode 100644 index 00000000..0cd1f4f5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsLogReaderTests.cs @@ -0,0 +1,187 @@ +using System.Text; +using VectorNNTPD.TransitService.Ninpaths; + +namespace VectorNNTPD.TransitService.Tests.Ninpaths; + +public sealed class NinpathsLogReaderTests +{ + [Fact] + public void EmptyFile_YieldsNoArticles() + { + var stats = Read(""); + Assert.Equal(0, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void SingleHopPath_ChopsTheOnlyElement() + { + var stats = Read("Path: only\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void MultiHopPath_RegistersAllButTheLastHop() + { + var stats = Read("Path: a!b!c\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("a", out var a)); + Assert.True(stats.TryGetSentTo("b", out var b)); + Assert.False(stats.TryGetSentTo("c", out _)); + Assert.Equal(1, a); + Assert.Equal(1, b); + Assert.True(stats.TryGetRelation("a", "b", out var tally)); + Assert.Equal(1, tally); + } + + [Fact] + public void NotForMail_IsDroppedAsTheFinalElement() + { + var stats = Read("Path: news.example!peer.example!not-for-mail\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("news.example", out _)); + Assert.True(stats.TryGetSentTo("peer.example", out _)); + Assert.False(stats.TryGetSentTo("not-for-mail", out _)); + } + + [Fact] + public void RepeatedSite_IncrementsSentTo_WithoutSelfRelation() + { + var stats = Read("Path: a!a!b\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(1, stats.SiteCount); + Assert.True(stats.TryGetSentTo("a", out var sent)); + Assert.Equal(2, sent); + Assert.False(stats.TryGetRelation("a", "a", out _)); + Assert.False(stats.TryGetSentTo("b", out _)); + } + + [Fact] + public void MultipleUnrelatedPaths_AccumulateSitesAndRelations() + { + var stats = Read("Path: a!b!z\nPath: c!d!z\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.Equal(4, stats.SiteCount); + Assert.True(stats.TryGetRelation("a", "b", out var ab)); + Assert.True(stats.TryGetRelation("c", "d", out var cd)); + Assert.Equal(1, ab); + Assert.Equal(1, cd); + Assert.False(stats.TryGetRelation("a", "c", out _)); + } + + [Fact] + public void WhitespaceTerminatesThePath_AndDoesNotSkipLeadingSpaceAfterPrefix() + { + var stats = Read("Path: a!b!c extra\nPath: skipped-as-empty!x\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("b", out _)); + Assert.False(stats.TryGetSentTo("c", out _)); + Assert.False(stats.TryGetSentTo("skipped-as-empty", out _)); + } + + [Fact] + public void PathPrefixIsCaseInsensitive() + { + var stats = Read("path: a!b!c\nPATH: d!e!f\n"); + Assert.Equal(2, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("d", out _)); + } + + [Fact] + public void EmptyPathLine_CountsAnArticleWithoutSites() + { + var stats = Read("Path: \n"); + Assert.Equal(1, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void LineWithoutNewlineAtEof_IsBogus() + { + var stats = Read("Path: a!b!c"); + Assert.Equal(0, stats.TotalArticles); + Assert.Equal(0, stats.SiteCount); + } + + [Fact] + public void ConsecutiveBangs_AreSkipped() + { + var stats = Read("Path: a!!b!z\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetRelation("a", "b", out var tally)); + Assert.Equal(1, tally); + Assert.False(stats.TryGetSentTo(string.Empty, out _)); + } + + [Fact] + public void OverlongHostToken_AbortsRestOfPath_ButStillCountsTheArticle() + { + var longHop = new string('x', NinpathsConstants.MaxHostChars + 1); + var stats = Read($"Path: keep!{longHop}!later!end\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("keep", out var sent)); + Assert.Equal(1, sent); + Assert.False(stats.TryGetSentTo("later", out _)); + Assert.False(stats.TryGetSentTo(longHop, out _)); + } + + [Fact] + public void MaxHostCharsToken_IsAccepted() + { + var hop = new string('y', NinpathsConstants.MaxHostChars); + var stats = Read($"Path: {hop}!end\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo(hop, out var sent)); + Assert.Equal(1, sent); + } + + [Fact] + public void SiteNamesAreCaseSensitive() + { + var stats = Read("Path: News!news!end\n"); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("News", out _)); + Assert.True(stats.TryGetSentTo("news", out _)); + } + + [Fact] + public void LargeSyntheticStream_DoesNotRetainPerArticleState() + { + var builder = new StringBuilder(capacity: 80 * 20_000); + for (var i = 0; i < 20_000; i++) + { + builder.Append("Path: site-one!site-two!not-for-mail\n"); + } + + var stats = Read(builder.ToString()); + Assert.Equal(20_000, stats.TotalArticles); + Assert.Equal(2, stats.SiteCount); + Assert.True(stats.TryGetSentTo("site-one", out var one)); + Assert.True(stats.TryGetSentTo("site-two", out var two)); + Assert.Equal(20_000, one); + Assert.Equal(20_000, two); + Assert.True(stats.TryGetRelation("site-one", "site-two", out var tally)); + Assert.Equal(20_000, tally); + } + + [Fact] + public void FgetsWindowWithoutWhitespace_SkipsTheNextWindowThatHasWhitespace() + { + var bogus = new string('z', NinpathsConstants.MaxFgetsChars); + var stats = Read(bogus + "\nPath: a!b!c\n"); + Assert.Equal(1, stats.TotalArticles); + Assert.True(stats.TryGetSentTo("a", out _)); + Assert.True(stats.TryGetSentTo("b", out _)); + } + + private static NinpathsStatistics Read(string text) + { + var bytes = Encoding.Latin1.GetBytes(text); + using var stream = new MemoryStream(bytes, writable: false); + return NinpathsLogReader.Read(stream); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs new file mode 100644 index 00000000..8e5bf924 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsProcessingServiceTests.cs @@ -0,0 +1,316 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Logging; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Email; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.Ninpaths; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace VectorNNTPD.TransitService.Tests.Ninpaths; + +public sealed class NinpathsProcessingServiceTests +{ + [Fact] + public void Top1000_WhitespaceOnly_IsDisabled() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = [" ", "\t"]; + Assert.False(NinpathsTop1000.IsEnabled(options)); + Assert.Empty(NinpathsTop1000.GetRecipients(options)); + } + + [Fact] + public void Top1000_DedupesAndKeepsValidMailboxes() + { + var options = TestHostFactory.CreateValidOptions(); + options.Top1000 = ["a@example.com", " ", "A@example.com", "b@example.com"]; + Assert.Equal(["a@example.com", "b@example.com"], NinpathsTop1000.GetRecipients(options)); + } + + [Fact] + public async Task Handler_ReturnsWithoutParsing_UntilTheWorkerRuns() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + + handler.OnCompletedFile(path); + + Assert.Equal(1, service.PendingCount); + Assert.Empty(email.Sent); + Assert.True(File.Exists(path)); + + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + + var message = Assert.Single(email.Sent); + Assert.Equal("inpaths nntpd01.usenet.ninja", message.Subject); + var body = Encoding.Latin1.GetString(message.Body.Span); + Assert.StartsWith("!!NINP 3.1.1 ", body, StringComparison.Ordinal); + Assert.Contains("!!NLREC", body, StringComparison.Ordinal); + Assert.Contains("!!NLEND", body, StringComparison.Ordinal); + Assert.DoesNotContain("survey@example.com", body, StringComparison.Ordinal); + } + + [Fact] + public async Task MultipleRecipients_ShareOneGeneratedReport() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["one@example.com", "two@example.com"]); + var handler = CreateHandler(service, ["one@example.com", "two@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, email.SendCount); + var message = Assert.Single(email.Sent); + Assert.Equal(["one@example.com", "two@example.com"], message.To.Select(static a => a.Address).ToArray()); + Assert.Equal("us-ascii", message.Charset); + Assert.Contains(message.Headers, static h => h.Name == "Auto-Submitted" && h.Value == "auto-generated"); + } + + [Fact] + public async Task EmailFailure_IsNonFatal_AndDoesNotFaultExecution() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService + { + Throw = new InvalidOperationException("smtp exploded"), + }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (service.PendingCount > 0 && !wait.IsCancellationRequested) + { + await Task.Delay(10, wait.Token); + } + + Assert.NotNull(service.Execution); + Assert.False(service.Execution.IsFaulted); + Assert.False(service.Execution.IsCompleted); + await service.StopAsync(CancellationToken.None); + Assert.False(service.Execution.IsFaulted); + } + + [Fact] + public async Task EmailDisabledStatus_IsNonFatal() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService { Status = EmailEnqueueStatus.Disabled }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Equal(EmailEnqueueStatus.Disabled, email.Status); + Assert.False(service.Execution!.IsFaulted); + } + + [Fact] + public async Task DisabledTop1000_DoesNotOpenOrEnqueue() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, []); + var handler = CreateHandler(service, []); + handler.OnCompletedFile(path); + Assert.Equal(0, service.PendingCount); + Assert.Empty(email.Sent); + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DuplicateCompletedFile_IsNotProcessedTwice() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + handler.OnCompletedFile(path); + handler.OnCompletedFile(path); + Assert.Equal(1, service.PendingCount); + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Equal(1, email.SendCount); + } + + [Fact] + public async Task CompletedSourceRemainsReadableAfterDelete_ViaShareDeleteHandle() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + handler.OnCompletedFile(path); + File.Delete(path); + Assert.False(File.Exists(path)); + await service.StartAsync(CancellationToken.None); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Single(email.Sent); + } + + [Fact] + public async Task GzipStillRuns_WhileNinpathsHoldsTheFile() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", "Path: a!b!c\n"); + var email = new RecordingEmailService { Hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + + handler.OnCompletedFile(path); + Assert.True(new GzipLogFileCompressor().TryCompressAndReplace(path)); + + Assert.False(File.Exists(path)); + var archive = Path.Join(dir.Path, NntpdFileLogging.GzipArchiveFileName(path)); + Assert.True(File.Exists(archive)); + + email.Hold.SetResult(); + await email.SentOnce.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.StopAsync(CancellationToken.None); + Assert.Single(email.Sent); + } + + [Fact] + public async Task EmptySurvey_DoesNotSendEmail() + { + using var dir = new TempDir(); + var path = dir.Write("inpaths-20260927.log", ""); + var email = new RecordingEmailService(); + var service = CreateService(email, ["survey@example.com"]); + var handler = CreateHandler(service, ["survey@example.com"]); + await service.StartAsync(CancellationToken.None); + handler.OnCompletedFile(path); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (service.PendingCount > 0 && !wait.IsCancellationRequested) + { + await Task.Delay(10, wait.Token); + } + + await service.StopAsync(CancellationToken.None); + Assert.Empty(email.Sent); + } + + [Fact] + public void Host_RegistersNinpathsHandlerAndService() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + Assert.IsType(host.Services.GetRequiredService()); + var services = host.Services.GetServices().Select(static s => s.GetType()).ToArray(); + // Transit does not host SMTP delivery; Ninpaths is registered right after the incoming spool writer. + Assert.DoesNotContain(typeof(EmailDeliveryService), services); + var spool = Array.IndexOf(services, typeof(IncomingSpoolWriterService)); + var ninpaths = Array.IndexOf(services, typeof(NinpathsProcessingService)); + Assert.True(spool >= 0); + Assert.Equal(spool + 1, ninpaths); + } + + [Fact] + public void ReportMail_DoesNotEmbedRecipientAddressesInTheBody() + { + var report = "!!NINP 3.1.1 1 2 0 1 2\n\n!!NLREC\n\n!!NLEND 0\n"u8.ToArray(); + var message = NinpathsReportMail.TryCreate( + report, + "nntpd01.usenet.ninja", + "noreply@usenet.ninja", + ["one@example.com", "two@example.com"]); + Assert.NotNull(message); + var body = Encoding.Latin1.GetString(message.Body.Span); + Assert.DoesNotContain("one@example.com", body, StringComparison.Ordinal); + Assert.DoesNotContain("two@example.com", body, StringComparison.Ordinal); + Assert.Equal("inpaths nntpd01.usenet.ninja", message.Subject); + } + + private static NinpathsProcessingService CreateService(RecordingEmailService email, string[] top1000) + { + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.Top1000 = top1000; + var emailOptions = new EmailOptions { DefaultFrom = "noreply@usenet.ninja" }; + return new NinpathsProcessingService( + email, + Options.Create(nntpd), + Options.Create(emailOptions), + NullLogger.Instance, + new UtcTimeProvider(new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero))); + } + + private static NinpathsCompletedFileHandler CreateHandler(NinpathsProcessingService service, string[] top1000) + { + var nntpd = TestHostFactory.CreateValidOptions(); + nntpd.Top1000 = top1000; + return new NinpathsCompletedFileHandler( + service, + Options.Create(nntpd), + NullLogger.Instance); + } + + private sealed class UtcTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utc; + + public UtcTimeProvider(DateTimeOffset utc) => _utc = utc; + + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + + public override DateTimeOffset GetUtcNow() => _utc; + } + + private sealed class TempDir : IDisposable + { + public TempDir() + { + Path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "ninpaths-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public string Write(string name, string content) + { + var file = System.IO.Path.Join(Path, name); + File.WriteAllBytes(file, Encoding.Latin1.GetBytes(content)); + return file; + } + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsReportFormatterTests.cs b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsReportFormatterTests.cs new file mode 100644 index 00000000..b3b22806 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Ninpaths/NinpathsReportFormatterTests.cs @@ -0,0 +1,139 @@ +using System.Text; +using VectorNNTPD.TransitService.Ninpaths; + +namespace VectorNNTPD.TransitService.Tests.Ninpaths; + +public sealed class NinpathsReportFormatterTests +{ + [Fact] + public void Format_EmptyTraffic_ReturnsEmptyBuffer() + { + var stats = Read(""); + Assert.Empty(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + } + + [Fact] + public void Format_WritesInn311CompactDump() + { + var stats = Read("Path: alpha!beta!omega\nPath: alpha!beta!omega\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 100, 200, 200)); + Assert.StartsWith("!!NINP 3.1.1 100 200 2 2 200\n", dump, StringComparison.Ordinal); + Assert.Contains("!!NLREC\n", dump, StringComparison.Ordinal); + Assert.EndsWith("!!NLEND 1\n", dump, StringComparison.Ordinal); + Assert.Contains("alpha 2", dump, StringComparison.Ordinal); + Assert.Contains("beta 2", dump, StringComparison.Ordinal); + Assert.DoesNotContain("omega", dump, StringComparison.Ordinal); + Assert.DoesNotContain("nntpd", dump, StringComparison.Ordinal); + Assert.DoesNotContain("ZCZC", dump, StringComparison.Ordinal); + + Assert.True(stats.TryGetSentTo("alpha", out _)); + var alpha = stats.Find("alpha"u8)!; + var beta = stats.Find("beta"u8)!; + Assert.Contains( + $":{alpha.Number}!{beta.Number},2", + dump, + StringComparison.Ordinal); + } + + [Fact] + public void Format_OmitsCountWhenTallyIsOne() + { + var stats = Read("Path: left!right!end\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + var left = stats.Find("left"u8)!; + var right = stats.Find("right"u8)!; + Assert.Contains($":{left.Number}!{right.Number}", dump, StringComparison.Ordinal); + Assert.DoesNotContain($":{left.Number}!{right.Number},", dump, StringComparison.Ordinal); + Assert.EndsWith("!!NLEND 1\n", dump, StringComparison.Ordinal); + } + + [Fact] + public void Format_NumbersSitesInInnHashWalkOrder() + { + var stats = Read("Path: a!b!z\nPath: c!d!z\n"); + _ = NinpathsReportFormatter.Format(stats, 1, 2, 2); + long expected = 0; + for (var i = 0; i < stats.Buckets.Length; i++) + { + for (var site = stats.Buckets[i]; site is not null; site = site.BucketNext) + { + Assert.Equal(expected, site.Number); + expected++; + } + } + + Assert.Equal(stats.SiteCount, expected); + } + + [Fact] + public void Format_RelationOrderIsInnPrependOrder() + { + var stats = Read("Path: hub!first!end\nPath: hub!second!end\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 1, 2, 2)); + var hub = stats.Find("hub"u8)!; + var first = stats.Find("first"u8)!; + var second = stats.Find("second"u8)!; + var expected = $":{hub.Number}!{second.Number}!{first.Number}"; + Assert.Contains(expected, dump, StringComparison.Ordinal); + } + + [Fact] + public void Format_DoesNotRereadTheSource() + { + var stats = Read("Path: a!b!c\n"); + var first = NinpathsReportFormatter.Format(stats, 5, 6, 6); + var second = NinpathsReportFormatter.Format(stats, 5, 6, 6); + Assert.Equal(first, second); + } + + [Fact] + public void ReportPeriod_UsesLocalMidnightFromInpathsFileName() + { + var time = new UtcTimeProvider(new DateTimeOffset(2026, 9, 28, 1, 0, 0, TimeSpan.Zero)); + var (start, end, average) = NinpathsReportPeriod.Resolve( + Path.Join("logs", "inpaths-20260927.log"), + time); + Assert.Equal( + new DateTimeOffset(2026, 9, 27, 0, 0, 0, TimeSpan.Zero).ToUnixTimeSeconds(), + start); + Assert.Equal(time.GetUtcNow().ToUnixTimeSeconds(), end); + Assert.Equal(end, average); + } + + [Fact] + public void ReportPeriod_UnknownFileName_UsesDumpTimeForAllFields() + { + var time = new UtcTimeProvider(new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero)); + var unix = time.GetUtcNow().ToUnixTimeSeconds(); + var (start, end, average) = NinpathsReportPeriod.Resolve("not-a-survey.txt", time); + Assert.Equal(unix, start); + Assert.Equal(unix, end); + Assert.Equal(unix, average); + } + + [Fact] + public void HeaderFields_AreSpaceSeparatedInInnOrder() + { + var stats = Read("Path: a!b!c\n"); + var dump = Encoding.Latin1.GetString(NinpathsReportFormatter.Format(stats, 10, 20, 15)); + var header = dump.Split('\n')[0].Split(' '); + Assert.Equal(["!!NINP", "3.1.1", "10", "20", "2", "1", "15"], header); + } + + private static NinpathsStatistics Read(string text) + { + using var stream = new MemoryStream(Encoding.Latin1.GetBytes(text), writable: false); + return NinpathsLogReader.Read(stream); + } + + private sealed class UtcTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utc; + + public UtcTimeProvider(DateTimeOffset utc) => _utc = utc; + + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + + public override DateTimeOffset GetUtcNow() => _utc; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs b/tests/VectorNNTPD.TransitService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs new file mode 100644 index 00000000..9ec4aec9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/NntpDb/MySqlNntpDbConnectionTests.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.TransitService.Tests.NntpDb; + +public sealed class MySqlNntpDbConnectionTests +{ + [Fact] + public async Task OpenAsync_MalformedConnectionString_ThrowsConfigurationException() + { + var factory = new MySqlNntpDbConnectionFactory(); + var ex = await Assert.ThrowsAsync( + () => factory.OpenAsync(TestHostFactory.MalformedNntpDbConnectionString, CancellationToken.None)); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Contains("initialization string", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.IsType(ex.InnerException); + } + + [Fact] + public async Task OpenAsync_MalformedConnectionString_DoesNotExposePassword() + { + var factory = new MySqlNntpDbConnectionFactory(); + var ex = await Assert.ThrowsAsync( + () => factory.OpenAsync( + TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + CancellationToken.None)); + + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.Reason, StringComparison.Ordinal); + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, ex.ToString(), StringComparison.Ordinal); + Assert.DoesNotContain("Password=", ex.ToString(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ConvertSelectOneScalar_AcceptsInt32() + { + Assert.Equal(1, MySqlNntpDbConnection.ConvertSelectOneScalar(1)); + } + + [Fact] + public void ConvertSelectOneScalar_AcceptsMySqlInt64() + { + Assert.Equal(1, MySqlNntpDbConnection.ConvertSelectOneScalar(1L)); + } + + [Theory] + [InlineData((ulong)4_294_967_295)] + public void ConvertUnsignedCount_PreservesUInt32Max(ulong expected) + { + Assert.Equal(expected, MySqlNntpDbConnection.ConvertUnsignedCount(uint.MaxValue, "count_high", "g")); + } + + [Fact] + public void ConvertUnsignedCount_PreservesUInt64() + { + Assert.Equal(ulong.MaxValue, MySqlNntpDbConnection.ConvertUnsignedCount(ulong.MaxValue, "count_high", "g")); + } + + [Fact] + public void ConvertUnsignedCount_RejectsNegativeAndFloatingPoint() + { + Assert.Throws( + () => MySqlNntpDbConnection.ConvertUnsignedCount(-1, "count_low", "g")); + Assert.Throws( + () => MySqlNntpDbConnection.ConvertUnsignedCount(1.5d, "count_low", "g")); + } + + [Theory] + [InlineData((byte)'y')] + [InlineData((byte)'n')] + [InlineData((byte)'m')] + [InlineData((byte)'x')] + [InlineData((byte)'j')] + public void ConvertPostingStatus_AcceptsSingleOctet(byte status) + { + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus(status, "g")); + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus(((char)status).ToString(), "g")); + Assert.Equal(status, MySqlNntpDbConnection.ConvertPostingStatus((char)status, "g")); + } + + [Theory] + [InlineData((byte)'z')] + [InlineData((byte)'Y')] + [InlineData((byte)'=')] + public void ConvertPostingStatus_RejectsUnknownAndEquals(byte status) + { + Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus(status, "g")); + Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus(((char)status).ToString(), "g")); + } + + [Fact] + public void ConvertPostingStatus_RejectsEqualsAliasString() + { + var ex = Assert.Throws( + () => MySqlNntpDbConnection.ConvertPostingStatus("=other.group", "g")); + Assert.Contains("=", ex.Message, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbOptionsTests.cs new file mode 100644 index 00000000..921ff008 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbOptionsTests.cs @@ -0,0 +1,188 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Hosting; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.NntpDb; + +public sealed class NntpDbOptionsTests +{ + [Fact] + public void Defaults_MatchProductionValues() + { + var options = new NntpDbOptions(); + Assert.Equal(TimeSpan.FromSeconds(15), options.StartupTimeout); + Assert.Equal("NntpDb", NntpDbOptions.SectionName); + Assert.Equal("NntpDB", NntpDbOptions.ConnectionStringName); + Assert.Equal("ConnectionStrings__NntpDB", NntpDbOptions.ConnectionStringEnvironmentVariable); + } + + [Fact] + public void Validate_Succeeds_ForDefaultsWithConnectionString() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Succeeded); + } + + [Fact] + public void Validate_Fails_WhenConnectionStringMissing() + { + var result = new NntpDbOptionsValidator().Validate(null, new NntpDbOptions()); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("ConnectionStrings:NntpDB", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("Password", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_Fails_WhenConnectionStringIsMalformed() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.MalformedNntpDbConnectionString }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("NntpDB connection string is invalid", StringComparison.Ordinal)); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("initialization string", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains(TestHostFactory.MalformedNntpDbConnectionString, StringComparison.Ordinal)); + } + + [Fact] + public void Validate_MalformedConnectionString_DoesNotExposePassword() + { + var options = new NntpDbOptions + { + ConnectionString = TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains(TestHostFactory.FakeNntpDbPassword, StringComparison.Ordinal)); + Assert.DoesNotContain( + result.Failures!, + static failure => failure.Contains("Password=", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_Succeeds_WhenConnectionStringIsAcceptedByMySqlConnector() + { + var options = new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Succeeded); + NntpDbConnectionString.Validate(options.ConnectionString); + } + + [Fact] + public void Validate_Fails_WhenStartupTimeoutNotPositive() + { + var options = new NntpDbOptions + { + ConnectionString = TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = TimeSpan.Zero, + }; + Assert.True(new NntpDbOptionsValidator().Validate(null, options).Failed); + } + + [Fact] + public void Bind_ReadsConnectionStringsNntpDB() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:NntpDB"] = TestHostFactory.TestNntpDbConnectionString, + ["NntpDb:StartupTimeout"] = "00:00:20", + }) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddNntpDbOptions(); + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(TestHostFactory.TestNntpDbConnectionString, options.ConnectionString); + Assert.Equal(TimeSpan.FromSeconds(20), options.StartupTimeout); + } + + [Fact] + public void ProductionAppsettings_DoesNotContainConnectionStrings() + { + var path = FindProductionAppsettings(); + using var document = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + Assert.False(document.RootElement.TryGetProperty("ConnectionStrings", out _)); + Assert.False(File.ReadAllText(path).Contains("GrabberDB", StringComparison.Ordinal)); + + var nntpDb = document.RootElement.GetProperty("NntpDb"); + Assert.Equal("00:00:15", nntpDb.GetProperty("StartupTimeout").GetString()); + Assert.False(nntpDb.TryGetProperty("ConnectionString", out _)); + Assert.False(nntpDb.TryGetProperty("MinimumPoolSize", out _)); + Assert.False(nntpDb.TryGetProperty("MaximumPoolSize", out _)); + Assert.False(nntpDb.TryGetProperty("MaximumIdleTime", out _)); + Assert.False(nntpDb.TryGetProperty("MaintenanceInterval", out _)); + Assert.False(nntpDb.TryGetProperty("AcquisitionTimeout", out _)); + } + + [Fact] + public void Same_connection_strings_nntpdb_value_targets_nntpdb_catalog() + { + const string connectionString = + "Server=198.18.0.70;Port=3306;Database=nntpdb;User ID=nntpd;Pooling=true;MinimumPoolSize=2;MaximumPoolSize=32;ConnectionIdleTimeout=300;"; + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + [$"ConnectionStrings:{NntpDbOptions.ConnectionStringName}"] = connectionString, + }) + .Build(); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddNntpDbOptions(); + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + Assert.Equal(connectionString, options.ConnectionString); + Assert.Equal("nntpdb", new MySqlConnector.MySqlConnectionStringBuilder(options.ConnectionString).Database); + Assert.Equal("ConnectionStrings__NntpDB", NntpDbOptions.ConnectionStringEnvironmentVariable); + } + + [Fact] + public void Factory_DoesNotOverrideSuppliedPoolingSetting() + { + using var disabled = MySqlNntpDbConnectionFactory.CreateConnection( + "Server=db;Database=nntpdb;User ID=nntpd;Pooling=false;"); + var disabledPooling = new MySqlConnector.MySqlConnectionStringBuilder(disabled.ConnectionString).Pooling; + Assert.False(disabledPooling); + + using var enabled = MySqlNntpDbConnectionFactory.CreateConnection( + "Server=db;Database=nntpdb;User ID=nntpd;Pooling=true;"); + var enabledPooling = new MySqlConnector.MySqlConnectionStringBuilder(enabled.ConnectionString).Pooling; + Assert.True(enabledPooling); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbServiceTests.cs new file mode 100644 index 00000000..97b2a1f6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpDbServiceTests.cs @@ -0,0 +1,422 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.NntpDb; + +public sealed class NntpDbServiceTests +{ + [Fact] + public async Task StartAsync_OpensConnection_SelectsOne_AndDisposesLogicalConnection() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenCount); + var connection = Assert.Single(factory.Connections); + Assert.Equal(1, connection.SelectOneCount); + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task StartAsync_Fails_WhenMySqlIsUnavailable() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbUnavailableException("down"), + }; + var service = CreateService(factory, startupTimeout: TimeSpan.FromMilliseconds(1)); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("startup", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_FailsImmediately_WithoutRetry() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbUnavailableException("should not be reached"), + }; + var clock = new ControllableTimeProvider(); + var logger = new CollectingLogger(); + var service = CreateService( + factory, + startupTimeout: TimeSpan.FromMinutes(1), + clock: clock, + connectionString: TestHostFactory.MalformedNntpDbConnectionString, + logger: logger); + + var starting = service.StartAsync(CancellationToken.None); + var ex = await Assert.ThrowsAsync( + () => starting.WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Contains("initialization string", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.Contains("index", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.IsType(ex.InnerException); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + Assert.Contains( + logger.Messages, + static message => message.Contains("NntpDB connection string is invalid", StringComparison.Ordinal)); + Assert.DoesNotContain( + logger.Messages, + static message => message.Contains("startup connectivity retry", StringComparison.Ordinal)); + Assert.Equal(1, logger.Messages.Count(static message => + message.Contains("NntpDB connection string is invalid", StringComparison.Ordinal))); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_DoesNotExposePassword() + { + var factory = new FakeNntpDbConnectionFactory(); + var logger = new CollectingLogger(); + var service = CreateService( + factory, + connectionString: TestHostFactory.MalformedNntpDbConnectionStringWithPassword, + logger: logger); + + var ex = await Assert.ThrowsAsync( + () => service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + AssertSecretNotExposed(ex.Message); + AssertSecretNotExposed(ex.Reason); + AssertSecretNotExposed(ex.ToString()); + foreach (var message in logger.Messages) + { + AssertSecretNotExposed(message); + } + + foreach (var logged in logger.Exceptions) + { + AssertSecretNotExposed(logged.ToString()); + } + + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + } + + [Fact] + public async Task StartAsync_ValidConnectionString_ProceedsPastParsing() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory, connectionString: TestHostFactory.TestNntpDbConnectionString); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.Equal(1, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenAttemptCount); + Assert.Equal(1, factory.OpenCount); + } + + [Fact] + public async Task StartAsync_CanceledToken_IsNotConvertedToConfigurationFailure() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory, connectionString: TestHostFactory.TestNntpDbConnectionString); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => service.StartAsync(cts.Token)); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.False(service.HasStarted); + } + + [Fact] + public async Task StartAsync_MalformedConnectionString_WithRealFactory_DoesNotOpen() + { + var clock = new ControllableTimeProvider(); + var service = new NntpDbService( + new MySqlNntpDbConnectionFactory(), + Options.Create(new NntpDbOptions + { + ConnectionString = TestHostFactory.MalformedNntpDbConnectionString, + StartupTimeout = TimeSpan.FromMinutes(1), + }), + NullLogger.Instance, + clock); + + var ex = await Assert.ThrowsAsync( + () => service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.False(service.HasStarted); + } + + [Fact] + public async Task Lifecycle_MalformedConnectionString_FailsStartupWithConfigurationException() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService( + factory, + startupTimeout: TimeSpan.FromMinutes(1), + connectionString: TestHostFactory.MalformedNntpDbConnectionString); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + var ex = await Assert.ThrowsAsync( + () => lifecycle.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2))); + + Assert.Contains("NntpDB connection string is invalid", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, service.StartupConnectAttempts); + Assert.Equal(0, factory.OpenAttemptCount); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + } + + [Fact] + public async Task StartAsync_FailsImmediately_OnAuthenticationFailure() + { + var factory = new FakeNntpDbConnectionFactory + { + OpenException = new NntpDbAuthenticationException("denied"), + }; + var service = CreateService(factory, startupTimeout: TimeSpan.FromMinutes(1)); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task StartAsync_FailsImmediately_WhenSelectOneReturnsWrongValue() + { + var factory = new FakeNntpDbConnectionFactory { NextHealthCheckResult = 0 }; + var service = CreateService(factory); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Contains("SELECT 1", ex.Message, StringComparison.Ordinal); + Assert.False(service.HasStarted); + Assert.Equal(1, Assert.Single(factory.Connections).DisposeCount); + } + + [Fact] + public async Task StartAsync_FailsImmediately_WhenSelectOneThrows() + { + var factory = new FakeNntpDbConnectionFactory + { + NextHealthCheckException = new NntpDbUnavailableException("MySQL health query SELECT 1 failed."), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasStarted); + Assert.Equal(1, Assert.Single(factory.Connections).DisposeCount); + } + + [Fact] + public async Task StartAsync_RetriesTransientOpenFailure_ThenSucceeds() + { + var factory = new FakeNntpDbConnectionFactory { RemainingOpenFailures = 1 }; + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + Assert.True(service.HasStarted); + Assert.Equal(0, factory.RemainingOpenFailures); + Assert.Equal(2, factory.OpenAttemptCount); + Assert.Equal(2, service.StartupConnectAttempts); + Assert.Equal(1, factory.OpenCount); + Assert.Equal(1, Assert.Single(factory.Connections).SelectOneCount); + Assert.Equal(1, factory.Connections[0].DisposeCount); + } + + [Fact] + public async Task StartAsync_CancellationDuringOpen_ResetsState_AndIsRetryable() + { + var factory = new FakeNntpDbConnectionFactory + { + BlockOpen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + OpenStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var starting = service.StartAsync(cts.Token); + await factory.OpenStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => starting); + factory.BlockOpen.TrySetResult(); + factory.BlockOpen = null; + factory.OpenStarted = null; + + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + Assert.Empty(factory.Connections); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, Assert.Single(factory.Connections).SelectOneCount); + } + + [Fact] + public async Task StartAsync_CancellationDuringSelectOne_DisposesConnection_AndRetriesFullStartup() + { + var factory = new FakeNntpDbConnectionFactory + { + BlockSelectOne = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + SelectOneStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var starting = service.StartAsync(cts.Token); + await factory.SelectOneStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => starting); + factory.BlockSelectOne.TrySetResult(); + factory.BlockSelectOne = null; + factory.SelectOneStarted = null; + + var first = Assert.Single(factory.Connections); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(0, first.SelectOneCount); + Assert.False(service.HasStarted); + Assert.False(service.IsAccepting); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.HasStarted); + Assert.True(service.IsAccepting); + Assert.Equal(1, factory.Connections.Sum(static connection => connection.SelectOneCount)); + Assert.All(factory.Connections, static connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task OpenAsync_CreatesLogicalConnection_AndDoesNotRetainIt() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.Equal(1, factory.OpenCount); + + await using (var connection = await service.OpenAsync()) + { + Assert.Equal(2, factory.OpenCount); + Assert.Equal(0, ((FakeNntpDbConnection)connection).DisposeCount); + } + + Assert.Equal(2, factory.OpenCount); + Assert.All(factory.Connections, static connection => Assert.Equal(1, connection.DisposeCount)); + } + + [Fact] + public async Task StopAsync_RejectsNewOpens() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + await service.StopAsync(CancellationToken.None); + + await Assert.ThrowsAsync(() => service.OpenAsync().AsTask()); + Assert.False(service.IsAccepting); + } + + [Fact] + public async Task Lifecycle_StartsServiceBeforeRunning_AndStopsIt() + { + var factory = new FakeNntpDbConnectionFactory(); + var service = CreateService(factory); + var lifecycle = TestHostFactory.CreateLifecycle([service]); + + await lifecycle.StartAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Running, lifecycle.State); + Assert.True(service.HasStarted); + Assert.Equal(1, factory.OpenCount); + + await lifecycle.StopAsync(CancellationToken.None); + Assert.Equal(ApplicationState.Stopped, lifecycle.State); + await Assert.ThrowsAsync(() => service.OpenAsync().AsTask()); + } + + [Fact] + public void Options_NeverExposeSecretPropertyNamesOnValidatorFailures() + { + var options = new NntpDbOptions + { + ConnectionString = "Server=db;Password=super-secret;User ID=nntpd;", + StartupTimeout = TimeSpan.Zero, + }; + var result = new NntpDbOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("super-secret", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static failure => failure.Contains("Password=", StringComparison.Ordinal)); + } + + private static NntpDbService CreateService( + FakeNntpDbConnectionFactory factory, + TimeSpan? startupTimeout = null, + TimeProvider? clock = null, + string? connectionString = null, + ILogger? logger = null) + { + var options = new NntpDbOptions + { + ConnectionString = connectionString ?? TestHostFactory.TestNntpDbConnectionString, + StartupTimeout = startupTimeout ?? TimeSpan.FromSeconds(15), + }; + return new NntpDbService( + factory, + Options.Create(options), + logger ?? NullLogger.Instance, + clock ?? TimeProvider.System); + } + + private static void AssertSecretNotExposed(string text) + { + Assert.DoesNotContain(TestHostFactory.FakeNntpDbPassword, text, StringComparison.Ordinal); + Assert.DoesNotContain("Password=", text, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(TestHostFactory.MalformedNntpDbConnectionStringWithPassword, text, StringComparison.Ordinal); + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public List Exceptions { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + if (exception is not null) + { + Exceptions.Add(exception); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs new file mode 100644 index 00000000..50b3fcbb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/NntpDb/NntpSharedConfigurationServiceTests.cs @@ -0,0 +1,265 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.NntpDb; + +public sealed class NntpSharedConfigurationServiceTests +{ + [Fact] + public async Task Start_publishes_one_shared_configuration_row() + { + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = + [ + new NntpSharedConfigurationCandidate( + 4096, + "news.example", + "http://prom.example", + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"), + ], + }; + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(4096, service.Current.MaxArticleBytes); + Assert.Equal("news.example", service.Current.SiteName); + Assert.Equal("http://prom.example", service.Current.PrometheusUrl); + await service.DisposeAsync(); + } + + [Fact] + public async Task Start_fails_when_the_row_is_invalid_and_does_not_publish() + { + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [], + }; + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + NullLogger.Instance, + TimeProvider.System, + TimeSpan.FromHours(1)); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + await service.DisposeAsync(); + } + + [Fact] + public async Task Start_copies_shared_credentials_and_does_not_log_them() + { + var nntp = Options.Create(new NntpdOptions + { + ServerId = 1, + BindPortTls = 0, + AcmeEmail = "from-env@example.test", + AcmeCertificatePassword = "from-env-pfx", + CloudFlareApiKey = "from-env-key", + }); + var rabbit = Options.Create(new RabbitMqOptions + { + Username = "from-env-user", + Password = "from-env-pass", + Hosts = ["127.0.0.1"], + VirtualHost = "/articles", + }); + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow()], + }); + var service = new NntpSharedConfigurationService( + database, + nntp, + logger, + TimeProvider.System, + TimeSpan.FromHours(1), + rabbit); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderAcmeAccount, nntp.Value.AcmeEmail); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderAcmeCertificatePassword, nntp.Value.AcmeCertificatePassword); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, nntp.Value.CloudFlareApiKey); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqUsername, rabbit.Value.Username); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, rabbit.Value.Password); + Assert.Equal("/articles", rabbit.Value.VirtualHost); + Assert.Equal("127.0.0.1", Assert.Single(rabbit.Value.Hosts!)); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderAcmeAccount, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderAcmeCertificatePassword, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain(NntpSharedConfigurationColumns.PlaceholderRabbitMqUsername, logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain("from-env-pfx", logger.Text, StringComparison.Ordinal); + Assert.DoesNotContain("from-env-pass", logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Invalid_shared_credentials_prevent_the_first_snapshot() + { + const string secret = "credential-must-not-be-logged-xxxxxxxxxxxxxxxxxxxx"; + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow() with { CloudFlareApiKey = secret }], + }); + var service = new NntpSharedConfigurationService( + database, + Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }), + logger, + TimeProvider.System, + TimeSpan.FromHours(1)); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.False(service.HasSnapshot); + Assert.Contains("cloudflareapikey", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(secret, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain(secret, logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Later_invalid_refresh_retains_the_previous_credentials() + { + const string secret = "refresh-credential-must-not-be-logged-xxxxxxxxxxxx"; + var factory = new FakeNntpDbConnectionFactory + { + SharedConfigurationRows = [ValidRow()], + }; + var clock = new ControllableTimeProvider(); + var nntp = Options.Create(new NntpdOptions { ServerId = 1, BindPortTls = 0 }); + var rabbit = Options.Create(new RabbitMqOptions()); + var logger = new CaptureLogger(); + await using var database = await StartDatabaseAsync(factory); + var service = new NntpSharedConfigurationService( + database, + nntp, + logger, + clock, + TimeSpan.FromSeconds(60), + rabbit); + await service.StartAsync(CancellationToken.None); + var published = service.Current; + + factory.SharedConfigurationRows = [ValidRow() with { RabbitMqPassword = secret }]; + await WaitUntilAsync(() => clock.HasScheduledTimers); + clock.Advance(TimeSpan.FromSeconds(60)); + await WaitUntilAsync(() => logger.Text.Contains("retaining the last known-good snapshot", StringComparison.Ordinal)); + + Assert.Equal(published, service.Current); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderRabbitMqPassword, rabbit.Value.Password); + Assert.Equal(NntpSharedConfigurationColumns.PlaceholderCloudFlareApiKey, nntp.Value.CloudFlareApiKey); + Assert.DoesNotContain(secret, logger.Text, StringComparison.Ordinal); + await service.DisposeAsync(); + } + + [Fact] + public async Task Open_before_startup_is_refused() + { + var database = new NntpDbService( + new FakeNntpDbConnectionFactory(), + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await Assert.ThrowsAsync(() => database.OpenAsync(CancellationToken.None).AsTask()); + await database.DisposeAsync(); + } + + private static NntpSharedConfigurationCandidate ValidRow() => + new( + 4096, + "news.example", + null, + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "0123456789abcdef0123456789abcdef", + "usenet.ninja"); + + private static async Task WaitUntilAsync(Func condition) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(10, timeout.Token); + } + } + + private static async Task StartDatabaseAsync(FakeNntpDbConnectionFactory factory) + { + var database = new NntpDbService( + factory, + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await database.StartAsync(CancellationToken.None); + return database; + } + + private sealed class CaptureLogger : ILogger + { + private readonly List _entries = []; + + public string Text + { + get + { + lock (_entries) + { + return string.Join('\n', _entries); + } + } + } + + public IDisposable BeginScope(TState state) + where TState : notnull => + NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var text = formatter(state, exception); + if (exception is not null) + { + text = text + " " + exception; + } + + lock (_entries) + { + _entries.Add(text); + } + } + + private sealed class NullScope : IDisposable + { + internal static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs new file mode 100644 index 00000000..294de63e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkAggregatePolicyTests.cs @@ -0,0 +1,36 @@ +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +/// +/// Regression: definitive provider 430 / +/// must not schedule another backbone attempt. +/// +public sealed class ArticleWorkAggregatePolicyTests +{ + [Fact] + public void ArticleNotFound_is_attempt_terminal_and_does_not_try_next_backbone() + { + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.ArticleNotFound)); + } + + [Fact] + public void InvalidArticle_and_InvalidRequest_may_try_next_backbone() + { + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.InvalidArticle)); + Assert.True(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.InvalidArticle)); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.InvalidRequest)); + Assert.True(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.InvalidRequest)); + } + + [Fact] + public void Success_completes_lookup_without_trying_next() + { + Assert.True(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.Success)); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.Success)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.Success)); + } + +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs new file mode 100644 index 00000000..497b63a6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcClientTests.cs @@ -0,0 +1,612 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkRpcClientTests +{ + private static readonly byte[] MessageIdBytes = "<12345@example.invalid>"u8.ToArray(); + private const string MessageId = "<12345@example.invalid>"; + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public async Task No_active_consumers_publishes_nothing() + { + await using var harness = CreateHarness(eligible: []); + var result = await harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Contains("No BackFiller", result.Error, StringComparison.OrdinalIgnoreCase); + Assert.Empty(harness.Publisher.Publications); + } + + [Fact] + public async Task One_active_backbone_receives_exactly_one_request() + { + await using var harness = CreateHarness(Eligible("Giganews", 10)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + AssertAmqpContract(publication); + Assert.Equal("Giganews", ReadBackbone(publication)); + harness.DeliverSuccess(publication, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Two_active_backbones_first_attempt_goes_to_exactly_one() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(1); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + harness.DeliverSuccess(harness.Publisher.Publications[0], "Giganews"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task First_not_found_ends_lookup_without_publishing_to_another_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("missing", result.Error); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => p.Exchange == "backfiller.eweka"); + } + + [Fact] + public async Task First_not_found_never_retries_same_or_other_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 1)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Single(harness.Publisher.Publications); + Assert.Equal("backfiller.giganews", harness.Publisher.Publications[0].Exchange); + } + + [Fact] + public async Task First_eligible_not_found_returns_not_found_without_second_publish() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 2), Eligible("Eweka", 2)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(1); + harness.DeliverOutcome(harness.Publisher.Publications[0], ArticleWorkOutcome.ArticleNotFound, "a"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("a", result.Error); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Zero_consumer_backbone_is_never_selected() + { + await using var harness = CreateHarness(Eligible("Eweka", 5)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + Assert.DoesNotContain(harness.Publisher.Publications, static p => p.Exchange.Contains("giganews", StringComparison.Ordinal)); + harness.DeliverSuccess(publication, "Eweka"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task Success_stops_immediately_without_further_publishes() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverSuccess(first, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Cancellation_stops_without_launching_another_attempt() + { + await using var harness = CreateHarness(Eligible("Giganews", 10)); + using var cts = new CancellationTokenSource(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, cts.Token); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await lookup); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public async Task Attempt_timeout_without_response_does_not_start_another_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(ArticleWorkRpcTiming.LookupDeadline); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True( + result.Error is not null + && (result.Error.Contains("timed out", StringComparison.OrdinalIgnoreCase) + || result.Error.Contains("deadline elapsed", StringComparison.OrdinalIgnoreCase)), + result.Error); + Assert.Single(harness.Publisher.Publications); + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Lookup_completes_by_five_second_scheduler_deadline() + { + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10)], + nextInt: static _ => 0); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(5)); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True(harness.Time.GetUtcNow() - started <= ArticleWorkRpcTiming.LookupDeadline); + Assert.Single(harness.Publisher.Publications); + Assert.DoesNotContain("absolute", result.Error, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("60", result.Error, StringComparison.Ordinal); + } + + [Fact] + public async Task Definitive_not_found_before_deadline_ends_lookup_immediately() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + harness.DeliverOutcome(first, ArticleWorkOutcome.ArticleNotFound, "missing"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Single(harness.Publisher.Publications); + Assert.True(harness.Time.GetUtcNow() - started < ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Invalid_article_before_deadline_advances_to_next_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + CreateSequentialRolls(0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad yenc"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverSuccess(second, "Eweka"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(2, harness.Publisher.Publications.Count); + Assert.True(harness.Time.GetUtcNow() - started < ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Success_before_deadline_terminates_immediately() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 100), Eligible("Eweka", 50)], + nextInt: static _ => 0); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromMilliseconds(250)); + harness.DeliverSuccess(first, "Giganews"); + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Single(harness.Publisher.Publications); + } + + [Fact] + public void Lookup_deadline_is_five_seconds_with_no_separate_sixty_second_ceiling() + { + Assert.Equal(TimeSpan.FromSeconds(5), ArticleWorkRpcTiming.LookupDeadline); + Assert.Null(typeof(ArticleWorkRpcTiming).GetField( + "AbsoluteLifetime", + System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic)); + } + + [Fact] + public void Weighted_selection_is_approximately_two_to_one_for_100_vs_50() + { + var roll = 0; + var selector = new WeightedBackboneSelector(exclusiveMax => + { + var value = roll % exclusiveMax; + roll++; + return value; + }); + var candidates = new[] + { + Eligible("Giganews", 100), + Eligible("Eweka", 50), + }; + var counts = new Dictionary(StringComparer.Ordinal); + const int iterations = 15_000; + for (var i = 0; i < iterations; i++) + { + var pick = selector.Select(candidates)!.Value.Backbone; + counts[pick] = counts.GetValueOrDefault(pick) + 1; + } + + var ratio = counts["Giganews"] / (double)counts["Eweka"]; + Assert.InRange(ratio, 1.7, 2.3); + } + + [Fact] + public async Task Invalid_article_is_attempt_terminal_and_tries_next() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10)], + CreateSequentialRolls(0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad yenc"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverSuccess(second, "Eweka"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + [Fact] + public async Task Repeated_lookups_with_single_active_backbone_never_touch_others() + { + await using var harness = CreateHarness(Eligible("Giganews", 25)); + for (var i = 0; i < 8; i++) + { + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + var publication = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews", skip: i); + Assert.Equal("backfiller.giganews", publication.Exchange); + harness.DeliverSuccess(publication, "Giganews"); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + Assert.Equal(8, harness.Publisher.Publications.Count); + Assert.All( + harness.Publisher.Publications, + static p => Assert.Equal("backfiller.giganews", p.Exchange)); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => + p.Exchange.Contains("eweka", StringComparison.Ordinal) + || p.Exchange.Contains("highwinds", StringComparison.Ordinal) + || p.Exchange.Contains("storage", StringComparison.Ordinal)); + } + + [Fact] + public async Task Zero_consumer_and_storage_never_receive_attempts_even_when_listed() + { + await using var harness = CreateHarness( + [ + Eligible("Giganews", 100), + Eligible("Eweka", 50), + Eligible("Highwinds", 0), + new BackfillEligibleBackbone( + "Storage", + StorageArticleRetrievalTopology.EntityName, + StorageArticleRetrievalTopology.EntityName, + StorageArticleRetrievalTopology.EntityName, + 0), + ], + CreateSequentialRolls(0, 0)); + + for (var i = 0; i < 6; i++) + { + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + await harness.Publisher.WaitForPublicationCountAsync(i + 1); + var publication = harness.Publisher.Publications[i]; + Assert.Contains( + publication.Exchange, + new[] { "backfiller.giganews", "backfiller.eweka" }, + StringComparer.Ordinal); + harness.DeliverSuccess(publication, ReadBackbone(publication)); + Assert.Equal(ArticleWorkOutcome.Success, (await lookup).Outcome); + } + + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => + p.Exchange.Equals("backfiller.highwinds", StringComparison.Ordinal) + || p.Exchange.Equals(StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal)); + } + + [Fact] + public async Task Three_backbone_invalid_article_chain_never_reuses_attempted_backbone() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10), Eligible("Highwinds", 10)], + CreateSequentialRolls(0, 0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad-g"); + var second = await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.DeliverOutcome(second, ArticleWorkOutcome.InvalidArticle, "bad-e"); + var third = await harness.Publisher.WaitForPublicationAsync("backfiller.highwinds"); + harness.DeliverSuccess(third, "Highwinds"); + + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal("Highwinds", result.Backbone); + Assert.Equal(3, harness.Publisher.Publications.Count); + Assert.Equal( + new[] { "backfiller.giganews", "backfiller.eweka", "backfiller.highwinds" }, + harness.Publisher.Publications.Select(static p => p.Exchange).ToArray()); + Assert.Equal( + Guid.Parse(first.RequestId), + Guid.Parse(second.RequestId)); + Assert.Equal( + Guid.Parse(first.RequestId), + Guid.Parse(third.RequestId)); + Assert.NotEqual(first.CorrelationId, second.CorrelationId); + Assert.NotEqual(second.CorrelationId, third.CorrelationId); + Assert.Equal(1, harness.Publisher.Publications.Count(static p => p.Exchange == "backfiller.giganews")); + } + + [Fact] + public async Task Global_deadline_bounds_total_lookup_not_per_attempt() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 10), Eligible("Eweka", 10), Eligible("Highwinds", 10)], + CreateSequentialRolls(0, 0, 0)); + var started = harness.Time.GetUtcNow(); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + var first = await harness.Publisher.WaitForPublicationAsync("backfiller.giganews"); + harness.Time.Advance(TimeSpan.FromSeconds(1)); + // InvalidArticle (not NotFound) may advance; NotFound would end after the first publish. + harness.DeliverOutcome(first, ArticleWorkOutcome.InvalidArticle, "bad"); + + await harness.Publisher.WaitForPublicationAsync("backfiller.eweka"); + harness.Time.Advance(TimeSpan.FromSeconds(4)); + var result = await lookup; + + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal(2, harness.Publisher.Publications.Count); + Assert.DoesNotContain( + harness.Publisher.Publications, + static p => p.Exchange == "backfiller.highwinds"); + Assert.True(harness.Time.GetUtcNow() - started <= ArticleWorkRpcTiming.LookupDeadline); + } + + [Fact] + public async Task Not_found_on_first_of_three_backbones_publishes_only_once() + { + await using var harness = CreateHarness( + [Eligible("Giganews", 5), Eligible("Eweka", 5), Eligible("Highwinds", 5)], + CreateSequentialRolls(0, 0, 0)); + var lookup = harness.Client.LookupByMessageIdAsync(MessageIdBytes, CancellationToken.None); + + await harness.Publisher.WaitForPublicationCountAsync(1); + harness.DeliverOutcome(harness.Publisher.Publications[0], ArticleWorkOutcome.ArticleNotFound, "a"); + + var result = await lookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal("a", result.Error); + Assert.Single(harness.Publisher.Publications); + } + + private static Func CreateSequentialRolls(params int[] rolls) + { + var index = 0; + return exclusiveMax => + { + _ = exclusiveMax; + var value = rolls[Math.Min(index, rolls.Length - 1)]; + index++; + return value; + }; + } + + private static BackfillEligibleBackbone Eligible(string backbone, int consumers) + { + var entity = BackfillArticleRetrievalTopology.BuildProviderEntityName(backbone); + return new BackfillEligibleBackbone(backbone, entity, entity, entity, consumers); + } + + private static Harness CreateHarness(params BackfillEligibleBackbone[] eligible) => + CreateHarness(eligible, nextInt: static _ => 0); + + private static Harness CreateHarness(BackfillEligibleBackbone[] eligible, Func nextInt) => + CreateHarness(eligible, nextInt, currentGeneration: null); + + private static Harness CreateHarness( + BackfillEligibleBackbone[] eligible, + Func? nextInt, + Func? currentGeneration) + { + var time = new FakeTimeProvider(); + var publisher = new RecordingArticleWorkRpcPublisher(); + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var availability = new StaticBackfillConsumerAvailability(eligible); + var selector = new WeightedBackboneSelector(nextInt ?? (static _ => 0)); + var client = new ArticleWorkRpcClient( + publisher, + router, + availability, + selector, + time, + NullLogger.Instance, + currentGeneration); + return new Harness(time, publisher, router, client, availability); + } + + private static void AssertAmqpContract(FakeRabbitMqRpcPublication publication) + { + Assert.False(string.IsNullOrWhiteSpace(publication.ReplyTo)); + Assert.Equal(ArticleWorkWireProtocol.JsonContentType, publication.ContentType); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + Assert.True(Guid.TryParse(publication.RequestId, out var requestId)); + Assert.NotEqual(Guid.Empty, requestId); + Assert.True(Guid.TryParse(publication.CorrelationId, out var correlationId)); + Assert.NotEqual(Guid.Empty, correlationId); + Assert.Equal(publication.Exchange, publication.RoutingKey); + using var document = JsonDocument.Parse(publication.Body); + Assert.Equal(requestId, document.RootElement.GetProperty("requestId").GetGuid()); + } + + private static string ReadBackbone(FakeRabbitMqRpcPublication publication) => + JsonDocument.Parse(publication.Body).RootElement.GetProperty("backbone").GetString()!; + + private static byte[] SuccessBody(Guid requestId, string backbone) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"Success","fqdn":"{{SuccessFqdn}}","vatpPort":{{SuccessVatpPort}},"articleId":"{{SuccessArticleIdHex}}"}"""); + } + + private static byte[] FailureBody(Guid requestId, ArticleWorkOutcome outcome, string error, string backbone) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"{{outcome}}","error":"{{error}}"}"""); + } + + private sealed class Harness : IAsyncDisposable + { + internal Harness( + FakeTimeProvider time, + RecordingArticleWorkRpcPublisher publisher, + ArticleWorkRpcResponseRouter router, + ArticleWorkRpcClient client, + StaticBackfillConsumerAvailability availability) + { + Time = time; + Publisher = publisher; + Router = router; + Client = client; + Availability = availability; + } + + internal FakeTimeProvider Time { get; } + + internal RecordingArticleWorkRpcPublisher Publisher { get; } + + internal ArticleWorkRpcResponseRouter Router { get; } + + internal ArticleWorkRpcClient Client { get; } + + internal StaticBackfillConsumerAvailability Availability { get; } + + internal void DeliverSuccess(FakeRabbitMqRpcPublication publication, string backbone, Guid? requestId = null) + { + var id = requestId ?? Guid.Parse(publication.RequestId); + Router.Dispatch(publication.CorrelationId, SuccessBody(id, backbone), id.ToString("D"), 0); + } + + internal void DeliverOutcome(FakeRabbitMqRpcPublication publication, ArticleWorkOutcome outcome, string error) + { + var requestId = Guid.Parse(publication.RequestId); + var backbone = ReadBackbone(publication); + Router.Dispatch( + publication.CorrelationId, + FailureBody(requestId, outcome, error, backbone), + requestId.ToString("D"), + 0); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class RecordingArticleWorkRpcPublisher : IArticleWorkRpcPublisher + { + private readonly SemaphoreSlim _publishedPulse = new(0, int.MaxValue); + + public string ReplyTo => "nntpd.01.test.rpc"; + + public List Publications { get; } = []; + + public Task PublishAsync( + string exchange, + string routingKey, + Guid requestId, + string correlationId, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + lock (Publications) + { + Publications.Add(new FakeRabbitMqRpcPublication( + exchange, + routingKey, + correlationId, + requestId.ToString("D"), + ReplyTo, + ArticleWorkWireProtocol.JsonContentType, + Expiration: string.Empty, + body.ToArray())); + } + + _publishedPulse.Release(); + return Task.CompletedTask; + } + + public async Task WaitForPublicationAsync(string exchange, int skip = 0) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + FakeRabbitMqRpcPublication? match; + lock (Publications) + { + match = Publications.Where(publication => publication.Exchange == exchange).Skip(skip).FirstOrDefault(); + } + + if (match is not null) + { + return match; + } + + await _publishedPulse.WaitAsync(cts.Token).ConfigureAwait(false); + } + } + + public async Task WaitForPublicationCountAsync(int count) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + lock (Publications) + { + if (Publications.Count >= count) + { + return; + } + } + + await _publishedPulse.WaitAsync(cts.Token).ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs new file mode 100644 index 00000000..2fc489fc --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcResponseRouterTests.cs @@ -0,0 +1,169 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkRpcResponseRouterTests +{ + private const string MessageId = "<12345@example.invalid>"; + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public void StaleGeneration_CannotCompleteNewerGenerationRequest() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 2); + + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 1); + Assert.False(operation.IsCompleted); + + Assert.Equal(1, router.OutstandingCount); + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 2); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal("cache.requests", result.SourceExchange); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void Success_RemovesAllLocalCorrelationsImmediately() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var first = Guid.NewGuid().ToString("D"); + var second = Guid.NewGuid().ToString("D"); + router.Register(first, operation, "cache.requests", generation: 0); + router.Register(second, operation, "backfiller.eweka", generation: 0); + Assert.Equal(2, router.OutstandingCount); + + router.Dispatch(first, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void ResponseRacingWithLocalCompletion_CannotResurrectLookup() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + operation.TryComplete(ArticleWorkRpcResult.NotFound(requestId, MessageId, "deadline")); + router.UnregisterAll(operation); + router.Register(Guid.NewGuid().ToString("D"), operation, "backfiller.eweka", generation: 0); + Assert.Equal(0, router.OutstandingCount); + + router.Dispatch(correlationId, SuccessBody(requestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void ConcurrentSuccess_CompletesExactlyOnce_AndClearsCorrelations() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var first = Guid.NewGuid().ToString("D"); + var second = Guid.NewGuid().ToString("D"); + var third = Guid.NewGuid().ToString("D"); + router.Register(first, operation, "backfiller.eweka", generation: 0); + router.Register(second, operation, "backfiller.giganews", generation: 0); + router.Register(third, operation, "backfiller.abavia", generation: 0); + + Parallel.Invoke( + () => router.Dispatch(first, SuccessBody(requestId, "Eweka"), requestId.ToString("D"), 0), + () => router.Dispatch(second, SuccessBody(requestId, "Giganews"), requestId.ToString("D"), 0), + () => router.Dispatch(third, SuccessBody(requestId, "Abavia"), requestId.ToString("D"), 0)); + + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.True( + result.SourceExchange is "backfiller.eweka" or "backfiller.giganews" or "backfiller.abavia", + result.SourceExchange); + Assert.Equal(0, router.OutstandingCount); + } + + [Fact] + public void MismatchedAmqpRequestId_IsIgnored() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(requestId), Guid.NewGuid().ToString("D"), deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + Assert.Equal(1, router.OutstandingCount); + } + + [Fact] + public void MissingAmqpRequestId_IsIgnored() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(requestId), amqpRequestId: null, deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + } + + [Fact] + public void JsonRequestIdMismatch_IsIgnoredEvenWhenAmqpMatchesAnotherValue() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var otherRequestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "cache.requests", generation: 0); + + router.Dispatch(correlationId, SuccessBody(otherRequestId), requestId.ToString("D"), deliveryGeneration: 0); + Assert.False(operation.IsCompleted); + } + + [Fact] + public void ArticleNotFound_CompletesAttempt_ButIsNotLookupSuccess() + { + var router = new ArticleWorkRpcResponseRouter(NullLogger.Instance); + var requestId = Guid.NewGuid(); + var operation = new ArticleWorkLookupOperation(requestId, MessageId); + var correlationId = Guid.NewGuid().ToString("D"); + router.Register(correlationId, operation, "backfiller.giganews", generation: 0); + + router.Dispatch(correlationId, FailureBody(requestId, "ArticleNotFound"), requestId.ToString("D"), 0); + Assert.True(operation.IsCompleted); + Assert.True(operation.TryGetResult(out var result)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, result.Outcome); + Assert.True(ArticleWorkAggregatePolicy.IsAttemptTerminal(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.ShouldTryNextBackbone(ArticleWorkOutcome.ArticleNotFound)); + Assert.False(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.ArticleNotFound)); + Assert.True(ArticleWorkAggregatePolicy.IsLookupSuccess(ArticleWorkOutcome.Success)); + } + + private static byte[] SuccessBody(Guid requestId, string backbone = "Storage") + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"{{backbone}}","outcome":"Success","fqdn":"{{SuccessFqdn}}","vatpPort":{{SuccessVatpPort}},"articleId":"{{SuccessArticleIdHex}}"}"""); + } + + private static byte[] FailureBody(Guid requestId, string outcome) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"{{MessageId}}","backbone":"Storage","outcome":"{{outcome}}","error":"missing"}"""); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs new file mode 100644 index 00000000..b2cf01de --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkRpcServiceTests.cs @@ -0,0 +1,287 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.RabbitMq.Management; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +[Collection(SerilogCollection.Name)] +public sealed class ArticleWorkRpcServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresExclusiveReplyQueue_AndConsumesOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + var rpc = CreateRpcService(rabbit); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Equal(2, connection.RpcChannels.Count); + var consume = connection.RpcChannels[1]; + var queue = Assert.Single(consume.Queues); + Assert.Equal(rpc.CurrentReplyTo, queue.Name); + Assert.False(queue.Durable); + Assert.True(queue.Exclusive); + Assert.True(queue.AutoDelete); + Assert.Equal(queue.Name, consume.ConsumedQueue); + Assert.Matches("^nntpd\\.01\\.[0-9a-f]{32}\\.rpc$", queue.Name); + Assert.Null(queue.Arguments); + + await rpc.StopAsync(CancellationToken.None); + Assert.Null(rpc.CurrentReplyTo); + Assert.Equal(1, connection.RpcChannels[0].DisposeCount); + Assert.Equal(1, consume.DisposeCount); + } + + [Fact] + public async Task ConnectionReplacement_DisposesStaleSession_AndDoesNotCorruptNewGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory()); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + var first = factory.Connections[0]; + var firstReplyTo = rpc.CurrentReplyTo; + Assert.Equal(1, rpc.CurrentSessionGeneration); + var firstPublish = first.RpcChannels[0]; + + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.SimulateLost(); + await factory.Connected.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitForAsync(() => rpc.CurrentSessionGeneration == 2 && firstPublish.DisposeCount == 1); + + Assert.Equal(2, rpc.CurrentSessionGeneration); + Assert.Equal(1, firstPublish.DisposeCount); + Assert.True(first.RpcChannels.TrueForAll(static channel => channel.DisposeCount == 1)); + var second = factory.LastConnection ?? throw new InvalidOperationException("Expected replacement connection."); + Assert.Equal(2, second.RpcChannels.Count); + Assert.NotNull(firstReplyTo); + Assert.False(string.IsNullOrWhiteSpace(rpc.CurrentReplyTo)); + await Assert.ThrowsAsync( + () => firstPublish.PublishAsync( + "cache.requests", + "cache.requests", + Guid.NewGuid().ToString("D"), + Guid.NewGuid().ToString("D"), + firstReplyTo!, + ArticleWorkWireProtocol.JsonContentType, + ArticleWorkRpcAmqp.ExpirationMilliseconds, + ReadOnlyMemory.Empty, + CancellationToken.None)); + Assert.Empty(second.RpcChannels[0].Publications); + + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StopAsync_CancelsPendingLookup_AndClearsCorrelations() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory()); + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var lookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => factory.LastConnection!.RpcChannels[0].Publications.Count == 1); + var publication = Assert.Single(factory.LastConnection!.RpcChannels[0].Publications); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + Assert.True(Guid.TryParse(publication.RequestId, out _)); + Assert.Equal(ArticleWorkWireProtocol.JsonContentType, publication.ContentType); + Assert.Equal(rpc.CurrentReplyTo, publication.ReplyTo); + Assert.True(rpc.OutstandingCorrelations >= 1); + + await rpc.StopAsync(CancellationToken.None); + await Assert.ThrowsAnyAsync(() => lookup); + Assert.Equal(0, rpc.OutstandingCorrelations); + } + + [Fact] + public async Task WorkerResponse_CarriesOneSecondExpiration_AndCompletesLookup() + { + var time = new FakeTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory(), time); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + + var lookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => factory.LastConnection!.RpcChannels[0].Publications.Count == 1); + var publication = Assert.Single(factory.LastConnection!.RpcChannels[0].Publications); + Assert.True(string.IsNullOrEmpty(publication.Expiration)); + + var consume = factory.LastConnection.RpcChannels[1]; + await consume.DeliverAsync( + publication.CorrelationId, + SuccessBody(publication.RequestId), + requestId: publication.RequestId, + expiration: ArticleWorkRpcAmqp.ResponseExpirationMilliseconds); + var result = await lookup; + + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(ArticleWorkRpcAmqp.ResponseExpirationMilliseconds, consume.LastDelivery?.Expiration); + Assert.Equal(0, rpc.OutstandingCorrelations); + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task StaleGenerationDelivery_CannotCompleteNewerLookup() + { + var time = new FakeTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var rabbit = CreateRabbitMqService(factory); + var rpc = CreateRpcService(rabbit, CreateActiveInventory(), time); + + await rabbit.StartAsync(CancellationToken.None); + await rpc.StartAsync(CancellationToken.None); + var first = factory.Connections[0]; + var firstLookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + await WaitForAsync(() => first.RpcChannels[0].Publications.Count == 1); + var firstPublication = Assert.Single(first.RpcChannels[0].Publications); + + factory.Connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.SimulateLost(); + await factory.Connected.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await WaitForAsync(() => rpc.CurrentSessionGeneration == 2); + + var secondLookup = rpc.LookupByMessageIdAsync("<12345@example.invalid>"u8.ToArray(), CancellationToken.None); + var second = factory.LastConnection ?? throw new InvalidOperationException("Expected replacement connection."); + await WaitForAsync(() => second.RpcChannels[0].Publications.Count == 1); + var secondPublication = Assert.Single(second.RpcChannels[0].Publications); + + await second.RpcChannels[1].DeliverAsync( + firstPublication.CorrelationId, + SuccessBody(firstPublication.RequestId), + requestId: firstPublication.RequestId); + Assert.False(secondLookup.IsCompleted); + + await first.RpcChannels[1].DeliverAsync( + secondPublication.CorrelationId, + SuccessBody(secondPublication.RequestId), + requestId: secondPublication.RequestId); + Assert.False(secondLookup.IsCompleted); + + await second.RpcChannels[1].DeliverAsync( + secondPublication.CorrelationId, + SuccessBody(secondPublication.RequestId), + requestId: secondPublication.RequestId); + var result = await secondLookup; + Assert.Equal(ArticleWorkOutcome.Success, result.Outcome); + Assert.Equal(secondPublication.RequestId, result.RequestId.ToString("D"), StringComparer.Ordinal); + + time.Advance(ArticleWorkRpcTiming.LookupDeadline); + var firstResult = await firstLookup; + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, firstResult.Outcome); + await rpc.StopAsync(CancellationToken.None); + } + + [Fact] + public void RpcChannel_DoesNotExposeQueueCleanup() + { + var methods = typeof(IRabbitMqRpcChannel).GetMethods().Select(static method => method.Name).ToHashSet(StringComparer.Ordinal); + Assert.DoesNotContain("QueueDeleteAsync", methods); + Assert.DoesNotContain("QueuePurgeAsync", methods); + Assert.DoesNotContain("ExchangeDeleteAsync", methods); + } + + [Fact] + public async Task Host_RegistersRpcService_ImmediatelyAfterTopology() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + // NntpDB and nntpsharedconfig are registered ahead of Cloudflare, Redis, and RabbitMQ. + Assert.Equal(typeof(RabbitMqTopologyService), services[5].GetType()); + Assert.Equal(typeof(VectorNNTPD.TransitService.Storage.StorageServerFleetConsumerService), services[6].GetType()); + Assert.Equal(typeof(VectorNNTPD.TransitService.Storage.StorageArticleLookupService), services[7].GetType()); + Assert.Equal(typeof(ArticleWorkRpcService), services[8].GetType()); + Assert.Same( + host.Services.GetRequiredService(), + services[8]); + } + + private static ArticleWorkRpcService CreateRpcService( + IRabbitMqService rabbit, + IRabbitMqManagementQueueInventory? inventory = null, + TimeProvider? time = null) => + new( + rabbit, + Options.Create(TestHostFactory.CreateValidOptions()), + inventory ?? new EmptyManagementInventory(), + NullLogger.Instance, + time); + + private static IRabbitMqManagementQueueInventory CreateActiveInventory() + { + var queues = BackfillArticleRetrievalTopology.Definitions + .Select(static definition => new RabbitMqManagementQueueInfo(definition.QueueName, 1)) + .ToArray(); + return new StaticManagementInventory(queues); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static async Task WaitForAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(10, cts.Token).ConfigureAwait(false); + } + } + + private static byte[] SuccessBody(string requestId) + { + return Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"{{requestId}}","messageId":"<12345@example.invalid>","backbone":"Storage","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + } + + private sealed class EmptyManagementInventory : IRabbitMqManagementQueueInventory + { + public Task> ListQueuesAsync(CancellationToken cancellationToken) => + Task.FromResult>([]); + } + + private sealed class StaticManagementInventory(IReadOnlyList queues) + : IRabbitMqManagementQueueInventory + { + public Task> ListQueuesAsync(CancellationToken cancellationToken) => + Task.FromResult(queues); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs new file mode 100644 index 00000000..1285278b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/ArticleWorkWireProtocolTests.cs @@ -0,0 +1,144 @@ +using System.Text; +using System.Text.Json; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +public sealed class ArticleWorkWireProtocolTests +{ + private static readonly Guid RequestId = Guid.Parse("7c1cb8a0-95f9-4c13-8e53-339773e3afaa"); + + [Fact] + public void SerializeRequestV1_IsCompactCanonicalJson_WithoutTransportFields() + { + var request = new ArticleWorkRequest(1, RequestId, "<12345@example.invalid>", "Giganews"); + var bytes = ArticleWorkWireProtocol.SerializeRequestV1(request); + var json = Encoding.UTF8.GetString(bytes); + + Assert.Equal( + """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews"}""", + json); + Assert.DoesNotContain('\n', json); + Assert.DoesNotContain("CorrelationId", json, StringComparison.Ordinal); + Assert.DoesNotContain("ReplyTo", json, StringComparison.Ordinal); + Assert.DoesNotContain("correlationId", json, StringComparison.Ordinal); + Assert.DoesNotContain("replyTo", json, StringComparison.Ordinal); + + using var document = JsonDocument.Parse(bytes); + Assert.Equal(1, document.RootElement.GetProperty("version").GetInt32()); + Assert.Equal(RequestId, document.RootElement.GetProperty("requestId").GetGuid()); + Assert.Equal("<12345@example.invalid>", document.RootElement.GetProperty("messageId").GetString()); + Assert.Equal("Giganews", document.RootElement.GetProperty("backbone").GetString()); + Assert.Equal(4, document.RootElement.EnumerateObject().Count()); + } + + private const string SuccessFqdn = "backfiller01.usenet.ninja"; + private const int SuccessVatpPort = 119; + private const string SuccessArticleIdHex = "dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"; + + [Fact] + public void TryParseResponseV1_AcceptsCanonicalSuccess() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""u8; + Assert.True(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Equal(string.Empty, reason); + Assert.NotNull(response); + Assert.Equal(ArticleWorkOutcome.Success, response.Outcome); + Assert.Equal(RequestId, response.RequestId); + Assert.Equal("<12345@example.invalid>", response.MessageId); + Assert.Equal("Giganews", response.Backbone); + Assert.Equal(SuccessFqdn, response.Fqdn); + Assert.Equal(SuccessVatpPort, response.VatpPort); + Assert.Equal( + VectorNNTPD.Common.Articles.ArticleId.ParseLowerHex(SuccessArticleIdHex), + response.ArticleId); + Assert.Null(response.Error); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithoutArticleId() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("articleId", reason, StringComparison.Ordinal); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithInvalidArticleId() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"NOT-HEX"}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("articleId", reason, StringComparison.Ordinal); + } + + [Theory] + [InlineData("nodots")] + [InlineData("")] + [InlineData("Backfiller01.usenet.ninja")] + public void TryParseResponseV1_RejectsInvalidFqdn(string fqdn) + { + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"{{fqdn}}","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("fqdn", reason, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(65536)] + public void TryParseResponseV1_RejectsInvalidVatpPort(int port) + { + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":{{port}},"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("vatpPort", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_AcceptsCanonicalArticleNotFound() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"ArticleNotFound","error":"No article with that message-id"}"""u8; + Assert.True(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out _)); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, response!.Outcome); + Assert.Null(response.Fqdn); + Assert.Null(response.VatpPort); + Assert.Null(response.ArticleId); + Assert.Equal("No article with that message-id", response.Error); + } + + [Fact] + public void TryParseResponseV1_RejectsLegacyUriField() + { + var legacy = "vatp" + "://" + "backfiller01.usenet.ninja:119/" + SuccessArticleIdHex; + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"{{SuccessArticleIdHex}}","uri":"{{legacy}}"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("uri", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_RejectsUriOnlySuccess() + { + var legacy = "vatp" + "://" + "backfiller01.usenet.ninja:119/" + SuccessArticleIdHex; + var json = Encoding.UTF8.GetBytes( + $$"""{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","uri":"{{legacy}}","articleId":"{{SuccessArticleIdHex}}"}"""); + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("uri", reason, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void TryParseResponseV1_RejectsSuccessWithError() + { + var json = """{"version":1,"requestId":"7c1cb8a0-95f9-4c13-8e53-339773e3afaa","messageId":"<12345@example.invalid>","backbone":"Giganews","outcome":"Success","fqdn":"backfiller01.usenet.ninja","vatpPort":119,"articleId":"dcab316ba0e91c6abbad8d5759bff207932dbe9168c88954c6dd9240b4a6da14","error":"nope"}"""u8; + Assert.False(ArticleWorkWireProtocol.TryParseResponseV1(json, out var response, out var reason)); + Assert.Null(response); + Assert.Contains("error", reason, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs new file mode 100644 index 00000000..9e52ca03 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/ArticleWork/BackfillConsumerAvailabilityTests.cs @@ -0,0 +1,475 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.TransitService.RabbitMq.Management; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq.ArticleWork; + +/// +/// Proves Management API–backed BackFiller availability discovery (~5s cache, +/// last-known-good on failure, no AMQP passive probing). +/// +public sealed class BackfillConsumerAvailabilityTests +{ + [Fact] + public async Task Cache_is_lazy_no_management_call_without_GetEligibleBackbones() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 10)); + await using var availability = CreateAvailability(inventory, time); + + await Task.Delay(20); + Assert.Equal(0, inventory.CallCount); + } + + [Fact] + public async Task First_GetEligible_refreshes_once_and_reuses_within_refresh_interval() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 100), + Queue("backfiller.eweka", 50), + Queue("backfiller.highwinds", 0)); + await using var availability = CreateAvailability(inventory, time); + + var first = availability.GetEligibleBackbones(); + Assert.Equal(2, first.Count); + Assert.Contains(first, static b => b.Backbone == "Giganews" && b.ConsumerCount == 100); + Assert.Contains(first, static b => b.Backbone == "Eweka" && b.ConsumerCount == 50); + Assert.DoesNotContain(first, static b => b.Backbone.Equals("Highwinds", StringComparison.OrdinalIgnoreCase)); + Assert.Equal(1, inventory.CallCount); + + for (var i = 0; i < 20; i++) + { + time.Advance(TimeSpan.FromMilliseconds(100)); + _ = availability.GetEligibleBackbones(); + } + + Assert.Equal(1, inventory.CallCount); + } + + [Fact] + public async Task After_refresh_interval_next_GetEligible_requeries_and_picks_up_changes() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 10)); + await using var availability = CreateAvailability(inventory, time); + + var initial = availability.GetEligibleBackbones(); + Assert.Single(initial); + Assert.Equal("Giganews", initial[0].Backbone); + + inventory.SetQueues( + Queue("backfiller.eweka", 25), + Queue("backfiller.giganews", 0)); + + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval - TimeSpan.FromMilliseconds(1)); + var stillCached = availability.GetEligibleBackbones(); + Assert.Single(stillCached); + Assert.Equal("Giganews", stillCached[0].Backbone); + Assert.Equal(1, inventory.CallCount); + + time.Advance(TimeSpan.FromMilliseconds(1)); + var refreshed = availability.GetEligibleBackbones(); + Assert.Single(refreshed); + Assert.Equal("Eweka", refreshed[0].Backbone); + Assert.Equal(25, refreshed[0].ConsumerCount); + Assert.Equal(2, inventory.CallCount); + } + + [Fact] + public async Task Missing_provider_queue_is_not_eligible() + { + var inventory = new FakeManagementInventory(Queue("backfiller.eweka", 7)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.DoesNotContain(eligible, static b => b.Backbone == "Abavia"); + Assert.DoesNotContain(eligible, static b => b.Backbone == "Giganews"); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 7); + } + + [Fact] + public async Task Existing_provider_queue_with_zero_consumers_is_not_eligible() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 0), + Queue("backfiller.eweka", 7)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.DoesNotContain(eligible, static b => b.Backbone == "Giganews"); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 7); + } + + [Fact] + public async Task Non_backfiller_queues_are_ignored() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 5), + Queue(StorageArticleRetrievalTopology.EntityName, 99), + Queue("overviewdb.queue", 3), + Queue("unrelated.queue", 12)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.Single(eligible); + Assert.Equal("Giganews", eligible[0].Backbone); + Assert.DoesNotContain( + eligible, + static b => b.QueueName.Equals(StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal)); + } + + [Fact] + public async Task All_twelve_queues_with_mixed_consumers_only_positive_are_eligible() + { + var rows = BackfillArticleRetrievalTopology.Definitions + .Select((definition, index) => Queue(definition.QueueName, (uint)(index % 3 == 0 ? 0 : index + 1))) + .ToArray(); + var inventory = new FakeManagementInventory(rows); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + + Assert.All(eligible, static b => Assert.True(b.ConsumerCount > 0)); + Assert.Equal( + BackfillArticleRetrievalTopology.Definitions.Count(static d => + { + var index = Array.IndexOf(BackfillArticleRetrievalTopology.Providers, d.Provider); + return index % 3 != 0; + }), + eligible.Count); + Assert.Equal(BackfillArticleRetrievalTopology.Definitions.Count, rows.Length); + } + + [Fact] + public async Task Provider_names_match_canonical_topology_entity_names() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.gtt", 4), + Queue("backfiller.usenetnode1", 2), + Queue("backfiller.baseip", 1)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + + var eligible = availability.GetEligibleBackbones(); + Assert.Contains(eligible, static b => b.Backbone == "GTT" && b.QueueName == "backfiller.gtt"); + Assert.Contains(eligible, static b => b.Backbone == "UsenetNode1" && b.QueueName == "backfiller.usenetnode1"); + Assert.Contains(eligible, static b => b.Backbone == "BaseIP" && b.QueueName == "backfiller.baseip"); + } + + [Fact] + public async Task Weighted_selection_receives_management_eligible_set() + { + var inventory = new FakeManagementInventory( + Queue("backfiller.giganews", 100), + Queue("backfiller.eweka", 50)); + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + var eligible = availability.GetEligibleBackbones(); + var selector = new WeightedBackboneSelector(static _ => 0); + + var selected = selector.Select(eligible); + + // Nullable value type: HasValue/GetValueOrDefault (Assert.NotNull does not narrow for CodeQL). + Assert.True(selected.HasValue); + var choice = selected.GetValueOrDefault(); + // Definitions order places Eweka before Giganews; roll 0 selects the first positive weight. + Assert.Equal("Eweka", choice.Backbone); + Assert.Equal(50, choice.ConsumerCount); + Assert.Equal(2, eligible.Count); + Assert.Contains(eligible, static b => b.Backbone == "Giganews" && b.ConsumerCount == 100); + Assert.Contains(eligible, static b => b.Backbone == "Eweka" && b.ConsumerCount == 50); + } + + [Fact] + public async Task Concurrent_GetEligible_does_not_issue_multiple_simultaneous_refreshes() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 3)) + { + Delay = TimeSpan.FromMilliseconds(100), + }; + await using var availability = CreateAvailability(inventory, time); + + var tasks = Enumerable.Range(0, 16) + .Select(_ => Task.Run(() => availability.GetEligibleBackbones())) + .ToArray(); + var results = await Task.WhenAll(tasks); + + Assert.Equal(1, inventory.CallCount); + Assert.All(results, static r => Assert.Single(r)); + Assert.Equal(1, inventory.MaxConcurrentCalls); + } + + [Fact] + public async Task Failed_refresh_retains_last_successful_snapshot() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 9)); + await using var availability = CreateAvailability(inventory, time); + + var first = availability.GetEligibleBackbones(); + Assert.Single(first); + Assert.Equal("Giganews", first[0].Backbone); + + inventory.Fault = new HttpRequestException("Management API returned 503 (ServiceUnavailable) for queue inventory."); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var retained = availability.GetEligibleBackbones(); + + Assert.Single(retained); + Assert.Equal("Giganews", retained[0].Backbone); + Assert.Equal(9, retained[0].ConsumerCount); + Assert.Equal(2, inventory.CallCount); + } + + [Fact] + public async Task Successful_refresh_replaces_previous_snapshot() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 1)); + await using var availability = CreateAvailability(inventory, time); + _ = availability.GetEligibleBackbones(); + + inventory.SetQueues(Queue("backfiller.eweka", 8)); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var next = availability.GetEligibleBackbones(); + + Assert.Single(next); + Assert.Equal("Eweka", next[0].Backbone); + Assert.Equal(8, next[0].ConsumerCount); + } + + [Fact] + public async Task Cancellation_during_refresh_propagates() + { + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 1)) + { + Delay = TimeSpan.FromSeconds(30), + }; + await using var availability = CreateAvailability(inventory, new FakeTimeProvider()); + using var cts = new CancellationTokenSource(); + var refresh = availability.RefreshNowAsync(cts.Token); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(async () => await refresh); + } + + [Fact] + public async Task Malformed_inventory_failure_does_not_fabricate_empty_snapshot_after_success() + { + var time = new FakeTimeProvider(); + var inventory = new FakeManagementInventory(Queue("backfiller.giganews", 4)); + await using var availability = CreateAvailability(inventory, time); + _ = availability.GetEligibleBackbones(); + + inventory.Fault = new InvalidOperationException( + "RabbitMQ Management API returned a malformed queue inventory payload."); + time.Advance(BackfillConsumerAvailabilityService.RefreshInterval); + var retained = availability.GetEligibleBackbones(); + + Assert.Single(retained); + Assert.Equal("Giganews", retained[0].Backbone); + } + + [Fact] + public async Task Availability_service_does_not_use_amqp_passive_queue_declare() + { + var source = await File.ReadAllTextAsync( + Path.Join( + RepoPaths.FindRepoRoot(), + "src", + "VectorNNTPD.TransitService", + "RabbitMq", + "ArticleWork", + "BackfillConsumerAvailability.cs")); + + Assert.DoesNotContain("QueueDeclarePassiveAsync", source, StringComparison.Ordinal); + Assert.DoesNotContain("CreateTopologyChannelAsync", source, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_lists_queues_for_encoded_vhost_with_basic_auth() + { + var handler = new CapturingHandler( + """ + [{"name":"backfiller.giganews","consumers":100},{"name":"backfiller.eweka","consumers":50}] + """); + var options = RabbitMqOptionsTests.CreateValid(); + options.VirtualHost = "/"; + options.Username = "mgmt-user"; + options.Password = "mgmt-secret"; + using var http = new HttpClient(handler) + { + BaseAddress = new Uri("http://127.0.0.1:15672/"), + }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var queues = await client.ListQueuesAsync(CancellationToken.None); + + Assert.Equal(2, queues.Count); + Assert.Equal("/api/queues/%2F", handler.LastPathAndQuery); + Assert.NotNull(handler.LastAuthorization); + Assert.Equal("Basic", handler.LastAuthorization!.Scheme); + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(handler.LastAuthorization.Parameter!)); + Assert.Equal("mgmt-user:mgmt-secret", decoded); + // codeql[cs/call-to-object-tostring] Assert default Object.ToString does not embed credentials. + Assert.DoesNotContain("mgmt-secret", handler.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_auth_failure_does_not_embed_credentials_in_exception() + { + var handler = new CapturingHandler("{}", HttpStatusCode.Unauthorized); + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "mgmt-user"; + options.Password = "super-secret-password"; + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://127.0.0.1:15672/") }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var ex = await Assert.ThrowsAsync( + () => client.ListQueuesAsync(CancellationToken.None)); + + Assert.DoesNotContain("super-secret-password", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("mgmt-user:super", ex.Message, StringComparison.Ordinal); + Assert.Contains("401", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_client_rejects_malformed_json() + { + var handler = new CapturingHandler("{not-json"); + var options = RabbitMqOptionsTests.CreateValid(); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://127.0.0.1:15672/") }; + var client = new RabbitMqManagementHttpClient(http, Options.Create(options)); + + var ex = await Assert.ThrowsAsync( + () => client.ListQueuesAsync(CancellationToken.None)); + Assert.Contains("malformed", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + private static BackfillConsumerAvailabilityService CreateAvailability( + FakeManagementInventory inventory, + FakeTimeProvider time) => + new(inventory, NullLogger.Instance, time); + + private static RabbitMqManagementQueueInfo Queue(string name, uint consumers) => + new(name, checked((int)consumers)); + + private sealed class FakeManagementInventory : IRabbitMqManagementQueueInventory + { + private readonly object _gate = new(); + private IReadOnlyList _queues; + private int _inflight; + + public FakeManagementInventory(params RabbitMqManagementQueueInfo[] queues) + { + _queues = queues; + } + + public int CallCount { get; private set; } + + public int MaxConcurrentCalls { get; private set; } + + public TimeSpan Delay { get; set; } + + public Exception? Fault { get; set; } + + public void SetQueues(params RabbitMqManagementQueueInfo[] queues) + { + lock (_gate) + { + _queues = queues; + Fault = null; + } + } + + public async Task> ListQueuesAsync( + CancellationToken cancellationToken) + { + var inflight = Interlocked.Increment(ref _inflight); + try + { + MaxConcurrentCalls = Math.Max(MaxConcurrentCalls, inflight); + CallCount++; + if (Delay > TimeSpan.Zero) + { + await Task.Delay(Delay, cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (Fault is not null) + { + throw Fault; + } + + return _queues; + } + } + finally + { + Interlocked.Decrement(ref _inflight); + } + } + } + + private sealed class CapturingHandler : HttpMessageHandler + { + private readonly string _body; + private readonly HttpStatusCode _status; + + public CapturingHandler(string body, HttpStatusCode status = HttpStatusCode.OK) + { + _body = body; + _status = status; + } + + public string? LastPathAndQuery { get; private set; } + + public System.Net.Http.Headers.AuthenticationHeaderValue? LastAuthorization { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + LastPathAndQuery = request.RequestUri?.PathAndQuery; + LastAuthorization = request.Headers.Authorization; + // codeql[cs/local-not-disposed] Ownership transfers to the scripted handler queue / HTTP pipeline for the caller to dispose. + var response = new HttpResponseMessage(_status) + { + Content = new StringContent(_body, Encoding.UTF8, "application/json"), + }; + return Task.FromResult(response); + } + } +} + +/// Resolves the repository root from the test assembly location. +file static class RepoPaths +{ + internal static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not locate VectorNNTPD.slnx from the test base directory."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs new file mode 100644 index 00000000..a39a55ba --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/BackfillArticleRetrievalTopologyTests.cs @@ -0,0 +1,123 @@ +using VectorNNTPD.TransitService.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class BackfillArticleRetrievalTopologyTests +{ + private static readonly string[] ExpectedProviders = + [ + "Abavia", + "Altopia", + "BaseIP", + "Eweka", + "Elbracht", + "Giganews", + "GTT", + "Highwinds", + "ItsHosted", + "Novia", + "UExpress", + "UsenetNode1", + ]; + + [Fact] + public void Providers_AreExactlyTheTwelveBackFillerIdentifiers() + { + Assert.Equal(ExpectedProviders, BackfillArticleRetrievalTopology.Providers); + Assert.Equal(12, BackfillArticleRetrievalTopology.Definitions.Count); + Assert.Equal(ExpectedProviders, BackfillArticleRetrievalTopology.Definitions.Select(static d => d.Provider)); + Assert.DoesNotContain("storage", BackfillArticleRetrievalTopology.Providers, StringComparer.OrdinalIgnoreCase); + Assert.DoesNotContain("storage.requests", BackfillArticleRetrievalTopology.Providers, StringComparer.Ordinal); + Assert.DoesNotContain( + BackfillArticleRetrievalTopology.Definitions, + static definition => + definition.ExchangeName == StorageArticleRetrievalTopology.EntityName + || definition.QueueName == StorageArticleRetrievalTopology.EntityName + || definition.ExchangeName.Contains("storage.requests", StringComparison.Ordinal) + || definition.ExchangeName.Contains("grabbers.", StringComparison.Ordinal)); + } + + [Fact] + public void Definitions_UseNormalizedBackFillerEntityNamesAndFanoutBinding() + { + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + var entityName = $"backfiller.{definition.Provider.ToLowerInvariant()}"; + Assert.Equal(entityName, definition.ExchangeName); + Assert.Equal(entityName, definition.QueueName); + Assert.Equal(entityName, definition.RoutingKey); + Assert.Equal(definition.ExchangeName, definition.QueueName); + Assert.Equal(definition.QueueName, definition.RoutingKey); + Assert.DoesNotContain("grabbers.", definition.ExchangeName, StringComparison.Ordinal); + Assert.Equal("fanout", definition.ExchangeType); + Assert.True(definition.ExchangeDurable); + Assert.False(definition.ExchangeAutoDelete); + Assert.True(definition.QueueDurable); + Assert.False(definition.QueueExclusive); + Assert.False(definition.QueueAutoDelete); + } + + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName("Abavia")); + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName(" Abavia ")); + Assert.Equal("backfiller.abavia", BackfillArticleRetrievalTopology.BuildProviderEntityName("AbAvIa")); + Assert.Equal("backfiller.gtt", BackfillArticleRetrievalTopology.BuildProviderEntityName("GTT")); + Assert.Equal("backfiller.usenetnode1", BackfillArticleRetrievalTopology.BuildProviderEntityName("UsenetNode1")); + Assert.Equal("backfiller.baseip", BackfillArticleRetrievalTopology.BuildProviderEntityName("BaseIP")); + Assert.Equal("backfiller.itshosted", BackfillArticleRetrievalTopology.BuildProviderEntityName("ItsHosted")); + Assert.Equal("backfiller.uexpress", BackfillArticleRetrievalTopology.BuildProviderEntityName("UExpress")); + } + + [Fact] + public void Definitions_DeclareQuorumQueuesWithQueueTypeArgument() + { + foreach (var definition in BackfillArticleRetrievalTopology.Definitions) + { + Assert.NotNull(definition.QueueArguments); + Assert.True(definition.QueueDurable); + Assert.False(definition.QueueExclusive); + Assert.False(definition.QueueAutoDelete); + Assert.True(definition.QueueArguments.TryGetValue( + BackfillArticleRetrievalTopology.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(BackfillArticleRetrievalTopology.QuorumQueueType, queueType); + Assert.False(definition.QueueArguments.ContainsKey("x-message-ttl")); + Assert.False(definition.QueueArguments.ContainsKey("x-expires")); + Assert.Single(definition.QueueArguments); + } + } + + [Fact] + public void BuildDefinitions_PreservesProviderCasing_AndNormalizesOnlyEntityNames() + { + var definition = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions(["Giganews"])); + Assert.Equal("Giganews", definition.Provider); + Assert.Equal("backfiller.giganews", definition.ExchangeName); + Assert.Equal("backfiller.giganews", definition.QueueName); + Assert.Equal("backfiller.giganews", definition.RoutingKey); + } + + [Fact] + public void BuildDefinitions_NormalizesWhitespaceAndMixedCaseProviderNames() + { + var definition = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions([" Abavia "])); + Assert.Equal("Abavia", definition.Provider); + Assert.Equal("backfiller.abavia", definition.ExchangeName); + Assert.Equal("backfiller.abavia", definition.QueueName); + Assert.Equal("backfiller.abavia", definition.RoutingKey); + + var mixed = Assert.Single(BackfillArticleRetrievalTopology.BuildDefinitions(["AbAvIa"])); + Assert.Equal("AbAvIa", mixed.Provider); + Assert.Equal("backfiller.abavia", mixed.ExchangeName); + } + + [Fact] + public void BuildDefinitions_SkipsBlanksAndDuplicateProviders() + { + var definitions = BackfillArticleRetrievalTopology.BuildDefinitions( + ["Giganews", " ", "giganews", "Eweka", "Giganews"]); + + Assert.Equal(2, definitions.Count); + Assert.Equal("Giganews", definitions[0].Provider); + Assert.Equal("Eweka", definitions[1].Provider); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqOptionsTests.cs new file mode 100644 index 00000000..73f620ef --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqOptionsTests.cs @@ -0,0 +1,359 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class RabbitMqOptionsTests +{ + [Fact] + public void SectionName_IsRabbitMQ() + { + Assert.Equal("RabbitMQ", RabbitMqOptions.SectionName); + } + + [Fact] + public void Defaults_MatchBackFillerContract() + { + var options = new RabbitMqOptions(); + + Assert.Equal(1024, options.WorkRequestMaxPayloadBytes); + Assert.Equal(60, options.ChannelLeaseTimeoutSeconds); + Assert.Equal(30, options.RpcTimeoutSeconds); + Assert.Equal(30, options.ConnectionBlockedTimeoutSeconds); + Assert.Empty(options.Hosts!); + Assert.Null(options.Username); + Assert.Null(options.Password); + Assert.Equal("/", options.VirtualHost); + Assert.True(options.EnableSsl); + Assert.Equal(5672, options.Port); + Assert.Equal(512, options.ChannelPoolSize); + Assert.Equal(4, options.MinConnections); + Assert.Equal(16, options.MaxConnections); + Assert.Equal(1024, options.MaxPendingLeaseWaiters); + Assert.Equal(300, options.ConnectionScaleDownIdleSeconds); + Assert.Equal(30, options.ScaleDownCooldownSeconds); + Assert.Equal(5, options.NetworkRecoveryIntervalSeconds); + Assert.Equal(250, options.PoolReconnectBaseDelayMs); + Assert.Equal(30000, options.PoolReconnectMaxDelayMs); + Assert.Equal(300, options.MinimumConnectionLifetimeSeconds); + Assert.Equal(10, options.PublishConfirmTimeoutSeconds); + Assert.Equal(30, options.MaximumShutdownDrainTimeoutSeconds); + Assert.Equal(0.75, options.DegradedThreshold); + Assert.Equal(5, options.UnhealthyThreshold); + Assert.Equal(60, options.RequestedHeartbeatSeconds); + Assert.Equal(30, options.SocketTimeoutSeconds); + Assert.Equal(2047, options.RequestedChannelMax); + Assert.Null(options.ConsumerPrefetchCount); + Assert.Null(options.DiagnosticPayloadCorrelationId); + Assert.NotNull(options.Management); + Assert.Null(options.Management.BaseUrl); + Assert.Equal(5, options.Management.RequestTimeoutSeconds); + } + + [Fact] + public void Bind_ReadsIdenticalShape() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["RabbitMQ:Hosts:0"] = "rabbit-01.example.net", + ["RabbitMQ:Hosts:1"] = "10.0.0.8", + ["RabbitMQ:Port"] = "5671", + ["RabbitMQ:Username"] = "nntparticles", + ["RabbitMQ:Password"] = "unit-test-password-not-real", + ["RabbitMQ:VirtualHost"] = "/articles", + ["RabbitMQ:EnableSsl"] = "true", + ["RabbitMQ:ChannelLeaseTimeoutSeconds"] = "90", + ["RabbitMQ:RpcTimeoutSeconds"] = "45", + ["RabbitMQ:ConnectionBlockedTimeoutSeconds"] = "40", + ["RabbitMQ:WorkRequestMaxPayloadBytes"] = "2048", + ["RabbitMQ:ChannelPoolSize"] = "256", + ["RabbitMQ:MinConnections"] = "2", + ["RabbitMQ:MaxConnections"] = "8", + ["RabbitMQ:MaxPendingLeaseWaiters"] = "64", + ["RabbitMQ:ConnectionScaleDownIdleSeconds"] = "120", + ["RabbitMQ:ScaleDownCooldownSeconds"] = "15", + ["RabbitMQ:NetworkRecoveryIntervalSeconds"] = "8", + ["RabbitMQ:PoolReconnectBaseDelayMs"] = "100", + ["RabbitMQ:PoolReconnectMaxDelayMs"] = "5000", + ["RabbitMQ:MinimumConnectionLifetimeSeconds"] = "90", + ["RabbitMQ:PublishConfirmTimeoutSeconds"] = "12", + ["RabbitMQ:MaximumShutdownDrainTimeoutSeconds"] = "20", + ["RabbitMQ:DegradedThreshold"] = "0.5", + ["RabbitMQ:UnhealthyThreshold"] = "4", + ["RabbitMQ:RequestedHeartbeatSeconds"] = "30", + ["RabbitMQ:SocketTimeoutSeconds"] = "15", + ["RabbitMQ:RequestedChannelMax"] = "1024", + ["RabbitMQ:ConsumerPrefetchCount"] = "50", + ["RabbitMQ:DiagnosticPayloadCorrelationId"] = "ee9f7a44-a5d6-4603-ba22-c5e0803fc834", + ["RabbitMQ:Management:BaseUrl"] = "http://rabbit-01.example.net:15672", + ["RabbitMQ:Management:RequestTimeoutSeconds"] = "5", + }) + .Build(); + + var options = new RabbitMqOptions(); + configuration.GetSection(RabbitMqOptions.SectionName).Bind(options); + + Assert.Equal(["rabbit-01.example.net", "10.0.0.8"], options.Hosts!); + Assert.Equal(5671, options.Port); + Assert.Equal("nntparticles", options.Username); + Assert.Equal("unit-test-password-not-real", options.Password); + Assert.Equal("/articles", options.VirtualHost); + Assert.True(options.EnableSsl); + Assert.Equal(90, options.ChannelLeaseTimeoutSeconds); + Assert.Equal(45, options.RpcTimeoutSeconds); + Assert.Equal(40, options.ConnectionBlockedTimeoutSeconds); + Assert.Equal(2048, options.WorkRequestMaxPayloadBytes); + Assert.Equal(256, options.ChannelPoolSize); + Assert.Equal(2, options.MinConnections); + Assert.Equal(8, options.MaxConnections); + Assert.Equal(64, options.MaxPendingLeaseWaiters); + Assert.Equal(120, options.ConnectionScaleDownIdleSeconds); + Assert.Equal(15, options.ScaleDownCooldownSeconds); + Assert.Equal(8, options.NetworkRecoveryIntervalSeconds); + Assert.Equal(100, options.PoolReconnectBaseDelayMs); + Assert.Equal(5000, options.PoolReconnectMaxDelayMs); + Assert.Equal(90, options.MinimumConnectionLifetimeSeconds); + Assert.Equal(12, options.PublishConfirmTimeoutSeconds); + Assert.Equal(20, options.MaximumShutdownDrainTimeoutSeconds); + Assert.Equal(0.5, options.DegradedThreshold); + Assert.Equal(4, options.UnhealthyThreshold); + Assert.Equal(30, options.RequestedHeartbeatSeconds); + Assert.Equal(15, options.SocketTimeoutSeconds); + Assert.Equal(1024, options.RequestedChannelMax); + Assert.Equal((ushort)50, options.ConsumerPrefetchCount); + Assert.Equal("ee9f7a44-a5d6-4603-ba22-c5e0803fc834", options.DiagnosticPayloadCorrelationId); + Assert.Equal("http://rabbit-01.example.net:15672", options.Management.BaseUrl); + Assert.Equal(5, options.Management.RequestTimeoutSeconds); + } + + [Fact] + public void Validate_Succeeds_ForValidHosts() + { + var result = new NntpdRabbitMqOptionsValidator().Validate(null, CreateValid()); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validate_Fails_WhenHostsMissing() + { + var result = new NntpdRabbitMqOptionsValidator().Validate(null, new RabbitMqOptions()); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Hosts", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenHostEntryEmpty() + { + var options = CreateValid(); + options.Hosts = [" "]; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenHostIncludesCredentials() + { + var options = CreateValid(); + options.Hosts = ["user:pass@broker.example.net"]; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("credentials", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenUsernameConfiguredWithoutPassword() + { + var options = CreateValid(); + options.Username = "nntparticles"; + options.Password = null; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Password", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(0)] + [InlineData(65536)] + public void Validate_Fails_WhenPortOutOfRange(int port) + { + var options = CreateValid(); + options.Port = port; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenMinConnectionsExceedMaxConnections() + { + var options = CreateValid(); + options.MinConnections = 8; + options.MaxConnections = 2; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenLeaseTimeoutLessThanRpcTimeout() + { + var options = CreateValid(); + options.ChannelLeaseTimeoutSeconds = 10; + options.RpcTimeoutSeconds = 30; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void ToRuntimeOptions_ProjectsValidatedSnapshot() + { + var options = CreateValid(); + options.Username = "nntparticles"; + options.Password = "unit-test-password-not-real"; + options.EnableSsl = false; + + var runtime = options.ToRuntimeOptions(); + + Assert.Equal(["127.0.0.1"], runtime.Hosts); + Assert.Equal(5672, runtime.Port); + Assert.Equal("nntparticles", runtime.Username); + Assert.Equal("unit-test-password-not-real", runtime.Password); + Assert.Equal("/", runtime.VirtualHost); + Assert.False(runtime.EnableSsl); + Assert.Equal(60, runtime.RequestedHeartbeatSeconds); + } + + [Fact] + public void ToRuntimeOptions_TrimsDropsBlankAndDeduplicatesHosts() + { + var options = CreateValid(); + options.Hosts = [" broker.example.net ", " ", "BROKER.example.net", "10.0.0.8", ""]; + + var runtime = options.ToRuntimeOptions(); + + Assert.Equal(["broker.example.net", "10.0.0.8"], runtime.Hosts); + } + + [Fact] + public void ToRuntimeOptions_TrimsUsername_AndConvertsBlankToNull() + { + var options = CreateValid(); + options.Username = " nntparticles "; + options.Password = "unit-test-password-not-real"; + + Assert.Equal("nntparticles", options.ToRuntimeOptions().Username); + + options.Username = " \t "; + Assert.Null(options.ToRuntimeOptions().Username); + + options.Username = null; + Assert.Null(options.ToRuntimeOptions().Username); + } + + [Fact] + public void ToRuntimeOptions_TrimsVirtualHost_AndConvertsBlankToSlash() + { + var options = CreateValid(); + options.VirtualHost = " /articles "; + Assert.Equal("/articles", options.ToRuntimeOptions().VirtualHost); + + options.VirtualHost = " \t "; + Assert.Equal("/", options.ToRuntimeOptions().VirtualHost); + + options.VirtualHost = null; + Assert.Equal("/", options.ToRuntimeOptions().VirtualHost); + } + + [Fact] + public void EnvironmentVariables_DocumentedCredentialNames_AreExact() + { + Assert.Equal("VECTOR__RABBITMQ__USERNAME", RabbitMqOptions.UsernameEnvironmentVariable); + Assert.Equal("VECTOR__RABBITMQ__PASSWORD", RabbitMqOptions.PasswordEnvironmentVariable); + Assert.Equal("Username", RabbitMqOptions.UsernameConfigurationKey); + Assert.Equal("Password", RabbitMqOptions.PasswordConfigurationKey); + } + + [Fact] + public void EnvironmentVariables_NntpdPrefixedCredentials_BindToRabbitMqSection() + { + const string username = "nntparticles"; + const string password = "unit-test-password-not-real"; + + var previousUser = Environment.GetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable); + var previousPass = Environment.GetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable); + + try + { + Environment.SetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable, username); + Environment.SetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable, password); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["RabbitMQ:Hosts:0"] = "127.0.0.1", + ["RabbitMQ:Username"] = "from-json-should-be-overridden", + ["RabbitMQ:Password"] = "from-json-should-be-overridden", + }) + .AddNntpdPrefixedEnvironmentVariables() + .Build(); + + var options = new RabbitMqOptions(); + configuration.GetSection(RabbitMqOptions.SectionName).Bind(options); + + Assert.Equal(username, options.Username); + Assert.Equal(password, options.Password); + Assert.Equal(username, configuration[$"{RabbitMqOptions.SectionName}:{RabbitMqOptions.UsernameConfigurationKey}"]); + Assert.Equal(password, configuration[$"{RabbitMqOptions.SectionName}:{RabbitMqOptions.PasswordConfigurationKey}"]); + } + finally + { + Environment.SetEnvironmentVariable(RabbitMqOptions.UsernameEnvironmentVariable, previousUser); + Environment.SetEnvironmentVariable(RabbitMqOptions.PasswordEnvironmentVariable, previousPass); + } + } + + [Fact] + public void Validate_Fails_WhenManagementBaseUrlMissing() + { + var options = CreateValid(); + options.Management.BaseUrl = null; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("Management:BaseUrl", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_Fails_WhenManagementBaseUrlIncludesApiPath() + { + var options = CreateValid(); + options.Management.BaseUrl = "http://127.0.0.1:15672/api"; + var result = new NntpdRabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures, static failure => failure.Contains("must not include a path", StringComparison.Ordinal)); + } + + [Fact] + public void GetDefaultConnectionName_UsesNntpdPrefix() + { + Assert.Equal( + "VectorNNTPD.TransitService:nntpd01.usenet.ninja", + RabbitMqRuntimeOptions.GetDefaultConnectionName("VectorNNTPD.TransitService", "nntpd01.usenet.ninja")); + } + + internal static RabbitMqOptions CreateValid() => + new() + { + Hosts = ["127.0.0.1"], + EnableSsl = false, + Username = "guest", + Password = "guest", + Management = new RabbitMqManagementOptions + { + BaseUrl = "http://127.0.0.1:15672", + RequestTimeoutSeconds = 5, + }, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs new file mode 100644 index 00000000..5c8c1c4b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceConcurrencyTests.cs @@ -0,0 +1,322 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class RabbitMqServiceConcurrencyTests +{ + [Fact] + public async Task SimultaneousLossSignals_StartOneReconnect() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + Parallel.For(0, 8, _ => first.SimulateLost()); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + // Eight signals coalesce onto one watch loop. A loss observed during the + // reconnect delay can schedule one follow-up replace after N+1 is installed. + Assert.InRange(factory.ConnectCount, 2, 3); + Assert.True(service.ConnectionGeneration >= 2); + Assert.True(service.IsReady); + Assert.False(service.Execution!.IsCompleted); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StaleConnectionLost_DoesNotAffectCurrentGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + var current = factory.LastConnection!; + + var extra = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = extra; + first.SimulateLost(); + + var extraConnect = extra.Task.WaitAsync(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync(() => extraConnect); + + Assert.Equal(2, factory.ConnectCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.Same(current, handle.Connection); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(0, current.DisposeCount); + + await service.DisposeAsync(); + Assert.Equal(1, current.DisposeCount); + } + + [Fact] + public async Task StaleConnectionDispose_DoesNotDisposeCurrent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + var current = factory.LastConnection!; + + await first.DisposeAsync(); + + Assert.Equal(2, first.DisposeCount); + Assert.Equal(0, current.DisposeCount); + Assert.True(service.IsReady); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.Same(current, handle.Connection); + + await service.DisposeAsync(); + Assert.Equal(1, current.DisposeCount); + } + + [Fact] + public async Task ShutdownDuringReconnect_DoesNotInstallAReplacement() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var connectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.ConnectStarted = connectStarted; + factory.BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await connectStarted.Task.WaitAsync(safety.Token); + + var stop = service.StopAsync(CancellationToken.None); + factory.BlockConnect.TrySetCanceled(); + await stop; + + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, service.ConnectionGeneration); + Assert.False(service.IsReady); + Assert.False(service.TryGetCurrent(out _)); + foreach (var connection in factory.Connections) + { + Assert.Equal(1, connection.DisposeCount); + } + } + + [Fact] + public async Task ConcurrentStopAndLoss_DisposesCurrentOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + + var stop = service.StopAsync(CancellationToken.None); + first.SimulateLost(); + await service.DisposeAsync(); + await stop; + + Assert.Equal(1, first.DisposeCount); + Assert.Equal(1, factory.ConnectCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task CapturedHandle_IsNotCurrentAfterLoss_BeforeConnectionIsDisposed() + { + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 30000; + options.PoolReconnectMaxDelayMs = 30000; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var handle)); + var first = factory.LastConnection!; + Assert.True(handle.IsCurrent); + + first.SimulateLost(); + + Assert.False(handle.IsCurrent); + Assert.False(handle.IsOpen); + Assert.Equal(1, handle.Generation); + Assert.Same(first, handle.Connection); + Assert.Equal(0, first.DisposeCount); + Assert.False(service.TryGetCurrent(out _)); + + await service.DisposeAsync(); + Assert.Equal(1, first.DisposeCount); + } + + [Fact] + public async Task HoldingAHandle_DoesNotPreventDisposeOfThatGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var generationOne)); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.False(generationOne.IsCurrent); + Assert.Equal(1, generationOne.Generation); + Assert.Same(first, generationOne.Connection); + Assert.Equal(1, first.DisposeCount); + Assert.True(service.TryGetCurrent(out var generationTwo)); + Assert.Equal(2, generationTwo.Generation); + Assert.True(generationTwo.IsCurrent); + Assert.NotSame(generationOne.Connection, generationTwo.Connection); + + await service.DisposeAsync(); + } + + [Fact] + public async Task DisposeOfCapturedGeneration_CanOverlapCallerStillHoldingTheHandle() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var handle)); + var first = factory.LastConnection!; + var disposeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.DisposeStarted = disposeStarted; + first.BlockDispose = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await disposeStarted.Task.WaitAsync(safety.Token); + + Assert.False(handle.IsCurrent); + Assert.Same(first, handle.Connection); + Assert.Equal(0, first.DisposeCount); + Assert.False(service.TryGetCurrent(out _)); + + first.BlockDispose.TrySetResult(); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(1, first.DisposeCount); + Assert.False(handle.IsCurrent); + Assert.True(service.TryGetCurrent(out var next)); + Assert.Equal(2, next.Generation); + + await service.DisposeAsync(); + } + + [Fact] + public async Task ConcurrentTryGetCurrent_DuringReplacement_NeverReturnsADisposedCurrentHandle() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + var stop = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var observed = new List(); + var sampler = Task.Run(async () => + { + while (!stop.Task.IsCompleted) + { + if (service.TryGetCurrent(out var handle)) + { + lock (observed) + { + observed.Add(handle); + } + } + + await Task.Yield(); + } + }); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + stop.TrySetResult(); + await sampler.WaitAsync(safety.Token); + + Assert.NotEmpty(observed); + Assert.True(service.TryGetCurrent(out var current)); + Assert.Equal(2, current.Generation); + Assert.Equal(1, first.DisposeCount); + lock (observed) + { + foreach (var handle in observed) + { + if (!handle.IsCurrent) + { + continue; + } + + Assert.Equal(service.ConnectionGeneration, handle.Generation); + Assert.Equal(0, ((FakeRabbitMqConnection)handle.Connection).DisposeCount); + } + } + + await service.DisposeAsync(); + } + + [Fact] + public async Task RepeatedLifecycleCalls_AreIdempotent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await Task.WhenAll( + service.StopAsync(CancellationToken.None), + service.StopAsync(CancellationToken.None), + service.DisposeAsync().AsTask(), + service.DisposeAsync().AsTask()); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, connection.DisposeCount); + Assert.False(service.IsReady); + } + + private static RabbitMqService CreateService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceTests.cs new file mode 100644 index 00000000..7630e4fb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqServiceTests.cs @@ -0,0 +1,549 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class RabbitMqServiceTests +{ + [Fact] + public void ConnectionHandle_DoesNotOwnTheConnection() + { + Assert.DoesNotContain(typeof(IDisposable), typeof(RabbitMqConnectionHandle).GetInterfaces()); + Assert.DoesNotContain(typeof(IAsyncDisposable), typeof(RabbitMqConnectionHandle).GetInterfaces()); + } + + [Fact] + public async Task StartAsync_ConnectsOnce_AndIsReady() + { + var factory = new FakeRabbitMqConnectionFactory(); + var replaced = new List(); + var service = CreateService(factory); + service.ConnectionReplaced += (_, args) => replaced.Add(args); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.NotNull(service.Execution); + Assert.True(service.TryGetCurrent(out var handle)); + Assert.True(handle.IsCurrent); + Assert.Equal(1, handle.Generation); + Assert.Same(factory.LastConnection, handle.Connection); + var generation = Assert.Single(replaced); + Assert.Equal(1, generation.ConnectionGeneration); + Assert.False(generation.IsReplacement); + Assert.Equal("VectorNNTPD.TransitService:nntpd01.usenet.ninja", factory.LastConnection!.ClientProvidedName); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectThrows() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectException = new InvalidOperationException("broker down"), + }; + var service = CreateService(factory); + + var ex = await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal("broker down", ex.Message); + Assert.Equal(0, service.ConnectionCount); + Assert.Equal(0, service.ConnectionGeneration); + Assert.False(service.IsReady); + Assert.Null(service.Execution); + Assert.False(service.TryGetCurrent(out _)); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectionIsNotUsable() + { + var factory = new FakeRabbitMqConnectionFactory { ReturnUnusableConnection = true }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + Assert.Equal(1, factory.LastConnection!.DisposeCount); + } + + [Fact] + public async Task StartAsync_Cancelled_DoesNotLeaveConnection() + { + var factory = new FakeRabbitMqConnectionFactory + { + BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + + var start = service.StartAsync(cts.Token); + cts.Cancel(); + factory.BlockConnect.TrySetCanceled(cts.Token); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + Assert.Null(service.Execution); + } + + [Fact] + public async Task StartAsync_CanRetry_AfterFailedConnect() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectException = new InvalidOperationException("broker down"), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + factory.ConnectException = null; + + await service.StartAsync(CancellationToken.None); + Assert.True(service.IsReady); + Assert.Equal(1, service.ConnectionGeneration); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StartAsync_IsIdempotent() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + await service.DisposeAsync(); + } + + [Fact] + public async Task ConnectionLost_ReplacesConnection_AndIncrementsGeneration() + { + var factory = new FakeRabbitMqConnectionFactory(); + var replaced = new List(); + var service = CreateService(factory); + service.ConnectionReplaced += (_, args) => replaced.Add(args); + + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var count = await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, count); + Assert.Equal(2, service.ConnectionCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1, first.DisposeCount); + Assert.Equal(2, replaced.Count); + Assert.True(replaced[1].IsReplacement); + Assert.Equal(2, replaced[1].ConnectionGeneration); + Assert.True(service.TryGetCurrent(out var current)); + Assert.Same(factory.LastConnection, current.Connection); + + await service.DisposeAsync(); + } + + [Fact] + public async Task ConnectionLost_TryGetCurrent_FailsUntilReplaced() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + Assert.True(service.TryGetCurrent(out var before)); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + Assert.False(service.IsReady); + Assert.False(service.TryGetCurrent(out _)); + Assert.False(before.IsCurrent); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + Assert.True(service.IsReady); + Assert.True(service.TryGetCurrent(out var after)); + Assert.Equal(2, after.Generation); + Assert.True(after.IsCurrent); + Assert.False(before.IsCurrent); + + await service.DisposeAsync(); + Assert.False(after.IsCurrent); + } + + [Fact] + public async Task Reconnect_FailsThenSucceeds() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + factory.RemainingConnectFailures = 2; + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = recovered; + + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await recovered.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1 + 2 + 1, factory.AttemptCount); + Assert.Equal(2, factory.ConnectCount); + + await service.DisposeAsync(); + } + + [Fact] + public async Task Reconnect_Continues_BeyondFormerAbandonThreshold() + { + const int formerAbandonThreshold = 5; + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var failures = formerAbandonThreshold + 3; + factory.RemainingConnectFailures = failures; + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = recovered; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await factory.WaitForAttemptsAsync(1 + formerAbandonThreshold + 1, safety.Token); + Assert.False(service.IsReady); + Assert.Equal(1, service.ConnectionGeneration); + Assert.False(service.Execution!.IsCompleted); + + await recovered.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.True(service.IsReady); + Assert.Equal(1 + failures + 1, factory.AttemptCount); + Assert.False(service.Execution!.IsCompleted); + + await service.DisposeAsync(); + Assert.True(service.Execution.IsCompleted); + } + + [Fact] + public async Task Reconnect_KeepsTrying_UntilShutdown() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + factory.RemainingConnectFailures = 100; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await factory.WaitForAttemptsAsync(8, safety.Token); + Assert.False(service.IsReady); + Assert.False(service.Execution!.IsCompleted); + + await service.StopAsync(CancellationToken.None); + + var attempts = factory.AttemptCount; + Assert.True(service.Execution.IsCompleted); + Assert.False(service.TryGetCurrent(out _)); + + using var noMore = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync( + () => factory.WaitForAttemptsAsync(attempts + 1, noMore.Token)); + Assert.Equal(attempts, factory.AttemptCount); + } + + [Fact] + public async Task SuccessfulReconnect_ResetsBackoffOnNextLoss() + { + var clock = new RecordingTimeProvider(); + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 800; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance, + clock); + + await service.StartAsync(CancellationToken.None); + factory.RemainingConnectFailures = 2; + var firstRecovery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = firstRecovery; + factory.LastConnection!.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await firstRecovery.Task.WaitAsync(safety.Token); + var afterFirstRecovery = clock.Delays.ToArray(); + Assert.Equal( + [TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(200)], + afterFirstRecovery); + + var secondRecovery = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondRecovery; + factory.LastConnection!.SimulateLost(); + await secondRecovery.Task.WaitAsync(safety.Token); + + Assert.Equal(3, service.ConnectionGeneration); + Assert.Equal(TimeSpan.FromMilliseconds(50), clock.Delays[^1]); + + await service.DisposeAsync(); + } + + [Fact] + public async Task StopAsync_InterruptsRecoveryDelay() + { + var factory = new FakeRabbitMqConnectionFactory(); + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 30000; + options.PoolReconnectMaxDelayMs = 30000; + var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.SimulateLost(); + + await service.StopAsync(CancellationToken.None); + + Assert.Equal(1, factory.AttemptCount); + Assert.True(service.Execution!.IsCompleted); + Assert.False(service.TryGetCurrent(out _)); + } + + [Fact] + public async Task StopAsync_DisposesConnectionOnce() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.Equal(1, connection.DisposeCount); + Assert.False(service.IsReady); + Assert.True(service.Execution is { IsCompleted: true }); + } + + [Fact] + public async Task DisposeAsync_IsSafe_WhenNeverStarted() + { + var service = CreateService(new FakeRabbitMqConnectionFactory()); + await service.DisposeAsync(); + await service.DisposeAsync(); + Assert.False(service.IsReady); + Assert.Equal(0, service.ConnectionGeneration); + } + + [Fact] + public async Task StopAsync_WhileReconnecting_CancelsAndDisposes() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var connectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.ConnectStarted = connectStarted; + factory.BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await connectStarted.Task.WaitAsync(safety.Token); + + await service.StopAsync(CancellationToken.None); + factory.BlockConnect.TrySetCanceled(); + + Assert.Equal(1, first.DisposeCount); + Assert.False(service.IsReady); + Assert.Equal(1, factory.ConnectCount); + foreach (var connection in factory.Connections) + { + Assert.Equal(1, connection.DisposeCount); + } + } + + [Fact] + public async Task StopAsync_DuringStartupConnect_LeavesNoConnection() + { + var factory = new FakeRabbitMqConnectionFactory + { + ConnectStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + BlockConnect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var service = CreateService(factory); + using var cts = new CancellationTokenSource(); + var start = service.StartAsync(cts.Token); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await factory.ConnectStarted.Task.WaitAsync(safety.Token); + cts.Cancel(); + factory.BlockConnect.TrySetCanceled(cts.Token); + + await Assert.ThrowsAnyAsync(() => start); + Assert.Equal(0, service.ConnectionCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task StopAsync_DoesNotReconnect() + { + var factory = new FakeRabbitMqConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + await service.StopAsync(CancellationToken.None); + + first.SimulateLost(); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, factory.AttemptCount); + Assert.False(service.IsReady); + } + + [Fact] + public async Task Logs_DoNotContainPassword() + { + const string secret = "unit-test-rabbitmq-password-not-real"; + var logger = new CollectingLogger(); + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "nntparticles"; + options.Password = secret; + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + + var service = new RabbitMqService( + new FakeRabbitMqConnectionFactory(), + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + logger); + + await service.StartAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.NotEmpty(logger.Messages); + Assert.Contains(logger.Messages, static message => message.Contains("Connecting to RabbitMQ", StringComparison.Ordinal)); + Assert.Contains(logger.Messages, static message => message.Contains("RabbitMQ connection established", StringComparison.Ordinal)); + Assert.Contains(logger.Messages, static message => message.Contains("RabbitMQ stopped", StringComparison.Ordinal)); + foreach (var message in logger.Messages) + { + Assert.DoesNotContain(secret, message, StringComparison.Ordinal); + Assert.DoesNotContain("Password=", message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("nntparticles:", message, StringComparison.Ordinal); + } + } + + [Fact] + public void CreateClientFactory_DisablesClientAutomaticRecovery() + { + var runtime = RabbitMqOptionsTests.CreateValid().ToRuntimeOptions(); + var factory = RabbitMqClientConnectionFactory.CreateClientFactory( + runtime, + "VectorNNTPD.TransitService:nntpd01.usenet.ninja"); + + Assert.False(factory.AutomaticRecoveryEnabled); + Assert.False(factory.TopologyRecoveryEnabled); + Assert.Equal(runtime.Port, factory.Port); + Assert.Equal(runtime.VirtualHost, factory.VirtualHost); + Assert.Equal((ushort)runtime.RequestedChannelMax, factory.RequestedChannelMax); + } + + [Fact] + public void CreateClientFactory_AppliesCredentialsWithoutExposingThemOnTheName() + { + var options = RabbitMqOptionsTests.CreateValid(); + options.Username = "nntparticles"; + options.Password = "unit-test-rabbitmq-password-not-real"; + var runtime = options.ToRuntimeOptions(); + const string name = "VectorNNTPD.TransitService:nntpd01.usenet.ninja"; + + var factory = RabbitMqClientConnectionFactory.CreateClientFactory(runtime, name); + + Assert.Equal("nntparticles", factory.UserName); + Assert.Equal("unit-test-rabbitmq-password-not-real", factory.Password); + Assert.Equal(name, factory.ClientProvidedName); + Assert.DoesNotContain(runtime.Password!, factory.ClientProvidedName, StringComparison.Ordinal); + } + + private static RabbitMqService CreateService( + FakeRabbitMqConnectionFactory factory, + int poolReconnectBaseDelayMs = 50) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = poolReconnectBaseDelayMs; + options.PoolReconnectMaxDelayMs = poolReconnectBaseDelayMs; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private sealed class RecordingTimeProvider : TimeProvider + { + private readonly TimeProvider _inner = TimeProvider.System; + + public List Delays { get; } = []; + + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + if (dueTime > TimeSpan.Zero && dueTime != Timeout.InfiniteTimeSpan) + { + Delays.Add(dueTime); + } + + return _inner.CreateTimer(callback, state, dueTime, period); + } + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs new file mode 100644 index 00000000..4c540bdc --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyNamesTests.cs @@ -0,0 +1,46 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class RabbitMqTopologyNamesTests +{ + [Theory] + [InlineData(" BackFiller.Abavia ", "backfiller.abavia")] + [InlineData(" cache.requests ", "cache.requests")] + [InlineData(" cache.requests ", "cache.requests")] + [InlineData("Abavia", "abavia")] + public void Normalize_TrimsAndUsesInvariantLowerCase(string input, string expected) + { + Assert.Equal(expected, RabbitMqTopologyNames.Normalize(input)); + } + + [Fact] + public void CreateFanoutQuorumBinding_NormalizesExchangeQueueAndRoutingKey() + { + var endpoint = RabbitMqArticleRetrievalEndpoints.CreateFanoutQuorumBinding(" BackFiller.Abavia "); + Assert.Equal("backfiller.abavia", endpoint.ExchangeName); + Assert.Equal("backfiller.abavia", endpoint.QueueName); + Assert.Equal("backfiller.abavia", endpoint.RoutingKey); + Assert.Equal(endpoint.ExchangeName, endpoint.QueueName); + Assert.Equal(endpoint.QueueName, endpoint.RoutingKey); + Assert.True(endpoint.QueueDurable); + Assert.False(endpoint.QueueExclusive); + Assert.False(endpoint.QueueAutoDelete); + Assert.True(endpoint.QueueArguments.TryGetValue( + RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(RabbitMqArticleRetrievalEndpoints.QuorumQueueType, queueType); + Assert.Single(endpoint.QueueArguments); + Assert.DoesNotContain("grabbers.", endpoint.ExchangeName, StringComparison.Ordinal); + } + + [Fact] + public void RequiredTopology_HasNoSharedCacheRequestsQuorumQueue() + { + Assert.Empty(ArticleRetrievalTopology.Required); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, CacheRequestsTopology.ExchangeName); + Assert.DoesNotContain("grabbers.", CacheRequestsTopology.ExchangeName, StringComparison.Ordinal); + Assert.DoesNotContain("backfiller.storage", CacheRequestsTopology.ExchangeName, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs new file mode 100644 index 00000000..83287934 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/RabbitMqTopologyServiceTests.cs @@ -0,0 +1,246 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.Core; +using VectorNNTPD.TransitService.Hosting; +using VectorNNTPD.TransitService.Logging; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +[Collection(SerilogCollection.Name)] +public sealed class RabbitMqTopologyServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresCacheFanoutExchanges_AndOverviewQueue_WithoutSharedRequestsQueue() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + var channel = Assert.Single(connection.TopologyChannels); + Assert.Equal(1, channel.DisposeCount); + AssertDeclaredTopology(connection, expectedPasses: 1); + + await topology.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DeclareRequiredTopologyAsync_IsIdempotent_AndDoesNotDeleteOrPurge() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await topology.DeclareRequiredTopologyAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Equal(2, connection.TopologyChannels.Count); + Assert.All(connection.TopologyChannels, static channel => Assert.Equal(1, channel.DisposeCount)); + AssertDeclaredTopology(connection, expectedPasses: 2); + var channelMethods = typeof(IRabbitMqTopologyChannel).GetMethods().Select(static m => m.Name).ToHashSet(); + Assert.Contains(nameof(IRabbitMqTopologyChannel.ExchangeDeclareAsync), channelMethods); + Assert.Contains(nameof(IRabbitMqTopologyChannel.QueueDeclareAsync), channelMethods); + Assert.Contains(nameof(IRabbitMqTopologyChannel.QueueBindAsync), channelMethods); + Assert.DoesNotContain("QueueDeleteAsync", channelMethods); + Assert.DoesNotContain("QueuePurgeAsync", channelMethods); + Assert.DoesNotContain("ExchangeDeleteAsync", channelMethods); + Assert.DoesNotContain("BasicPublishAsync", channelMethods); + Assert.DoesNotContain("BasicConsumeAsync", channelMethods); + } + + [Fact] + public async Task StartAsync_SecondCall_DoesNotRedeclare() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + await topology.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + Assert.Single(connection.TopologyChannels); + } + + [Fact] + public async Task StartAsync_Fails_WhenRabbitMqHasNotStarted() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + var ex = await Assert.ThrowsAsync( + () => topology.StartAsync(CancellationToken.None)); + + Assert.Contains("not ready", ex.Message, StringComparison.Ordinal); + Assert.Null(factory.LastConnection); + } + + [Fact] + public async Task StartAsync_Fails_WhenQueueDeclareIsRejected() + { + var factory = new FakeRabbitMqConnectionFactory + { + QueueDeclareException = new InvalidOperationException("PRECONDITION_FAILED - inequivalent arg 'x-queue-type' for queue 'overviewdb.queue'"), + }; + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + var ex = await Assert.ThrowsAsync( + () => topology.StartAsync(CancellationToken.None)); + + Assert.Contains("x-queue-type", ex.Message, StringComparison.Ordinal); + Assert.Single(factory.LastConnection!.TopologyChannels); + Assert.Equal(1, factory.LastConnection.TopologyChannels[0].DisposeCount); + } + + [Fact] + public async Task ApplicationServiceManager_TopologyFailure_PreventsStartup_AndRollsBackRabbitMq() + { + var factory = new FakeRabbitMqConnectionFactory + { + QueueDeclareException = new InvalidOperationException("classic queue cannot become quorum"), + }; + await using var rabbit = CreateRabbitMqService(factory); + var topology = new RabbitMqTopologyService(rabbit, NullLogger.Instance); + var manager = TestHostFactory.CreateServiceManager([rabbit, topology]); + + var ex = await Assert.ThrowsAsync( + () => manager.StartAsync(CancellationToken.None)); + + Assert.Contains("quorum", ex.Message, StringComparison.Ordinal); + Assert.Empty(manager.StartedServices); + Assert.False(rabbit.IsReady); + Assert.False(rabbit.TryGetCurrent(out _)); + } + + [Fact] + public async Task Host_Start_DeclaresArticleRetrievalTopology_BeforeRunning() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + await host.StartAsync(); + + Assert.Equal(ApplicationState.Running, host.Services.GetRequiredService().State); + var factory = Assert.IsType( + host.Services.GetRequiredService()); + var connection = factory.LastConnection ?? throw new InvalidOperationException("Expected a connected fake."); + AssertDeclaredTopology(connection, expectedPasses: 1); + + await host.StopAsync(); + } + + [Fact] + public void Host_RegistersTopologyServiceOnce_ImmediatelyAfterRabbitMqConnection() + { + var builder = Host.CreateApplicationBuilder(); + TestHostFactory.ConfigureNntpdTestHost(builder); + builder.ConfigureNntpdLogging(static lc => lc.MinimumLevel.Fatal()); + builder.Services.AddNntpTransitHosting(includePlaceholderService: false); + + using var host = builder.Build(); + var services = host.Services.GetServices().ToArray(); + + // NntpDB and nntpsharedconfig are registered ahead of Cloudflare, Redis, and RabbitMQ. + Assert.Equal(typeof(RabbitMqService), services[4].GetType()); + Assert.Equal(typeof(RabbitMqTopologyService), services[5].GetType()); + Assert.Equal(typeof(StorageServerFleetConsumerService), services[6].GetType()); + Assert.Equal(typeof(StorageArticleLookupService), services[7].GetType()); + Assert.Equal(typeof(VectorNNTPD.TransitService.RabbitMq.ArticleWork.ArticleWorkRpcService), services[8].GetType()); + Assert.Equal(1, services.Count(static s => s is RabbitMqTopologyService)); + Assert.Same( + host.Services.GetRequiredService(), + services[5]); + } + + private static RabbitMqService CreateRabbitMqService(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static void AssertDeclaredTopology(FakeRabbitMqConnection connection, int expectedPasses) + { + Assert.Empty(ArticleRetrievalTopology.Required); + // Per pass: cache.requests + cache.broadcast exchanges; overviewdb.queue only. + Assert.Equal(2 * expectedPasses, connection.ExchangeDeclarations.Count); + Assert.Equal(1 * expectedPasses, connection.QueueDeclarations.Count); + Assert.Empty(connection.BindingDeclarations); + + for (var pass = 0; pass < expectedPasses; pass++) + { + var exchangeOffset = pass * 2; + var requests = connection.ExchangeDeclarations[exchangeOffset]; + Assert.Equal(CacheRequestsTopology.ExchangeName, requests.Name); + Assert.Equal("fanout", requests.Type); + Assert.True(requests.Durable); + Assert.False(requests.AutoDelete); + + var broadcast = connection.ExchangeDeclarations[exchangeOffset + 1]; + Assert.Equal(CacheBroadcastTopology.ExchangeName, broadcast.Name); + Assert.Equal("fanout", broadcast.Type); + Assert.True(broadcast.Durable); + Assert.False(broadcast.AutoDelete); + + var overview = connection.QueueDeclarations[pass]; + Assert.Equal(OverviewDbTopology.QueueName, overview.Name); + Assert.True(overview.Durable); + Assert.False(overview.Exclusive); + Assert.False(overview.AutoDelete); + AssertQuorumQueueArguments(overview.Arguments); + } + + Assert.DoesNotContain( + connection.QueueDeclarations, + static queue => queue.Name == CacheFleetTopology.RequestsExchangeName); + Assert.DoesNotContain( + connection.QueueDeclarations, + static queue => + queue.Name == CacheFleetTopology.RequestsExchangeName + && queue.Durable + && queue.Arguments is not null + && queue.Arguments.TryGetValue("x-queue-type", out var type) + && Equals(type, "quorum")); + Assert.DoesNotContain( + connection.ExchangeDeclarations, + static exchange => exchange.Name is "backfiller.giganews" or "backfiller.eweka" or "backfiller.abavia"); + } + + private static void AssertQuorumQueueArguments( + IReadOnlyDictionary? arguments) + { + Assert.NotNull(arguments); + Assert.True(arguments.TryGetValue( + RabbitMqArticleRetrievalEndpoints.QueueTypeArgumentName, + out var queueType)); + Assert.Equal(RabbitMqArticleRetrievalEndpoints.QuorumQueueType, queueType); + Assert.False(arguments.ContainsKey("x-message-ttl")); + Assert.False(arguments.ContainsKey("x-expires")); + Assert.Single(arguments); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs new file mode 100644 index 00000000..e68e2d3f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/RabbitMq/StorageArticleRetrievalTopologyTests.cs @@ -0,0 +1,49 @@ +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.RabbitMq; + +public sealed class StorageArticleRetrievalTopologyTests +{ + [Fact] + public void CacheRequests_IsFanoutExchangeOnly_NotSharedQuorumQueue() + { + Assert.Equal("cache.requests", StorageArticleRetrievalTopology.EntityName); + Assert.Equal("cache.requests", StorageArticleRetrievalTopology.ExchangeName); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, CacheRequestsTopology.ExchangeName); + Assert.Equal("fanout", CacheRequestsTopology.ExchangeTypeName); + Assert.True(CacheRequestsTopology.ExchangeDurable); + Assert.False(CacheRequestsTopology.ExchangeAutoDelete); + Assert.Equal("1000", CacheRequestsTopology.ExpirationMilliseconds); + Assert.Empty(ArticleRetrievalTopology.Required); + Assert.DoesNotContain("backfiller.storage", StorageArticleRetrievalTopology.EntityName, StringComparison.Ordinal); + } + + [Fact] + public void StorageEndpoint_IsNotABackFillerProvider() + { + Assert.Equal(12, BackfillArticleRetrievalTopology.Providers.Length); + Assert.DoesNotContain("storage", BackfillArticleRetrievalTopology.Providers, StringComparer.OrdinalIgnoreCase); + Assert.Equal("Storage", StorageArticleRetrievalTopology.Backbone); + Assert.All( + BackfillArticleRetrievalTopology.Definitions, + static definition => + Assert.NotEqual(CacheFleetTopology.RequestsExchangeName, definition.ExchangeName)); + } + + [Fact] + public void StorageServerRequestQueues_ArePerInstance_AndIndependent() + { + var q1 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache01.usenet.ninja"); + var q2 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache02.usenet.ninja"); + var q3 = CacheFleetTopology.BuildStorageServerRequestQueueName("cache03.usenet.ninja"); + Assert.Equal("cache.cache01.usenet.ninja", q1); + Assert.Equal("cache.cache02.usenet.ninja", q2); + Assert.Equal("cache.cache03.usenet.ninja", q3); + Assert.NotEqual(q1, q2); + Assert.NotEqual(q2, q3); + Assert.NotEqual( + CacheFleetTopology.BuildNntpdBroadcastQueueName("nntpd01.usenet.ninja"), + q1); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Redis/RedisOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Redis/RedisOptionsTests.cs new file mode 100644 index 00000000..bcaa9aab --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Redis/RedisOptionsTests.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Configuration; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Redis; + +public sealed class RedisOptionsTests +{ + [Fact] + public void DefaultPort_Is6379() + { + Assert.Equal(6379, RedisOptions.DefaultPort); + Assert.Equal(6379, new RedisOptions().Port); + } + + [Fact] + public void Bind_ReadsHostArrayAndPort() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "redis-01.example.net", + ["Redis:Host:1"] = "redis-02.example.net", + ["Redis:Port"] = "6380", + }) + .Build(); + + var options = new RedisOptions(); + configuration.GetSection(RedisOptions.SectionName).Bind(options); + + Assert.Equal(["redis-01.example.net", "redis-02.example.net"], options.Host); + Assert.Equal(6380, options.Port); + } + + [Fact] + public void Bind_OmitsPort_DefaultsTo6379() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["Redis:Host:0"] = "127.0.0.1", + }) + .Build(); + + var options = new RedisOptions(); + configuration.GetSection(RedisOptions.SectionName).Bind(options); + Assert.Equal(6379, options.Port); + } + + [Fact] + public void Validate_Fails_WhenHostMissing() + { + var result = new RedisOptionsValidator().Validate(null, new RedisOptions()); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Fails_WhenHostEntryEmpty() + { + var result = new RedisOptionsValidator().Validate(null, new RedisOptions { Host = [" "] }); + Assert.True(result.Failed); + } + + [Theory] + [InlineData(0)] + [InlineData(65536)] + public void Validate_Fails_WhenPortOutOfRange(int port) + { + var result = new RedisOptionsValidator().Validate( + null, + new RedisOptions { Host = ["127.0.0.1"], Port = port }); + Assert.True(result.Failed); + } + + [Fact] + public void Validate_Succeeds_ForValidHosts() + { + var result = new RedisOptionsValidator().Validate( + null, + new RedisOptions { Host = ["redis-01.example.net", "10.0.0.8"], Port = 6379 }); + Assert.True(result.Succeeded); + } + +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Redis/RedisServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Redis/RedisServiceTests.cs new file mode 100644 index 00000000..894f841b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Redis/RedisServiceTests.cs @@ -0,0 +1,238 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Redis; + +public sealed class RedisServiceTests +{ + [Fact] + public async Task StartAsync_ConnectsOnce_AndPings() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + + await service.StartAsync(CancellationToken.None); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, service.ConnectionCount); + Assert.Equal(1, factory.LastConnection!.Database.PingCount); + Assert.Same(factory.LastConnection.Database, service.Database); + } + + [Fact] + public async Task StartAsync_DoesNotCreatePerRequestConnections() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + + _ = await service.KeyExistsAsync("a"u8.ToArray()); + _ = await service.KeyExistsAsync("b"u8.ToArray()); + _ = await service.GetAsync("c"u8.ToArray()); + await service.SetAsync("c"u8.ToArray(), "d"u8.ToArray(), TimeSpan.FromMinutes(1)); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(2, factory.LastConnection!.Database.KeyExistsCount); + Assert.Equal(1, factory.LastConnection.Database.GetCount); + Assert.Equal(1, factory.LastConnection.Database.SetCount); + } + + [Fact] + public async Task StartAsync_Fails_WhenConnectThrows() + { + var factory = new FakeRedisConnectionFactory + { + ConnectException = new RedisUnavailableException("down"), + }; + var service = CreateService(factory); + + await Assert.ThrowsAsync(() => service.StartAsync(CancellationToken.None)); + Assert.Equal(0, service.ConnectionCount); + } + + [Fact] + public async Task StopAsync_DisposesConnectionOnce() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var connection = factory.LastConnection!; + + await service.StopAsync(CancellationToken.None); + await service.DisposeAsync(); + + Assert.Equal(1, connection.DisposeCount); + } + + [Fact] + public async Task KeyExistsAsync_PropagatesInfrastructureFailure() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.Database.ExistsException = new RedisUnavailableException("timeout"); + + await Assert.ThrowsAsync( + () => service.KeyExistsAsync("k"u8.ToArray()).AsTask()); + } + + [Fact] + public async Task Circuit_BlocksUntilCooldown_ThenRecovers() + { + var clock = new ManualTimeProvider(); + var factory = new FakeRedisConnectionFactory(); + var service = new RedisService( + factory, + Options.Create(new RedisOptions { Host = ["127.0.0.1"], Port = 6379 }), + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(1)); + await service.StartAsync(CancellationToken.None); + factory.LastConnection!.Database.ExistsException = new RedisUnavailableException("timeout"); + + var history = new VectorNNTPD.TransitService.History.HistoryDb( + service, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new VectorNNTPD.TransitService.History.HistoryWriteQueue()); + + Assert.Equal( + VectorNNTPD.TransitService.History.HistoryLookupResult.Unavailable, + await history.LookupAsync(""u8.ToArray())); + Assert.True(service.IsUnavailable); + var exists = factory.LastConnection.Database.KeyExistsCount; + + Assert.Equal( + VectorNNTPD.TransitService.History.HistoryLookupResult.Unavailable, + await history.LookupAsync(""u8.ToArray())); + Assert.Equal(exists, factory.LastConnection.Database.KeyExistsCount); + + clock.Advance(TimeSpan.FromSeconds(1)); + factory.LastConnection.Database.ExistsException = null; + Assert.Equal( + VectorNNTPD.TransitService.History.HistoryLookupResult.Unseen, + await history.LookupAsync(""u8.ToArray())); + Assert.False(service.IsUnavailable); + Assert.Equal(exists + 1, factory.LastConnection.Database.KeyExistsCount); + + await service.DisposeAsync(); + } + + [Fact] + public async Task CancelledLookup_DoesNotOpenCooldown_AndNextLookupHitsRedis() + { + var factory = new FakeRedisConnectionFactory(); + var service = CreateService(factory); + await service.StartAsync(CancellationToken.None); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.LastConnection!.Database.ExistsStarted = started; + factory.LastConnection.Database.BlockExists = block; + + var history = new VectorNNTPD.TransitService.History.HistoryDb( + service, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new VectorNNTPD.TransitService.History.HistoryWriteQueue()); + + using var cts = new CancellationTokenSource(); + var lookup = history.LookupAsync(""u8.ToArray(), cts.Token); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await started.Task.WaitAsync(safety.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => lookup.AsTask()); + Assert.False(service.IsUnavailable); + + factory.LastConnection.Database.BlockExists = null; + block.TrySetCanceled(); + Assert.Equal( + VectorNNTPD.TransitService.History.HistoryLookupResult.Unseen, + await history.LookupAsync(""u8.ToArray())); + Assert.False(service.IsUnavailable); + Assert.Equal(1, factory.LastConnection.Database.KeyExistsCount); + + await service.DisposeAsync(); + } + + [Fact] + public void RecoveryProbe_Alone_CanClearCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var healthyGrant)); + Assert.False(healthyGrant); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out var becameUnavailable, out _); + Assert.True(becameUnavailable); + Assert.True(circuit.IsUnavailable); + Assert.False(circuit.TryBegin(out _)); + + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + Assert.False(circuit.TryBegin(out var other)); + Assert.False(other); + + circuit.Complete(isRecoveryProbe: false, succeeded: true, out _, out var recoveredByOther); + Assert.False(recoveredByOther); + Assert.False(circuit.TryBegin(out _)); + + circuit.Complete(isRecoveryProbe: true, succeeded: true, out _, out var recoveredByProbe); + Assert.True(recoveredByProbe); + Assert.False(circuit.IsUnavailable); + Assert.True(circuit.TryBegin(out var after)); + Assert.False(after); + } + + [Fact] + public void RecoveryProbe_Failure_RestartsCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out _)); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out _, out _); + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + circuit.Complete(isRecoveryProbe: true, succeeded: false, out var becameUnavailable, out var recovered); + Assert.False(recovered); + Assert.False(becameUnavailable); + Assert.True(circuit.IsUnavailable); + Assert.False(circuit.TryBegin(out _)); + } + + [Fact] + public void AbandonProbe_DoesNotOpenCooldown() + { + var clock = new ManualTimeProvider(); + var circuit = new RedisAvailability(clock, TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out _)); + circuit.Complete(isRecoveryProbe: false, succeeded: false, out _, out _); + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(circuit.TryBegin(out var probe)); + Assert.True(probe); + circuit.Abandon(isRecoveryProbe: true); + Assert.False(circuit.IsUnavailable); + Assert.True(circuit.TryBegin(out var next)); + Assert.True(next); + } + + private static RedisService CreateService(FakeRedisConnectionFactory factory) => + new( + factory, + Options.Create(new RedisOptions { Host = ["127.0.0.1"], Port = 6379 }), + NullLogger.Instance); + + private sealed class ManualTimeProvider : TimeProvider + { + private DateTimeOffset _utc = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => _utc; + + public void Advance(TimeSpan delta) => _utc += delta; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/ApprovedHeaderParserTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/ApprovedHeaderParserTests.cs new file mode 100644 index 00000000..581315ec --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/ApprovedHeaderParserTests.cs @@ -0,0 +1,93 @@ +using System.Text; +using VectorNNTPD.TransitService.Session.Commands.Posting; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class ApprovedHeaderParserTests +{ + [Theory] + [InlineData("moderator@example.com", "moderator@example.com")] + [InlineData(" Moderator@example.com ", "Moderator@example.com")] + [InlineData("Moderator Name ", "moderator@example.com")] + public void TryParseMailboxList_ExtractsIdentity(string value, string expected) + { + Assert.True(ApprovedHeaderParser.TryParseMailboxList(Encoding.ASCII.GetBytes(value), out var identities, out _)); + Assert.Equal(expected, Assert.Single(identities)); + } + + [Theory] + [InlineData("1")] + [InlineData("true")] + [InlineData("")] + [InlineData("not-a-mailbox")] + [InlineData(",moderator@example.com")] + public void TryParseMailboxList_RejectsMalformed(string value) + { + Assert.False(ApprovedHeaderParser.TryParseMailboxList(Encoding.ASCII.GetBytes(value), out _, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidApproved, failure.Category); + } + + [Fact] + public void TryRead_NoApproved_ReturnsEmpty() + { + var article = Parse(Build()); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Empty(identities); + } + + [Fact] + public void TryRead_HeaderNameIsCaseInsensitive() + { + var article = Parse(Build("aPpRoVeD: moderator@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal("moderator@example.com", Assert.Single(identities)); + } + + [Fact] + public void TryRead_IdenticalDuplicates_Collapse() + { + var article = Parse(Build("Approved: moderator@example.com\r\nApproved: Moderator@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal("moderator@example.com", Assert.Single(identities)); + } + + [Fact] + public void TryRead_ConflictingDuplicates_KeepBoth() + { + var article = Parse(Build("Approved: a@example.com\r\nApproved: b@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal(["a@example.com", "b@example.com"], identities); + } + + [Fact] + public void TryRead_MailboxList_CollectsDistinctIdentities() + { + var article = Parse(Build("Approved: a@example.com, b@example.com\r\n")); + Assert.True(ApprovedHeaderParser.TryRead(article, out var identities, out _)); + Assert.Equal(["a@example.com", "b@example.com"], identities); + } + + [Fact] + public void IdentitiesEqual_IsCaseInsensitive() + { + Assert.True(ApprovedHeaderParser.IdentitiesEqual("Moderator@Example.COM", "moderator@example.com")); + Assert.False(ApprovedHeaderParser.IdentitiesEqual("a@example.com", "b@example.com")); + } + + private static ParsedPostArticle Parse(string article) + { + Assert.True(PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build(string extraHeaders = "") => + "Date: 25 Sep 2026 12:00:00 +0000\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: misc.test\r\n" + + "Subject: test\r\n" + + "Message-ID: \r\n" + + extraHeaders + + "\r\n" + + "body\r\n"; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs new file mode 100644 index 00000000..d1329be1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Authentication/NewsmasterNntpAuthenticationProviderTests.cs @@ -0,0 +1,65 @@ +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Session.Authentication; + +public sealed class NewsmasterNntpAuthenticationProviderTests +{ + [Fact] + public void Create_WithoutCredentials_IsDenyAll() + { + var provider = NewsmasterNntpAuthenticationProvider.Create(TestHostFactory.CreateValidOptions()); + Assert.Same(DenyAllNntpAuthenticationProvider.Instance, provider); + } + + [Fact] + public async Task Authenticate_MatchingCredentials_GrantsControlCancel() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("newsmaster", "unit-test-newsmaster-password"); + Assert.True(result.Succeeded); + Assert.NotNull(result.Authorization); + Assert.True(result.Authorization!.ControlCancelPermitted); + Assert.True(result.Authorization.PostingPermitted); + Assert.False(result.Authorization.AuthorizedTransit); + Assert.True(result.Authorization.With(isAuthenticated: true).ControlCancelPermitted); + } + + [Fact] + public async Task Authenticate_WrongPassword_FailsWithoutPrivileges() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("newsmaster", "wrong"); + Assert.False(result.Succeeded); + Assert.Null(result.Authorization); + } + + [Fact] + public async Task Authenticate_WrongUser_Fails() + { + var options = TestHostFactory.CreateValidOptions(); + options.NewsmasterUser = "newsmaster"; + options.NewsmasterPassword = "unit-test-newsmaster-password"; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + var result = await provider.AuthenticateAsync("other", "unit-test-newsmaster-password"); + Assert.False(result.Succeeded); + } + + [Fact] + public void Unauthenticated_DoesNotPermitControlCancel() + { + Assert.False(NntpAuthorization.Unauthenticated.ControlCancelPermitted); + Assert.False(NntpAuthorization.TrustedTransitPeer.ControlCancelPermitted); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/BenchItCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/BenchItCommandTests.cs new file mode 100644 index 00000000..d3b58c3a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/BenchItCommandTests.cs @@ -0,0 +1,211 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// Internal BENCHIT benchmark facility contracts (not a production protocol feature). +public sealed class BenchItCommandTests +{ + [Fact] + public void StaticPayload_IsExactTargetSize_AndWireIsReusable() + { + Assert.Equal(750 * 1024, BenchIt.TargetArticleContentBytes); + Assert.Equal(BenchIt.TargetArticleContentBytes, BenchIt.ArticleContentBytes); + Assert.True(BenchIt.WireResponseBytes > BenchIt.ArticleContentBytes); + Assert.True(BenchIt.WireResponse.Span.StartsWith("220 0 \r\n"u8)); + Assert.True(BenchIt.WireResponse.Span.EndsWith("\r\n.\r\n"u8)); + Assert.True(BenchIt.WireResponse.Span.IndexOf("..DOTSTUFF-CHECK\r\n"u8) >= 0); + } + + [Fact] + public void Registry_ResolvesBenchIt_AsPublic() + { + var parsed = NntpCommandTestParse.ParseCommand("BENCHIT"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.BenchIt, parsed.Verb); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(parsed.Verb, parsed.Qualifier)); + Assert.Equal("BENCHIT", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + } + + [Fact] + public async Task Handle_WritesPrecomputedWire_WithoutAuth() + { + await using var duplex = await BenchDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + // Consume concurrently: production pipe pause threshold would otherwise stall a 750 KiB write. + var readTask = duplex.ReadExactClientBytesAsync(BenchIt.WireResponseBytes); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "BENCHIT"); + var received = await readTask; + + Assert.Equal(BenchIt.WireResponse.ToArray(), received); + } + + [Fact] + public async Task Capabilities_DoesNotAdvertiseBenchIt() + { + await using var duplex = await BenchDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadUntilTerminatorAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + var text = Encoding.ASCII.GetString(await readTask); + Assert.DoesNotContain("BENCHIT", text, StringComparison.OrdinalIgnoreCase); + } + + private sealed class BenchDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new BenchDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadExactClientBytesAsync(int count) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var buffer = new byte[count]; + var filled = 0; + while (filled < count) + { + var result = await _serverToClient.Reader.ReadAsync(cts.Token).ConfigureAwait(false); + var readable = result.Buffer; + if (readable.Length == 0 && result.IsCompleted) + { + throw new EndOfStreamException("Pipe completed before expected BENCHIT wire bytes arrived."); + } + + var toCopy = (int)Math.Min(readable.Length, count - filled); + readable.Slice(0, toCopy).CopyTo(buffer.AsSpan(filled)); + filled += toCopy; + var consumed = readable.GetPosition(toCopy); + _serverToClient.Reader.AdvanceTo(consumed, readable.End); + } + + return buffer; + } + + public async Task ReadUntilTerminatorAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var ms = new System.IO.MemoryStream(); + while (true) + { + var result = await _serverToClient.Reader.ReadAsync(cts.Token); + foreach (var segment in result.Buffer) + { + ms.Write(segment.Span); + } + + _serverToClient.Reader.AdvanceTo(result.Buffer.End); + var bytes = ms.GetBuffer().AsSpan(0, (int)ms.Length); + if (bytes.IndexOf("\r\n.\r\n"u8) >= 0) + { + return ms.ToArray(); + } + + if (result.IsCompleted) + { + return ms.ToArray(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs new file mode 100644 index 00000000..46b83606 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/CatalogueNewsgroupPostingPolicyTests.cs @@ -0,0 +1,287 @@ +using System.Text; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class CatalogueNewsgroupPostingPolicyTests +{ + [Fact] + public void Allowed_IsAccepted() + { + var evaluation = Evaluate(["misc.test"], Status("misc.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Null(evaluation.Failure); + } + + [Fact] + public void Prohibited_IsRejected() + { + AssertRejected(["no.post"], "posting prohibited", Status("no.post", NewsgroupPostingStatus.Prohibited)); + } + + [Fact] + public void Moderated_IsRequiresModeration_ApprovedIsNotPolicyInput() + { + var policy = Policy(Status("mod.test", NewsgroupPostingStatus.Moderated)); + var evaluation = policy.Evaluate(["mod.test"]); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Null(evaluation.Failure); + Assert.Equal(["mod.test"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Closed_X_IsRejected() + { + AssertRejected(["closed.test"], "newsgroup closed", Status("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)); + } + + [Fact] + public void PeerOnly_J_IsRejected() + { + AssertRejected(["peer.only"], "local posting not accepted", Status("peer.only", NewsgroupPostingStatus.PeerOnly)); + } + + [Fact] + public void UnknownGroup_IsRejected() + { + AssertRejected(["unknown.group"], "unknown newsgroup", Status("misc.test", NewsgroupPostingStatus.Allowed)); + } + + [Fact] + public void Multiple_AllAllowed_IsAccepted() + { + var evaluation = Evaluate( + ["misc.test", "comp.test"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + } + + [Fact] + public void Multiple_OneUnknown_IsRejected() + { + AssertRejected( + ["misc.test", "unknown.group"], + "unknown newsgroup", + Status("misc.test", NewsgroupPostingStatus.Allowed)); + } + + [Fact] + public void Multiple_OneProhibited_IsRejected() + { + AssertRejected( + ["misc.test", "no.post"], + "posting prohibited", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("no.post", NewsgroupPostingStatus.Prohibited)); + } + + [Fact] + public void Multiple_OneModerated_IsRequiresModeration() + { + var evaluation = Evaluate( + ["misc.test", "mod.test"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("mod.test", NewsgroupPostingStatus.Moderated)); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Equal(["mod.test"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Multiple_TwoModerated_PreservesNewsgroupsOrder() + { + var evaluation = Evaluate( + ["mod.b", "misc.test", "mod.a"], + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("mod.a", NewsgroupPostingStatus.Moderated), + Status("mod.b", NewsgroupPostingStatus.Moderated)); + Assert.Equal(NewsgroupCatalogStatus.RequiresModeration, evaluation.Status); + Assert.Equal(["mod.b", "mod.a"], evaluation.ModeratedGroupNames); + } + + [Fact] + public void Multiple_OneClosed_IsRejected() + { + AssertRejected( + ["misc.test", "closed.test"], + "newsgroup closed", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("closed.test", NewsgroupPostingStatus.NoPostingOrPeerArticles)); + } + + [Fact] + public void Multiple_OnePeerOnly_IsRejected() + { + AssertRejected( + ["misc.test", "peer.only"], + "local posting not accepted", + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("peer.only", NewsgroupPostingStatus.PeerOnly)); + } + + [Fact] + public void MixedCase_MatchesSnapshotIgnoreCase() + { + var evaluation = Evaluate(["MISC.TEST"], Status("misc.test", NewsgroupPostingStatus.Allowed)); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + } + + [Fact] + public void CaseVariantDuplicateTokens_AreBothLookedUpOnOneSnapshot() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var evaluation = policy.Evaluate(["misc.test", "Misc.Test"]); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Equal(1, catalogue.CurrentReadCount); + } + + [Fact] + public void Evaluate_CapturesCurrentOnce() + { + var first = Snapshot( + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + var catalogue = new PublishOnReadCatalogue(first, NewsgroupSnapshot.Empty); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var evaluation = policy.Evaluate(["misc.test", "comp.test"]); + Assert.Equal(NewsgroupCatalogStatus.Accepted, evaluation.Status); + Assert.Equal(1, catalogue.Reads); + } + + [Fact] + public void EmptyGroupList_IsRejected() + { + var evaluation = Policy(Status("misc.test", NewsgroupPostingStatus.Allowed)) + .Evaluate([]); + Assert.Equal(NewsgroupCatalogStatus.Rejected, evaluation.Status); + Assert.True(evaluation.Failure.HasValue); + var emptyFailure = evaluation.Failure.GetValueOrDefault(); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, emptyFailure.Category); + } + + [Fact] + public void TryValidate_MalformedNewsgroups_DoesNotConsultCatalogue() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("misc.test,,comp.test")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_EmptyNewsgroups_DoesNotConsultCatalogue() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_WhitespaceSeparatedGroups_AreAcceptedWhenBothAllowed() + { + var policy = Policy( + Status("misc.test", NewsgroupPostingStatus.Allowed), + Status("comp.test", NewsgroupPostingStatus.Allowed)); + var parsed = Parse(Build("misc.test, comp.test")); + Assert.True(PostArticleValidator.TryValidate(parsed, Now, policy, out _)); + Assert.Equal(["misc.test", "comp.test"], parsed.Newsgroups); + } + + [Fact] + public void TryValidate_InvalidCharacters_AreRejectedBeforeLookup() + { + var catalogue = new StaticNewsgroupCatalogue(Snapshot(Status("misc.test", NewsgroupPostingStatus.Allowed))); + var policy = new CatalogueNewsgroupPostingPolicy(catalogue); + var parsed = Parse(Build("misc test")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.InvalidNewsgroups, failure.Category); + Assert.Equal(0, catalogue.CurrentReadCount); + } + + [Fact] + public void TryValidate_UnknownGroup_IsPolicyRejected() + { + var policy = Policy(Status("misc.test", NewsgroupPostingStatus.Allowed)); + var parsed = Parse(Build("unknown.group")); + Assert.False(PostArticleValidator.TryValidate(parsed, Now, policy, out var failure)); + Assert.Equal(PostingFailureCategory.PolicyRejected, failure.Category); + Assert.Equal("unknown newsgroup", failure.Detail); + } + + private static readonly DateTimeOffset Now = new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + + private static void AssertRejected(string[] groups, string detail, params NewsgroupDefinition[] definitions) + { + var evaluation = Evaluate(groups, definitions); + Assert.Equal(NewsgroupCatalogStatus.Rejected, evaluation.Status); + Assert.True(evaluation.Failure.HasValue); + var failure = evaluation.Failure.GetValueOrDefault(); + Assert.Equal(PostingFailureCategory.PolicyRejected, failure.Category); + Assert.Equal(detail, failure.Detail); + } + + private static NewsgroupPostingEvaluation Evaluate(string[] groups, params NewsgroupDefinition[] definitions) => + Policy(definitions).Evaluate(groups); + + private static CatalogueNewsgroupPostingPolicy Policy(params NewsgroupDefinition[] definitions) => + new(new StaticNewsgroupCatalogue(Snapshot(definitions))); + + private static NewsgroupSnapshot Snapshot(params NewsgroupDefinition[] definitions) => + NewsgroupSnapshot.Create(definitions); + + private static NewsgroupDefinition Status(string name, NewsgroupPostingStatus status) => + new(name, string.Empty, 2, 1, status); + + private static ParsedPostArticle Parse(string article) + { + Assert.True( + PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), + failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build(string newsgroups) + { + return + "Date: 25 Sep 2026 12:00:00 +0000\r\n" + + "From: poster@example.com\r\n" + + "Newsgroups: " + newsgroups + "\r\n" + + "Subject: test\r\n" + + "Message-ID: \r\n" + + "\r\n" + + "body\r\n"; + } + + private sealed class PublishOnReadCatalogue : INewsgroupCatalogue + { + private readonly StaticNewsgroupCatalogue _inner; + private readonly NewsgroupSnapshot _replacement; + + public PublishOnReadCatalogue(NewsgroupSnapshot first, NewsgroupSnapshot replacement) + { + _inner = new StaticNewsgroupCatalogue(first); + _replacement = replacement; + } + + public int Reads => _inner.CurrentReadCount; + + public NewsgroupSnapshot Current + { + get + { + var snapshot = _inner.Current; + _inner.Publish(_replacement); + return snapshot; + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/CheckCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/CheckCommandTests.cs new file mode 100644 index 00000000..f6087876 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/CheckCommandTests.cs @@ -0,0 +1,340 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// RFC 4644 CHECK stub authorization — must match TAKETHIS +/// ( / ). +/// +public sealed class CheckCommandTests +{ + private static readonly IPAddress TransitPeer = IPAddress.Parse("198.18.0.70"); + private static readonly IPAddress OtherPeer = IPAddress.Parse("198.18.0.71"); + + [Fact] + public void Catalog_CheckAndTakeThis_ShareRequiresTransit() + { + var (check, checkLine) = NntpCommandTestParse.Parse("CHECK "); + var takeThis = NntpCommandTestParse.ParseCommand("TAKETHIS "); + Assert.True(check.IsValid); + Assert.True(takeThis.IsValid); + Assert.Equal(NntpVerb.Check, check.Verb); + Assert.Equal(NntpVerb.TakeThis, takeThis.Verb); + Assert.Equal("CHECK", DefaultNntpCommandCatalog.DisplayName(check.Verb, check.Qualifier)); + Assert.Equal("TAKETHIS", DefaultNntpCommandCatalog.DisplayName(takeThis.Verb, takeThis.Qualifier)); + Assert.Equal(NntpCommandAccess.RequiresTransit, DefaultNntpCommandCatalog.GetAccess(check.Verb, check.Qualifier)); + Assert.Equal( + DefaultNntpCommandCatalog.GetAccess(takeThis.Verb, takeThis.Qualifier), + DefaultNntpCommandCatalog.GetAccess(check.Verb, check.Qualifier)); + Assert.Equal("", Encoding.ASCII.GetString(check.ArgumentSpan(checkLine))); + } + + [Fact] + public async Task TransitPeer_Check_Returns238_WithoutUserAuthentication() + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + await using var duplex = await CheckDuplex.CreateAsync(TransitPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientLineAsync("CHECK " + id); + Assert.Equal("238 " + id + " send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.False(session.Authorization.IsAuthenticated); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_CheckDoesNotRequireModeStream() + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + await using var duplex = await CheckDuplex.CreateAsync(TransitPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_CheckThenTakeThis_SameAuthorization() + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await CheckDuplex.CreateAsync(TransitPeer); + var session = duplex.CreateSession(peers, queue); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientAsync( + "TAKETHIS \r\n" + CanonicalArticleText.Destuffed("") + ".\r\n"); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NonAuthorizedPeer_CheckAndTakeThis_BothReturn480() + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + await using var duplex = await CheckDuplex.CreateAsync(OtherPeer); + var session = duplex.CreateSession(peers); + Assert.False(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task UnauthenticatedDefault_CheckAndTakeThis_BothReturn480() + { + await using var duplex = await CheckDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AuthenticatedWithoutTransit_CheckAndTakeThis_BothReturn502() + { + await using var duplex = await CheckDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_MissingMessageId_Returns501AfterAuthorization() + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + await using var duplex = await CheckDuplex.CreateAsync(TransitPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + /// + /// Syntax is rejected before authorization gates. Old string parser treated + /// CHECK as a valid command and returned 480; the byte parser is the + /// syntactic boundary, so missing message-id is 501. + /// + [Fact] + public async Task NonAuthorizedPeer_MissingMessageId_Returns501SyntaxError() + { + await using var duplex = await CheckDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static void AssertNamedTransitPeer(NntpSession session) + { + Assert.False(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedTransit); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + Assert.NotNull(session.Authorization.TransitPeerPolicy); + Assert.Equal(TransitPeer, session.ClientAddress); + } + + private sealed class CheckDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly IPAddress _clientAddress; + + private CheckDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress? clientAddress = null) => + Task.FromResult(new CheckDuplex(clientAddress ?? IPAddress.Loopback)); + + public NntpSession CreateSession( + ITransitPeerAuthorization? peers = null, + IArticleIngestionQueue? queue = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientAddress, 40000))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTcpTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTcpTests.cs new file mode 100644 index 00000000..b7288ba7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTcpTests.cs @@ -0,0 +1,103 @@ +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Networking.Transport; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// CHECK over the production accept/transport path ( + +/// ), not PipelineTestConnection. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class CheckPipelineTcpTests +{ + [Fact] + public async Task Tcp_PipelinedAndSegmentedChecks_PreserveOrderAndCrlf() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + _ = await history.LookupAsync(""u8.ToArray()); + redis.Database.KeyExistsCount = 0; + + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var session = new NntpSession( + server, + NullLogger.Instance, + historyDb: history); + session.SetAuthorization(NntpAuthorization.TrustedTransitPeer); + var run = session.RunAsync(); + var rx = new SocketLineBuffer(); + + var greeting = await rx.ReadLineAsync(client); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + + await client.SendAsync("CHECK \r\nCHECK \r\nCHECK \r\n"u8.ToArray()); + + Assert.Equal("238 send article to be transferred", await rx.ReadLineAsync(client)); + Assert.Equal("438 ", await rx.ReadLineAsync(client)); + Assert.Equal("238 send article to be transferred", await rx.ReadLineAsync(client)); + + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await rx.ReadLineAsync(client)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + client.Shutdown(SocketShutdown.Both); + Assert.Equal(0, session.Pipeline!.Occupied); + Assert.True(session.Pipeline.PeakOccupied <= CheckPipeline.Depth); + } + + private sealed class SocketLineBuffer + { + private readonly List _bytes = []; + + public async Task ReadLineAsync(Socket socket) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + if (TryReadLine(out var line)) + { + return line; + } + + var scratch = new byte[512]; + var n = await socket.ReceiveAsync(scratch, safety.Token); + Assert.True(n > 0, "Socket closed while reading CHECK response."); + _bytes.AddRange(scratch.AsSpan(0, n)); + } + } + + private bool TryReadLine(out string line) + { + line = string.Empty; + for (var i = 0; i + 1 < _bytes.Count; i++) + { + if (_bytes[i] != (byte)'\r' || _bytes[i + 1] != (byte)'\n') + { + continue; + } + + line = Encoding.ASCII.GetString(_bytes.GetRange(0, i).ToArray()); + _bytes.RemoveRange(0, i + 2); + return true; + } + + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTests.cs new file mode 100644 index 00000000..6e4f6886 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/CheckPipelineTests.cs @@ -0,0 +1,849 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class CheckPipelineTests +{ + private static readonly IPAddress TransitPeer = IPAddress.Parse("198.18.0.70"); + private static readonly byte[] IdA = ""u8.ToArray(); + private static readonly byte[] IdB = ""u8.ToArray(); + private static readonly byte[] IdC = ""u8.ToArray(); + + [Fact] + public async Task PipelinedChecks_AreInFlight_BeforeRedisCompletes() + { + var redis = new FakeRedisService(); + redis.Database.BlockExists = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.NotifyExistsStartedAt = 3; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + "CHECK \r\nCHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(3, redis.Database.ExistsStartedCount); + Assert.Equal(0, redis.Database.KeyExistsCount); + Assert.True(session.Pipeline!.Occupied >= 1); + + redis.Database.BlockExists.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CompletionOrderBac_EmitsAbc() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var blockC = BlockKey(redis, IdC); + var history = CreateHistory(redis); + _ = await history.LookupAsync(IdB); + redis.Database.KeyExistsCount = 0; + // LookupAsync() already counted one Redis start. Threshold 3 is that warmup plus CHECK and CHECK . + redis.Database.NotifyExistsStartedAt = 3; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + "CHECK \r\nCHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.True(session.Pipeline!.Occupied >= 2); + + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + blockC.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task LocalHitBehindDelayedRedis_DoesNotEmitEarly() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + _ = await history.LookupAsync(IdB); + redis.Database.KeyExistsCount = 0; + + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (session.Pipeline is null || session.Pipeline.Occupied < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + var early = duplex.TryReadClientLine(); + Assert.Null(early); + + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task PartialCommandSegmentation_CompletesAfterSecondWrite() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\n"); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task StreamScratchReuse_DoesNotCorruptEarlierMessageId() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (session.Pipeline is null || session.Pipeline.Occupied < 1) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await duplex.WriteClientLineAsync("CHECK "); + while (session.Pipeline.Occupied < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ReaderScratchReuse_DoesNotCorruptEarlierMessageId() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + session.SetMode(NntpSessionMode.Reader); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (session.Pipeline is null || session.Pipeline.Occupied < 1) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + await duplex.WriteClientLineAsync("CHECK "); + while (session.Pipeline.Occupied < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CheckThenTakeThis_DrainsChecksFirst() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history, queue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\nCHECK \r\n"); + await duplex.WriteClientAsync( + "TAKETHIS \r\n" + CanonicalArticleText.Destuffed("") + ".\r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.ExistsStartedCount < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Null(duplex.TryReadClientLine()); + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task WindowFull_StopsRxUntilCapacityFrees() + { + var redis = new FakeRedisService(); + redis.Database.BlockExists = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.NotifyExistsStartedAt = 16; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var payload = new StringBuilder(); + for (var i = 1; i <= 16; i++) + { + payload.Append(System.Globalization.CultureInfo.InvariantCulture, $"CHECK \r\n"); + } + + await duplex.WriteClientAsync(payload.ToString()); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(16, redis.Database.ExistsStartedCount); + Assert.Equal(16, session.Pipeline!.Occupied); + + redis.Database.NotifyExistsStartedAt = 17; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await duplex.WriteClientLineAsync("CHECK "); + await Task.Yield(); + Assert.Equal(16, redis.Database.ExistsStartedCount); + Assert.False(redis.Database.ExistsReached.Task.IsCompleted); + + redis.Database.BlockExists.TrySetResult(); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(17, redis.Database.ExistsStartedCount); + + for (var i = 1; i <= 17; i++) + { + var line = await duplex.ReadClientLineAsync(); + Assert.StartsWith("238 + { + enqueueReached.TrySetResult(); + await enqueue.Task; + }; + + await duplex.WriteClientLineAsync("CHECK "); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (session.Pipeline.Occupied < 1) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Null(duplex.TryReadClientLine()); + blockA.TrySetResult(); + await enqueueReached.Task.WaitAsync(safety.Token); + Assert.Equal(1, session.Pipeline.Occupied); + Assert.Null(duplex.TryReadClientLine()); + + await duplex.WriteClientLineAsync("CHECK "); + while (session.Pipeline.Occupied < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Null(duplex.TryReadClientLine()); + enqueue.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task SlowTx_FastRedis_SeventeenthWaitsOnOccupancy() + { + var redis = new FakeRedisService(); + redis.Database.BlockExists = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.NotifyExistsStartedAt = 16; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var payload = new StringBuilder(); + for (var i = 1; i <= 16; i++) + { + payload.Append(System.Globalization.CultureInfo.InvariantCulture, $"CHECK \r\n"); + } + + await duplex.WriteClientAsync(payload.ToString()); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(16, session.Pipeline!.Occupied); + Assert.Equal(16, redis.Database.ExistsStartedCount); + + var enqueue = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var enqueueReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.Pipeline.BeforeEnqueueProbe = async () => + { + enqueueReached.TrySetResult(); + await enqueue.Task; + }; + + redis.Database.BlockExists.TrySetResult(); + await enqueueReached.Task.WaitAsync(safety.Token); + Assert.Equal(16, session.Pipeline.Occupied); + Assert.True(session.Pipeline.InFlightCompletions <= CheckPipeline.Depth); + Assert.Null(duplex.TryReadClientLine()); + + redis.Database.NotifyExistsStartedAt = 17; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await duplex.WriteClientLineAsync("CHECK "); + await Task.Yield(); + Assert.Equal(16, redis.Database.ExistsStartedCount); + Assert.Equal(16, session.Pipeline.Occupied); + Assert.False(redis.Database.ExistsReached.Task.IsCompleted); + + enqueue.TrySetResult(); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(17, redis.Database.ExistsStartedCount); + + for (var i = 1; i <= 17; i++) + { + var line = await duplex.ReadClientLineAsync(); + Assert.StartsWith("238 \r\n"); + } + + await duplex.WriteClientAsync(payload.ToString()); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + Assert.Equal(16, session.Pipeline!.Occupied); + + sessionCts.Cancel(); + redis.Database.BlockExists.TrySetCanceled(); + await run.WaitAsync(safety.Token); + Assert.Equal(0, session.Pipeline.Occupied); + Assert.Equal(0, session.Pipeline.InFlightCompletions); + Assert.False(redis.IsUnavailable); + Assert.Null(duplex.TryReadClientLine()); + } + + [Fact] + public async Task FortyPipelinedChecks_DoNotRetainCompletions() + { + var redis = new FakeRedisService(); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var payload = new StringBuilder(); + for (var i = 1; i <= 40; i++) + { + payload.Append(System.Globalization.CultureInfo.InvariantCulture, $"CHECK \r\n"); + } + + await duplex.WriteClientAsync(payload.ToString()); + for (var i = 1; i <= 40; i++) + { + var line = await duplex.ReadClientLineAsync(); + Assert.StartsWith("238 \r\nCHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + + sessionCts.Cancel(); + redis.Database.BlockExists.TrySetCanceled(); + await run.WaitAsync(safety.Token); + Assert.False(redis.IsUnavailable); + Assert.Null(duplex.TryReadClientLine()); + + redis.Database.BlockExists = null; + await using var duplex2 = new CheckPipelineDuplex(); + var session2 = duplex2.CreateSession(history); + var run2 = session2.RunAsync(); + _ = await duplex2.ReadClientLineAsync(); + var existsBefore = redis.Database.KeyExistsCount; + await duplex2.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex2.ReadClientLineAsync()); + Assert.Equal(existsBefore + 1, redis.Database.KeyExistsCount); + await duplex2.WriteClientLineAsync("QUIT"); + _ = await duplex2.ReadClientLineAsync(); + await run2; + } + + [Fact] + public async Task RedisFailure_UnderPipeline_OpensCooldown_NoFalse438() + { + var redis = new FakeRedisService(); + redis.Database.BlockExists = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.NotifyExistsStartedAt = 3; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + "CHECK \r\nCHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + while (session.Pipeline is null || session.Pipeline.Occupied < 3) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + redis.Database.ExistsException = new RedisUnavailableException("down"); + redis.Database.BlockExists.TrySetResult(); + + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.True(redis.IsUnavailable); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(IdA))); + var exists = redis.Database.KeyExistsCount; + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.Equal(exists, redis.Database.KeyExistsCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task OneLookupCanceled_DoesNotOpenCooldown_OthersContinue() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var blockB = BlockKey(redis, IdB); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.ExistsStartedCount < 2) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + blockA.TrySetCanceled(); + blockB.TrySetResult(); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.False(redis.IsUnavailable); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task SameMessageId_ConcurrentChecks_RemainAdvisory() + { + var redis = new FakeRedisService(); + redis.Database.BlockExists = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.NotifyExistsStartedAt = 2; + redis.Database.ExistsReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\nCHECK \r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await redis.Database.ExistsReached.Task.WaitAsync(safety.Token); + redis.Database.BlockExists.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(IdA))); + Assert.Equal(0, history.Writes.Count); + Assert.Equal(0, redis.Database.SetCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void Depth_IsArchitecturalSixteen_AndNotConfigurable() + { + Assert.Equal(16, CheckPipeline.Depth); + Assert.Null(typeof(NntpdOptions).GetProperty("CheckPipelineDepth")); + Assert.Null(typeof(NntpSession).GetProperty("CheckPipelineDepth")); + } + + [Theory] + [InlineData("DATE", "111 ")] + [InlineData("AUTHINFO USER peer", "381 ")] + [InlineData("MODE READER", "502 Command not available on this server")] + [InlineData("MODE STREAM", "203 ")] + [InlineData("STARTTLS", "502 TLS provider unavailable")] + [InlineData("QUIT", "205 Connection closing")] + public async Task NonCheckBarrier_EmitsCheckBeforeFollowingCommand(string command, string followingPrefix) + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\n" + command + "\r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.ExistsStartedCount < 1) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Null(duplex.TryReadClientLine()); + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + var following = await duplex.ReadClientLineAsync(); + Assert.StartsWith(followingPrefix, following, StringComparison.Ordinal); + if (!command.StartsWith("QUIT", StringComparison.Ordinal)) + { + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + } + + await run; + } + + [Fact] + public async Task CompressBarrier_EmitsCheckBefore206() + { + var redis = new FakeRedisService(); + var blockA = BlockKey(redis, IdA); + var history = CreateHistory(redis); + await using var duplex = new CheckPipelineDuplex(); + var session = duplex.CreateSession(history); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("CHECK \r\nCOMPRESS DEFLATE\r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (redis.Database.ExistsStartedCount < 1) + { + safety.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.Null(duplex.TryReadClientLine()); + blockA.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("206 Compression active", await duplex.ReadClientLineAsync()); + await run.WaitAsync(safety.Token); + } + + private static HistoryDb CreateHistory(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private static TaskCompletionSource BlockKey(FakeRedisService redis, ReadOnlySpan messageId) + { + var key = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(messageId)); + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.BlockExistsByKey[Convert.ToHexString(key)] = block; + return block; + } + + private sealed class CheckPipelineDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + IHistoryDb history, + IArticleIngestionQueue? ingestion = null) + { + var peers = TransitTestPeers.ForAllowFrom(TransitPeer); + var connection = new PipelineTestConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(TransitPeer, 40000))); + return new NntpSession( + connection, + NullLogger.Instance, + transitPeerAuthorization: peers, + historyDb: history, + articleIngestion: ingestion); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public string? TryReadClientLine() + { + if (!_serverToClient.Reader.TryRead(out var result)) + { + return null; + } + + var buffer = result.Buffer; + if (!NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + _serverToClient.Reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + return null; + } + + var text = Encoding.ASCII.GetString(System.Buffers.BuffersExtensions.ToArray(in lineBytes)); + _serverToClient.Reader.AdvanceTo(buffer.Start, buffer.Start); + return text; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipelineTestConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs new file mode 100644 index 00000000..0677ffca --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleBodyTxIntegrationTests.cs @@ -0,0 +1,451 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// +/// ARTICLE/BODY wire shapes on the shared TX path from externally supplied destuffed bytes. +/// Command handlers return RFC lookup-failure codes; storage/catalog is still out of scope. +/// +public sealed class NntpArticleBodyTxIntegrationTests +{ + private static readonly string IncomingRoot = @"C:\Temp\Incoming"; + + [Fact] + public async Task Article_Success_FramingAndRestuff() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\nhello\r\n"); + await harness.Writer.WriteCustomerArticleAsync(stored, ""); + var wire = await harness.DrainAllAsync(); + Assert.Equal(BuildArticleOracle(stored, "", 0), wire); + Assert.StartsWith("220 0 \r\n", Encoding.ASCII.GetString(wire), StringComparison.Ordinal); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(wire), StringComparison.Ordinal); + } + + [Fact] + public async Task Article_WithArticleNumber_InStatusLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: x\r\n\r\nbody\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(stored, "", articleNumber: 3000234); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.StartsWith("220 3000234 \r\n", wire, StringComparison.Ordinal); + } + + [Fact] + public async Task Head_Success_HeadersOnly() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\nFrom: a@b\r\n\r\nbody-line\r\n"); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out var headers, out _)); + await harness.Writer.WriteCustomerHeadAsync(headers.ToArray(), ""); + var text = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.StartsWith("221 0 \r\n", text, StringComparison.Ordinal); + Assert.Contains("Subject: t\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("body-line", text, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task Body_Success_BodyOnly_NoHeaders() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\nFrom: a@b\r\n\r\nbody-line\r\n"); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + var wire = await harness.DrainAllAsync(); + var expected = BuildBodyOracle("body-line\r\n"u8, "", 0); + Assert.Equal(expected, wire); + var text = Encoding.ASCII.GetString(wire); + Assert.StartsWith("222 0 \r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("Subject:", text, StringComparison.Ordinal); + Assert.EndsWith(".\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task Body_EmptyBody_WhenNoBlankLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: only-headers\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(ReadOnlySpan.Empty, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_EmptyBody_AfterBlankLine() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = "Subject: t\r\n\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(ReadOnlySpan.Empty, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_DotLeadingLines_Restuffed() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\n.example\r\n..two\r\n"); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + Assert.Equal(BuildBodyOracle(".example\r\n..two\r\n"u8, "", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task Body_FromSeparatedBody_MatchesSplitPath() + { + await using var a = await TxHarness.CreateAsync(); + await using var b = await TxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes("H: 1\r\n\r\n.body\r\n"); + await a.Writer.WriteCustomerBodyAsync(stored, "", 42); + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out _, out var body)); + await b.Writer.WriteCustomerBodyFromBodyAsync(body.ToArray(), "", 42); + Assert.Equal(await a.DrainAllAsync(), await b.DrainAllAsync()); + } + + [Fact] + public async Task ArticleAndBody_PreserveMessageId() + { + await using var harness = await TxHarness.CreateAsync(); + const string mid = ""; + var stored = "Subject: t\r\n\r\nx\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(stored, mid); + await harness.Writer.WriteCustomerBodyAsync(stored, mid, 7); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.Contains("220 0 " + mid, wire, StringComparison.Ordinal); + Assert.Contains("222 7 " + mid, wire, StringComparison.Ordinal); + } + + [Theory] + [InlineData(NntpArticleTxChunkBudget.DefaultBytes)] + [InlineData(128 * 1024)] + [InlineData(NntpArticleTxChunkBudget.MaxBytes)] + public async Task Article_Large_ChunkBudgets_ByteIdentical(int chunkBytes) + { + await using var harness = await TxHarness.CreateAsync(); + var stored = BuildStoredArticle(740 * 1024, leadingDotEvery: 40); + var framing = NntpArticleTxFraming.CustomerArticle("", 0); + await harness.Writer.WriteArticleAsync(stored, framing, chunkBytes); + var actual = await harness.DrainAllAsync(); + Assert.Equal(BuildArticleOracle(stored, "", 0), actual); + Assert.True(harness.Writer.ChannelEnqueueCount < 100); + Assert.True(harness.Writer.ChannelEnqueueCount >= 2); + } + + [Fact] + public async Task Body_Large_UsesChunkedPath_NotPerLine() + { + await using var harness = await TxHarness.CreateAsync(); + var body = BuildStoredArticle(750 * 1024, leadingDotEvery: 50); + var stored = Encoding.ASCII.GetBytes("Subject: t\r\n\r\n").Concat(body).ToArray(); + await harness.Writer.WriteCustomerBodyAsync(stored, ""); + _ = await harness.DrainAllAsync(); + // ~750 KiB body → O(10) chunks, not tens of thousands of lines + Assert.InRange(harness.Writer.ChannelEnqueueCount, 2, 64); + Assert.InRange(harness.Writer.PipeFlushCount, 2, 128); + } + + [Fact] + public async Task Article_TwoMiB_IfSynthetic() + { + await using var harness = await TxHarness.CreateAsync(); + var stored = BuildStoredArticle(2 * 1024 * 1024, leadingDotEvery: 80); + await harness.Writer.WriteCustomerArticleAsync(stored, "<2m@id>"); + Assert.Equal(BuildArticleOracle(stored, "<2m@id>", 0), await harness.DrainAllAsync()); + } + + [Fact] + public async Task ConsecutiveArticles_RemainOrdered() + { + await using var harness = await TxHarness.CreateAsync(); + var a = "Subject: a\r\n\r\nA\r\n"u8.ToArray(); + var b = "Subject: b\r\n\r\n.B\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerArticleAsync(a, "", 1); + await harness.Writer.WriteCustomerArticleAsync(b, "", 2); + var expected = Concat( + BuildArticleOracle(a, "", 1), + BuildArticleOracle(b, "", 2)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task ConsecutiveBodies_RemainOrdered() + { + await using var harness = await TxHarness.CreateAsync(); + var a = "H: a\r\n\r\na\r\n"u8.ToArray(); + var b = "H: b\r\n\r\n.b\r\n"u8.ToArray(); + await harness.Writer.WriteCustomerBodyAsync(a, ""); + await harness.Writer.WriteCustomerBodyAsync(b, "", 9); + var expected = Concat( + BuildBodyOracle("a\r\n"u8, "", 0), + BuildBodyOracle(".b\r\n"u8, "", 9)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task Cancellation_DuringLargeArticle() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 4 * 1024, + resumeWriterThreshold: 2 * 1024, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var stored = BuildStoredArticle(300 * 1024, 0); + using var cts = new CancellationTokenSource(); + var task = writer.WriteCustomerArticleAsync(stored, "", cancellationToken: cts.Token); + await Task.Delay(20); + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => await task); + pipe.Reader.CancelPendingRead(); + await pipe.Reader.CompleteAsync(); + await pipe.Writer.CompleteAsync(); + } + + [Fact] + public async Task Corpus_ArticleAndBody_IfPresent() + { + if (!Directory.Exists(IncomingRoot)) + { + return; + } + + var file = Directory.EnumerateFiles(IncomingRoot, "*", SearchOption.AllDirectories) + .Select(static f => new FileInfo(f)) + .Where(static f => f.Exists && f.Length is > 1024 and < 2_000_000) + .OrderBy(static f => Math.Abs(f.Length - 750 * 1024)) + .Select(static f => f.FullName) + .FirstOrDefault(); + if (file is null) + { + return; + } + + var stored = await File.ReadAllBytesAsync(file); + await using var articleHarness = await TxHarness.CreateAsync(); + await articleHarness.Writer.WriteCustomerArticleAsync(stored, ""); + Assert.Equal(BuildArticleOracle(stored, "", 0), await articleHarness.DrainAllAsync()); + + await using var bodyHarness = await TxHarness.CreateAsync(); + await bodyHarness.Writer.WriteCustomerBodyAsync(stored, ""); + ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out _, out var body); + Assert.Equal(BuildBodyOracle(body, "", 0), await bodyHarness.DrainAllAsync()); + } + + [Fact] + public void ArticleHandlers_WireMessageIdThroughWriterHelpers() + { + var src = File.ReadAllText( + Path.Join( + FindRepoRoot(), + "src", + "VectorNNTPD.TransitService", + "Session", + "Commands", + "Article.cs")); + Assert.DoesNotContain("NntpCommandNotImplemented", src, StringComparison.Ordinal); + Assert.Contains("NoArticleWithMessageId", src, StringComparison.Ordinal); + Assert.Contains("NoArticleWithNumber", src, StringComparison.Ordinal); + Assert.Contains("NoNewsgroupSelected", src, StringComparison.Ordinal); + Assert.Contains("CurrentArticleNumberInvalid", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerArticleAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerHeadAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerBodyFromBodyAsync", src, StringComparison.Ordinal); + Assert.Contains("WriteCustomerStatAsync", src, StringComparison.Ordinal); + Assert.Contains("IVatpArticleClient", src, StringComparison.Ordinal); + Assert.Contains("InboundArticleProducer.BackFiller", src, StringComparison.Ordinal); + } + + [Fact] + public void SplitHeadersAndBody_FindsBlankLine() + { + var stored = "A: 1\r\nB: 2\r\n\r\n.body\r\n"u8; + Assert.True(ArticleWireReconstructor.TrySplitHeadersAndBody(stored, out var headers, out var body)); + Assert.Equal("A: 1\r\nB: 2\r\n\r\n"u8.ToArray(), headers.ToArray()); + Assert.Equal(".body\r\n"u8.ToArray(), body.ToArray()); + } + + private static byte[] BuildArticleOracle(ReadOnlySpan stored, string mid, long number) + { + var framing = NntpArticleTxFraming.CustomerArticle(mid, number); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(stored); + return Concat(header, body); + } + + private static byte[] BuildBodyOracle(ReadOnlySpan storedBody, string mid, long number) + { + var framing = NntpArticleTxFraming.CustomerBody(mid, number); + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(storedBody); + return Concat(header, body); + } + + private static byte[] Concat(params byte[][] parts) + { + var len = parts.Sum(static p => p.Length); + var result = new byte[len]; + var o = 0; + foreach (var p in parts) + { + p.CopyTo(result.AsSpan(o)); + o += p.Length; + } + + return result; + } + + private static byte[] Concat(byte[] a, byte[] b) + { + var result = new byte[a.Length + b.Length]; + a.CopyTo(result.AsSpan()); + b.CopyTo(result.AsSpan(a.Length)); + return result; + } + + private static byte[] BuildStoredArticle(int sizeHint, int leadingDotEvery) + { + using var ms = new MemoryStream(sizeHint + 256); + ms.Write("Subject: synthetic\r\n\r\n"u8); + var n = 0; + while (ms.Length < sizeHint) + { + if (leadingDotEvery > 0 && n % leadingDotEvery == 0) + { + ms.Write("."u8); + } + + ms.Write(Encoding.ASCII.GetBytes($"line-{n}-xxxxxxxx\r\n")); + n++; + } + + return ms.ToArray(); + } + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not locate repository root."); + } + + private sealed class TxHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private readonly MemoryStream _capture = new(); + private readonly CancellationTokenSource _cts = new(); + private readonly Task _drainTask; + private int _disposed; + + private TxHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + _drainTask = DrainLoopAsync(_cts.Token); + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new TxHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task DrainAllAsync() + { + Interlocked.Exchange(ref _disposed, 1); + await Writer.DisposeAsync().ConfigureAwait(false); + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + await _drainTask.ConfigureAwait(false); + lock (_capture) + { + return _capture.ToArray(); + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + await Writer.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + await _cts.CancelAsync().ConfigureAwait(false); + try + { + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + try + { + await _drainTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + _cts.Dispose(); + } + + private async Task DrainLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var result = await _pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + foreach (var segment in buffer) + { + lock (_capture) + { + _capture.Write(segment.Span); + } + } + + _pipe.Reader.AdvanceTo(buffer.End); + if (result.IsCompleted) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + await _pipe.Reader.CompleteAsync().ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleTxWriterTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleTxWriterTests.cs new file mode 100644 index 00000000..17f607a6 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpArticleTxWriterTests.cs @@ -0,0 +1,608 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; +using Xunit.Abstractions; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// Phase 1 shared article TX primitive — correctness, ownership, cancellation. +public sealed class NntpArticleTxWriterTests +{ + private readonly ITestOutputHelper _output; + + public NntpArticleTxWriterTests(ITestOutputHelper output) => _output = output; + + private static readonly string IncomingRoot = @"C:\Temp\Incoming"; + + [Fact] + public async Task WriteArticle_Empty_EmitsTerminatorOnlyAfterHeader() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, ReadOnlySpan.Empty); + + await harness.Writer.WriteArticleAsync(ReadOnlyMemory.Empty, framing); + + Assert.Equal(oracle, await harness.DrainAllAsync()); + Assert.Equal(1, harness.Writer.ChannelEnqueueCount); // header+terminator fit one 64KiB chunk + } + + [Fact] + public async Task WriteArticle_Small_CustomerFraming() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "hello\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, stored); + + await harness.Writer.WriteArticleAsync(stored, framing); + + Assert.Equal(oracle, await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_PeerTakeThisFraming() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "body\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.PeerTakeThis(""); + var oracle = BuildOracle(framing, stored); + + await harness.Writer.WriteArticleAsync(stored, framing); + + var wire = await harness.DrainAllAsync(); + Assert.Equal(oracle, wire); + Assert.StartsWith("TAKETHIS \r\n", Encoding.ASCII.GetString(wire)); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(wire)); + } + + [Fact] + public async Task WriteArticle_LeadingDot_Restuffed() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = ".example\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_MultipleLeadingDots() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = "..example\r\n.\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_TerminatorPresentAndLoneDotContentDoesNotEndEarly() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = ".\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var wire = BuildOracle(framing, stored); + await harness.Writer.WriteArticleAsync(stored, framing); + var actual = await harness.DrainAllAsync(); + Assert.Equal(wire, actual); + + var header = framing.BuildHeaderBytes(); + var bodyWire = actual.AsMemory(header.Length); + var result = await NntpMultilineDataReader.ReadArticleAsync( + PipeReader.Create(new ReadOnlySequence(bodyWire)), + 1024, + CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(stored, result.Payload.ToArray()); + } + + [Fact] + public async Task WriteArticle_PreservesCrlf_AndBareLfInsideLine() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + // Bare LF is not a line separator for restuff (CRLF-based); preserved as content. + var stored = "Path: host\nnot-for-mail\r\nSubject: x\r\n\r\nbody\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Theory] + [InlineData(8)] + [InlineData(16)] + [InlineData(64)] + [InlineData(100)] + public async Task WriteArticle_ChunkBoundary_InsideBody(int chunkBytes) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = Encoding.ASCII.GetBytes(new string('A', 250) + "\r\n" + new string('B', 250) + "\r\n"); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleForTestsAsync(stored, framing, chunkBytes); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + Assert.True(harness.Writer.ChannelEnqueueCount >= 2); + } + + [Fact] + public async Task WriteArticle_ChunkBoundary_OnCrlfAndDotLine() + { + var stored = Encoding.ASCII.GetBytes( + new string('x', 20) + "\r\n" + + ".dotted\r\n" + + new string('y', 20) + "\r\n"); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var oracle = BuildOracle(framing, stored); + foreach (var chunk in new[] { 7, 11, 17, 23, 32 }) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + await harness.Writer.WriteArticleForTestsAsync(stored, framing, chunk); + Assert.Equal(oracle, await harness.DrainAllAsync()); + } + } + + [Theory] + [InlineData(NntpArticleTxChunkBudget.DefaultBytes)] + [InlineData(128 * 1024)] + [InlineData(NntpArticleTxChunkBudget.MaxBytes)] + public async Task WriteArticle_ProductionChunkSizes_TypicalSizedArticle(int chunkBytes) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = BuildStoredArticleBytes(sizeHint: 740 * 1024, leadingDotEvery: 50); + var framing = NntpArticleTxFraming.CustomerArticle("<740k@test>"); + await harness.Writer.WriteArticleAsync(stored, framing, chunkBytes); + var actual = await harness.DrainAllAsync(); + Assert.Equal(BuildOracle(framing, stored), actual); + var expectedChunks = (int)Math.Ceiling(actual.Length / (double)chunkBytes); + Assert.Equal(expectedChunks, harness.Writer.ChannelEnqueueCount); + } + + [Fact] + public async Task WriteArticle_TwoMiB_Article() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = BuildStoredArticleBytes(sizeHint: 2 * 1024 * 1024, leadingDotEvery: 100); + var framing = NntpArticleTxFraming.CustomerArticle("<2m@test>"); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_MultipleConsecutive_PreserveOrder() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var a = "aaa\r\n"u8.ToArray(); + var b = ".bbb\r\n"u8.ToArray(); + var c = "ccc\r\n"u8.ToArray(); + var fa = NntpArticleTxFraming.CustomerArticle(""); + var fb = NntpArticleTxFraming.PeerTakeThis(""); + var fc = NntpArticleTxFraming.CustomerArticle(""); + + await harness.Writer.WriteArticleAsync(a, fa); + await harness.Writer.WriteArticleAsync(b, fb); + await harness.Writer.WriteArticleAsync(c, fc); + + var expected = Concat(BuildOracle(fa, a), BuildOracle(fb, b), BuildOracle(fc, c)); + Assert.Equal(expected, await harness.DrainAllAsync()); + } + + [Fact] + public async Task WriteArticle_RejectsChunkOutsideProductionRange() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAsync(async () => + await harness.Writer.WriteArticleAsync("a\r\n"u8.ToArray(), framing, 1024)); + await Assert.ThrowsAsync(async () => + await harness.Writer.WriteArticleAsync("a\r\n"u8.ToArray(), framing, 512 * 1024)); + } + + [Fact] + public async Task WriteArticle_Cancellation_BeforeEnqueue_Throws() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAnyAsync(async () => + await harness.Writer.WriteArticleAsync("body\r\n"u8.ToArray(), framing, cts.Token)); + } + + [Fact] + public async Task WriteArticle_Cancellation_WhileQueued_ReleasesWaiters() + { + // Capacity 1 + unread pipe → first chunk flush blocks pump; second enqueue waits; cancel producer. + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 4 * 1024, + resumeWriterThreshold: 2 * 1024, + minimumSegmentSize: 1024, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var stored = BuildStoredArticleBytes(sizeHint: 200 * 1024, leadingDotEvery: 0); + using var cts = new CancellationTokenSource(); + + var writeTask = writer.WriteArticleAsync(stored, framing, cts.Token); + // Allow first chunk to enter channel / pipe. + await Task.Delay(50); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(async () => await writeTask); + pipe.Reader.CancelPendingRead(); + await pipe.Reader.CompleteAsync(); + await pipe.Writer.CompleteAsync(); + } + + [Fact] + public async Task WriteArticle_DisposeWriter_CancelsInFlight() + { + var pipe = new Pipe(); + var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + var framing = NntpArticleTxFraming.CustomerArticle(""); + var stored = BuildStoredArticleBytes(180 * 1024, 0); + var writeTask = writer.WriteArticleAsync(stored, framing); + await Task.Delay(30); + await writer.DisposeAsync(); + await Assert.ThrowsAnyAsync(async () => await writeTask); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task WriteArticle_CompletedPipe_SurfacesFailure() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await pipe.Reader.CompleteAsync(); // send path sees completed reader → FlushAsync IsCompleted + await pipe.Writer.CompleteAsync(); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await Assert.ThrowsAnyAsync(async () => + await writer.WriteArticleAsync("x\r\n"u8.ToArray(), framing)); + } + + [Fact] + public async Task WriteArticle_DoesNotUseSessionMode() + { + // Primitive has no mode parameter; both framings work on the same writer instance. + await using var harness = await ArticleTxHarness.CreateAsync(); + await harness.Writer.WriteArticleAsync("r\r\n"u8.ToArray(), NntpArticleTxFraming.CustomerArticle("")); + await harness.Writer.WriteArticleAsync("s\r\n"u8.ToArray(), NntpArticleTxFraming.PeerTakeThis("")); + var wire = Encoding.ASCII.GetString(await harness.DrainAllAsync()); + Assert.Contains("220 0 ", wire, StringComparison.Ordinal); + Assert.Contains("TAKETHIS ", wire, StringComparison.Ordinal); + } + + [Fact] + public async Task WriteArticle_ChannelItemsLessThanPerLineModel() + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var lineCount = 2000; + var sb = new StringBuilder(); + for (var i = 0; i < lineCount; i++) + { + sb.Append("line-").Append(i).Append("\r\n"); + } + + var stored = Encoding.ASCII.GetBytes(sb.ToString()); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + _ = await harness.DrainAllAsync(); + // Per-line model would be ~2002 Channel items; chunked must be far fewer. + Assert.True(harness.Writer.ChannelEnqueueCount < lineCount / 10); + } + + [Fact] + public async Task ProductionRestuff_MatchesRoundTripWithMultilineReader() + { + var originalWire = Encoding.ASCII.GetBytes( + "Subject: t\r\n\r\n..leading\r\n...three\r\nnormal\r\n.\r\n"); + var reader = PipeReader.Create(new ReadOnlySequence(originalWire)); + var destuffed = await NntpMultilineDataReader + .ReadArticleAsync(reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + var reconstructed = ArticleWireReconstructor.RestuffArticle(destuffed.Payload.Span); + Assert.Equal(originalWire, reconstructed); + } + + [Fact] + public async Task Corpus_Oracle_IfPresent() + { + if (!Directory.Exists(IncomingRoot)) + { + return; // environment without corpus — synthetic tests still cover Phase 1 + } + + var samples = PickCorpusSamples(IncomingRoot, maxFiles: 8); + if (samples.Count == 0) + { + return; + } + + foreach (var path in samples) + { + await using var harness = await ArticleTxHarness.CreateAsync(); + var stored = await File.ReadAllBytesAsync(path); + var framing = NntpArticleTxFraming.CustomerArticle(""); + await harness.Writer.WriteArticleAsync(stored, framing); + Assert.Equal(BuildOracle(framing, stored), await harness.DrainAllAsync()); + } + } + + [Fact] + public async Task FocusedPerf_ChunkedVsPerLine_TypicalArticle() + { + var stored = BuildStoredArticleBytes(750 * 1024, leadingDotEvery: 40); + var framing = NntpArticleTxFraming.CustomerArticle(""); + + // Chunked path + await using var chunked = await ArticleTxHarness.CreateAsync(); + PrepareForAllocationSample(); + var before = GC.GetTotalAllocatedBytes(precise: true); + var sw = System.Diagnostics.Stopwatch.StartNew(); + await chunked.Writer.WriteArticleAsync(stored, framing); + sw.Stop(); + var chunkedAlloc = GC.GetTotalAllocatedBytes(precise: true) - before; + var chunkedMs = sw.Elapsed.TotalMilliseconds; + var chunkedEnq = chunked.Writer.ChannelEnqueueCount; + var chunkedFlush = chunked.Writer.PipeFlushCount; + _ = await chunked.DrainAllAsync(); + + // Legacy per-line path + await using var lines = await ArticleTxHarness.CreateAsync(); + PrepareForAllocationSample(); + before = GC.GetTotalAllocatedBytes(precise: true); + sw.Restart(); + await lines.Writer.WriteMultilineStartAsync(220, "0 "); + + // Deterministic line emission matching restuff semantics via oracle body without header: + var body = ArticleWireReconstructor.RestuffArticle(stored); + // Strip terminator for per-line write then end: + var bodyWithoutTerm = body.AsSpan(0, body.Length - 3); // remove .\r\n + var text = Encoding.ASCII.GetString(bodyWithoutTerm); + var parts = text.Split("\r\n"); + // last split empty if body ended with CRLF before terminator + for (var i = 0; i < parts.Length; i++) + { + if (i == parts.Length - 1 && parts[i].Length == 0) + { + break; + } + + await lines.Writer.WriteMultilineDataAsync(parts[i]); + } + + await lines.Writer.WriteMultilineEndAsync(); + sw.Stop(); + var lineAlloc = GC.GetTotalAllocatedBytes(precise: true) - before; + var lineMs = sw.Elapsed.TotalMilliseconds; + var lineEnq = lines.Writer.ChannelEnqueueCount; + var lineFlush = lines.Writer.PipeFlushCount; + + Assert.True(chunkedEnq < lineEnq, $"chunked enq {chunkedEnq} should be < line enq {lineEnq}"); + Assert.True(chunkedFlush < lineFlush, $"chunked flush {chunkedFlush} should be < line flush {lineFlush}"); + // Soft throughput check: chunked should not be dramatically slower on in-memory pipe. + Assert.True(chunkedMs < lineMs * 5 + 50, $"chunked {chunkedMs:F1}ms vs lines {lineMs:F1}ms"); + + _output.WriteLine( + $"PERF chunked: {chunkedMs:F2}ms alloc={chunkedAlloc} enq={chunkedEnq} flush={chunkedFlush}; " + + $"per-line: {lineMs:F2}ms alloc={lineAlloc} enq={lineEnq} flush={lineFlush}"); + Assert.True(chunkedAlloc > 0 && lineAlloc > 0); + } + + private static byte[] BuildOracle(NntpArticleTxFraming framing, ReadOnlySpan stored) + { + var header = framing.BuildHeaderBytes(); + var body = ArticleWireReconstructor.RestuffArticle(stored); + var result = new byte[header.Length + body.Length]; + header.CopyTo(result.AsSpan()); + body.CopyTo(result.AsSpan(header.Length)); + return result; + } + + /// Forces a full GC cycle before allocation sampling in the focused perf probe. + private static void PrepareForAllocationSample() + { + // codeql[cs/call-to-gc] Perf probe resets GC before allocation sampling. + GC.Collect(); + GC.WaitForPendingFinalizers(); + // codeql[cs/call-to-gc] Second collect after finalizers for stable allocation baseline. + GC.Collect(); + } + + private static byte[] Concat(params byte[][] parts) + { + var len = parts.Sum(static p => p.Length); + var result = new byte[len]; + var o = 0; + foreach (var p in parts) + { + p.CopyTo(result.AsSpan(o)); + o += p.Length; + } + + return result; + } + + private static byte[] BuildStoredArticleBytes(int sizeHint, int leadingDotEvery) + { + using var ms = new MemoryStream(sizeHint + 1024); + var n = 0; + while (ms.Length < sizeHint) + { + if (leadingDotEvery > 0 && n % leadingDotEvery == 0) + { + ms.Write("."u8); + } + + var line = Encoding.ASCII.GetBytes($"line-{n}-payload-xxxxxxxx\r\n"); + ms.Write(line); + n++; + } + + return ms.ToArray(); + } + + private static List PickCorpusSamples(string root, int maxFiles) + { + var files = Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .Where(static f => !f.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase)) + .Take(2000) + .Select(static f => new FileInfo(f)) + .Where(static f => f.Exists && f.Length > 0) + .OrderBy(static f => f.Length) + .ToList(); + if (files.Count == 0) + { + return []; + } + + var picks = new List(); + void Add(FileInfo? f) + { + if (f is not null && picks.Count < maxFiles && !picks.Contains(f.FullName)) + { + picks.Add(f.FullName); + } + } + + Add(files.First()); + Add(files[files.Count / 2]); + Add(files.FirstOrDefault(static f => Math.Abs(f.Length - 750 * 1024) < 50 * 1024)); + Add(files[(int)(files.Count * 0.95)]); + Add(files[(int)(files.Count * 0.99)]); + Add(files.Last()); + foreach (var f in files.Where(static x => + { + try + { + var head = File.ReadAllBytes(x.FullName).AsSpan(0, Math.Min(4096, (int)x.Length)); + return head.IndexOf("\r\n."u8) >= 0; + } + // codeql[cs/catch-of-all-exceptions] Test process-boundary helper must tolerate any failure while releasing resources. + catch (Exception ex) + { + _ = ex.GetType(); // observed: unreadable sample skipped + return false; + } + }).Take(2)) + { + Add(f); + } + + return picks; + } + + private sealed class ArticleTxHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private readonly MemoryStream _capture = new(); + private readonly CancellationTokenSource _cts = new(); + private readonly Task _drainTask; + private int _disposed; + + private ArticleTxHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + _drainTask = DrainLoopAsync(_cts.Token); + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + return Task.FromResult(new ArticleTxHarness(pipe, writer)); + } + + public async Task DrainAllAsync() + { + Interlocked.Exchange(ref _disposed, 1); + await Writer.DisposeAsync().ConfigureAwait(false); + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + await _drainTask.ConfigureAwait(false); + + lock (_capture) + { + return _capture.ToArray(); + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + await Writer.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + await _cts.CancelAsync().ConfigureAwait(false); + try + { + await _pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Test duplex teardown must complete pipes after any protocol or cancel fault. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + try + { + await _drainTask.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Test duplex teardown must complete pipes after any protocol or cancel fault. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // ignore + } + + _cts.Dispose(); + } + + private async Task DrainLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var result = await _pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + var buffer = result.Buffer; + foreach (var segment in buffer) + { + lock (_capture) + { + _capture.Write(segment.Span); + } + } + + _pipe.Reader.AdvanceTo(buffer.End); + if (result.IsCompleted) + { + break; + } + } + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + await _pipe.Reader.CompleteAsync().ConfigureAwait(false); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpPhaseCResponseTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpPhaseCResponseTests.cs new file mode 100644 index 00000000..0d201c57 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpPhaseCResponseTests.cs @@ -0,0 +1,449 @@ +using System.Buffers; +using System.Globalization; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Session; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// +/// Phase C: byte-native dynamic replies and one-write static multiline bodies. +/// +public sealed class NntpPhaseCResponseTests +{ + private static readonly NntpAuthorization TransitAuth = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Theory] + [InlineData("")] + [InlineData("")] + [InlineData("")] + public void Check_Compose_ExactWire_FromMessageIdBytes(string messageId) + { + var mid = Encoding.ASCII.GetBytes(messageId); + var wire = NntpResponseCompose.Concat( + NntpResponses.CheckPrefix.Span, + mid, + NntpResponses.CheckSuffix.Span); + Assert.Equal( + "238 " + messageId + " send article to be transferred\r\n", + Encoding.ASCII.GetString(wire)); + } + + [Fact] + public void Check_Compose_NearMaximumMessageId() + { + var mid = NearMaxMessageId(); + Assert.Equal(VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.MaxMessageIdLength, mid.Length); + Assert.True(VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.IsValidMessageId(mid)); + + var wire = NntpResponseCompose.Concat( + NntpResponses.CheckPrefix.Span, + mid, + NntpResponses.CheckSuffix.Span); + var expected = "238 " + Encoding.ASCII.GetString(mid) + " send article to be transferred\r\n"; + Assert.Equal(expected, Encoding.ASCII.GetString(wire)); + Assert.Equal(4 + mid.Length + NntpResponses.CheckSuffix.Length, wire.Length); + } + + [Theory] + [InlineData("")] + [InlineData("")] + public void TakeThis_Compose_Exact239And439_FromMessageIdBytes(string messageId) + { + var mid = Encoding.ASCII.GetBytes(messageId); + var ok = NntpResponseCompose.Concat( + NntpResponses.ArticleTransferredOkPrefix.Span, + mid, + NntpResponses.Crlf.Span); + var rejected = NntpResponseCompose.Concat( + NntpResponses.TransferRejectedPrefix.Span, + mid, + NntpResponses.Crlf.Span); + Assert.Equal("239 " + messageId + "\r\n", Encoding.ASCII.GetString(ok)); + Assert.Equal("439 " + messageId + "\r\n", Encoding.ASCII.GetString(rejected)); + } + + [Fact] + public void Date_FormatUtcDateLine_ExactStatusAndStamp() + { + var utc = new DateTime(2026, 9, 24, 1, 2, 3, DateTimeKind.Utc); + var wire = Date.FormatUtcDateLine(utc); + Assert.Equal("111 20260924010203\r\n", Encoding.ASCII.GetString(wire)); + Assert.Equal(20, wire.Length); + Assert.Equal( + utc.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture), + Encoding.ASCII.GetString(wire, 4, 14)); + } + + [Fact] + public void StaticSingleLine_ConnectionClosing_RemainsByteIdentical() + { + Assert.True(NntpResponses.ConnectionClosing.Span.SequenceEqual("205 Connection closing\r\n"u8)); + } + + [Fact] + public async Task Help_Dispatch_OneEnqueue_ExactCompleteWire() + { + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + var expected = ExpectedHelpWire(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Equal(Help.BodyLines.Count + 2, CountCrlf(expected)); + } + + [Fact] + public async Task Capabilities_DefaultSession_OneEnqueue_ExactWire() + { + var expected = Concat( + NntpResponses.CapabilityListFollows, + NntpResponses.CapabilityVersion2, + NntpResponses.CapabilityImplementation, + NntpResponses.CapabilityAuthinfoUser, + NntpResponses.CapabilityStartTls, + NntpResponses.CapabilityCompressDeflate, + NntpResponses.CapabilityIhave, + NntpResponses.CapabilityStreaming, + NntpResponses.CapabilitySpeedTest, + NntpResponses.MultilineTerminator); + + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Contains("VERSION 2", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.Contains("SPEEDTEST", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.Contains("IMPLEMENTATION VectorNNTPD.TransitService", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("READER", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("LIST", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("NEWNEWS", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("NEWGROUPS", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.True(expected.AsSpan().EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task Capabilities_Authenticated_OmitsAuthinfo_OneEnqueue() + { + var expected = Concat( + NntpResponses.CapabilityListFollows, + NntpResponses.CapabilityVersion2, + NntpResponses.CapabilityImplementation, + NntpResponses.CapabilityStartTls, + NntpResponses.CapabilityCompressDeflate, + NntpResponses.CapabilityIhave, + NntpResponses.CapabilityStreaming, + NntpResponses.CapabilitySpeedTest, + NntpResponses.MultilineTerminator); + + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.ApplySuccessfulAuthentication( + "user", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + Assert.Equal(Encoding.ASCII.GetString(expected), await read); + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.DoesNotContain("AUTHINFO", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + Assert.DoesNotContain("MODE-READER", Encoding.ASCII.GetString(expected), StringComparison.Ordinal); + } + + [Fact] + public async Task Check_Dispatch_ExactResponseBytes_OneEnqueue() + { + const string id = ""; + var expected = "238 " + id + " send article to be transferred\r\n"; + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK " + id); + Assert.Equal(expected, await read); + Assert.Equal(1, response.ChannelEnqueueCount); + } + + [Fact] + public async Task Check_Dispatch_NearMaximumMessageId_ExactBytes() + { + var mid = Encoding.ASCII.GetString(NearMaxMessageId()); + var expected = "238 " + mid + " send article to be transferred\r\n"; + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK " + mid); + Assert.Equal(expected, await read); + } + + [Fact] + public async Task TakeThis_Dispatch_Exact239_FromOwnedCompose() + { + const string id = ""; + var expected = "239 " + id + "\r\n"; + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await duplex.WriteClientAsync(CanonicalArticleText.Destuffed(id) + ".\r\n"); + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "TAKETHIS " + id); + Assert.Equal(1, response.ChannelEnqueueCount); + await response.FlushCoalescedAsync(); + Assert.Equal(expected, await read); + Assert.Equal(1, queue.Count); + } + + [Fact] + public async Task TakeThis_Dispatch_Exact439_FromOwnedCompose() + { + const string id = ""; + var expected = "439 " + id + "\r\n"; + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await duplex.WriteClientAsync("Subject: oversized-payload-here\r\n\r\nbody\r\n.\r\n"); + var read = duplex.ReadExactAsciiAsync(expected.Length); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "TAKETHIS " + id); + Assert.Equal(1, response.ChannelEnqueueCount); + await response.FlushCoalescedAsync(); + Assert.Equal(expected, await read); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task Date_Dispatch_ExactStatusAndDigitStamp_OneEnqueue() + { + await using var duplex = await PhaseCDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + var read = duplex.ReadExactAsciiAsync(20); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "DATE"); + var wire = await read; + Assert.Equal(20, wire.Length); + Assert.StartsWith("111 ", wire, StringComparison.Ordinal); + Assert.EndsWith("\r\n", wire, StringComparison.Ordinal); + Assert.All(wire[4..18], static c => Assert.True(char.IsAsciiDigit(c))); + Assert.Equal(1, response.ChannelEnqueueCount); + } + + private static byte[] ExpectedHelpWire() + { + var builder = new StringBuilder(); + builder.Append("100 Help text follows\r\n"); + foreach (var line in Help.BodyLines) + { + builder.Append(line).Append("\r\n"); + } + + builder.Append(".\r\n"); + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + private static byte[] Concat(params ReadOnlyMemory[] parts) => + NntpResponseCompose.Concatenate(parts); + + private static byte[] NearMaxMessageId() + { + var mid = new byte[VectorNNTPD.Common.Articles.Validation.NntpMessageIdValidation.MaxMessageIdLength]; + mid[0] = (byte)'<'; + for (var i = 1; i < mid.Length - 3; i++) + { + mid[i] = (byte)'a'; + } + + mid[^3] = (byte)'@'; + mid[^2] = (byte)'x'; + mid[^1] = (byte)'>'; + return mid; + } + + private static int CountCrlf(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + + private sealed class PhaseCDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new PhaseCDuplex()); + + public NntpSession CreateSession( + IArticleIngestionQueue? queue = null, + bool allowCleartextAuth = true) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + allowCleartextAuth: allowCleartextAuth); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _serverToClient.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _serverToClient.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs new file mode 100644 index 00000000..51060455 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpResponseWriterCoalescingTests.cs @@ -0,0 +1,259 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// +/// Bounded TX coalescing: logical response bytes stay identical; only Pipe flush granularity changes. +/// +public sealed class NntpResponseWriterCoalescingTests +{ + [Fact] + public async Task TxChannel_AllowsSynchronousContinuations() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + Assert.True(writer.ChannelAllowsSynchronousContinuations); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task SingleWriteLine_DeliversExactBytes() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(200, "Hello"); + Assert.Equal("200 Hello\r\n", await harness.ReadExactAsciiAsync("200 Hello\r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task TwoWriteLines_ExactBytesAndOrder() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(200, "one"); + await harness.Writer.WriteLineAsync(201, "two"); + Assert.Equal("200 one\r\n201 two\r\n", await harness.ReadExactAsciiAsync("200 one\r\n201 two\r\n".Length)); + } + + [Fact] + public async Task EightEnqueueLines_OneFlush_ExactBytesAndOrder() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 1; i <= NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + var text = $""; + expected.Append("239 ").Append(text).Append("\r\n"); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, text); + } + + var wire = expected.ToString(); + Assert.Equal(wire, await harness.ReadExactAsciiAsync(wire.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task NineEnqueueLines_FirstEightFlush_NinthOrderedAfterBarrier() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 1; i <= 9; i++) + { + var text = $""; + expected.Append("239 ").Append(text).Append("\r\n"); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, text); + } + + var firstEight = expected.ToString()[..^("239 \r\n".Length)]; + Assert.Equal(firstEight, await harness.ReadExactAsciiAsync(firstEight.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.True(harness.Writer.HasCoalescedUnflushed); + + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal("239 \r\n", await harness.ReadExactAsciiAsync("239 \r\n".Length)); + Assert.Equal(2, harness.Writer.PipeFlushCount); + Assert.Equal(expected.ToString(), firstEight + "239 \r\n"); + } + + [Fact] + public async Task PartialBatch_FlushCoalesced_DeliversHeldLines() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + Assert.Equal(0, harness.Writer.PipeFlushCount); + + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal( + "239 \r\n239 \r\n239 \r\n", + await harness.ReadExactAsciiAsync("239 \r\n239 \r\n239 \r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task WriteLineBarrier_FlushesQueuedEnqueueLinesFirst() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.WriteLineAsync(382, "Continue with TLS negotiation"); + + Assert.Equal( + "239 \r\n239 \r\n382 Continue with TLS negotiation\r\n", + await harness.ReadExactAsciiAsync( + "239 \r\n239 \r\n382 Continue with TLS negotiation\r\n".Length)); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task Cancellation_PendingPartialBatch_DoesNotHang() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + await writer.EnqueueLineAsync(239, ""); + await writer.EnqueueLineAsync(239, ""); + + using var cts = new CancellationTokenSource(); + var flush = writer.FlushCoalescedAsync(cts.Token).AsTask(); + cts.Cancel(); + await writer.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + var ex = await Assert.ThrowsAnyAsync(() => flush); + Assert.True( + ex is OperationCanceledException or ObjectDisposedException, + $"Unexpected {ex.GetType().FullName}: {ex.Message}"); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task Disconnect_DisposeTerminatesPumpWithHeldBatch() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var writer = new NntpResponseWriter(pipe.Writer); + await writer.EnqueueLineAsync(239, ""); + await writer.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task Backpressure_EnqueueWaitsWhenChannelAndPipeAreSaturated() + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 2); + + for (var i = 0; i < NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + await writer.EnqueueLineAsync(239, $""); + } + + // Batch of 8 flushes into a paused pipe. Two more fill the bounded channel; the next waits. + await writer.EnqueueLineAsync(239, ""); + await writer.EnqueueLineAsync(239, ""); + var blocked = writer.EnqueueLineAsync(239, "").AsTask(); + await Assert.ThrowsAsync(() => blocked.WaitAsync(TimeSpan.FromMilliseconds(200))); + + var read = await pipe.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + pipe.Reader.AdvanceTo(read.Buffer.End); + await blocked.WaitAsync(TimeSpan.FromSeconds(5)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + pipe.Reader.Complete(); + } + + [Fact] + public async Task NonTakethisWriteLine_IsNotHeldForBatch() + { + await using var harness = await CoalesceHarness.CreateAsync(); + var write = harness.Writer.WriteLineAsync(111, "Server date-time").AsTask(); + await write.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal("111 Server date-time\r\n", await harness.ReadExactAsciiAsync("111 Server date-time\r\n".Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public async Task ExactBytes_CrlfAndFramingPreserved() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.EnqueueLineAsync(439, ""); + await harness.Writer.FlushCoalescedAsync(); + var bytes = Encoding.ASCII.GetBytes(await harness.ReadExactAsciiAsync("239 \r\n439 \r\n".Length)); + Assert.Equal("239 \r\n439 \r\n"u8.ToArray(), bytes); + } + + [Fact] + public async Task FlushCoalesced_WhenEmpty_IsNoOp() + { + await using var harness = await CoalesceHarness.CreateAsync(); + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal(0, harness.Writer.PipeFlushCount); + } + + private sealed class CoalesceHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private int _disposed; + + private CoalesceHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new CoalesceHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _pipe.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _pipe.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await Writer.DisposeAsync(); + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStaticResponseTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStaticResponseTests.cs new file mode 100644 index 00000000..e02a0f49 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStaticResponseTests.cs @@ -0,0 +1,310 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Runtime.InteropServices; +using System.Text; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// +/// Phase B: immortal static NNTP replies are encoded once, CRLF-terminated, and written +/// without a per-call string interpolation / ASCII encode. +/// +public sealed class NntpStaticResponseTests +{ + [Theory] + [MemberData(nameof(SingleLineResponses))] + public void StaticResponse_IsAscii_CrlfOnce_AndImmutable(string name, ReadOnlyMemory wire, string expected) + { + _ = name; + var expectedBytes = Encoding.ASCII.GetBytes(expected); + Assert.True(wire.Span.SequenceEqual(expectedBytes), Encoding.ASCII.GetString(wire.Span)); + Assert.True(wire.Span.EndsWith("\r\n"u8)); + Assert.Equal(1, CountCrlf(wire.Span)); + + Assert.True(MemoryMarshal.TryGetArray(wire, out ArraySegment segment)); + Assert.NotNull(segment.Array); + var snapshot = wire.ToArray(); + Assert.True(wire.Span.SequenceEqual(snapshot)); + Assert.True(wire.Span.SequenceEqual(expectedBytes)); + } + + [Fact] + public async Task WriteLine_Static_DeliversExactBytes_Repeatedly() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(NntpResponses.ConnectionClosing); + await harness.Writer.WriteLineAsync(NntpResponses.ConnectionClosing); + var expected = "205 Connection closing\r\n205 Connection closing\r\n"; + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + Assert.Equal(2, harness.Writer.PipeFlushCount); + Assert.True(NntpResponses.ConnectionClosing.Span.SequenceEqual("205 Connection closing\r\n"u8)); + } + + [Fact] + public async Task EnqueueLine_Static_PreservesCoalesceBatchOfEight() + { + await using var harness = await WriterHarness.CreateAsync(); + var expected = new StringBuilder(); + for (var i = 0; i < NntpResponseWriter.CoalesceResponseBatchSize; i++) + { + expected.Append("205 Connection closing\r\n"); + await harness.Writer.EnqueueLineAsync(NntpResponses.ConnectionClosing); + } + + var wire = expected.ToString(); + Assert.Equal(wire, await harness.ReadExactAsciiAsync(wire.Length)); + Assert.Equal(1, harness.Writer.PipeFlushCount); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task StaticBarrier_FlushesQueuedEnqueueLinesFirst() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(239, ""); + await harness.Writer.WriteLineAsync(NntpResponses.ContinueWithTls); + Assert.Equal( + "239 \r\n382 Continue with TLS negotiation\r\n", + await harness.ReadExactAsciiAsync( + "239 \r\n382 Continue with TLS negotiation\r\n".Length)); + Assert.False(harness.Writer.HasCoalescedUnflushed); + } + + [Fact] + public async Task DynamicCheck_StillUsesStringPath_AndIncludesMessageId() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync( + NntpReplyCodes.SendArticleToBeTransferred, + " send article to be transferred"); + Assert.Equal( + "238 send article to be transferred\r\n", + await harness.ReadExactAsciiAsync( + "238 send article to be transferred\r\n".Length)); + } + + [Fact] + public async Task DynamicTakethis_EnqueueLine_StillEncodesMessageIdPerCall() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.ArticleTransferredOk, ""); + await harness.Writer.EnqueueLineAsync(NntpReplyCodes.TransferRejected, ""); + await harness.Writer.FlushCoalescedAsync(); + Assert.Equal( + "239 \r\n439 \r\n", + await harness.ReadExactAsciiAsync("239 \r\n439 \r\n".Length)); + } + + [Fact] + public async Task WriteBytesAndFlush_ReusesCallerMemory_WithoutCopyingIntoNewArray() + { + await using var harness = await WriterHarness.CreateAsync(); + var payload = NntpResponses.CompressionActive; + Assert.True(MemoryMarshal.TryGetArray(payload, out var before)); + await harness.Writer.WriteBytesAndFlushAsync(payload); + Assert.True(MemoryMarshal.TryGetArray(payload, out var after)); + Assert.Same(before.Array, after.Array); + Assert.Equal("206 Compression active\r\n", await harness.ReadExactAsciiAsync("206 Compression active\r\n".Length)); + } + + [Fact] + public void HelpBody_MatchesExistingStringLines_AndNoneNeedDotStuffing() + { + Assert.Equal(Help.BodyLines.Count, NntpResponses.HelpBodyLines.Length); + for (var i = 0; i < Help.BodyLines.Count; i++) + { + Assert.False(Help.BodyLines[i].StartsWith('.')); + var expected = Encoding.ASCII.GetBytes(Help.BodyLines[i] + "\r\n"); + Assert.True(NntpResponses.HelpBodyLines[i].Span.SequenceEqual(expected)); + } + } + + [Fact] + public void HelpComplete_IsExactConcatenationOfStatusBodyAndTerminator() + { + var expected = ExpectedHelpWire(); + Assert.True(NntpResponses.HelpComplete.Span.SequenceEqual(expected)); + Assert.True(NntpResponses.HelpComplete.Span.EndsWith(".\r\n"u8)); + Assert.Equal(Help.BodyLines.Count + 2, CountCrlf(NntpResponses.HelpComplete.Span)); + Assert.False(NntpResponses.HelpComplete.Span.EndsWith("\r\n.\r\n.\r\n"u8)); + } + + [Fact] + public async Task HelpComplete_WriteOnce_OneChannelItem_ExactBytes() + { + await using var harness = await WriterHarness.CreateAsync(); + await harness.Writer.WriteLineAsync(NntpResponses.HelpComplete); + var expected = Encoding.ASCII.GetString(ExpectedHelpWire()); + Assert.Equal(expected, await harness.ReadExactAsciiAsync(expected.Length)); + Assert.Equal(1, harness.Writer.ChannelEnqueueCount); + Assert.Equal(1, harness.Writer.PipeFlushCount); + } + + [Fact] + public void StaticResponseTable_Access_DoesNotAllocate() + { + _ = NntpResponses.ConnectionClosing; + _ = NntpResponses.HelpBodyLines; + var sink = 0; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 256; i++) + { + sink += NntpResponses.ConnectionClosing.Length; + sink += NntpResponses.GreetingPostingProhibited.Length; + sink += NntpResponses.HelpTextFollows.Length; + sink += NntpResponses.HelpBodyLines.Length; + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + Assert.True(sink > 0); + } + + [Fact] + public void ResponseEncoding_StaticTableIsZeroAlloc_StringPathAllocates() + { + const int iterations = 256; + var sink = 0; + + for (var i = 0; i < 8; i++) + { + sink += Encoding.ASCII.GetBytes($"{205} Connection closing\r\n").Length; + sink += NntpResponses.ConnectionClosing.Length; + } + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < iterations; i++) + { + sink += Encoding.ASCII.GetBytes($"{205} Connection closing\r\n").Length; + } + + var stringEncodeAlloc = GC.GetAllocatedBytesForCurrentThread() - before; + + before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < iterations; i++) + { + sink += NntpResponses.ConnectionClosing.Length; + } + + var staticEncodeAlloc = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.Equal(0, staticEncodeAlloc); + Assert.True( + stringEncodeAlloc >= iterations * NntpResponses.ConnectionClosing.Length, + $"string encode allocated {stringEncodeAlloc} B over {iterations} lines."); + Assert.NotEqual(int.MinValue, sink); + } + + public static TheoryData, string> SingleLineResponses() => + new() + { + { "greeting-200", NntpResponses.GreetingPostingPermitted, "200 VectorNNTPD.TransitService ready, posting permitted\r\n" }, + { "greeting-201", NntpResponses.GreetingPostingProhibited, "201 VectorNNTPD.TransitService ready, posting prohibited\r\n" }, + { "quit", NntpResponses.ConnectionClosing, "205 Connection closing\r\n" }, + { "compress-206", NntpResponses.CompressionActive, "206 Compression active\r\n" }, + { "auth-281", NntpResponses.AuthenticationAccepted, "281 Authentication accepted\r\n" }, + { "auth-381", NntpResponses.PasswordRequired, "381 Password required\r\n" }, + { "starttls-382", NntpResponses.ContinueWithTls, "382 Continue with TLS negotiation\r\n" }, + { "240", NntpResponses.ArticleReceivedOk, "240 Article received OK\r\n" }, + { "340", NntpResponses.PostSendArticle, "340 Input article; end with .\r\n" }, + { "400", NntpResponses.ServiceTemporarilyUnavailable, "400 Service temporarily unavailable\r\n" }, + { "440", NntpResponses.PostingProhibited, "440 Posting not permitted\r\n" }, + { "441", NntpResponses.PostingFailed, "441 Posting failed\r\n" }, + { "480", NntpResponses.AuthenticationRequired, "480 Authentication required\r\n" }, + { "500-unknown", NntpResponses.UnknownCommand, "500 Unknown command\r\n" }, + { "500-stub", NntpResponses.CommandNotImplemented, "500 Command not implemented\r\n" }, + { "501", NntpResponses.SyntaxError, "501 Syntax error\r\n" }, + { "503-list-data", NntpResponses.ListDataItemNotStored, "503 Data item not stored\r\n" }, + { "215-overview-fmt", NntpResponses.ListOverviewFmtFollows, "215 Order of fields in overview database.\r\n" }, + { "215-headers", NntpResponses.ListHeadersFollows, "215 headers and metadata items supported:\r\n" }, + { "412", NntpResponses.NoNewsgroupSelected, "412 No newsgroup selected\r\n" }, + { "420", NntpResponses.CurrentArticleNumberInvalid, "420 Current article number is invalid\r\n" }, + { "423", NntpResponses.NoArticleWithNumber, "423 No article with that number\r\n" }, + { "430", NntpResponses.NoArticleWithMessageId, "430 No article with that message-id\r\n" }, + { "help-100", NntpResponses.HelpTextFollows, "100 Help text follows\r\n" }, + { "caps-101", NntpResponses.CapabilityListFollows, "101 Capability list:\r\n" }, + { "multiline-end", NntpResponses.MultilineTerminator, ".\r\n" }, + }; + + private static byte[] ExpectedHelpWire() + { + var builder = new StringBuilder(); + builder.Append("100 Help text follows\r\n"); + foreach (var line in Help.BodyLines) + { + builder.Append(line).Append("\r\n"); + } + + builder.Append(".\r\n"); + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + private static int CountCrlf(ReadOnlySpan bytes) + { + var count = 0; + for (var i = 0; i < bytes.Length - 1; i++) + { + if (bytes[i] == (byte)'\r' && bytes[i + 1] == (byte)'\n') + { + count++; + } + } + + return count; + } + + private sealed class WriterHarness : IAsyncDisposable + { + private readonly Pipe _pipe; + private int _disposed; + + private WriterHarness(Pipe pipe, NntpResponseWriter writer) + { + _pipe = pipe; + Writer = writer; + } + + public NntpResponseWriter Writer { get; } + + public static Task CreateAsync() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + return Task.FromResult(new WriterHarness(pipe, new NntpResponseWriter(pipe.Writer))); + } + + public async Task ReadExactAsciiAsync(int byteCount) + { + var buffer = new byte[byteCount]; + var copied = 0; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (copied < byteCount) + { + var result = await _pipe.Reader.ReadAsync(timeout.Token); + var unread = result.Buffer; + var take = (int)Math.Min(unread.Length, byteCount - copied); + unread.Slice(0, take).CopyTo(buffer.AsSpan(copied, take)); + copied += take; + _pipe.Reader.AdvanceTo(unread.GetPosition(take)); + if (result.IsCompleted && copied < byteCount) + { + throw new InvalidOperationException($"Pipe completed after {copied} of {byteCount} bytes."); + } + } + + return Encoding.ASCII.GetString(buffer); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) + { + return; + } + + await Writer.DisposeAsync(); + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs new file mode 100644 index 00000000..a64d96bf --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Commands/NntpStreamArticleTxSchedulerTests.cs @@ -0,0 +1,839 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Tests.Session.Commands; + +/// +/// Phase 3: bounded STREAM outstanding-depth scheduler (no storage / no catalog). +/// +public sealed class NntpStreamArticleTxSchedulerTests +{ + [Fact] + public void DefaultDepth_IsEight() + { + using var scheduler = new SyncDispose(new NntpStreamArticleTxScheduler()); + Assert.Equal(8, scheduler.Inner.Depth); + Assert.Equal(8, NntpStreamArticleTxScheduler.DefaultDepth); + } + + [Theory] + [InlineData(4)] + [InlineData(8)] + [InlineData(16)] + public void ValidDepths_Accepted(int depth) + { + using var scheduler = new SyncDispose(new NntpStreamArticleTxScheduler(depth)); + Assert.Equal(depth, scheduler.Inner.Depth); + Assert.True(NntpStreamArticleTxScheduler.IsValidDepth(depth)); + } + + [Theory] + [InlineData(3)] + [InlineData(0)] + [InlineData(-1)] + [InlineData(17)] + [InlineData(64)] + public void InvalidDepths_Rejected(int depth) + { + Assert.False(NntpStreamArticleTxScheduler.IsValidDepth(depth)); + Assert.Throws(() => new NntpStreamArticleTxScheduler(depth)); + Assert.Throws(() => + NntpStreamArticleTxScheduler.ThrowIfNotValidDepth(depth)); + } + + [Fact] + public async Task AtMostN_ConcurrentlyAdmitted() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var started = new TaskCompletionSource[depth + 1]; + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + for (var i = 0; i < started.Length; i++) + { + started[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var tasks = new Task[depth + 1]; + for (var i = 0; i < tasks.Length; i++) + { + var index = i; + tasks[i] = scheduler.ExecuteAsync( + async ct => + { + started[index].TrySetResult(); + await release.Task.WaitAsync(ct).ConfigureAwait(false); + }, + CancellationToken.None) + .AsTask(); + } + + var firstWave = started.Take(depth).Select(s => s.Task); + await Task.WhenAll(firstWave).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(depth, scheduler.Outstanding); + Assert.False(started[depth].Task.IsCompleted); + + release.TrySetResult(); + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(5)); + await started[depth].Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CompletedOperation_ReleasesSlot_ForWaiter() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var blockers = new TaskCompletionSource[depth]; + for (var i = 0; i < depth; i++) + { + blockers[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var holders = new Task[depth]; + for (var i = 0; i < depth; i++) + { + var index = i; + holders[i] = scheduler.ExecuteAsync( + async ct => await blockers[index].Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask(); + } + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + var admitted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waiter = scheduler.ExecuteAsync( + _ => + { + admitted.TrySetResult(); + return ValueTask.CompletedTask; + }, + CancellationToken.None) + .AsTask(); + + Assert.False(admitted.Task.IsCompleted); + blockers[0].TrySetResult(); + await admitted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + for (var i = 1; i < depth; i++) + { + blockers[i].TrySetResult(); + } + + await Task.WhenAll(holders.Append(waiter)).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWhileWaiting_DoesNotLeakSlot() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holders = Enumerable.Range(0, depth) + .Select(_ => scheduler.ExecuteAsync( + async ct => await hold.Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask()) + .ToArray(); + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + using var cts = new CancellationTokenSource(); + var waiting = scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, cts.Token).AsTask(); + await WaitUntilAsync(() => scheduler.Outstanding == depth && !waiting.IsCompleted, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => waiting); + Assert.Equal(depth, scheduler.Outstanding); + + hold.TrySetResult(); + await Task.WhenAll(holders).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + + await scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, CancellationToken.None); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelAfterAcquisition_ReleasesSlot() + { + await using var scheduler = new NntpStreamArticleTxScheduler(4); + using var cts = new CancellationTokenSource(); + var acquired = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var task = scheduler.ExecuteAsync( + async ct => + { + acquired.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + }, + cts.Token) + .AsTask(); + + await acquired.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => task); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task ExceptionDuringOperation_ReleasesSlot() + { + await using var scheduler = new NntpStreamArticleTxScheduler(4); + await Assert.ThrowsAsync(() => + scheduler.ExecuteAsync( + _ => throw new InvalidOperationException("boom"), + CancellationToken.None) + .AsTask()); + Assert.Equal(0, scheduler.Outstanding); + + await scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, CancellationToken.None); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task RepeatedAcquireRelease_NeverExceedsBound() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var peak = 0; + var tasks = Enumerable.Range(0, 40) + .Select(_ => scheduler.ExecuteAsync( + _ => + { + var o = scheduler.Outstanding; + while (true) + { + var observed = Volatile.Read(ref peak); + if (o <= observed || Interlocked.CompareExchange(ref peak, o, observed) == observed) + { + break; + } + } + + Assert.True(o <= depth); + return ValueTask.CompletedTask; + }, + CancellationToken.None) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.True(peak <= depth); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWaiting_UnblocksWithoutAcquire() + { + const int depth = 4; + await using var scheduler = new NntpStreamArticleTxScheduler(depth); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holders = Enumerable.Range(0, depth) + .Select(_ => scheduler.ExecuteAsync( + async ct => await hold.Task.WaitAsync(ct).ConfigureAwait(false), + CancellationToken.None) + .AsTask()) + .ToArray(); + + await WaitUntilAsync(() => scheduler.Outstanding == depth, TimeSpan.FromSeconds(5)); + + using var cts = new CancellationTokenSource(); + var waiting = scheduler.ExecuteAsync(_ => ValueTask.CompletedTask, cts.Token).AsTask(); + await WaitUntilAsync(() => !waiting.IsCompleted, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waiting); + + hold.TrySetResult(); + await Task.WhenAll(holders).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task OverlappingWriteArticle_PreservesFifoWireOrder() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var bodies = new[] + { + "AAA\r\n"u8.ToArray(), + "BBB\r\n"u8.ToArray(), + "CCC\r\n"u8.ToArray(), + "DDD\r\n"u8.ToArray(), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + var framing = NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"); + tasks[index] = scheduler + .WriteArticleAsync(writer, bodies[index], framing, CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + + var buffer = new ArrayBufferWriter(); + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + buffer.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + var wire = Encoding.ASCII.GetString(buffer.WrittenSpan); + var expected = new StringBuilder(); + for (var i = 0; i < bodies.Length; i++) + { + var framing = NntpArticleTxFraming.PeerTakeThis($"<{i}@id>"); + expected.Append(Encoding.ASCII.GetString(framing.BuildHeaderBytes())); + expected.Append(Encoding.ASCII.GetString(ArticleWireReconstructor.RestuffArticle(bodies[i]))); + } + + Assert.Equal(expected.ToString(), wire); + } + + [Fact] + public async Task WriteArticle_FramingRemainsByteCorrect() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var stored = ".dot\r\nbody\r\n"u8.ToArray(); + var framing = NntpArticleTxFraming.CustomerArticle("", 42); + await scheduler.WriteArticleAsync(writer, stored, framing); + + await writer.DisposeAsync(); + pipe.Writer.Complete(); + + var read = await pipe.Reader.ReadAsync(); + var wire = read.Buffer.ToArray(); + pipe.Reader.AdvanceTo(read.Buffer.End); + + var expected = framing.BuildHeaderBytes() + .Concat(ArticleWireReconstructor.RestuffArticle(stored)) + .ToArray(); + Assert.Equal(expected, wire); + } + + [Fact] + public async Task CancelDuringBackpressuredWrite_ReleasesSlot() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + using var cts = new CancellationTokenSource(); + var large = Encoding.ASCII.GetBytes(new string('x', 256 * 1024) + "\r\n"); + + var write = scheduler + .WriteArticleAsync( + writer, + large, + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await WaitUntilAsync( + () => scheduler.Outstanding > 0 || write.IsCompleted, + TimeSpan.FromSeconds(5)); + + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task ThreeConcurrentArticles_NeverInterleaveChunks() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var bodies = new[] + { + Encoding.ASCII.GetBytes("ONE\r\n" + new string('1', 3000) + "\r\n"), + Encoding.ASCII.GetBytes("TWO\r\n" + new string('2', 3000) + "\r\n"), + Encoding.ASCII.GetBytes("THREE\r\n" + new string('3', 3000) + "\r\n"), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleForTestsAsync( + writer, + bodies[index], + NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"), + chunkBytes: 1024, + CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + var wire = await DrainPipeAsync(pipe, writer); + Assert.Equal(BuildOracle(bodies), wire); + } + + [Fact] + public async Task Preparation_CanOverlap_UnderDepth() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var afterPrepare = new TaskCompletionSource[4]; + var releasePrepare = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + for (var i = 0; i < 4; i++) + { + afterPrepare[i] = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + var seen = 0; + scheduler.TestAfterPrepare = async ct => + { + var n = Interlocked.Increment(ref seen); + afterPrepare[n - 1].TrySetResult(); + await releasePrepare.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + var bodies = Enumerable.Range(0, 4) + .Select(i => Encoding.ASCII.GetBytes($"P{i}\r\n")) + .ToArray(); + var tasks = new Task[4]; + for (var i = 0; i < 4; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleAsync(writer, bodies[index], NntpArticleTxFraming.PeerTakeThis($"")) + .AsTask(); + } + + await Task.WhenAll(afterPrepare.Select(t => t.Task)).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(scheduler.PeakPreparing >= 2); + Assert.Equal(4, scheduler.Outstanding); + + releasePrepare.TrySetResult(); + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task Enqueue_RemainsArticleAtomic_UnderSlowConsumer() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 2); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var capture = new ArrayBufferWriter(); + using var drainCts = new CancellationTokenSource(); + var drain = DrainSlowCapturingAsync(pipe, capture, drainCts.Token); + + var bodies = new[] + { + Encoding.ASCII.GetBytes("AAAA\r\n" + new string('a', 20_000) + "\r\n"), + Encoding.ASCII.GetBytes("BBBB\r\n" + new string('b', 20_000) + "\r\n"), + Encoding.ASCII.GetBytes("CCCC\r\n" + new string('c', 20_000) + "\r\n"), + }; + + var tasks = new Task[bodies.Length]; + for (var i = 0; i < bodies.Length; i++) + { + var index = i; + tasks[index] = scheduler + .WriteArticleForTestsAsync( + writer, + bodies[index], + NntpArticleTxFraming.PeerTakeThis($"<{index}@id>"), + chunkBytes: 4096, + CancellationToken.None) + .AsTask(); + } + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(30)); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + await drainCts.CancelAsync(); + try + { + await drain.WaitAsync(TimeSpan.FromSeconds(10)); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + // Finish draining any remaining bytes after cancel. + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + capture.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + Assert.Equal(0, scheduler.Enqueuing); + Assert.Equal(0, scheduler.Outstanding); + Assert.Equal(BuildOracle(bodies), Encoding.ASCII.GetString(capture.WrittenSpan)); + } + + [Fact] + public async Task SlowConsumer_DoesNotLeakDepthPermits() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var drain = DrainInBackgroundAsync(pipe); + + var body = Encoding.ASCII.GetBytes(new string('z', 80_000) + "\r\n"); + var tasks = Enumerable.Range(0, 8) + .Select(i => scheduler + .WriteArticleForTestsAsync( + writer, + body, + NntpArticleTxFraming.PeerTakeThis($""), + chunkBytes: 8192) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(60)); + Assert.Equal(0, scheduler.Outstanding); + + await writer.DisposeAsync(); + pipe.Writer.Complete(); + await drain; + } + + [Fact] + public async Task CancelDuringPreparation_ReleasesDepthPermit() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var prepared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestAfterPrepare = async ct => + { + prepared.TrySetResult(); + await hold.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + using var cts = new CancellationTokenSource(); + var write = scheduler + .WriteArticleAsync( + writer, + "body\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await prepared.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + hold.TrySetCanceled(); + } + + [Fact] + public async Task CancelWhileWaitingForEnqueueGate_ReleasesDepthPermit() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestWhileHoldingEnqueueGate = async ct => + { + holding.TrySetResult(); + await releaseGate.Task.WaitAsync(CancellationToken.None).ConfigureAwait(false); + }; + + var holder = scheduler + .WriteArticleAsync( + writer, + "hold\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis("")) + .AsTask(); + + await holding.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, scheduler.Enqueuing); + + using var cts = new CancellationTokenSource(); + var waiter = scheduler + .WriteArticleAsync( + writer, + "wait\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis(""), + cts.Token) + .AsTask(); + + await WaitUntilAsync(() => scheduler.Outstanding == 2, TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => waiter); + Assert.Equal(1, scheduler.Outstanding); + + releaseGate.TrySetResult(); + await holder.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, scheduler.Outstanding); + } + + [Fact] + public async Task CancelWhileAwaitingTxFlush_ReleasesDepthPermit() + { + var pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 1, resumeWriterThreshold: 1)); + await using var writer = new NntpResponseWriter(pipe.Writer, channelCapacity: 1); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + var enteredFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var holdFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + scheduler.TestBeforeAwaitFlush = async ct => + { + enteredFlush.TrySetResult(); + await holdFlush.Task.WaitAsync(ct).ConfigureAwait(false); + }; + + using var cts = new CancellationTokenSource(); + var large = Encoding.ASCII.GetBytes(new string('q', 128 * 1024) + "\r\n"); + var write = scheduler + .WriteArticleForTestsAsync( + writer, + large, + NntpArticleTxFraming.PeerTakeThis(""), + chunkBytes: 4096, + cts.Token) + .AsTask(); + + // Drain so enqueue can complete and reach the flush-await hook. + var drain = DrainInBackgroundAsync(pipe); + await enteredFlush.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(1, scheduler.Outstanding); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => write); + Assert.Equal(0, scheduler.Outstanding); + holdFlush.TrySetCanceled(); + await writer.DisposeAsync(); + pipe.Writer.Complete(); + try + { + await drain; + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + [Fact] + public async Task ExceptionDuringEnqueue_ReleasesDepthAndGate() + { + var pipe = new Pipe(); + var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(4); + + await writer.DisposeAsync(); + + await Assert.ThrowsAnyAsync(() => + scheduler + .WriteArticleAsync( + writer, + "x\r\n"u8.ToArray(), + NntpArticleTxFraming.PeerTakeThis("")) + .AsTask()); + + Assert.Equal(0, scheduler.Outstanding); + Assert.Equal(0, scheduler.Enqueuing); + pipe.Writer.Complete(); + } + + [Fact] + public async Task EnqueueGate_NeverAllowsConcurrentEnqueuers() + { + var pipe = new Pipe(); + await using var writer = new NntpResponseWriter(pipe.Writer); + await using var scheduler = new NntpStreamArticleTxScheduler(8); + + var peakEnq = 0; + scheduler.TestWhileHoldingEnqueueGate = _ => + { + var e = scheduler.Enqueuing; + while (true) + { + var observed = Volatile.Read(ref peakEnq); + if (e <= observed || Interlocked.CompareExchange(ref peakEnq, e, observed) == observed) + { + break; + } + } + + Assert.Equal(1, e); + return ValueTask.CompletedTask; + }; + + var tasks = Enumerable.Range(0, 8) + .Select(i => scheduler + .WriteArticleAsync( + writer, + Encoding.ASCII.GetBytes($"E{i}\r\n"), + NntpArticleTxFraming.PeerTakeThis($"")) + .AsTask()) + .ToArray(); + + await Task.WhenAll(tasks).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(1, peakEnq); + } + + private static string BuildOracle(IReadOnlyList bodies) + { + var expected = new StringBuilder(); + for (var i = 0; i < bodies.Count; i++) + { + var framing = NntpArticleTxFraming.PeerTakeThis($"<{i}@id>"); + expected.Append(Encoding.ASCII.GetString(framing.BuildHeaderBytes())); + expected.Append(Encoding.ASCII.GetString(ArticleWireReconstructor.RestuffArticle(bodies[i]))); + } + + return expected.ToString(); + } + + private static async Task DrainPipeAsync(Pipe pipe, NntpResponseWriter writer) + { + await writer.DisposeAsync(); + pipe.Writer.Complete(); + var buffer = new ArrayBufferWriter(); + while (true) + { + var read = await pipe.Reader.ReadAsync(); + foreach (var segment in read.Buffer) + { + buffer.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + + return Encoding.ASCII.GetString(buffer.WrittenSpan); + } + + private static async Task DrainSlowCapturingAsync( + Pipe pipe, + ArrayBufferWriter capture, + CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + ReadResult read; + try + { + read = await pipe.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + if (!read.Buffer.IsEmpty) + { + var take = Math.Min(4096, (int)read.Buffer.Length); + var examined = read.Buffer.Slice(0, take); + foreach (var segment in examined) + { + capture.Write(segment.Span); + } + + pipe.Reader.AdvanceTo(examined.End, read.Buffer.End); + } + else + { + pipe.Reader.AdvanceTo(read.Buffer.End); + } + + if (read.IsCompleted) + { + break; + } + + await Task.Yield(); + } + } + + private static async Task DrainInBackgroundAsync(Pipe pipe) + { + try + { + while (true) + { + var read = await pipe.Reader.ReadAsync().ConfigureAwait(false); + pipe.Reader.AdvanceTo(read.Buffer.End); + if (read.IsCompleted) + { + break; + } + } + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Reader completed. + } + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var start = Environment.TickCount64; + while (!condition()) + { + if (Environment.TickCount64 - start > timeout.TotalMilliseconds) + { + throw new TimeoutException("Condition not met within timeout."); + } + + await Task.Yield(); + } + } + + /// Sync dispose wrapper for constructor-only tests. + private sealed class SyncDispose : IDisposable + { + public SyncDispose(NntpStreamArticleTxScheduler inner) => Inner = inner; + + public NntpStreamArticleTxScheduler Inner { get; } + + public void Dispose() => Inner.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ArticleWireReconstructorTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ArticleWireReconstructorTests.cs new file mode 100644 index 00000000..be777f41 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ArticleWireReconstructorTests.cs @@ -0,0 +1,146 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +public sealed class ArticleWireReconstructorTests +{ + [Fact] + public void Restuff_NormalLine_Unchanged() + { + var stored = "hello\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("hello\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_LeadingDot_GetsStuffed() + { + var stored = ".example\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("..example\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_MultipleLeadingDots() + { + var stored = "..example\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("...example\r\n.\r\n"u8.ToArray(), wire); + } + + [Fact] + public async Task Restuff_LoneDotContentLine_DoesNotTerminateEarly() + { + var stored = ".\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal("..\r\n.\r\n"u8.ToArray(), wire); + + var reader = PipeReader.Create(new ReadOnlySequence(wire)); + var result = await NntpMultilineDataReader + .ReadArticleAsync(reader, 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(".\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public void Restuff_EmptyArticle() + { + var wire = ArticleWireReconstructor.RestuffArticle(ReadOnlySpan.Empty); + Assert.Equal(".\r\n"u8.ToArray(), wire); + } + + [Fact] + public void Restuff_WithoutTerminator_MatchesIngestQueueContract() + { + var stored = "Subject: t\r\n\r\n.hidden\r\n"u8.ToArray(); + var wire = ArticleWireReconstructor.RestuffArticle(stored, includeTerminator: false); + Assert.Equal("Subject: t\r\n\r\n..hidden\r\n"u8.ToArray(), wire); + Assert.False(wire.AsSpan().EndsWith("\r\n.\r\n"u8)); + } + + [Fact] + public async Task RoundTrip_DestuffThenRestuff_MatchesOriginalWire() + { + var originalWire = Encoding.ASCII.GetBytes( + "Subject: t\r\n\r\n..leading\r\n...three\r\nnormal\r\n.\r\n"); + var reader = PipeReader.Create(new ReadOnlySequence(originalWire)); + var destuffed = await NntpMultilineDataReader + .ReadArticleAsync(reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + var reconstructed = ArticleWireReconstructor.RestuffArticle(destuffed.Payload.Span); + Assert.Equal(originalWire, reconstructed); + } + + [Fact] + public void BuildTakeThisTransaction_PrefixesCommand() + { + var stored = "body\r\n"u8.ToArray(); + var txn = FramingWireFactory.BuildTakeThisTransaction("", stored); + Assert.StartsWith("TAKETHIS \r\n", Encoding.ASCII.GetString(txn)); + Assert.EndsWith(".\r\n", Encoding.ASCII.GetString(txn)); + } + + [Fact] + public async Task ContinuousParser_DoesNotTreatCommandsInsideArticleAsControl() + { + var stored = Encoding.ASCII.GetBytes("QUIT\r\nCHECK x\r\nTAKETHIS y\r\n"); + var txn = FramingWireFactory.BuildTakeThisTransaction("", stored); + var next = FramingWireFactory.BuildTakeThisTransaction("", "x\r\n"u8); + var stream = txn.Concat(next).ToArray(); + var result = await ProductionTakethisStream.ParseAsync(stream); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal(0, result.Quits); + Assert.Equal(0, result.Checks); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } + + [Fact] + public void Restuff_IsDeterministic_AndMatchesByteEstimate() + { + var stored = Encoding.ASCII.GetBytes(".a\r\nbb\r\n..c\r\n"); + var first = ArticleWireReconstructor.RestuffArticle(stored); + var second = ArticleWireReconstructor.RestuffArticle(stored); + Assert.Equal(first, second); + Assert.Equal(first.Length, ArticleWireReconstructor.EstimateRestuffedWireBytes(stored)); + } +} + +public sealed class MessageIdDigestTests +{ + [Fact] + public void Blake3_WithAngleBrackets_MatchesKnownCorpusFilename() + { + const string mid = "<3810724$5fcf702$2a675b8@82ecaa5ff0.e67ff>"; + const string expected = "00024a9fbd6824b3529c34dabdb6af28505b5be684936bdca64279279736e7ad"; + Assert.Equal(expected, MessageIdDigest.ComputeHexLower(mid)); + Assert.Equal( + Path.Join("00", "02", expected), + MessageIdDigest.BuildRelativePath(expected)); + } + + [Fact] + public void WithoutAngleBrackets_DoesNotMatch() + { + const string mid = "<3810724$5fcf702$2a675b8@82ecaa5ff0.e67ff>"; + const string expected = "00024a9fbd6824b3529c34dabdb6af28505b5be684936bdca64279279736e7ad"; + Assert.NotEqual(expected, MessageIdDigest.ComputeHexLower(mid.Trim('<', '>'))); + } + + [Fact] + public void ExtractMessageId_AcceptsBareLfAfterPath() + { + var stored = Encoding.ASCII.GetBytes( + "Path: host!not-for-mail\n" + + "Message-ID: \r\n" + + "Subject: x\r\n" + + "\r\n" + + "body\r\n"); + Assert.True(StoredArticleReader.TryExtractMessageId(stored, out var mid)); + Assert.Equal("", mid); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/BulkMultilineFramerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/BulkMultilineFramerTests.cs new file mode 100644 index 00000000..565cf732 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/BulkMultilineFramerTests.cs @@ -0,0 +1,167 @@ +using System.Text; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// +/// Correctness tests for production destuff/terminator rules. +/// +public sealed class BulkMultilineFramerTests +{ + [Fact] + public async Task EmptyArticle() + { + var (status, payload) = await ReadRecording(FramingWireFactory.EmptyArticle, segmentSize: 1); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal(0, payload.Length); + } + + [Fact] + public async Task FiveByteDelimiterIsDetected() + { + var (status, payload) = await ReadRecording(FramingWireFactory.DataThenDelimiter(), segmentSize: 64); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal("data\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task HelloDotIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("Hello.\r\nSomething else.\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 8); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal("Hello.\r\nSomething else.\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task DotStuffedLineIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("..foo\r\nbar\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 3); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Equal(".foo\r\nbar\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task TerminatorAtEndOfSegment() + { + var wire = FramingWireFactory.WithTerminator("abc\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 5); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("abc\r\n", Encoding.ASCII.GetString(result.Payload.Span)); + } + + [Fact] + public async Task TerminatorSplitAcrossSegments() + { + var wire = FramingWireFactory.WithTerminator("split-term\r\n"); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBackBytes: 2); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("split-term\r\n", Encoding.ASCII.GetString(result.Payload.Span)); + } + + [Fact] + public async Task TerminatorSplitAcrossReads() + { + var wire = FramingWireFactory.WithTerminator("across-reads\r\n"); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(FramingWireFactory.ExpectedDestuffedLength(wire), result.Payload.Length); + } + + [Fact] + public async Task MultiSegmentArticle() + { + var wire = FramingWireFactory.SmallArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 17); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task PipelinedSecondArticleRemainsUnread() + { + var wire = FramingWireFactory.PipelinedTwoArticles(); + var pipe = new System.IO.Pipelines.Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.Equal("article-one\r\n", Encoding.ASCII.GetString(first.Payload.Span)); + Assert.Equal("article-two\r\n", Encoding.ASCII.GetString(second.Payload.Span)); + } + + [Fact] + public async Task LargeArticle() + { + var wire = FramingWireFactory.Large768KiBArticle(); + var expectedLen = FramingWireFactory.ExpectedDestuffedLength(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 64); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(expectedLen, result.Payload.Length); + } + + [Fact] + public async Task MultipleCRLFs() + { + var wire = FramingWireFactory.ManyCrlfArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 11); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task MultipleDotStuffedLines() + { + var wire = FramingWireFactory.ManyDotStuffedLinesArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 9); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.Contains(".stuffed-line-0\r\n"u8, payload.Span); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + } + + [Fact] + public async Task TrailingPeriodsDoNotTerminate() + { + var wire = FramingWireFactory.TrailingPeriodLinesArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var (status, payload) = await ReadRecording(wire, segmentSize: 13); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.True(expected.AsSpan().SequenceEqual(payload.Span)); + Assert.EndsWith("period.\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task TerminatorNotDeliveredToSink() + { + var wire = FramingWireFactory.WithTerminator("keep-me\r\n"); + var (status, payload) = await ReadRecording(wire, segmentSize: 2); + Assert.Equal(NntpMultilineReadStatus.Completed, status); + Assert.False(payload.Span.EndsWith(".\r\n"u8)); + Assert.Equal("keep-me\r\n", Encoding.ASCII.GetString(payload.Span)); + } + + [Fact] + public async Task PipeReader_ChunkedFeed_MatchesExpected() + { + var wire = FramingWireFactory.MediumArticle(); + var expected = FramingWireFactory.ExpectedDestuffedLength(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 32); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(expected, result.Payload.Length); + } + + private static async Task<(NntpMultilineReadStatus Status, ReadOnlyMemory Payload)> ReadRecording( + byte[] wire, + int segmentSize) + { + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + return (result.Status, result.Payload); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ContinuousTakethisParserTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ContinuousTakethisParserTests.cs new file mode 100644 index 00000000..21b98fce --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/ContinuousTakethisParserTests.cs @@ -0,0 +1,113 @@ +using System.Text; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +public sealed class ContinuousTakethisParserTests +{ + [Fact] + public async Task ParsesMultipleTakethis_Contiguous() + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(articleBytes: 128, articleCount: 5); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(5, result.Articles); + Assert.Equal(5 * 128, result.ArticleBytes); + } + + [Fact] + public async Task ParsesAcrossSmallSegments() + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(256, 3); + var result = await ProductionTakethisStream.ParseAsync(wire, chunkSize: 3); + Assert.True(result.IsComplete); + Assert.Equal(3, result.Articles); + } + + [Fact] + public async Task DotStuffedAndTrailingPeriod_NotTerminator() + { + var body = Encoding.ASCII.GetBytes("Hello.\r\n..foo\r\n...\r\n"); + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + ms.Write("TAKETHIS \r\n"u8); + ms.Write("x\r\n"u8); + ms.Write(".\r\n"u8); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + var destuffedFirst = FramingWireFactory.DestuffPayload(body); + Assert.Equal(destuffedFirst, result.Payloads[0].ToArray()); + Assert.Equal("x\r\n"u8.ToArray(), result.Payloads[1].ToArray()); + Assert.Equal(destuffedFirst.Length + result.Payloads[1].Length, result.ArticleBytes); + } + + [Fact] + public async Task CheckAndQuit_EscapeToControlPlane() + { + using var ms = new MemoryStream(); + ms.Write("CHECK \r\n"u8); + ms.Write("TAKETHIS \r\n"u8); + ms.Write("hi\r\n"u8); + ms.Write(".\r\n"u8); + ms.Write("QUIT\r\n"u8); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal(1, result.Checks); + Assert.Equal(1, result.Quits); + } + + [Fact] + public async Task EmptyArticle() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n.\r\n"); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal(0, result.ArticleBytes); + } + + [Fact] + public async Task IncompleteArticle_ReturnsIncomplete() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nno-terminator"); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.False(result.IsComplete); + } + + [Theory] + [InlineData(1)] + [InlineData(17)] + public async Task ProductionReader_AgreesOnMultiArticleStream_AcrossChunkSizes(int chunkSize) + { + var wire = FramingWireFactory.BuildFixedSizeTakethisStream(512, 8); + var result = await ProductionTakethisStream.ParseAsync(wire, chunkSize: chunkSize); + Assert.True(result.IsComplete); + Assert.Equal(8, result.Articles); + Assert.Equal(8 * 512, result.ArticleBytes); + + var stuffed = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\nbar\r\n"); + var segmented = await FramingPipe.ReadSegmentedAsync(stuffed, chunkSize); + Assert.Equal(NntpMultilineReadStatus.Completed, segmented.Status); + Assert.Equal(FramingWireFactory.ExpectedDestuffed(stuffed), segmented.Payload.ToArray()); + } + + [Fact] + public async Task InnWire7_SingleArticleStream() + { + var article = InnArticleCorpus.ReadAllBytes("wire-7"); + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(article); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(1, result.Articles); + Assert.Equal( + FramingWireFactory.DestuffCompleteWire(article).Length, + result.ArticleBytes); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/DataPlanePipelineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/DataPlanePipelineTests.cs new file mode 100644 index 00000000..d8805a65 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/DataPlanePipelineTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +public sealed class DataPlanePipelineTests +{ + [Fact] + public async Task Pipeline_Synthetic_CheckQuitAndPayloadCommands() + { + var aStored = Encoding.ASCII.GetBytes("CHECK ignored\r\nQUIT ignored\r\nTAKETHIS ignored\r\n"); + var cStored = Encoding.ASCII.GetBytes("hello\r\n"); + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", aStored)); + ms.Write("CHECK \r\n"u8); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", cStored)); + ms.Write("QUIT\r\n"u8); + ms.Write("TAKETHIS \r\nbody\r\n.\r\n"u8); + + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray(), stopAfterQuit: true); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal(1, result.Checks); + Assert.Equal(1, result.Quits); + Assert.Equal("", result.MessageIds[0]); + Assert.Equal("", result.MessageIds[1]); + Assert.Equal(aStored, result.Payloads[0].ToArray()); + Assert.Equal(cStored, result.Payloads[1].ToArray()); + } + + [Fact] + public async Task Pipeline_ControlPlane_CheckAndQuit() + { + var aStored = Encoding.ASCII.GetBytes("CHECK ignored\r\nQUIT ignored\r\nTAKETHIS ignored\r\n"); + var cStored = Encoding.ASCII.GetBytes("hello\r\n"); + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", aStored)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("", cStored)); + + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(2, result.Articles); + Assert.Equal("", result.MessageIds[0]); + Assert.Equal("", result.MessageIds[1]); + } + + [Fact] + public async Task Pipeline_CommandsInsideArticle_ArePayload() + { + var stored = Encoding.ASCII.GetBytes("line\r\nCHECK x\r\nQUIT\r\nTAKETHIS y\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(0, result.Checks); + Assert.Equal(0, result.Quits); + Assert.Equal(1, result.Articles); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } + + [Fact] + public async Task Pipeline_ResponseOrdering_Preserved() + { + using var ms = new MemoryStream(); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<1@t>", "a\r\n"u8)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<2@t>", "b\r\n"u8)); + ms.Write(FramingWireFactory.BuildTakeThisTransaction("<3@t>", "c\r\n"u8)); + var result = await ProductionTakethisStream.ParseAsync(ms.ToArray()); + Assert.True(result.IsComplete); + Assert.Equal(3, result.MessageIds.Count); + Assert.Equal("<1@t>", result.MessageIds[0]); + Assert.Equal("<2@t>", result.MessageIds[1]); + Assert.Equal("<3@t>", result.MessageIds[2]); + } + + [Fact] + public async Task Pipeline_RoundTrip_StoredEqualsDestuffedCurrent() + { + var stored = Encoding.ASCII.GetBytes(".leading\r\n..two\r\nnormal\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var result = await ProductionTakethisStream.ParseAsync(wire); + Assert.True(result.IsComplete); + Assert.Equal(stored.Length, result.ArticleBytes); + Assert.Equal(stored, result.Payloads[0].ToArray()); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/FramingTestSupport.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/FramingTestSupport.cs new file mode 100644 index 00000000..f08b490d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/FramingTestSupport.cs @@ -0,0 +1,665 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Blake3; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// Test-only helpers for production / restuff coverage. +internal static class FramingPipe +{ + public static async Task ReadOneAsync( + byte[] wire, + int maxArticleBytes = 8 * 1024 * 1024, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(wire, cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + + public static async Task ReadSegmentedAsync( + byte[] wire, + int segmentSize, + int maxArticleBytes = 8 * 1024 * 1024, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, segmentSize), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + + var read = NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, maxArticleBytes, cancellationToken) + .AsTask(); + await WriteChunkedAsync(pipe.Writer, wire, segmentSize, cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await read.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken).ConfigureAwait(false); + } + + public static async Task WriteChunkedAsync( + PipeWriter writer, + ReadOnlyMemory wire, + int chunkSize, + CancellationToken cancellationToken = default) + { + var offset = 0; + while (offset < wire.Length) + { + var n = Math.Min(chunkSize, wire.Length - offset); + await writer.WriteAsync(wire.Slice(offset, n), cancellationToken).ConfigureAwait(false); + await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + offset += n; + } + } + + public static async Task ReadSplitTerminatorAsync( + byte[] wire, + int holdBackBytes = 1, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(holdBackBytes, 1); + if (wire.Length <= holdBackBytes) + { + throw new ArgumentException("Wire is shorter than the split.", nameof(wire)); + } + + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: 8, + pauseWriterThreshold: 1024 * 1024, + resumeWriterThreshold: 1024 * 1024, + useSynchronizationContext: false)); + var read = NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, 8 * 1024 * 1024, cancellationToken) + .AsTask(); + await pipe.Writer.WriteAsync(wire.AsMemory()[..^holdBackBytes], cancellationToken).ConfigureAwait(false); + await pipe.Writer.FlushAsync(cancellationToken).ConfigureAwait(false); + await pipe.Writer.WriteAsync(wire.AsMemory()[^holdBackBytes..], cancellationToken).ConfigureAwait(false); + await pipe.Writer.FlushAsync(cancellationToken).ConfigureAwait(false); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + return await read.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); + } +} + +/// Synthetic complete-wire articles for production destuff tests. +internal static class FramingWireFactory +{ + public static byte[] EmptyArticle { get; } = ".\r\n"u8.ToArray(); + + public static byte[] WithTerminator(string bodyWithoutTerminator) => + Encoding.ASCII.GetBytes(bodyWithoutTerminator + ".\r\n"); + + public static byte[] DataThenDelimiter() => WithTerminator("data\r\n"); + + public static byte[] SmallArticle() + { + var sb = new StringBuilder(1600); + for (var i = 0; i < 40; i++) + { + sb.Append("Subject: line-").Append(i).Append("\r\n"); + } + + sb.Append("\r\n"); + for (var i = 0; i < 20; i++) + { + sb.Append("body text line ").Append(i).Append(" with some padding abcdefghijklmnop\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] MediumArticle() => BuildFilledArticle(128 * 1024, 64); + + public static byte[] Large768KiBArticle() => BuildFilledArticle(768 * 1024, 128); + + public static byte[] ManyCrlfArticle() + { + var sb = new StringBuilder(70_000); + for (var i = 0; i < 8000; i++) + { + sb.Append('x').Append(i % 10).Append("\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] ManyDotStuffedLinesArticle() + { + var sb = new StringBuilder(40_000); + for (var i = 0; i < 2000; i++) + { + sb.Append("..stuffed-line-").Append(i).Append("\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] TrailingPeriodLinesArticle() + { + var sb = new StringBuilder(40_000); + sb.Append("Hello.\r\n"); + sb.Append("Something else.\r\n"); + sb.Append("This line has a trailing period.\r\n"); + for (var i = 0; i < 1000; i++) + { + sb.Append("Line ").Append(i).Append(" ends with a period.\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] ManyDotsHostileArticle() + { + var sb = new StringBuilder(280_000); + for (var i = 0; i < 4000; i++) + { + sb.Append("Hello.\r\n"); + sb.Append("....not-a-terminator.\r\n"); + sb.Append("..\r\n"); + sb.Append("x.y.z.\r\n"); + } + + return WithTerminator(sb.ToString()); + } + + public static byte[] PipelinedTwoArticles() + { + var a1 = WithTerminator("article-one\r\n"); + var a2 = WithTerminator("article-two\r\n"); + var combined = new byte[a1.Length + a2.Length]; + a1.CopyTo(combined, 0); + a2.CopyTo(combined, a1.Length); + return combined; + } + + public static byte[] ExpectedDestuffed(ReadOnlySpan completeWire) => + DestuffCompleteWire(completeWire); + + public static int ExpectedDestuffedLength(ReadOnlySpan completeWire) => + DestuffCompleteWire(completeWire).Length; + + public static byte[] DestuffCompleteWire(ReadOnlySpan completeWire) + { + if (completeWire.Length == 3 + && completeWire[0] == (byte)'.' + && completeWire[1] == (byte)'\r' + && completeWire[2] == (byte)'\n') + { + return []; + } + + if (completeWire.Length < 3 + || completeWire[^3] != (byte)'.' + || completeWire[^2] != (byte)'\r' + || completeWire[^1] != (byte)'\n') + { + throw new ArgumentException("Expected a complete wire article ending in .\\r\\n.", nameof(completeWire)); + } + + return DestuffPayload(completeWire[..^3]); + } + + public static byte[] DestuffPayload(ReadOnlySpan wirePayloadWithoutTerminator) + { + if (wirePayloadWithoutTerminator.IsEmpty) + { + return []; + } + + var output = new ArrayBufferWriter(wirePayloadWithoutTerminator.Length); + var offset = 0; + while (offset < wirePayloadWithoutTerminator.Length) + { + var remaining = wirePayloadWithoutTerminator[offset..]; + var crlf = remaining.IndexOf("\r\n"u8); + if (crlf < 0) + { + throw new InvalidOperationException("Expected CRLF-framed wire payload."); + } + + var line = remaining[..crlf]; + if (line.Length > 0 && line[0] == (byte)'.') + { + line = line[1..]; + } + + var span = output.GetSpan(line.Length + 2); + line.CopyTo(span); + span[line.Length] = (byte)'\r'; + span[line.Length + 1] = (byte)'\n'; + output.Advance(line.Length + 2); + offset += crlf + 2; + } + + return output.WrittenSpan.ToArray(); + } + + public static byte[] BuildTakeThisTransaction(string messageId, ReadOnlySpan storedDestuffed) + { + var header = Encoding.ASCII.GetBytes("TAKETHIS " + messageId + "\r\n"); + var body = ArticleWireReconstructor.RestuffArticle(storedDestuffed); + var wire = new byte[header.Length + body.Length]; + header.CopyTo(wire, 0); + body.CopyTo(wire, header.Length); + return wire; + } + + public static byte[] BuildFixedSizeTakethisStream(int articleBytes, int articleCount, int seed = 42) + { + var rng = new Random(seed); + using var ms = new MemoryStream(); + var body = new byte[articleBytes]; + for (var i = 0; i < articleCount; i++) + { + ms.Write(Encoding.ASCII.GetBytes($"TAKETHIS \r\n")); + for (var b = 0; b < body.Length; b++) + { + if (b > 0 && b % 256 == 255) + { + body[b] = (byte)'\n'; + body[b - 1] = (byte)'\r'; + } + else + { + var v = (byte)rng.Next(1, 250); + body[b] = v == (byte)'.' ? (byte)'x' : v; + } + } + + if (body.Length >= 2) + { + body[^2] = (byte)'\r'; + body[^1] = (byte)'\n'; + } + + if (body.Length > 16) + { + body[8] = (byte)(i & 0xFF); + body[9] = (byte)((i >> 8) & 0xFF); + } + + ms.Write(body); + ms.Write(".\r\n"u8); + } + + return ms.ToArray(); + } + + private static byte[] BuildFilledArticle(int targetBodyBytes, int lineLength) + { + var sb = new StringBuilder(targetBodyBytes + 64); + var n = 0; + while (sb.Length < targetBodyBytes) + { + var pad = new string('a', Math.Max(1, lineLength - 8)); + sb.Append(n).Append(' ').Append(pad).Append("\r\n"); + n++; + } + + return WithTerminator(sb.ToString()); + } +} + +/// Production command-line + multiline reader over a TAKETHIS/CHECK/QUIT stream. +internal static class ProductionTakethisStream +{ + public static async Task ParseAsync( + byte[] wire, + bool stopAfterQuit = false, + int? chunkSize = null, + CancellationToken cancellationToken = default) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, chunkSize ?? 4096), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + var parse = ParseFromReaderAsync(pipe.Reader, stopAfterQuit, cancellationToken); + if (chunkSize is int size) + { + await FramingPipe.WriteChunkedAsync(pipe.Writer, wire, size, cancellationToken).ConfigureAwait(false); + } + else + { + await pipe.Writer.WriteAsync(wire, cancellationToken).ConfigureAwait(false); + } + + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + + return await parse.ConfigureAwait(false); + } + + private static async Task ParseFromReaderAsync( + PipeReader reader, + bool stopAfterQuit, + CancellationToken cancellationToken) + { + var articles = 0; + var articleBytes = 0L; + var checks = 0; + var quits = 0; + var payloads = new List>(); + var messageIds = new List(); + + while (true) + { + var line = await NntpCommandLineReader + .ReadLineAsync(reader, cancellationToken) + .ConfigureAwait(false); + if (line is null) + { + break; + } + + if (line.StartsWith("TAKETHIS ", StringComparison.OrdinalIgnoreCase)) + { + var id = line["TAKETHIS ".Length..]; + var article = await NntpMultilineDataReader + .ReadArticleAsync(reader, 8 * 1024 * 1024, cancellationToken) + .ConfigureAwait(false); + if (article.Status != NntpMultilineReadStatus.Completed) + { + return new TakethisStreamResult( + IsComplete: false, + articles, + articleBytes, + checks, + quits, + payloads, + messageIds); + } + + articles++; + articleBytes += article.Payload.Length; + payloads.Add(article.Payload); + messageIds.Add(id); + continue; + } + + if (line.StartsWith("CHECK ", StringComparison.OrdinalIgnoreCase)) + { + checks++; + continue; + } + + if (line.Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + quits++; + if (stopAfterQuit) + { + break; + } + } + } + + return new TakethisStreamResult(true, articles, articleBytes, checks, quits, payloads, messageIds); + } +} + +internal readonly record struct TakethisStreamResult( + bool IsComplete, + int Articles, + long ArticleBytes, + int Checks, + int Quits, + IReadOnlyList> Payloads, + IReadOnlyList MessageIds); + +/// INN fixture loader for the corpus already stored under tests/Data/InnArticles. +internal static class InnArticleCorpus +{ + public const string UpstreamCommit = "6843efceb8a15774d23e4e2892d08daf54574314"; + + private static readonly object Gate = new(); + private static InnArticleManifest? _manifest; + private static string? _root; + + public static string ResolveRoot() + { + if (_root is not null) + { + return _root; + } + + lock (Gate) + { + if (_root is not null) + { + return _root; + } + + var candidates = new[] + { + Path.Join(AppContext.BaseDirectory, "Data", "InnArticles"), + Path.GetFullPath(Path.Join(AppContext.BaseDirectory, "..", "..", "..", "Data", "InnArticles")), + }; + + foreach (var candidate in candidates) + { + if (File.Exists(Path.Join(candidate, "inn-articles.manifest.json"))) + { + _root = candidate; + return _root; + } + } + + throw new DirectoryNotFoundException( + "INN article corpus not found. Expected Data/InnArticles/inn-articles.manifest.json."); + } + } + + public static InnArticleManifest LoadManifest() + { + if (_manifest is not null) + { + return _manifest; + } + + lock (Gate) + { + if (_manifest is not null) + { + return _manifest; + } + + var json = File.ReadAllText(Path.Join(ResolveRoot(), "inn-articles.manifest.json")); + var manifest = JsonSerializer.Deserialize(json, InnArticleManifestJsonContext.Default.InnArticleManifest) + ?? throw new InvalidOperationException("Failed to deserialize inn-articles.manifest.json."); + _manifest = manifest; + return _manifest; + } + } + + public static byte[] ReadAllBytes(string relativePath) + { + var full = Path.Join(ResolveRoot(), relativePath); + if (!File.Exists(full)) + { + throw new FileNotFoundException($"INN fixture missing: {relativePath}", full); + } + + return File.ReadAllBytes(full); + } + + public static void VerifySha256(InnArticleManifestEntry entry) + { + var bytes = ReadAllBytes(entry.RelativePath); + var hash = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + // Hash the raw file bytes. InnArticles is marked binary in .gitattributes so + // core.autocrlf cannot rewrite LF fixtures before this comparison. + if (!string.Equals(hash, entry.Sha256, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"SHA-256 mismatch for {entry.RelativePath}: disk={hash}, manifest={entry.Sha256}."); + } + + if (bytes.Length != entry.Length) + { + throw new InvalidOperationException( + $"Length mismatch for {entry.RelativePath}: disk={bytes.Length}, manifest={entry.Length}."); + } + } + + public static IReadOnlyList CompleteWireArticles() => + LoadManifest().Files + .Where(static f => f.FramingClassification == "complete_multiline_wire") + .ToArray(); + + public static bool ContainsFiveByteDelimiter(ReadOnlySpan data) => + data.IndexOf("\r\n.\r\n"u8) >= 0; +} + +internal sealed class InnArticleManifest +{ + [JsonPropertyName("repository")] + public string Repository { get; set; } = ""; + + [JsonPropertyName("sourcePath")] + public string SourcePath { get; set; } = ""; + + [JsonPropertyName("upstreamCommit")] + public string UpstreamCommit { get; set; } = ""; + + [JsonPropertyName("fileCount")] + public int FileCount { get; set; } + + [JsonPropertyName("totalBytes")] + public long TotalBytes { get; set; } + + [JsonPropertyName("files")] + public List Files { get; set; } = []; +} + +internal sealed class InnArticleManifestEntry +{ + [JsonPropertyName("relativePath")] + public string RelativePath { get; set; } = ""; + + [JsonPropertyName("length")] + public int Length { get; set; } + + [JsonPropertyName("sha256")] + public string Sha256 { get; set; } = ""; + + [JsonPropertyName("framingClassification")] + public string FramingClassification { get; set; } = ""; + + [JsonPropertyName("containsLeadingDotLines")] + public bool ContainsLeadingDotLines { get; set; } +} + +[JsonSerializable(typeof(InnArticleManifest))] +[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] +internal partial class InnArticleManifestJsonContext : JsonSerializerContext; + +internal static class MessageIdDigest +{ + public static string ComputeHexLower(string messageIdWithBrackets) + { + var utf8 = Encoding.UTF8.GetBytes(messageIdWithBrackets); + var hash = Hasher.Hash(utf8); + return Convert.ToHexString(hash.AsSpan()).ToLowerInvariant(); + } + + public static string BuildRelativePath(string digestHexLower) => + Path.Join(digestHexLower[..2], digestHexLower.Substring(2, 2), digestHexLower); +} + +internal static class StoredArticleReader +{ + public static bool TryExtractMessageId(ReadOnlySpan storedHead, out string messageIdWithBrackets) + { + messageIdWithBrackets = string.Empty; + var offset = 0; + while (offset < storedHead.Length) + { + var remaining = storedHead[offset..]; + var eol = remaining.IndexOf((byte)'\n'); + ReadOnlySpan physical; + if (eol < 0) + { + physical = remaining; + offset = storedHead.Length; + } + else + { + physical = remaining[..eol]; + offset += eol + 1; + } + + if (!physical.IsEmpty && physical[^1] == (byte)'\r') + { + physical = physical[..^1]; + } + + if (physical.Length == 0) + { + break; + } + + if (!StartsWithIgnoreCase(physical, "Message-ID:"u8)) + { + continue; + } + + var colon = physical.IndexOf((byte)':'); + var value = physical[(colon + 1)..]; + while (!value.IsEmpty && value[0] is (byte)' ' or (byte)'\t') + { + value = value[1..]; + } + + var lt = value.IndexOf((byte)'<'); + var gt = value.LastIndexOf((byte)'>'); + if (lt >= 0 && gt > lt) + { + messageIdWithBrackets = Encoding.ASCII.GetString(value[lt..(gt + 1)]); + return true; + } + + return false; + } + + return false; + } + + private static bool StartsWithIgnoreCase(ReadOnlySpan line, ReadOnlySpan asciiPrefix) + { + if (line.Length < asciiPrefix.Length) + { + return false; + } + + for (var i = 0; i < asciiPrefix.Length; i++) + { + var a = line[i]; + var b = asciiPrefix[i]; + if (a is >= (byte)'A' and <= (byte)'Z') + { + a = (byte)(a + 32); + } + + if (b is >= (byte)'A' and <= (byte)'Z') + { + b = (byte)(b + 32); + } + + if (a != b) + { + return false; + } + } + + return true; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/IHaveArticleReaderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/IHaveArticleReaderTests.cs new file mode 100644 index 00000000..ad488bc5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/IHaveArticleReaderTests.cs @@ -0,0 +1,218 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +public sealed class IHaveArticleReaderTests +{ + private static Pipe CreatePipe() => + new(new PipeOptions(pauseWriterThreshold: 4 * 1024 * 1024, resumeWriterThreshold: 1024 * 1024, useSynchronizationContext: false)); + + [Fact] + public async Task OrdinaryLines_SurviveByteForByte_AndLeaveNextCommand() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: hi\r\n\r\nhello\r\n.\r\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: hi\r\n\r\nhello\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal(result.Payload.Length, result.Metrics.ArticleSize); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task DotStuffedLines_RemainStuffed_InOwnedPayload() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: d\r\n\r\n..foo\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: d\r\n\r\n..foo\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.DoesNotContain("\r\n.foo\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task DotStuffing_WithinDestuffedLimit_IsRetainedWhenStuffedWireExceedsIt() + { + var pipe = CreatePipe(); + var stuffed = string.Concat(Enumerable.Repeat("..\r\n", 10)); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffed + ".\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 35, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(40, result.Payload.Length); + Assert.Equal(stuffed, Encoding.ASCII.GetString(result.Payload.Span)); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task DotStuffing_OverDestuffedLimit_ReturnsTooLarge_AndLeavesNextCommand() + { + var pipe = CreatePipe(); + var stuffed = string.Concat(Enumerable.Repeat("..\r\n", 10)); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffed + ".\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 29, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.TooLarge, result.Status); + Assert.Equal(0, result.Payload.Length); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task Terminator_IsNotStored() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("From: a\r\n\r\nbody\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal("From: a\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.False(result.Payload.Span.EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task TerminatorAndNextCommand_SameBuffer_LeavesCommand() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: n\r\n\r\nx\r\n.\r\nQUIT\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("QUIT\r\n", await ReadLeftoverAsync(pipe.Reader)); + Assert.DoesNotContain("QUIT"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task TerminatorSplitAcrossReads_Completes() + { + var pipe = CreatePipe(); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await pipe.Writer.WriteAsync("Subject: s\r\n\r\nbody\r\n.\r"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: s\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task PartialPipeSegments_ReassembleWire() + { + var pipe = CreatePipe(); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await pipe.Writer.WriteAsync("Sub"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("ject: split\r\n\r\nbo"u8.ToArray()); + await pipe.Writer.FlushAsync(); + await Task.Yield(); + await pipe.Writer.WriteAsync("dy\r\n.\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Subject: split\r\n\r\nbody\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.True(result.Metrics.PipeReads >= 2); + } + + [Fact] + public async Task PrematureEof_IsIncomplete() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Subject: x\r\n\r\nno-terminator"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + } + + [Fact] + public async Task TooLarge_ThenTerminator_ReturnsTooLarge_AndLeavesCommand() + { + var pipe = CreatePipe(); + var body = new string('A', 64); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes($"Subject: big\r\n\r\n{body}\r\n.\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, maxArticleBytes: 16, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.TooLarge, result.Status); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + [Fact] + public async Task LargeArticle_IsOwnedAfterAdvance() + { + var body = new string('B', 120 * 1024); + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes("Subject: big\r\n\r\n" + body + "\r\n.\r\nDATE\r\n")); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 2 * 1024 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var leftover = await ReadLeftoverAsync(pipe.Reader); + Assert.Equal("DATE\r\n", leftover); + Assert.StartsWith("Subject: big\r\n\r\n", Encoding.ASCII.GetString(result.Payload.Span), StringComparison.Ordinal); + Assert.True(result.Payload.Length >= 120 * 1024); + _ = result.Payload.Span[0]; + _ = result.Payload.Span[^1]; + } + + [Fact] + public async Task StreamingFourKiBChunks_LeaveNextCommandIntact() + { + var body = new string('S', 12 * 1024); + var framed = Encoding.ASCII.GetBytes($"Subject: stream\r\n\r\n{body}\r\n.\r\nDATE\r\n"); + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 8 * 1024, + resumeWriterThreshold: 4 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false)); + var read = IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + await Task.Yield(); + for (var offset = 0; offset < framed.Length; offset += 4096) + { + var n = Math.Min(4096, framed.Length - offset); + await pipe.Writer.WriteAsync(framed.AsMemory(offset, n)); + await pipe.Writer.FlushAsync(); + } + + var result = await read; + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + Assert.True(result.Metrics.PipeReads >= 2); + } + + [Fact] + public async Task DeclaredSizes_DoNotConsumePastTerminator() + { + var pipe = CreatePipe(); + await pipe.Writer.WriteAsync("Content-Length: 999999\r\n\r\nshort\r\n.\r\nDATE\r\n"u8.ToArray()); + await pipe.Writer.FlushAsync(); + + var result = await IHaveArticleReader.ReadAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Content-Length: 999999\r\n\r\nshort\r\n"u8.ToArray(), result.Payload.ToArray()); + Assert.Equal("DATE\r\n", await ReadLeftoverAsync(pipe.Reader)); + } + + private static async Task ReadLeftoverAsync(PipeReader reader) + { + var result = await reader.ReadAsync(); + var text = Encoding.ASCII.GetString(result.Buffer.ToArray()); + reader.AdvanceTo(result.Buffer.End); + return text; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs new file mode 100644 index 00000000..eee6cf56 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/InnArticleCorpusFramerTests.cs @@ -0,0 +1,237 @@ +using System.IO.Pipelines; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// +/// Correctness coverage for the INN tests/data/articles corpus against production +/// destuff/terminator rules. +/// +public sealed class InnArticleCorpusFramerTests +{ + public static IEnumerable AllFixtureNames() + { + foreach (var entry in InnArticleCorpus.LoadManifest().Files) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable CompleteWireFixtureNames() + { + foreach (var entry in InnArticleCorpus.CompleteWireArticles()) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable SegmentSizes() + { + foreach (var size in new[] { 1, 2, 4, 16, 4096 }) + { + yield return [size]; + } + } + + [Fact] + public void Manifest_MatchesPinnedCommit_AndFixtureCount() + { + var manifest = InnArticleCorpus.LoadManifest(); + Assert.Equal(InnArticleCorpus.UpstreamCommit, manifest.UpstreamCommit); + Assert.Equal("InterNetNews/inn", manifest.Repository); + Assert.Equal("tests/data/articles", manifest.SourcePath); + Assert.Equal(25, manifest.FileCount); + Assert.Equal(25, manifest.Files.Count); + Assert.Equal(19544, manifest.TotalBytes); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public void EveryFixture_Sha256MatchesManifest(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + InnArticleCorpus.VerifySha256(entry); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public async Task EveryFixture_BulkFramer_StatusMatchesClassification(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize: 7); + + if (entry.FramingClassification == "complete_multiline_wire") + { + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.Equal(expected.Length, result.Payload.Length); + Assert.NotEqual(wire.Length, result.Payload.Length); + } + else + { + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + // Production does not emit an incomplete payload (retired bulk prototype forwarded the buffer). + Assert.True(result.Payload.IsEmpty); + Assert.Equal(wire, InnArticleCorpus.ReadAllBytes(relativePath)); + } + } + + [Theory] + [MemberData(nameof(CompleteWireFixtureNames))] + public async Task CompleteWire_PreservesDotStuffing_AndExcludesTerminator(string relativePath) + { + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + Assert.True(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + var stuffedWithoutTerminator = wire.AsSpan()[..^3]; + Assert.Contains("..\r\n"u8, stuffedWithoutTerminator); + + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().StartsWith("."u8)); + Assert.Contains("..\r\n"u8, expected); + Assert.NotEqual(wire.Length, expected.Length); + Assert.False(wire.AsSpan()[..^3].SequenceEqual(expected)); + + var result = await FramingPipe.ReadSegmentedAsync(wire, 3); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.True(result.Payload.Span.StartsWith("."u8)); + } + + [Theory] + [MemberData(nameof(SegmentSizes))] + public async Task Wire7_IdenticalPayload_AcrossSegmentSizes(int segmentSize) + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task Wire7_FiveByteDelimiter_SplitAtEveryBoundary() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(wire.AsSpan().EndsWith("\r\n.\r\n"u8)); + + var whole = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, whole.Status); + Assert.True(expected.AsSpan().SequenceEqual(whole.Payload.Span)); + + for (var holdBack = 1; holdBack <= 5; holdBack++) + { + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBack); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + } + + [Fact] + public async Task Wire7_DelimiterSplitAcrossReads() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task Wire7_HelloDotStyleTrailingPeriod_IsNotTerminator() + { + // Upstream wire-7 body includes lines that end with '.' before CRLF inside content; + // terminator is only the final .\r\n line. Prove payload retains those periods. + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.Contains(".\r\n"u8, expected); + + var result = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + Assert.Contains("parser.\r\n"u8, result.Payload.Span); + } + + [Fact] + public async Task Wire7_CurrentUnstuffs_BulkPreservesWire() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var stuffedWithoutTerminator = wire[..^3]; + var destuffed = FramingWireFactory.DestuffCompleteWire(wire); + + var current = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, current.Status); + Assert.True(destuffed.AsSpan().SequenceEqual(current.Payload.Span)); + + // Retired bulk prototype kept stuffed wire; production destuffs. + Assert.NotEqual(current.Payload.Length, stuffedWithoutTerminator.Length); + Assert.True(current.Payload.Span.SequenceEqual(FramingWireFactory.DestuffPayload(stuffedWithoutTerminator))); + } + + [Fact] + public async Task WireTruncated_IsIncomplete() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-truncated"); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + var result = await FramingPipe.ReadOneAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + } + + [Fact] + public async Task WireStrange_IsIncomplete_PreservesExactBytesIncludingNul() + { + var wire = InnArticleCorpus.ReadAllBytes("wire-strange"); + Assert.Contains((byte)0, wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, 1); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + // Fixture integrity: production does not echo incomplete bytes, but the NUL fixture is unchanged. + Assert.Contains((byte)0, InnArticleCorpus.ReadAllBytes("wire-strange")); + Assert.Equal(477, wire.Length); + } + + [Theory] + [InlineData("5")] + [InlineData("6")] + [InlineData("7")] + public async Task LeadingDotSpoolFixtures_DoNotFalseComplete(string name) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == name); + Assert.True(entry.ContainsLeadingDotLines); + Assert.Equal("spool_or_non_wire_no_nntp_terminator", entry.FramingClassification); + var wire = InnArticleCorpus.ReadAllBytes(name); + var result = await FramingPipe.ReadSegmentedAsync(wire, 2); + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + Assert.False(InnArticleCorpus.ContainsFiveByteDelimiter(wire)); + } + + [Fact] + public async Task Pipelined_TwoWire7_SecondRemains() + { + var one = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(one); + var combined = new byte[one.Length * 2]; + one.CopyTo(combined.AsSpan()); + one.CopyTo(combined.AsSpan(one.Length)); + + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(combined); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var leftover = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.True(expected.AsSpan().SequenceEqual(first.Payload.Span)); + Assert.True(expected.AsSpan().SequenceEqual(second.Payload.Span)); + Assert.Equal(NntpMultilineReadStatus.Incomplete, leftover.Status); + Assert.True(leftover.Payload.IsEmpty); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpContinuousRxTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpContinuousRxTests.cs new file mode 100644 index 00000000..97c5581b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpContinuousRxTests.cs @@ -0,0 +1,677 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// +/// Continuous STREAM scanner: command/article units, ownership, splits, and control-plane rules. +/// TAKETHIS payload is framed wire bytes (terminator omitted, no destuff). +/// +public sealed class NntpContinuousRxTests +{ + [Fact] + public async Task OneTakeThis_CopiesFramedWireAndOmitsTerminator() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n..foo\r\nbar\r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("..foo\r\nbar\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task ManyTakeThis_SamePipeBuffer_NoExtraTransportRead() + { + var wire = Encoding.ASCII.GetBytes( + "TAKETHIS \r\none\r\n.\r\nTAKETHIS \r\ntwo\r\n.\r\nTAKETHIS \r\nthree\r\n.\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(3, units.Count); + Assert.All(units, u => Assert.Equal(NntpContinuousRxKind.TakeThis, u.Kind)); + Assert.Equal(["", "", ""], units.Select(u => u.MessageId)); + Assert.Equal("one\r\n"u8.ToArray(), units[0].Article.Payload.ToArray()); + Assert.Equal("two\r\n"u8.ToArray(), units[1].Article.Payload.ToArray()); + Assert.Equal("three\r\n"u8.ToArray(), units[2].Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(5)] + [InlineData(17)] + [InlineData(64)] + public async Task ArbitrarySegmentBoundaries_MatchFramedWireOracle(int segmentSize) + { + var article = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\n...\r\n"); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var expected = article[..^3]; + var unit = await ReadOneSegmentedAsync(wire, segmentSize, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(expected, unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public async Task CommandCrlfSplit_StillRecognizesTakeThis(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nbody\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + Assert.Equal("body\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(5)] + public async Task ArticleTerminatorSplit_Completes(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nkeep\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("keep\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task CommandLikeTextInsideArticle_IsPayload() + { + var stored = Encoding.ASCII.GetBytes("QUIT\r\nCHECK x\r\nTAKETHIS y\r\n"); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true); + Assert.Single(units); + Assert.Equal(NntpContinuousRxKind.TakeThis, units[0].Kind); + Assert.Equal(stored, units[0].Article.Payload.ToArray()); + } + + [Fact] + public async Task CheckAndQuit_AreControlCommands() + { + var wire = Encoding.ASCII.GetBytes( + "CHECK \r\nTAKETHIS \r\nhi\r\n.\r\nQUIT\r\nTAKETHIS \r\nz\r\n.\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true, stopAfterQuit: true); + Assert.Equal(3, units.Count); + Assert.Equal(NntpContinuousRxKind.Command, units[0].Kind); + Assert.Equal("CHECK ", units[0].CommandLine); + Assert.Equal(NntpContinuousRxKind.TakeThis, units[1].Kind); + Assert.Equal("", units[1].MessageId); + Assert.Equal(NntpContinuousRxKind.Command, units[2].Kind); + Assert.Equal("QUIT", units[2].CommandLine); + } + + [Fact] + public async Task IncompleteCommand_NeedMoreThenCompletes() + { + var parser = new NntpContinuousRxParser(); + var partial = new ReadOnlySequence("TAKETH"u8.ToArray()); + Assert.Equal(NntpContinuousRxKind.NeedMore, parser.TryConsume(ref partial, true, 1024).Kind); + + var unit = await ReadOneAsync("TAKETHIS \r\nok\r\n.\r\n"u8.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal("", unit.MessageId); + } + + [Fact] + public async Task IncompleteArticle_Eof_IsIncomplete() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nno-terminator"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.TakeThis, unit.Kind); + Assert.Equal(NntpMultilineReadStatus.Incomplete, unit.Article.Status); + Assert.True(unit.Article.Payload.IsEmpty); + } + + [Fact] + public async Task EmptyArticle_IsCompletedEmptyPayload() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(0, unit.Article.Payload.Length); + } + + [Fact] + public async Task Eof_EmptyPipe_IsNeedMore() + { + var unit = await ReadOneAsync([], consumeTakeThisArticle: true); + Assert.Equal(NntpContinuousRxKind.NeedMore, unit.Kind); + } + + [Fact] + public async Task Cancellation_Throws() + { + var pipe = new Pipe(); + var parser = new NntpContinuousRxParser(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => + NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, true, 1024, cts.Token) + .AsTask()); + } + + [Fact] + public async Task LargeArticle_MatchesOracle() + { + var article = FramingWireFactory.Large768KiBArticle(); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var expected = article[..^3]; + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true, maxArticleBytes: 4 * 1024 * 1024); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(expected.Length, unit.Article.Payload.Length); + Assert.True(expected.AsSpan().SequenceEqual(unit.Article.Payload.Span)); + } + + [Fact] + public async Task BinaryNulArticle_PreservedInFramedPayload() + { + var body = new byte[] { (byte)'a', 0, (byte)'b', (byte)'\r', (byte)'\n' }; + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + var unit = await ReadOneAsync(ms.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal(body, unit.Article.Payload.ToArray()); + Assert.Contains((byte)0, unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task TooLarge_ConsumesTerminator_EmptyPayload() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n0123456789\r\n.\r\nQUIT\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: true, maxArticleBytes: 4); + Assert.Equal(2, units.Count); + Assert.Equal(NntpMultilineReadStatus.TooLarge, units[0].Article.Status); + Assert.True(units[0].Article.Payload.IsEmpty); + Assert.Equal("QUIT", units[1].CommandLine); + } + + [Fact] + public async Task WithoutConsumeArticle_TakeThisIsCommandLineOnly() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nQUIT\r\n"); + var units = await ReadAllAsync(wire, consumeTakeThisArticle: false); + Assert.Equal(2, units.Count); + Assert.Equal(NntpContinuousRxKind.Command, units[0].Kind); + Assert.Equal("TAKETHIS ", units[0].CommandLine); + Assert.Equal("QUIT", units[1].CommandLine); + } + + [Fact] + public async Task OwnedPayload_SurvivesSubsequentAdvance() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nkeep-me\r\n.\r\nTAKETHIS \r\nx\r\n.\r\n"); + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var first = await NntpContinuousRxReader.ReadUnitAsync(pipe.Reader, parser, true, 64 * 1024, CancellationToken.None); + var second = await NntpContinuousRxReader.ReadUnitAsync(pipe.Reader, parser, true, 64 * 1024, CancellationToken.None); + Assert.Equal("keep-me\r\n"u8.ToArray(), first.Article.Payload.ToArray()); + Assert.Equal("x\r\n"u8.ToArray(), second.Article.Payload.ToArray()); + } + + [Fact] + public async Task WriterComplete_ThenReaderComplete_DoesNotReadAfterCompletion() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + await pipe.Writer.WriteAsync("DATE\r\n"u8.ToArray()); + await pipe.Writer.CompleteAsync(); + + var date = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + Assert.Equal(NntpVerb.Date, date.Command.Verb); + + var eof = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + Assert.Equal(NntpContinuousRxKind.NeedMore, eof.Kind); + + await pipe.Reader.CompleteAsync(); + var completed = false; + try + { + _ = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task CancelPendingRead_ThenCompleteWriterAndReader_DoesNotReadAfterCompletion() + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + using var cts = new CancellationTokenSource(); + var pending = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle: false, 1024, cts.Token) + .AsTask(); + + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => pending); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + + var completed = false; + try + { + _ = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle: false, + maxArticleBytes: 1024, + CancellationToken.None); + } + catch (InvalidOperationException ex) + { + completed = ex.Message.Contains( + "Reading is not allowed after reader was completed", + StringComparison.Ordinal); + } + + Assert.True(completed); + } + + [Fact] + public async Task StreamScanner_PreservesStuffedWire_WhileMultilineReaderDestuffs() + { + var article = FramingWireFactory.WithTerminator("Hello.\r\n..foo\r\nbar\r\n"); + var header = "TAKETHIS \r\n"u8.ToArray(); + var wire = header.Concat(article).ToArray(); + var stream = await ReadOneAsync(wire, consumeTakeThisArticle: true); + var reader = await FramingPipe.ReadOneAsync(article); + Assert.Equal(NntpMultilineReadStatus.Completed, stream.Article.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, reader.Status); + Assert.Equal("Hello.\r\n..foo\r\nbar\r\n"u8.ToArray(), stream.Article.Payload.ToArray()); + Assert.Equal("Hello.\r\n.foo\r\nbar\r\n"u8.ToArray(), reader.Payload.ToArray()); + Assert.NotEqual(reader.Payload.ToArray(), stream.Article.Payload.ToArray()); + } + + [Fact] + public async Task ManyCrlfLines_PreserveExactPayload() + { + var body = new StringBuilder(); + for (var i = 0; i < 200; i++) + { + body.Append("line-").Append(i).Append("\r\n"); + } + + var stored = Encoding.ASCII.GetBytes(body.ToString()); + var wire = FramingWireFactory.BuildTakeThisTransaction("", stored); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal(stored, unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task StuffedDotLines_ArePayloadNotTerminator() + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\n..one\r\n..two\r\n...three\r\n.\r\n"); + var unit = await ReadOneAsync(wire, consumeTakeThisArticle: true); + Assert.Equal("..one\r\n..two\r\n...three\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + public async Task FiveByteTerminator_SplitAcrossEveryByteBoundary(int holdBack) + { + var wire = Encoding.ASCII.GetBytes("TAKETHIS \r\nbody-line\r\n.\r\n"); + var unit = await ReadSplitAsync(wire, holdBack, consumeTakeThisArticle: true); + Assert.Equal(NntpMultilineReadStatus.Completed, unit.Article.Status); + Assert.Equal("body-line\r\n"u8.ToArray(), unit.Article.Payload.ToArray()); + } + + [Fact] + public async Task BinaryCrLfInsideLine_DoesNotPrematurelyTerminate() + { + // RFC 3977 forbids isolated CR/LF in a line; STREAM still treats only \r\n.\r\n as the end. + var body = new byte[] { (byte)'a', (byte)'\r', (byte)'b', (byte)'\n', (byte)'c', (byte)'\r', (byte)'\n' }; + using var ms = new MemoryStream(); + ms.Write("TAKETHIS \r\n"u8); + ms.Write(body); + ms.Write(".\r\n"u8); + var unit = await ReadOneAsync(ms.ToArray(), consumeTakeThisArticle: true); + Assert.Equal(body, unit.Article.Payload.ToArray()); + } + + [Fact] + public void TryConsume_MultipleUnitsFromOneSequence() + { + var wire = Encoding.ASCII.GetBytes( + "TAKETHIS <1@t>\r\na\r\n.\r\nTAKETHIS <2@t>\r\nb\r\n.\r\n"); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + var first = Capture(parser, parser.TryConsume(ref buffer, true, 1024)); + var second = Capture(parser, parser.TryConsume(ref buffer, true, 1024)); + Assert.Equal(NntpContinuousRxKind.TakeThis, first.Kind); + Assert.Equal(NntpContinuousRxKind.TakeThis, second.Kind); + Assert.Equal("<1@t>", first.MessageId); + Assert.Equal("<2@t>", second.MessageId); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_ExactlyMaxCommandLine_ParsesTheWholeLine() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes); + var wire = WithCrlf(line); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + var unit = Capture(parser, parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024)); + + Assert.Equal(NntpContinuousRxKind.Command, unit.Kind); + Assert.True(unit.Unit.Command.IsValid); + Assert.Equal(NntpVerb.Group, unit.Unit.Command.Verb); + Assert.Equal(NntpCommandLineReader.MaxCommandLineBytes, parser.CurrentCommandLine.Length); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongLine_IsSyntaxErrorAndDoesNotParseThePrefix() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + + Assert.Equal(NntpContinuousRxKind.Command, unit.Kind); + Assert.False(unit.Command.IsValid); + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.Equal(NntpVerb.None, unit.Command.Verb); + Assert.Equal(0, parser.CurrentCommandLine.Length); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_ValidPrefixPlusExtraPastLimit_IsNotDispatchedAsThePrefix() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 6]; + "STAT "u8.CopyTo(line); + line.AsSpan("STAT "u8.Length, NntpCommandLineReader.MaxCommandLineBytes - "STAT "u8.Length) + .Fill((byte)' '); + " extra"u8.CopyTo(line.AsSpan(NntpCommandLineReader.MaxCommandLineBytes)); + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.NotEqual(NntpVerb.Stat, unit.Command.Verb); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongInvalidPrefix_IsTooLongNotAParsedVerb() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan(0, NntpCommandLineReader.MaxCommandLineBytes).Fill((byte)'!'); + line[^1] = (byte)'x'; + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.NotEqual(NntpParseStatus.UnknownVerb, unit.Command.Status); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_OverlongTakeThisThenQuit_ConsumesTheLineAndParsesTheNextCommand() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 8]; + "TAKETHIS "u8.CopyTo(line); + line.AsSpan("TAKETHIS "u8.Length).Fill((byte)'x'); + var wire = new byte[line.Length + 2 + "QUIT\r\n"u8.Length]; + WithCrlf(line).CopyTo(wire); + "QUIT\r\n"u8.CopyTo(wire.AsSpan(line.Length + 2)); + var buffer = new ReadOnlySequence(wire); + var parser = new NntpContinuousRxParser(); + + var first = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + var second = Capture(parser, parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024)); + + Assert.Equal(NntpContinuousRxKind.Command, first.Kind); + Assert.Equal(NntpParseStatus.CommandLineTooLong, first.Command.Status); + Assert.NotEqual(NntpContinuousRxKind.TakeThis, first.Kind); + Assert.Equal(NntpVerb.Quit, second.Unit.Command.Verb); + Assert.True(second.Unit.Command.IsValid); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_EmbeddedLfIsNotTheDelimiter() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan().Fill((byte)'a'); + line[10] = (byte)'\n'; + line[20] = (byte)'\r'; + var buffer = new ReadOnlySequence(WithCrlf(line)); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpParseStatus.CommandLineTooLong, unit.Command.Status); + Assert.True(buffer.IsEmpty); + } + + [Fact] + public void TryConsume_NoCrlf_StaysNeedMore() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 100]; + line.AsSpan().Fill((byte)'a'); + var buffer = new ReadOnlySequence(line); + var parser = new NntpContinuousRxParser(); + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: false, maxArticleBytes: 1024); + + Assert.Equal(NntpContinuousRxKind.NeedMore, unit.Kind); + Assert.Equal(line.Length, buffer.Length); + } + + [Fact] + public async Task TryConsume_OverlongSplitAcrossReads_RejectsOnceTheCrlfArrives() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = WithCrlf(line); + var pipe = new Pipe(); + var parser = new NntpContinuousRxParser(); + await pipe.Writer.WriteAsync(wire.AsMemory(0, 1000)); + var first = await ReadAvailableAsync(pipe.Reader, parser); + Assert.Equal(NntpContinuousRxKind.NeedMore, first.Kind); + + await pipe.Writer.WriteAsync(wire.AsMemory(1000)); + var second = await ReadAvailableAsync(pipe.Reader, parser); + Assert.Equal(NntpParseStatus.CommandLineTooLong, second.Command.Status); + Assert.Equal(NntpVerb.None, second.Command.Verb); + } + + private static byte[] GroupLine(int length) + { + var line = new byte[length]; + "GROUP "u8.CopyTo(line); + line.AsSpan("GROUP "u8.Length).Fill((byte)'a'); + return line; + } + + private static byte[] WithCrlf(byte[] line) + { + var wire = new byte[line.Length + 2]; + line.CopyTo(wire); + wire[^2] = (byte)'\r'; + wire[^1] = (byte)'\n'; + return wire; + } + + private static async Task ReadAvailableAsync(PipeReader reader, NntpContinuousRxParser parser) + { + var read = await reader.ReadAsync(); + var buffer = read.Buffer; + var unit = parser.TryConsume(ref buffer, consumeTakeThisArticle: true, maxArticleBytes: 1024); + reader.AdvanceTo(buffer.Start, buffer.End); + return unit; + } + + private static CapturedRxUnit Capture(NntpContinuousRxParser parser, NntpContinuousRxUnit unit) + { + var line = parser.CurrentCommandLine.Span; + return new CapturedRxUnit( + unit, + Encoding.ASCII.GetString(line), + Encoding.ASCII.GetString(unit.Command.ArgumentSpan(line))); + } + + private static async Task ReadOneAsync( + byte[] wire, + bool consumeTakeThisArticle, + int maxArticleBytes = 8 * 1024 * 1024) + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + if (wire.Length > 0) + { + await pipe.Writer.WriteAsync(wire); + } + + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var unit = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle, + maxArticleBytes, + CancellationToken.None); + return Capture(parser, unit); + } + + private static async Task ReadOneSegmentedAsync( + byte[] wire, + int segmentSize, + bool consumeTakeThisArticle) + { + var pipe = new Pipe(new PipeOptions( + minimumSegmentSize: Math.Max(1, segmentSize), + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + var read = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle, 8 * 1024 * 1024, CancellationToken.None) + .AsTask(); + await FramingPipe.WriteChunkedAsync(pipe.Writer, wire, segmentSize, CancellationToken.None); + await pipe.Writer.CompleteAsync(); + return Capture(parser, await read.WaitAsync(TimeSpan.FromSeconds(10))); + } + + private static async Task ReadSplitAsync( + byte[] wire, + int holdBack, + bool consumeTakeThisArticle) + { + var pipe = new Pipe(new PipeOptions(useSynchronizationContext: false)); + var parser = new NntpContinuousRxParser(); + var read = NntpContinuousRxReader + .ReadUnitAsync(pipe.Reader, parser, consumeTakeThisArticle, 8 * 1024 * 1024, CancellationToken.None) + .AsTask(); + await pipe.Writer.WriteAsync(wire.AsMemory()[..^holdBack]); + await pipe.Writer.FlushAsync(); + await pipe.Writer.WriteAsync(wire.AsMemory()[^holdBack..]); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + return Capture(parser, await read.WaitAsync(TimeSpan.FromSeconds(5))); + } + + private static async Task> ReadAllAsync( + byte[] wire, + bool consumeTakeThisArticle, + bool stopAfterQuit = false, + int maxArticleBytes = 8 * 1024 * 1024) + { + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 16 * 1024 * 1024, + resumeWriterThreshold: 8 * 1024 * 1024, + useSynchronizationContext: false)); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var parser = new NntpContinuousRxParser(); + var units = new List(); + while (true) + { + var unit = await NntpContinuousRxReader.ReadUnitAsync( + pipe.Reader, + parser, + consumeTakeThisArticle, + maxArticleBytes, + CancellationToken.None); + if (unit.Kind == NntpContinuousRxKind.NeedMore) + { + break; + } + + var captured = Capture(parser, unit); + units.Add(captured); + if (stopAfterQuit + && captured.Kind == NntpContinuousRxKind.Command + && captured.Unit.Command.Verb == NntpVerb.Quit) + { + break; + } + } + + return units; + } + + private readonly struct CapturedRxUnit + { + public CapturedRxUnit(NntpContinuousRxUnit unit, string commandLine, string argument) + { + Unit = unit; + CommandLine = commandLine; + Argument = argument; + } + + public NntpContinuousRxUnit Unit { get; } + + public string CommandLine { get; } + + public string Argument { get; } + + public string MessageId => Argument; + + public NntpContinuousRxKind Kind => Unit.Kind; + + public NntpMultilineReadResult Article => Unit.Article; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpDelimiterSearchTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpDelimiterSearchTests.cs new file mode 100644 index 00000000..ecc66d8b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/NntpDelimiterSearchTests.cs @@ -0,0 +1,100 @@ +using System.Buffers; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// +/// STREAM article terminator location: one \r\n.\r\n search, not per-CRLF line walks. +/// +public sealed class NntpDelimiterSearchTests +{ + [Fact] + public void EmptyArticle_AtStart_IsThreeByteTerminator() + { + var buffer = new ReadOnlySequence(".\r\nQUIT\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out var consumed)); + Assert.Equal(0, payload); + Assert.Equal(3, consumed); + } + + [Fact] + public void EmptyArticle_NotAtStart_IsNotThreeByteTerminator() + { + var buffer = new ReadOnlySequence(".\r\n"u8.ToArray()); + Assert.False(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: false, out _, out _)); + } + + [Fact] + public void FiveByteTerminator_PayloadIncludesLeadingCrlf() + { + var buffer = new ReadOnlySequence("data\r\n.\r\nNEXT\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: false, out var payload, out var consumed)); + Assert.Equal("data\r\n"u8.Length, payload); + Assert.Equal("data\r\n.\r\n"u8.Length, consumed); + } + + [Fact] + public void HelloDot_IsNotTerminator() + { + var buffer = new ReadOnlySequence("Hello.\r\nSomething else.\r\n.\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out _)); + Assert.Equal("Hello.\r\nSomething else.\r\n"u8.Length, payload); + } + + [Fact] + public void StuffedDotLine_IsNotTerminator() + { + var buffer = new ReadOnlySequence("..foo\r\nbar\r\n.\r\n"u8.ToArray()); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out _)); + Assert.Equal("..foo\r\nbar\r\n"u8.Length, payload); + } + + [Fact] + public void IncompleteFiveByte_ReturnsFalse() + { + var buffer = new ReadOnlySequence("keep\r\n.\r"u8.ToArray()); + Assert.False(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out _, out _)); + } + + [Fact] + public void MultiSegment_FindsTerminatorSpanningSegments() + { + var first = "keep\r\n."u8.ToArray(); + var second = "\r\nNEXT"u8.ToArray(); + var segment1 = new SequenceSegment(first); + var segment2 = new SequenceSegment(second); + segment1.SetNext(segment2); + var buffer = new ReadOnlySequence(segment1, 0, segment2, second.Length); + Assert.True(NntpDelimiterSearch.TryFindArticleTerminator(buffer, atArticleStart: true, out var payload, out var consumed)); + Assert.Equal("keep\r\n"u8.Length, payload); + Assert.Equal("keep\r\n.\r\n"u8.Length, consumed); + } + + [Fact] + public void CrlfSplitAcrossSegments_StillFindsLine() + { + var first = "DATE\r"u8.ToArray(); + var second = "\n"u8.ToArray(); + var segment1 = new SequenceSegment(first); + var segment2 = new SequenceSegment(second); + segment1.SetNext(segment2); + var buffer = new ReadOnlySequence(segment1, 0, segment2, second.Length); + Assert.True(NntpDelimiterSearch.TryReadLine(ref buffer, out var line)); + Assert.Equal("DATE"u8.ToArray(), line.ToArray()); + Assert.True(buffer.IsEmpty); + } + + private sealed class SequenceSegment : ReadOnlySequenceSegment + { + public SequenceSegment(byte[] memory) + { + Memory = memory; + } + + public void SetNext(SequenceSegment next) + { + Next = next; + next.RunningIndex = RunningIndex + Memory.Length; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs new file mode 100644 index 00000000..ff143472 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/Framing/SimdBulkMultilineFramerTests.cs @@ -0,0 +1,159 @@ +using VectorNNTPD.TransitService.Session.Framing; +using Xunit; + +namespace VectorNNTPD.TransitService.Tests.Session.Framing; + +/// +/// Segment-boundary destuff coverage for production . +/// +public sealed class SimdBulkMultilineFramerTests +{ + public static IEnumerable AllFixtureNames() + { + foreach (var entry in InnArticleCorpus.LoadManifest().Files) + { + yield return [entry.RelativePath]; + } + } + + public static IEnumerable SegmentSizes() + { + foreach (var size in new[] { 1, 2, 4, 16, 4096 }) + { + yield return [size]; + } + } + + [Fact] + public async Task EmptyArticle() + { + var result = await FramingPipe.ReadSegmentedAsync(FramingWireFactory.EmptyArticle, 1); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(0, result.Payload.Length); + } + + [Fact] + public async Task FiveByteDelimiterIsDetected() + { + var result = await FramingPipe.ReadSegmentedAsync(FramingWireFactory.DataThenDelimiter(), 64); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("data\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task HelloDotIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("Hello.\r\nSomething else.\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, 8); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal("Hello.\r\nSomething else.\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Fact] + public async Task DotStuffedLineIsNotTerminator() + { + var wire = FramingWireFactory.WithTerminator("..foo\r\nbar\r\n"); + var result = await FramingPipe.ReadSegmentedAsync(wire, 3); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(".foo\r\nbar\r\n"u8.ToArray(), result.Payload.ToArray()); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + public async Task DelimiterSplitAtEveryBoundary(int split) + { + var wire = FramingWireFactory.WithTerminator("split-term\r\n"); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var holdBack = split == 0 ? 5 : Math.Max(1, 5 - split); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire, holdBack); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Theory] + [MemberData(nameof(SegmentSizes))] + public async Task Wire7_IdenticalAcrossSegmentSizes(int segmentSize) + { + var wire = InnArticleCorpus.ReadAllBytes("wire-7"); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + var result = await FramingPipe.ReadSegmentedAsync(wire, segmentSize); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Theory] + [MemberData(nameof(AllFixtureNames))] + public async Task EveryInnFixture_MatchesBulkSemantics(string relativePath) + { + var entry = InnArticleCorpus.LoadManifest().Files.Single(f => f.RelativePath == relativePath); + var wire = InnArticleCorpus.ReadAllBytes(relativePath); + var result = await FramingPipe.ReadOneAsync(wire); + + if (entry.FramingClassification == "complete_multiline_wire") + { + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + var expected = FramingWireFactory.DestuffCompleteWire(wire); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + else + { + Assert.Equal(NntpMultilineReadStatus.Incomplete, result.Status); + Assert.True(result.Payload.IsEmpty); + } + } + + [Fact] + public async Task ManyDotsHostile_DoesNotFalseTerminate() + { + var wire = FramingWireFactory.ManyDotsHostileArticle(); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + Assert.True(expected.AsSpan().IndexOf("Hello.\r\n"u8) >= 0); + + var result = await FramingPipe.ReadSegmentedAsync(wire, 17); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task IndexOfFiveByteDelimiter_RejectsTrailingPeriodLines() + { + var content = "Hello.\r\nSomething else.\r\n"; + var withoutTerm = await FramingPipe.ReadOneAsync(System.Text.Encoding.ASCII.GetBytes(content)); + Assert.Equal(NntpMultilineReadStatus.Incomplete, withoutTerm.Status); + + var withTerm = FramingWireFactory.WithTerminator(content); + var complete = await FramingPipe.ReadOneAsync(withTerm); + Assert.Equal(NntpMultilineReadStatus.Completed, complete.Status); + Assert.Equal(content, System.Text.Encoding.ASCII.GetString(complete.Payload.Span)); + } + + [Fact] + public async Task DelimiterSplitAcrossReads() + { + var wire = FramingWireFactory.WithTerminator("across-reads\r\n"); + var expected = FramingWireFactory.ExpectedDestuffed(wire); + var result = await FramingPipe.ReadSplitTerminatorAsync(wire); + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.True(expected.AsSpan().SequenceEqual(result.Payload.Span)); + } + + [Fact] + public async Task PipelinedSecondArticleRemains() + { + var wire = FramingWireFactory.PipelinedTwoArticles(); + var pipe = new System.IO.Pipelines.Pipe(); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + var first = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + var second = await NntpMultilineDataReader.ReadArticleAsync(pipe.Reader, 64 * 1024, CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, first.Status); + Assert.Equal(NntpMultilineReadStatus.Completed, second.Status); + Assert.Equal("article-one\r\n"u8.ToArray(), first.Payload.ToArray()); + Assert.Equal("article-two\r\n"u8.ToArray(), second.Payload.ToArray()); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/HelpCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/HelpCommandTests.cs new file mode 100644 index 00000000..9369aaa7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/HelpCommandTests.cs @@ -0,0 +1,304 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// HELP command — static syntax listing (RFC 3977 §7.2) with ABNF-consistent notation. +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class HelpCommandTests +{ + [Fact] + public async Task Help_Returns100_AndStaticSyntaxLines() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadMultilineResponseAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + var (status, body) = await readTask; + + Assert.Equal(1, response.ChannelEnqueueCount); + Assert.Equal("100 Help text follows", status); + Assert.Equal(Help.BodyLines, body); + Assert.DoesNotContain(body, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain("NEWGROUPS", body); + Assert.DoesNotContain("NEWNEWS", body); + AssertUnsupportedListVariantsAbsent(body); + AssertHelpBodyStructure(body); + } + + [Fact] + public async Task Help_IsIdenticalAcrossSessionStates() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + session.SetMode(NntpSessionMode.Reader); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var readTask = duplex.ReadMultilineResponseAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP"); + var (_, body) = await readTask; + + Assert.Equal(Help.BodyLines, body); + } + + [Fact] + public async Task Help_TrailingArgs_Returns501() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "HELP MORE"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task Help_SessionRemainsOpen_ForSubsequentCommand() + { + await using var duplex = await HelpDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); // greeting + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("100 Help text follows", await duplex.ReadClientLineAsync()); + var body = await duplex.ReadMultilineBodyAsync(); + Assert.Equal(Help.BodyLines, body); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + Assert.Equal(14, date.Length - 4); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void SyntaxLines_AreDeterministicAndUseAbnfConsistentNotation() + { + Assert.Equal(Help.SyntaxLines, Help.SyntaxLines.ToArray()); + Assert.Equal(Help.BodyLines, Help.BodyLines.ToArray()); + Assert.Contains("HELP", Help.SyntaxLines); + Assert.Contains("COMPRESS DEFLATE", Help.SyntaxLines); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + AssertUnsupportedListVariantsAbsent(Help.SyntaxLines); + Assert.Contains("LIST", Help.SyntaxLines); + Assert.Contains("LIST ACTIVE [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST COUNTS [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST HEADERS [MSGID / RANGE]", Help.SyntaxLines); + Assert.Contains("LIST NEWSGROUPS [wildmat]", Help.SyntaxLines); + Assert.Contains("LIST OVERVIEW.FMT", Help.SyntaxLines); + Assert.Contains("LIST MOTD", Help.SyntaxLines); + + // ABNF-consistent: optional args use [], alternatives use /; no {…} or |. + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('{', StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('}', StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Contains('|', StringComparison.Ordinal)); + Assert.Contains("ARTICLE [message-id / article-number]", Help.SyntaxLines); + Assert.Contains("AUTHINFO USER username", Help.SyntaxLines); + Assert.Contains("AUTHINFO PASS password", Help.SyntaxLines); + Assert.Contains("AUTHINFO SASL mechanism [initial-response]", Help.SyntaxLines); + Assert.Contains("GROUP newsgroup", Help.SyntaxLines); + Assert.DoesNotContain(Help.SyntaxLines, l => l.Equals("GROUP [newsgroup]", StringComparison.Ordinal)); + Assert.DoesNotContain(Help.SyntaxLines, l => l.StartsWith("GROUP [", StringComparison.Ordinal)); + Assert.Contains("HDR header [range / message-id]", Help.SyntaxLines); + Assert.Contains("CHECK message-id", Help.SyntaxLines); + Assert.Contains("IHAVE message-id", Help.SyntaxLines); + Assert.Contains("TAKETHIS message-id", Help.SyntaxLines); + Assert.Contains("SPEEDTEST ", Help.SyntaxLines); + Assert.Contains("LISTGROUP [newsgroup [range]]", Help.SyntaxLines); + Assert.Contains("XOVER [range]", Help.SyntaxLines); + + Assert.Equal("ARTICLE [message-id / article-number]", Help.SyntaxLines[0]); + Assert.Equal("XOVER [range]", Help.SyntaxLines[^1]); + AssertHelpBodyStructure(Help.BodyLines); + } + + private static void AssertHelpBodyStructure(IReadOnlyList body) + { + Assert.Equal(Help.BodyLines, body); + + var syntaxEnd = Help.SyntaxLines.Count; + Assert.Equal(string.Empty, body[syntaxEnd]); + Assert.Equal("Range formats:", body[syntaxEnd + 1]); + Assert.Equal(Help.RangeHelpLines, body.Skip(syntaxEnd + 1).Take(Help.RangeHelpLines.Count).ToArray()); + + var afterRangeBlank = syntaxEnd + 1 + Help.RangeHelpLines.Count; + Assert.Equal(string.Empty, body[afterRangeBlank]); + Assert.Equal("Wildmat formats:", body[afterRangeBlank + 1]); + Assert.Equal(Help.WildmatHelpLines, body.Skip(afterRangeBlank + 1).Take(Help.WildmatHelpLines.Count).ToArray()); + + Assert.Contains("123 a single article number", body); + Assert.Contains("123-456 articles from 123 through 456 inclusive", body); + Assert.Contains("123- article 123 and all following article numbers", body); + Assert.Contains("* matches zero or more characters", body); + Assert.Contains("? matches exactly one character", body); + Assert.Contains("Examples: a* ; a*,!*b ; *.recovery", body); + Assert.DoesNotContain(body, l => l.Contains("regex", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(body, l => l.Contains("[^", StringComparison.Ordinal)); + } + + private static void AssertUnsupportedListVariantsAbsent(IReadOnlyList lines) + { + Assert.DoesNotContain(lines, l => l.Contains("ACTIVE.TIMES", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("DISTRIB.PATS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("DISTRIBUTIONS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("MODERATORS", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("SUBSCRIPTIONS", StringComparison.Ordinal)); + } + + private sealed class HelpDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new HelpDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async Task<(string Status, List Body)> ReadMultilineResponseAsync() + { + // Caller must start this before DispatchAsync so FlushAsync backpressure can drain. + var status = await ReadClientLineAsync().ConfigureAwait(false); + var body = await ReadMultilineBodyAsync().ConfigureAwait(false); + return (status, body); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _compressed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/IHaveCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/IHaveCommandTests.cs new file mode 100644 index 00000000..0917c267 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/IHaveCommandTests.cs @@ -0,0 +1,863 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class IHaveCommandTests +{ + private static NntpAuthorization TransitAuth { get; } = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Fact] + public async Task ValidIhave_AcceptsArticle_EnqueuesWire_AndReturns235() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("", ".body\r\n") + ".\r\n"); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.NotNull(inbound); + Assert.Equal("", inbound!.MessageId); + Assert.Equal(InboundArticleProducer.IHave, inbound.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + var text = Encoding.ASCII.GetString(inbound.Record.ArtData.Span); + Assert.Contains("\r\n\r\n..body\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("\r\n\r\n.body\r\n", text, StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ValidIhave_UnknownNewsgroup_WantTrash_Returns235() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + CanonicalArticleText.Stuffed("", newsgroups: "unknown.un.carried") + ".\r\n"); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + + using var dequeueCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.Equal("", inbound!.MessageId); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.Equal("unknown.un.carried", Encoding.ASCII.GetString(inbound.Record.Newsgroups)); + Assert.Contains("Newsgroups: unknown.un.carried"u8, inbound.Payload.Span); + Assert.DoesNotContain("junk"u8, inbound.Record.Newsgroups); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ValidIhave_DoesNotEvaluatePostFilter() + { + var filter = new CountingPostFilter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, postFilter: filter); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + Assert.Equal(0, filter.EvaluateCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task HistoryHit_Returns435_AndDoesNotReadArticleBytes() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + history.Remember(""u8.ToArray()); + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 2 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, history, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("435 Article not wanted", await duplex.ReadClientLineAsync()); + Assert.Empty(news.Events); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task RedisUnavailable_Returns436() + { + var redis = new FakeRedisService { IsUnavailable = true }; + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 2 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("436 Transfer not possible; try again later", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task MalformedMessageId_Is501() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE not-an-id"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task QueueUnavailable_Returns436BeforeArticle() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(DisabledArticleIngestionQueue.Instance); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("436 Transfer not possible; try again later", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ArticleLargerThanQueueBudget_Returns437() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { MaxArticleBytes = 1024 }, + transitQueueMemoryLimit: 16); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(0, queue.QueuedBytes); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(IngressNewsReasons.QueueCapacityExceeded, Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task BudgetExhaustedBefore335_Returns436Immediately_AndDoesNotWait() + { + var held = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + Assert.False(queue.TryProbeCapacity()); + + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var response = duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("436 Transfer not possible; try again later", await response.WaitAsync(TimeSpan.FromSeconds(2))); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task PostReceiveCapacityFailure_Returns436_Not235_AndReleasesOwnership() + { + var held = FillArticle(""); + var incoming = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave); + Assert.True(held.Payload.Length < incoming.Payload.Length); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { MaxArticleBytes = 64 * 1024 }, + transitQueueMemoryLimit: incoming.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + Assert.True(queue.TryProbeCapacity()); + + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + var article = duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("436 Transfer failed; try again later", await article.WaitAsync(TimeSpan.FromSeconds(2))); + Assert.Equal(held.Payload.Length, queue.QueuedBytes); + Assert.Equal(1, queue.Count); + Assert.Equal(0, queue.DebugWaiterCount); + + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + Assert.Equal("", (await queue.DequeueAsync(CancellationToken.None))!.MessageId); + Assert.Equal(0, queue.QueuedBytes); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task QueueCompleteAfter335_Returns436_AndLeavesAccounting() + { + var sample = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: sample.Payload.Length); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + queue.Complete(); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("436 Transfer failed; try again later", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.QueuedBytes); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void BudgetExhausted_WarningIsRateLimited() + { + IHaveAdmissionLog.ResetForTests(); + IHaveAdmissionLog.WarningInterval = TimeSpan.FromHours(1); + var recording = new RecordingLogger(); + var held = FillArticle(""); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + transitQueueMemoryLimit: held.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, queue.TryAdmit(held)); + + IHaveAdmissionLog.BudgetExhausted(recording, queue); + IHaveAdmissionLog.BudgetExhausted(recording, queue); + var first = Assert.Single(recording.Warnings); + Assert.Contains("TransitQueueMemoryLimit exhausted", first, StringComparison.Ordinal); + Assert.Contains($"{held.Payload.Length}/{held.Payload.Length}", first, StringComparison.Ordinal); + Assert.DoesNotContain("Subject:", first, StringComparison.Ordinal); + + IHaveAdmissionLog.ResetForTests(); + IHaveAdmissionLog.BudgetExhausted(recording, queue); + Assert.Equal(2, recording.Warnings.Count); + IHaveAdmissionLog.ResetForTests(); + } + + [Fact] + public async Task TooLarge_Returns437_AndLeavesNextCommand() + { + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = 2, MaxArticleBytes = 16 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync("Subject: big\r\n\r\n" + new string('Z', 64) + "\r\n.\r\nDATE\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TooLarge_WritesRejectedNewsWith437() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions { QueueCapacity = 2, MaxArticleBytes = 16 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync("Subject: big\r\n\r\n" + new string('Z', 64) + "\r\n.\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(437, evt.ResponseCode); + Assert.Equal(IngressNewsReasons.ArticleTooLarge, Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.DoesNotContain("437", Encoding.ASCII.GetString(evt.Reason.Span), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NewsWriterFailure_On437_DoesNotChangeRejectionResponse() + { + var news = new ThrowingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync("Subject: hi\r\n\r\n..body\r\n.\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(1, news.WriteCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Capabilities_AdvertisesIhave() + { + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("CAPABILITIES"); + var lines = new List(); + string line; + do + { + line = await duplex.ReadClientLineAsync(); + lines.Add(line); + } + while (line != "."); + + Assert.Contains("IHAVE", lines); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task IncompleteArticleAfter335_Returns437_AndDoesNotEnqueue() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync("Subject: hi\r\n\r\n..body\r\n.\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(437, evt.ResponseCode); + var incompleteReason = Encoding.ASCII.GetString(evt.Reason.Span); + Assert.DoesNotContain("rejected article record", incompleteReason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(incompleteReason)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task WantTrashFalse_UnknownGroup_Returns437_AndWritesRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession( + queue, + newsLog: news, + transit: new TransitOptions { WantTrash = false, LogTrash = false }, + catalogue: CarriedCatalogue("alt.test")); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + CanonicalArticleText.Stuffed("", newsgroups: "unknown.un.carried") + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(437, evt.ResponseCode); + Assert.Equal( + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, ["unknown.un.carried"]), + Encoding.ASCII.GetString(evt.Reason.Span)); + Assert.True(evt.MessageId.Span.SequenceEqual(""u8)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task InvalidMessageIdAfter335_WritesRejectedNewsWithParseReason() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + CanonicalArticleText.Stuffed("not-a-message-id") + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(437, evt.ResponseCode); + Assert.Equal(IngressNewsReasons.MessageIdInvalid, Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task InvalidDateAfter335_WritesRejectedNewsWithParseReason() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + var article = CanonicalArticleText.Destuffed("") + .Replace(CanonicalArticleText.Date, "not-a-date", StringComparison.Ordinal); + await duplex.WriteClientAsync(article.Replace("\r\n.", "\r\n..", StringComparison.Ordinal) + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(IngressNewsReasons.DateInvalid, Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task YEncValidationFailureAfter335_WritesRejectedNewsWithYEncReason() + { + await AssertYEncRejectionAsync( + "", + "=ybegin line=128 size=abc name=t.bin\r\n.\r\n=yend size=1 crc32=00000000\r\n"); + } + + [Fact] + public async Task YEncCrcMismatchAfter335_WritesRejectedNewsWithYEncReason() + { + await AssertYEncRejectionAsync( + "", + "=ybegin line=128 size=1 name=t.bin\r\nk\r\n=yend size=1 crc32=00000000\r\n"); + } + + [Fact] + public async Task NamedPeer_YEncReject_NewsLineHasInboundPeerSizeAndOutboundPlaceholder() + { + const string commandId = ""; + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true, + transitPeerName: "BlueWorldHosting")); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE " + commandId); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync( + CanonicalArticleText.Stuffed( + commandId, + "=ybegin line=128 size=1 name=t.bin\r\nk\r\n=yend size=1 crc32=00000000\r\n") + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + var evt = Assert.Single(news.Events); + Assert.True(evt.Feed.Span.SequenceEqual("BlueWorldHosting"u8)); + Assert.True(evt.Size > 0); + var rendered = FormatNews(in evt); + Assert.Equal( + FormatNews(new NewsLogEvent( + NewsLogDisposition.Rejected, + Encoding.ASCII.GetBytes(commandId), + "BlueWorldHosting"u8.ToArray(), + timestamp: evt.Timestamp, + responseCode: 437, + reason: "yEncoding invalid"u8.ToArray(), + size: evt.Size)), + rendered); + Assert.Contains("- BlueWorldHosting " + commandId + " " + evt.Size + " yEncoding invalid", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("- ? ", rendered, StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task CommandMessageIdNeedNotMatchArticleMessageId() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed("") + ".\r\n"); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + + using var dequeueTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var inbound = await queue.DequeueAsync(dequeueTimeout.Token); + Assert.Equal("", inbound!.MessageId); + Assert.True(inbound.Record.MessageId.SequenceEqual(""u8)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private static InboundArticle FillArticle(string messageId) => + CanonicalArticleText.CreateQueued(messageId, InboundArticleProducer.IHave); + + private static INewsgroupCatalogue CarriedCatalogue(string group) => + new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition(group, string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)])); + + private static string FormatNews(in NewsLogEvent evt) + { + var timestamp = evt.Timestamp == default + ? new DateTimeOffset(2024, 8, 25, 13, 37, 54, 638, TimeSpan.Zero) + : evt.Timestamp; + Span buffer = stackalloc byte[NewsLogLineFormatter.RequiredLength(in evt) + 16]; + var written = NewsLogLineFormatter.Write(buffer, in evt, timestamp); + return Encoding.ASCII.GetString(buffer[..written]); + } + + private static async Task AssertYEncRejectionAsync(string commandId, string body) + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await IHaveDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE " + commandId); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(CanonicalArticleText.Stuffed(commandId, body) + ".\r\n"); + Assert.Equal("437 Transfer rejected; do not retry", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(437, evt.ResponseCode); + Assert.Equal(IngressNewsReasons.YEncodingInvalid, Encoding.ASCII.GetString(evt.Reason.Span)); + var rendered = FormatNews(in evt); + Assert.EndsWith( + "- ? " + commandId + " " + evt.Size + " yEncoding invalid\n", + rendered, + StringComparison.Ordinal); + Assert.DoesNotContain("437", rendered, StringComparison.Ordinal); + Assert.DoesNotContain("YEncDecodingFailed", rendered, StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + private sealed class RecordingLogger : ILogger + { + public List Warnings { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) + { + Warnings.Add(formatter(state, exception)); + } + } + + private sealed class NullScope : IDisposable + { + public static NullScope Instance { get; } = new(); + + public void Dispose() + { + } + } + } + + private sealed class IHaveDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + + public static Task CreateAsync() => Task.FromResult(new IHaveDuplex()); + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + IHistoryDb? history = null, + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null, + TransitOptions? transit = null, + INewsgroupCatalogue? catalogue = null) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: history, + postFilter: postFilter, + newsgroupCatalogue: catalogue, + newsLog: newsLog, + transit: transit); + } + + public async Task WriteClientLineAsync(string line) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(line + "\r\n")); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/IHaveTcpTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/IHaveTcpTests.cs new file mode 100644 index 00000000..7b2741c2 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/IHaveTcpTests.cs @@ -0,0 +1,106 @@ +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// IHAVE over the production accept/transport path. +[Collection(nameof(TransportTestHostCollection))] +public sealed class IHaveTcpTests +{ + [Fact] + public async Task Tcp_SegmentedArticle_ThenNextCommandInSameReceive_LeavesCommandIntact() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var session = new NntpSession( + server, + NullLogger.Instance, + articleIngestion: queue); + session.SetAuthorization(NntpAuthorization.TrustedTransitPeer); + var run = session.RunAsync(); + var rx = new SocketLineBuffer(); + + Assert.StartsWith("201 ", await rx.ReadLineAsync(client), StringComparison.Ordinal); + + await client.SendAsync("IHAVE \r\n"u8.ToArray()); + Assert.Equal("335 Send article to be transferred", await rx.ReadLineAsync(client)); + + var body = string.Concat(Enumerable.Repeat("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT\r\n", 65)); + var destuffed = CanonicalArticleText.Destuffed("", body); + var headerEnd = destuffed.IndexOf("\r\n\r\n", StringComparison.Ordinal); + await client.SendAsync(Encoding.ASCII.GetBytes(destuffed[..headerEnd])); + await Task.Yield(); + await client.SendAsync("\r\n\r\n"u8.ToArray()); + await Task.Yield(); + await client.SendAsync(Encoding.ASCII.GetBytes(destuffed[(headerEnd + 4)..] + ".\r\nDATE\r\n")); + + Assert.Equal("235 Article transferred OK", await rx.ReadLineAsync(client)); + Assert.StartsWith("111 ", await rx.ReadLineAsync(client), StringComparison.Ordinal); + + using var dequeueCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var inbound = await queue.DequeueAsync(dequeueCts.Token); + Assert.NotNull(inbound); + Assert.Equal(InboundArticleProducer.IHave, inbound!.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, inbound.Record.ParseStatus); + Assert.True(inbound.Payload.Equals(inbound.Record.ArtData)); + Assert.True(inbound.Payload.Length >= 4096); + Assert.DoesNotContain("DATE"u8.ToArray(), inbound.Payload.ToArray()); + + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await rx.ReadLineAsync(client)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + client.Shutdown(SocketShutdown.Both); + } + + private sealed class SocketLineBuffer + { + private readonly List _bytes = []; + + public async Task ReadLineAsync(Socket socket) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + if (TryReadLine(out var line)) + { + return line; + } + + var scratch = new byte[512]; + var n = await socket.ReceiveAsync(scratch, safety.Token); + Assert.True(n > 0, "Socket closed while reading IHAVE response."); + _bytes.AddRange(scratch.AsSpan(0, n)); + } + } + + private bool TryReadLine(out string line) + { + line = string.Empty; + for (var i = 0; i + 1 < _bytes.Count; i++) + { + if (_bytes[i] != (byte)'\r' || _bytes[i + 1] != (byte)'\n') + { + continue; + } + + line = Encoding.ASCII.GetString(_bytes.GetRange(0, i).ToArray()); + _bytes.RemoveRange(0, i + 2); + return true; + } + + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs new file mode 100644 index 00000000..1700edee --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NewNewsAndNewGroupsRemovedTests.cs @@ -0,0 +1,247 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// NEWNEWS and NEWGROUPS are intentionally unregistered. They must take the generic +/// unknown-command path and must not appear as capability tokens. +/// +public sealed class NewNewsAndNewGroupsRemovedTests +{ + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + [InlineData("newnews")] + [InlineData("NewGroups")] + [InlineData("NEWNEWS * 19990624 000000 GMT")] + [InlineData("NEWGROUPS 19990624 000000 GMT")] + public void Parser_TreatsCommandsAsUnknownVerb(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.False(parsed.IsValid); + Assert.Equal(NntpVerb.Unknown, parsed.Verb); + Assert.Equal(NntpParseStatus.UnknownVerb, parsed.Status); + } + + [Fact] + public void Catalog_DoesNotRegisterCommands() + { + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.GetRegisteredKeys()); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.GetRegisteredKeys()); + Assert.Contains("LIST NEWSGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + } + + [Fact] + public void ListNewsgroups_RemainsARegisteredListKeyword() + { + var parsed = NntpCommandTestParse.ParseCommand("LIST NEWSGROUPS"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.List, parsed.Verb); + Assert.Equal(NntpVerb.Newsgroups, parsed.Qualifier); + Assert.Equal("LIST NEWSGROUPS", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + } + + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + [InlineData("NEWNEWS * 19990624 000000 GMT")] + [InlineData("NEWGROUPS 19990624 000000 GMT")] + public async Task Dispatch_Unauthenticated_ReturnsGenericUnknownCommand(string commandLine) + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + var line = await duplex.ReadClientLineAsync(); + + Assert.Equal("500 Unknown command", line); + Assert.DoesNotContain("Command not implemented", line, StringComparison.Ordinal); + Assert.DoesNotContain("480", line, StringComparison.Ordinal); + } + + [Theory] + [InlineData("NEWNEWS")] + [InlineData("NEWGROUPS")] + public async Task Dispatch_FullyAuthorized_StillReturnsGenericUnknownCommand(string commandLine) + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + var line = await duplex.ReadClientLineAsync(); + + Assert.Equal("500 Unknown command", line); + Assert.DoesNotContain("Command not implemented", line, StringComparison.Ordinal); + } + + [Fact] + public async Task Capabilities_DoesNotAdvertiseEitherCommand() + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + var read = duplex.ReadUntilTerminatorAsync(); + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CAPABILITIES"); + var body = await read; + var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal) + .Split('\n', StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal("101 Capability list:", lines[0]); + Assert.Contains("IHAVE", lines, StringComparer.Ordinal); + Assert.Contains("STREAMING", lines, StringComparer.Ordinal); + Assert.Contains("SPEEDTEST", lines, StringComparer.Ordinal); + Assert.DoesNotContain(lines, static l => l.StartsWith("LIST", StringComparison.Ordinal)); + Assert.DoesNotContain("READER", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWNEWS", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWGROUPS", lines, StringComparer.Ordinal); + Assert.DoesNotContain("NEWNEWS", body, StringComparison.Ordinal); + Assert.DoesNotContain("NEWGROUPS", body, StringComparison.Ordinal); + } + + [Fact] + public async Task ListNewsgroups_OnTransit_ReturnsCommandNotAvailable() + { + await using var duplex = await RemovedDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST NEWSGROUPS"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + } + + private sealed class RemovedDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new RemovedDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task ReadUntilTerminatorAsync() + { + var builder = new StringBuilder(); + while (true) + { + var line = await ReadClientLineAsync(); + builder.Append(line); + builder.Append('\n'); + if (line == ".") + { + return builder.ToString(); + } + } + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpArticleNumberTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpArticleNumberTests.cs new file mode 100644 index 00000000..764bfc06 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpArticleNumberTests.cs @@ -0,0 +1,51 @@ +using System.Text; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// RFC 3977 §9.8 article-number = 1*16DIGIT. No CLR integer parse exists. +/// +public sealed class NntpArticleNumberTests +{ + [Theory] + [InlineData("1")] + [InlineData("0")] + [InlineData("0001")] + [InlineData("2147483647")] + [InlineData("2147483648")] + [InlineData("9999999999999999")] + public void Syntax_AcceptsRfcDigitTokensThrough16Digits(string token) + { + Assert.True(NntpArticleNumber.IsSyntax(Encoding.ASCII.GetBytes(token))); + Assert.True(NntpCommandTestParse.ParseCommand("ARTICLE " + token).IsValid); + } + + [Theory] + [InlineData("")] + [InlineData("10000000000000000")] + [InlineData("12345678901234567")] + [InlineData("12a")] + [InlineData("-1")] + [InlineData("+1")] + [InlineData("1.0")] + public void Syntax_RejectsEmptySeventeenDigitsAndNonDigits(string token) + { + Assert.False(NntpArticleNumber.IsSyntax(Encoding.ASCII.GetBytes(token))); + if (token.Length == 0) + { + return; + } + + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandTestParse.ParseCommand("ARTICLE " + token).Status); + } + + [Fact] + public void MaxDigits_IsRfc16() + { + Assert.Equal(16, NntpArticleNumber.MaxDigits); + Assert.True(NntpArticleNumber.IsSyntax("9999999999999999"u8)); + Assert.False(NntpArticleNumber.IsSyntax("10000000000000000"u8)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpAuthinfoTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpAuthinfoTests.cs new file mode 100644 index 00000000..4dcd83b0 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpAuthinfoTests.cs @@ -0,0 +1,687 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpAuthinfoTests +{ + [Fact] + public async Task AuthinfoUser_WithoutArgument_Returns501() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER"); + Assert.StartsWith("501 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Null(session.PendingAuthUsername); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextAllowed_AuthinfoUserPass_SucceedsWithoutTls() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal("fred", session.PendingAuthUsername); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("fred", session.Authentication.Username); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoUser_Repeated_ReplacesPendingUsername() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO USER barney"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal("barney", session.PendingAuthUsername); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_WithoutUser_Returns482() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("482 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_WrongPassword_Returns481_LeavesUnauthenticated() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.StartsWith("481 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.StreamingPermitted); + Assert.Equal("fred", session.PendingAuthUsername); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Success_AuthenticatedWithoutImpliedPrivileges() + { + var provider = new ScriptedAuthenticationProvider( + (user, pass) => user == "fred" && pass == "flintstone" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("fred", session.Authentication.Username); + Assert.Null(session.PendingAuthUsername); + Assert.True(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.StreamingPermitted); + Assert.False(session.Authorization.ControlCancelPermitted); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Newsmaster_GrantsControlCancelOnly() + { + var options = new NntpdOptions + { + NewsmasterUser = "newsmaster", + NewsmasterPassword = "unit-test-newsmaster-password", + }; + var provider = NewsmasterNntpAuthenticationProvider.Create(options); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER newsmaster"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS unit-test-newsmaster-password"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.PostingPermitted); + Assert.True(session.Authorization.ControlCancelPermitted); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.StreamingPermitted); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_Success_AppliesProviderAuthorizationIndependently() + { + var provider = new ScriptedAuthenticationProvider( + (user, pass) => user == "reader1" && pass == "pw" + ? NntpAuthenticationResult.Success( + "reader1", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER reader1"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS pw"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + Assert.True(session.Authorization.AuthorizedReader); + Assert.True(session.Authorization.PostingPermitted); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.StreamingPermitted); + Assert.False(session.Authorization.ControlCancelPermitted); + + // Transit role: reader verbs are unavailable even for an authenticated reader-authorized user. + await duplex.WriteClientLineAsync("ARTICLE"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Authinfo_AfterSuccess_Returns502_AndCapabilitiesOmitAuthinfo() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("AUTHINFO USER other"); + Assert.StartsWith("502 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.DoesNotContain(caps, static c => c.StartsWith("AUTHINFO", StringComparison.Ordinal)); + Assert.DoesNotContain(caps, static c => c.Equals("MODE-READER", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithoutTls_AuthinfoUser_Returns483_DoesNotCallProvider() + { + var provider = new CountingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("483 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Null(session.PendingAuthUsername); + Assert.Equal(0, provider.CallCount); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithoutTls_AuthinfoPass_Returns483_DoesNotCallProvider() + { + var provider = new CountingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(allowCleartextAuth: false); + // Simulate a pending USER that somehow existed before policy change — PASS must still gate first. + session.SetPendingAuthUsername("fred"); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.StartsWith("483 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal(0, provider.CallCount); + Assert.False(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task CleartextDisabled_WithTls_AuthinfoUserPass_Succeeds() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider(), isTls: true); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS flintstone"); + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Authentication.IsAuthenticated); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextAllowed_AdvertisesAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO USER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextDisabled_WithoutTls_DoesNotAdvertiseAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider()); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO", caps); + Assert.DoesNotContain(caps, static c => c.Equals("AUTHINFO USER", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task Capabilities_CleartextDisabled_WithTls_AdvertisesAuthinfoUser() + { + await using var duplex = await AuthinfoDuplex.CreateAsync(CreateAcceptingProvider(), isTls: true); + var session = duplex.CreateSession(allowCleartextAuth: false); + var sessionTask = session.RunAsync(); + await duplex.ReadGreetingAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = await duplex.ReadMultilineBodyAsync(); + Assert.Contains("AUTHINFO USER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + await duplex.ReadClientLineAsync(); + await sessionTask; + } + + [Fact] + public async Task AuthinfoPass_DoesNotLogPassword_OnHandlerFailure() + { + var recording = new RecordingLoggerFactory(); + var provider = new ThrowingAuthenticationProvider(); + await using var duplex = await AuthinfoDuplex.CreateAsync(provider); + var session = duplex.CreateSession(provider, allowCleartextAuth: true, loggerFactory: recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS super-secret-password-xyz"); + Assert.StartsWith("403 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("super-secret-password-xyz", joined, StringComparison.Ordinal); + Assert.Contains(recording.Messages, m => m.Contains("AUTHINFO PASS", StringComparison.Ordinal)); + Assert.Contains( + recording.Categories, + c => c == "VectorNNTPD.TransitService.Session.Commands.AuthInfo"); + } + + private static ScriptedAuthenticationProvider CreateAcceptingProvider() => + new( + (user, pass) => user == "fred" && pass == "flintstone" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + + private sealed class ScriptedAuthenticationProvider : INntpAuthenticationProvider + { + private readonly Func _authenticate; + + public ScriptedAuthenticationProvider(Func authenticate) + { + _authenticate = authenticate; + } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(_authenticate(username, password)); + } + + private sealed class CountingAuthenticationProvider : INntpAuthenticationProvider + { + public int CallCount { get; private set; } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + CallCount++; + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class ThrowingAuthenticationProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + throw new InvalidOperationException("provider boom"); + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + if (exception is not null) + { + _messages.Add(exception.ToString()); + } + } + } + + private sealed class RecordingLogger : ILogger + { + public ConcurrentBag Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add(formatter(state, exception)); + if (exception is not null) + { + Messages.Add(exception.ToString()); + } + } + } + + private sealed class AuthinfoDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly INntpAuthenticationProvider _provider; + private readonly bool _isTls; + + private AuthinfoDuplex(INntpAuthenticationProvider provider, bool isTls) + { + _provider = provider; + _isTls = isTls; + } + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync( + INntpAuthenticationProvider provider, + bool isTls = false) => + Task.FromResult(new AuthinfoDuplex(provider, isTls)); + + public NntpSession CreateSession( + INntpAuthenticationProvider? provider = null, + bool allowCleartextAuth = true, + ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)), + isTls: _isTls); + return new NntpSession( + connection, + loggerFactory?.CreateLogger() ?? NullLogger.Instance, + authenticationProvider: provider ?? _provider, + allowCleartextAuth: allowCleartextAuth, + loggerFactory: loggerFactory); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("201 ", line, StringComparison.Ordinal); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection( + PipeReader input, + PipeWriter output, + ConnectionClientIdentity identity, + bool isTls = false) + { + Input = input; + Output = output; + ClientIdentity = identity; + IsTls = isTls; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls { get; } + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandInventoryTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandInventoryTests.cs new file mode 100644 index 00000000..951f1428 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandInventoryTests.cs @@ -0,0 +1,341 @@ +using System.IO.Pipelines; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class NntpCommandInventoryTests +{ + [Fact] + public void Catalog_RegistersCompleteInventory() + { + var keys = DefaultNntpCommandCatalog.GetRegisteredKeys(); + + // Production inventory remains complete; BENCHIT is an internal extra (not in InventoryKeys). + foreach (var key in DefaultNntpCommandCatalog.InventoryKeys) + { + Assert.Contains(key, keys); + } + + Assert.Contains("BENCHIT", keys); + Assert.DoesNotContain("BENCHIT", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWGROUPS", DefaultNntpCommandCatalog.InventoryKeys); + Assert.DoesNotContain("NEWNEWS", keys); + Assert.DoesNotContain("NEWGROUPS", keys); + Assert.Equal(DefaultNntpCommandCatalog.InventoryKeys.Count + 1, keys.Count); + } + + [Theory] + [InlineData("CAPABILITIES")] + [InlineData("DATE")] + [InlineData("HELP")] + [InlineData("QUIT")] + [InlineData("STARTTLS")] + [InlineData("LIST")] + [InlineData("LIST ACTIVE")] + [InlineData("LIST COUNTS")] + [InlineData("LIST COUNTS misc.*")] + [InlineData("LIST HEADERS")] + [InlineData("LIST HEADERS MSGID")] + [InlineData("LIST HEADERS RANGE")] + [InlineData("LIST MOTD")] + [InlineData("LIST NEWSGROUPS")] + [InlineData("LIST OVERVIEW.FMT")] + [InlineData("GROUP misc.test")] + [InlineData("LISTGROUP")] + [InlineData("LISTGROUP misc.test")] + [InlineData("ARTICLE")] + [InlineData("COMPRESS DEFLATE")] + [InlineData("MODE READER")] + [InlineData("MODE STREAM")] + [InlineData("AUTHINFO USER x")] + [InlineData("AUTHINFO PASS y")] + [InlineData("AUTHINFO SASL PLAIN")] + [InlineData("IHAVE ")] + [InlineData("CHECK ")] + [InlineData("TAKETHIS ")] + [InlineData("SPEEDTEST GIGANEWS")] + public void Parser_AcceptsInventoryCommands(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.True(parsed.IsValid); + Assert.NotEqual(NntpVerb.Unknown, parsed.Verb); + Assert.NotEqual(NntpVerb.None, parsed.Verb); + } + + [Theory] + [InlineData("LIST ACTIVE.TIMES")] + [InlineData("LIST DISTRIB.PATS")] + [InlineData("LIST DISTRIBUTIONS")] + [InlineData("LIST MODERATORS")] + [InlineData("LIST SUBSCRIPTIONS")] + [InlineData("LIST SUBSCRIPTIONS *")] + public void Parser_RejectsUnsupportedListVariants(string commandLine) + { + var parsed = NntpCommandTestParse.ParseCommand(commandLine); + Assert.Equal(NntpVerb.List, parsed.Verb); + Assert.Equal(NntpParseStatus.UnknownQualifier, parsed.Status); + Assert.False(parsed.IsValid); + } + + [Fact] + public void Inventory_ExcludesUnsupportedListVariants() + { + var keys = DefaultNntpCommandCatalog.InventoryKeys; + Assert.Contains("LIST", keys); + Assert.Contains("LIST ACTIVE", keys); + Assert.Contains("LIST COUNTS", keys); + Assert.DoesNotContain("LIST ACTIVE.TIMES", keys); + Assert.DoesNotContain("LIST DISTRIB.PATS", keys); + Assert.DoesNotContain("LIST DISTRIBUTIONS", keys); + Assert.DoesNotContain("LIST MODERATORS", keys); + Assert.DoesNotContain("LIST SUBSCRIPTIONS", keys); + + var registered = DefaultNntpCommandCatalog.GetRegisteredKeys(); + Assert.DoesNotContain("LIST ACTIVE.TIMES", registered); + Assert.Contains("LIST COUNTS", registered); + Assert.DoesNotContain("LIST DISTRIB.PATS", registered); + Assert.DoesNotContain("LIST DISTRIBUTIONS", registered); + Assert.DoesNotContain("LIST MODERATORS", registered); + Assert.DoesNotContain("LIST SUBSCRIPTIONS", registered); + } + + [Fact] + public void Registry_ExactMatch_RejectsPrefixConfusion() + { + var articleX = NntpCommandTestParse.ParseCommand("ARTICLEX"); + Assert.Equal(NntpVerb.Unknown, articleX.Verb); + Assert.Equal(NntpParseStatus.UnknownVerb, articleX.Status); + + var userXyz = NntpCommandTestParse.ParseCommand("AUTHINFO USERXYZ"); + Assert.Equal(NntpVerb.AuthInfo, userXyz.Verb); + Assert.Equal(NntpParseStatus.UnknownQualifier, userXyz.Status); + } + + [Fact] + public void Parser_Date_IsValidPublicCommand() + { + var parsed = NntpCommandTestParse.ParseCommand("DATE"); + Assert.True(parsed.IsValid); + Assert.Equal(NntpVerb.Date, parsed.Verb); + Assert.Equal(0, parsed.TokenCount); + Assert.Equal("DATE", DefaultNntpCommandCatalog.DisplayName(parsed.Verb, parsed.Qualifier)); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(parsed.Verb, parsed.Qualifier)); + } + + [Fact] + public async Task List_FullyAuthorizedReader_OnTransit_Returns502() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task List_WithoutAuth_OnTransit_Returns502BeforeAuthentication() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + } + + /// + /// Unknown LIST keywords stay UnknownQualifier / 501 Unknown command variant, + /// matching the old registry (not a new parser invention). + /// + [Theory] + [InlineData("LIST ACTIVE.TIMES")] + [InlineData("LIST DISTRIB.PATS")] + [InlineData("LIST DISTRIBUTIONS")] + [InlineData("LIST MODERATORS")] + [InlineData("LIST SUBSCRIPTIONS")] + [InlineData("LIST SUBSCRIPTIONS *")] + public async Task UnsupportedListVariants_Return501UnknownCommandVariant(string commandLine) + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.Equal("501 Unknown command variant", await duplex.ReadClientLineAsync()); + } + + [Theory] + [InlineData("LIST HEADERS")] + [InlineData("LIST OVERVIEW.FMT")] + public async Task ReaderStaticListKeywords_OnTransit_Return502(string commandLine) + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task ListMotd_OnTransit_Returns502EvenWhenAuthorizedReader() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "LIST MOTD"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task CompressDeflate_IsRegisteredPublic_Returns206() + { + await using var duplex = await InventoryDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Contains("206 Compression active", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(session.Connection.IsCompressed); + } + + private sealed class InventoryDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new InventoryDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLineLengthTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLineLengthTests.cs new file mode 100644 index 00000000..27d7ed64 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLineLengthTests.cs @@ -0,0 +1,182 @@ +using System.IO.Pipelines; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// Reader-mode command lines longer than +/// are consumed through CRLF and rejected as 501 Syntax error without parsing the prefix. +/// +public sealed class NntpCommandLineLengthTests +{ + [Fact] + public async Task ExactlyMaxBytes_IsCopiedAndParses() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.MaxCommandLineBytes, length); + var command = NntpCommandParser.Parse(scratch.AsSpan(0, length)); + Assert.True(command.IsValid); + Assert.Equal(NntpVerb.Group, command.Verb); + } + + [Fact] + public async Task OneByteOverMax_IsConsumedAndNotCopiedAsACommand() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.True((await pipe.Reader.ReadAsync()).Buffer.IsEmpty); + } + + [Fact] + public async Task ValidPrefixPlusExtra_DoesNotReturnThePrefix() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 6]; + "STAT "u8.CopyTo(line); + line.AsSpan("STAT "u8.Length, NntpCommandLineReader.MaxCommandLineBytes - "STAT "u8.Length) + .Fill((byte)' '); + " extra"u8.CopyTo(line.AsSpan(NntpCommandLineReader.MaxCommandLineBytes)); + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + scratch.AsSpan().Fill((byte)'Z'); + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.Equal((byte)'Z', scratch[0]); + } + + [Fact] + public async Task OverlongInvalidPrefix_IsTheSameSentinel() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan(0, NntpCommandLineReader.MaxCommandLineBytes).Fill((byte)'!'); + line[^1] = (byte)'x'; + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + } + + [Fact] + public async Task OverlongLineThenQuit_SecondLineParses() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = new byte[line.Length + 2 + "QUIT\r\n"u8.Length]; + WithCrlf(line).CopyTo(wire); + "QUIT\r\n"u8.CopyTo(wire.AsSpan(line.Length + 2)); + await using var pipe = await CommandPipe.CreateAsync(wire); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var first = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + var second = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, first); + Assert.Equal(4, second); + Assert.True(NntpCommandParser.Parse(scratch.AsSpan(0, second)).IsValid); + Assert.Equal(NntpVerb.Quit, NntpCommandParser.Parse(scratch.AsSpan(0, second)).Verb); + } + + [Fact] + public async Task OverlongSplitAcrossReads_StillOverlong() + { + var line = GroupLine(NntpCommandLineReader.MaxCommandLineBytes + 1); + var wire = WithCrlf(line); + var pipe = new Pipe(NntpPipeOptions.Create()); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + var read = NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None).AsTask(); + + await pipe.Writer.WriteAsync(wire.AsMemory(0, 1000)); + await pipe.Writer.WriteAsync(wire.AsMemory(1000)); + var length = await read.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + await pipe.Writer.CompleteAsync(); + await pipe.Reader.CompleteAsync(); + } + + [Fact] + public async Task EmbeddedLf_IsNotADelimiter() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 1]; + line.AsSpan().Fill((byte)'a'); + line[10] = (byte)'\n'; + line[20] = (byte)'\r'; + await using var pipe = await CommandPipe.CreateAsync(WithCrlf(line)); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(NntpCommandLineReader.OverlongLine, length); + Assert.True((await pipe.Reader.ReadAsync()).Buffer.IsEmpty); + } + + [Fact] + public async Task NoCrlfBeforeClose_ReturnsEof() + { + var line = new byte[NntpCommandLineReader.MaxCommandLineBytes + 100]; + line.AsSpan().Fill((byte)'a'); + var pipe = new Pipe(NntpPipeOptions.Create()); + await pipe.Writer.WriteAsync(line); + await pipe.Writer.CompleteAsync(); + var scratch = new byte[NntpCommandLineReader.MaxCommandLineBytes]; + + var length = await NntpCommandLineReader.ReadLineBytesAsync(pipe.Reader, scratch, CancellationToken.None); + + Assert.Equal(-1, length); + await pipe.Reader.CompleteAsync(); + } + + private static byte[] GroupLine(int length) + { + var line = new byte[length]; + "GROUP "u8.CopyTo(line); + line.AsSpan("GROUP "u8.Length).Fill((byte)'a'); + return line; + } + + private static byte[] WithCrlf(byte[] line) + { + var wire = new byte[line.Length + 2]; + line.CopyTo(wire); + wire[^2] = (byte)'\r'; + wire[^1] = (byte)'\n'; + return wire; + } + + private sealed class CommandPipe : IAsyncDisposable + { + private readonly Pipe _pipe; + + private CommandPipe(Pipe pipe) => _pipe = pipe; + + public PipeReader Reader => _pipe.Reader; + + public static async Task CreateAsync(byte[] wire) + { + var pipe = new Pipe(NntpPipeOptions.Create()); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.CompleteAsync(); + return new CommandPipe(pipe); + } + + public async ValueTask DisposeAsync() + { + await _pipe.Reader.CompleteAsync(); + await _pipe.Writer.CompleteAsync(); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggerCollection.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggerCollection.cs new file mode 100644 index 00000000..89f5d248 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggerCollection.cs @@ -0,0 +1,8 @@ +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// Serializes tests that configure the process-wide NntpCommandLoggers factory +/// so recording loggers are not cleared by concurrent catalog creation. +/// +[CollectionDefinition(nameof(NntpCommandLoggerCollection), DisableParallelization = true)] +public sealed class NntpCommandLoggerCollection; diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggingTests.cs new file mode 100644 index 00000000..973fc164 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandLoggingTests.cs @@ -0,0 +1,928 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// Command RX/TX logging contract (ownership, redaction, timing shape). +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpCommandLoggingTests +{ + private static readonly Regex TxPattern = new( + @"TX:\s+(?.+?)(?:\s+\[(?[^\]]*)\])?\s+executed in (?\d+\.\d{3})s", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + [Fact] + public async Task Capabilities_ProducesExactlyOneRxAndOneTx_UnderCapabilitiesLogger() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); // greeting + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + _ = await duplex.ReadMultilineBodyAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var rx = recording.Messages.Where(m => m.Contains("RX:", StringComparison.Ordinal)).ToArray(); + var tx = recording.Messages.Where(m => m.Contains("TX:", StringComparison.Ordinal)).ToArray(); + + Assert.Contains(rx, m => m.Contains("RX: CAPABILITIES", StringComparison.Ordinal)); + Assert.Equal(1, tx.Count(m => m.Contains("TX: CAPABILITIES [101 Capability list:] executed in", StringComparison.Ordinal))); + Assert.Contains(tx, m => TxPattern.IsMatch(m) && m.Contains("CAPABILITIES", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("VERSION 2", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("IMPLEMENTATION", StringComparison.Ordinal)); + Assert.Contains( + recording.Categories, + c => c == typeof(Capabilities).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command CAPABILITIES failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task Help_ProducesExactlyOneRxAndOneTx_UnderHelpLogger() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("HELP"); + Assert.Equal("100 Help text follows", await duplex.ReadClientLineAsync()); + var body = await duplex.ReadMultilineBodyAsync(); + Assert.Equal(Help.BodyLines, body); + Assert.DoesNotContain(body, l => l.Contains("BENCHIT", StringComparison.OrdinalIgnoreCase)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var rx = recording.Messages.Where(m => m.Contains("RX:", StringComparison.Ordinal)).ToArray(); + var tx = recording.Messages.Where(m => m.Contains("TX:", StringComparison.Ordinal)).ToArray(); + + Assert.Contains(rx, m => m.Contains("RX: HELP", StringComparison.Ordinal)); + Assert.Equal(1, tx.Count(m => m.Contains("TX: HELP [100 Help text follows] executed in", StringComparison.Ordinal))); + Assert.DoesNotContain(tx, m => m.Contains("ARTICLE [message-id / article-number]", StringComparison.Ordinal)); + Assert.DoesNotContain(tx, m => m.Contains("CHECK message-id", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(Help).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command HELP failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task StartTls_CompletionOwnedByStartTlsLogger_NotDispatcher() + { + var recording = new RecordingLoggerFactory(); + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var session = new NntpSession( + server, + recording.CreateLogger(), + certificateProvider: host.CertificateProvider, + loggerFactory: recording); + + var run = session.RunAsync(); + _ = await ReadPlainLineAsync(clientSocket); + + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await server.CompleteAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: STARTTLS", StringComparison.Ordinal)); + var startTlsTx = recording.Messages + .Where(m => m.Contains("TX: STARTTLS [382 Continue with TLS negotiation] executed in", StringComparison.Ordinal)) + .ToArray(); + Assert.Single(startTlsTx); + Assert.Matches(TxPattern, startTlsTx[0]); + Assert.Contains("TlsVersion=", startTlsTx[0], StringComparison.Ordinal); + Assert.Contains("Cipher=", startTlsTx[0], StringComparison.Ordinal); + Assert.DoesNotContain("[failed]", startTlsTx[0], StringComparison.Ordinal); + + Assert.True(server.TryGetNegotiatedTlsParameters(out var tlsVersion, out var cipher)); + Assert.Contains($"TlsVersion={tlsVersion}", startTlsTx[0], StringComparison.Ordinal); + Assert.Contains($"Cipher={cipher}", startTlsTx[0], StringComparison.Ordinal); + + Assert.Contains(recording.Categories, c => c == typeof(StartTls).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command STARTTLS failed", StringComparison.Ordinal)); + } + + [Fact] + public async Task Post_OnTransit_Gated502_ProducesExactlyOneTxCompletion() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: POST [502 Command not available on this server] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task Quit_ProducesExactlyOneTxCompletion() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: QUIT [205 Connection closing] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task AuthinfoPass_RxIsRedacted_AndPasswordNeverLogged() + { + var recording = new RecordingLoggerFactory(); + var provider = new AcceptFredProvider(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording, authenticationProvider: provider, allowCleartextAuth: true); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS super-secret-password-xyz"); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("super-secret-password-xyz", joined, StringComparison.Ordinal); + Assert.Contains(recording.Messages, m => m.Contains("RX: AUTHINFO PASS ", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: AUTHINFO PASS [281 Authentication accepted] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task EveryInventoryCommand_ProducesRxWhenIssued() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var commands = new (string Line, string RxNeedle)[] + { + ("CAPABILITIES", "RX: CAPABILITIES"), + ("DATE", "RX: DATE"), + ("HELP", "RX: HELP"), + ("MODE READER", "RX: MODE READER"), + ("LIST", "RX: LIST"), + ("GROUP", "RX: GROUP"), + ("LISTGROUP", "RX: LISTGROUP"), + ("ARTICLE", "RX: ARTICLE"), + ("HEAD", "RX: HEAD"), + ("BODY", "RX: BODY"), + ("STAT", "RX: STAT"), + ("LAST", "RX: LAST"), + ("NEXT", "RX: NEXT"), + ("OVER", "RX: OVER"), + ("HDR", "RX: HDR"), + ("POST", "RX: POST"), + ("IHAVE", "RX: IHAVE"), + ("CHECK", "RX: CHECK"), + ("MODE STREAM", "RX: MODE STREAM"), + ("COMPRESS DEFLATE", "RX: COMPRESS DEFLATE"), + ("AUTHINFO SASL", "RX: AUTHINFO SASL"), + }; + + foreach (var (line, _) in commands) + { + await duplex.WriteClientLineAsync(line); + await DrainResponseAsync(duplex); + if (line == "POST") + { + await duplex.WriteClientBytesAsync(".\r\n"u8.ToArray()); + await DrainResponseAsync(duplex); + } + } + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + foreach (var (_, needle) in commands) + { + Assert.Contains(recording.Messages, m => m.Contains(needle, StringComparison.Ordinal)); + } + } + + [Fact] + public async Task TakeThis_ProducesRxAndTxCompletion_Logs() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true), + articleIngestion: queue); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + await duplex.WriteClientBytesAsync(Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("") + ".\r\n")); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + // Authorized STREAM TAKETHIS is admitted by NntpStreamDataPlaneRx and never + // enters DispatchCommandAsync, so there is no RX: TAKETHIS line. TX completion + // (previously hot-path suppressed) is restored. + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("RX: TAKETHIS", StringComparison.Ordinal)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: TAKETHIS [239 ] executed in", StringComparison.Ordinal) + && m.Contains("[accepted]", StringComparison.Ordinal)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: DATE", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: DATE [111 ", StringComparison.Ordinal) + && m.Contains("] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task Date_StillProducesRxAndTx_WhenTakeThisSuppressed() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + Assert.Contains(recording.Messages, m => m.Contains("RX: DATE", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: DATE [111 ", StringComparison.Ordinal) + && m.Contains("] executed in", StringComparison.Ordinal))); + } + + [Fact] + public async Task StartTls_HandshakeFailure_TxHasFailed_WithoutFakeTlsDetails() + { + var recording = new RecordingLoggerFactory(); + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var session = new NntpSession( + server, + recording.CreateLogger(), + certificateProvider: host.CertificateProvider, + loggerFactory: recording); + + var run = session.RunAsync(); + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: STARTTLS [382 Continue with TLS negotiation] executed in", StringComparison.Ordinal)); + Assert.Contains("[failed]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("TlsVersion=", tx, StringComparison.Ordinal); + Assert.DoesNotContain("Cipher=", tx, StringComparison.Ordinal); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP command STARTTLS failed", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(StartTls).FullName); + } + + [Fact] + public async Task Check_Wanted_LogsExactStatusLineFromWriter() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Matches( + @"^\[198\.18\.0\.70:49860\] TX: CHECK \[238 send article to be transferred\] executed in \d+\.\d{3}s$", + tx); + Assert.DoesNotContain("438", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Check_Duplicate_LogsActual438StatusLine() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + historyDb: new SeenHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("438 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Contains("TX: CHECK [438 ] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("238", tx, StringComparison.Ordinal); + Assert.DoesNotContain("send article to be transferred", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Check_Unavailable_Logs431StatusLine() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + historyDb: new UnavailableHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: CHECK", StringComparison.Ordinal)); + Assert.Contains("TX: CHECK [431 ] executed in", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task TakeThis_RejectedTooLarge_Logs439StatusLine() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + articleIngestion: queue); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + await duplex.WriteClientBytesAsync("Subject: oversized-payload-for-439\r\n\r\nbody\r\n.\r\n"u8.ToArray()); + Assert.Equal("439 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: TAKETHIS", StringComparison.Ordinal)); + Assert.Contains("TX: TAKETHIS [439 ] executed in", tx, StringComparison.Ordinal); + Assert.Contains("[rejected too large]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("Subject: oversized", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task TakeThis_HistoryUnavailable_Logs400StatusLine() + { + var recording = new RecordingLoggerFactory(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: TransitAuthorization(), + articleIngestion: queue, + historyDb: new UnavailableHistoryDb()); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + await duplex.WriteClientBytesAsync("Subject: t\r\n\r\nbody\r\n.\r\n"u8.ToArray()); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: TAKETHIS", StringComparison.Ordinal)); + Assert.Contains( + "TX: TAKETHIS [400 Service temporarily unavailable] executed in", + tx, + StringComparison.Ordinal); + Assert.Contains("[temporary failure]", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task List_OnTransit_Logs502StatusLine_WithoutServingSnapshot() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await LoggingDuplex.CreateAsync(); + var session = duplex.CreateSession( + recording, + authorization: new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false), + newsgroupCatalogue: new StaticNewsgroupCatalogue(NewsgroupSnapshot.Empty)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("LIST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: LIST", StringComparison.Ordinal)); + Assert.Contains( + "TX: LIST [502 Command not available on this server] executed in", + tx, + StringComparison.Ordinal); + } + + [Fact] + public void RedactRxLine_ProtectsPassAndSasl() + { + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxLine("AUTHINFO PASS secret")); + Assert.Equal("AUTHINFO SASL ", NntpCommandLogFormat.RedactRxLine("AUTHINFO SASL PLAIN abc")); + Assert.Equal("CAPABILITIES", NntpCommandLogFormat.RedactRxLine("CAPABILITIES")); + } + + private static NntpAuthorization TransitAuthorization() => + new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + private static async Task DrainResponseAsync(LoggingDuplex duplex) + { + var line = await duplex.ReadClientLineAsync(); + // Multi-line responses (CAPABILITIES 101, HELP 100, LIST 215, …). + if (line.StartsWith("100 ", StringComparison.Ordinal) || + line.StartsWith("101 ", StringComparison.Ordinal) || + line.StartsWith("215 ", StringComparison.Ordinal) || + line.StartsWith("224 ", StringComparison.Ordinal) || + line.StartsWith("225 ", StringComparison.Ordinal) || + line.StartsWith("220 ", StringComparison.Ordinal) || + line.StartsWith("221 ", StringComparison.Ordinal) || + line.StartsWith("222 ", StringComparison.Ordinal) || + line.StartsWith("230 ", StringComparison.Ordinal) || + line.StartsWith("231 ", StringComparison.Ordinal)) + { + _ = await duplex.ReadMultilineBodyAsync(); + } + } + + private static async Task ReadPlainLineAsync(System.Net.Sockets.Socket socket) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = System.Text.Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private static async Task WriteSslLineAsync(System.Net.Security.SslStream ssl, string line) + { + var bytes = System.Text.Encoding.ASCII.GetBytes(line + "\r\n"); + await ssl.WriteAsync(bytes); + } + + private static async Task ReadSslLineAsync(System.Net.Security.SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = System.Text.Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private static System.Net.Security.SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private sealed class AcceptFredProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + new( + username == "fred" + ? NntpAuthenticationResult.Success( + "fred", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)) + : NntpAuthenticationResult.Failed); + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + } + } + + private sealed class LoggingDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new LoggingDuplex()); + + public NntpSession CreateSession( + ILoggerFactory loggerFactory, + INntpAuthenticationProvider? authenticationProvider = null, + bool allowCleartextAuth = true, + NntpAuthorization? authorization = null, + IArticleIngestionQueue? articleIngestion = null, + VectorNNTPD.TransitService.History.IHistoryDb? historyDb = null, + INewsgroupCatalogue? newsgroupCatalogue = null) + { + var connection = new PipeConnection(_clientToServer.Reader, _serverToClient.Writer); + var session = new NntpSession( + connection, + loggerFactory.CreateLogger(), + authenticationProvider: authenticationProvider, + allowCleartextAuth: allowCleartextAuth, + loggerFactory: loggerFactory, + articleIngestion: articleIngestion, + historyDb: historyDb, + newsgroupCatalogue: newsgroupCatalogue); + if (authorization is not null) + { + session.SetAuthorization(authorization); + } + + return session; + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientBytesAsync(ReadOnlyMemory bytes) + { + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task> ReadMultilineBodyAsync() + { + var lines = new List(); + while (true) + { + var line = await ReadClientLineAsync(); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output) + { + Input = input; + Output = output; + ClientIdentity = ConnectionClientIdentity.Direct( + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 49860)); + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } + } + + private sealed class UnavailableHistoryDb : VectorNNTPD.TransitService.History.IHistoryDb + { + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.TransitService.History.HistoryLookupResult.Unavailable); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.TransitService.History.HistoryLookupResult.Unavailable); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in VectorNNTPD.TransitService.History.HistoryDigest digest) => false; + } + + private sealed class SeenHistoryDb : VectorNNTPD.TransitService.History.IHistoryDb + { + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.TransitService.History.HistoryLookupResult.Seen); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + new(VectorNNTPD.TransitService.History.HistoryLookupResult.Seen); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in VectorNNTPD.TransitService.History.HistoryDigest digest) => true; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandParserTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandParserTests.cs new file mode 100644 index 00000000..0a8b05f8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandParserTests.cs @@ -0,0 +1,513 @@ +using System.IO.Pipelines; +using System.Text; +using VectorNNTPD.Common.Articles.Validation; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// Full-surface byte parser contracts: classification, syntax status, and reject-before-handler. +/// +public sealed class NntpCommandParserTests +{ + [Theory] + [InlineData("ARTICLE", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 123", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 0", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 0001", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE 9999999999999999", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE ", NntpVerb.Article, NntpVerb.None)] + [InlineData("ARTICLE <123@example.com>", NntpVerb.Article, NntpVerb.None)] + [InlineData("article", NntpVerb.Article, NntpVerb.None)] + [InlineData("ArTiClE", NntpVerb.Article, NntpVerb.None)] + [InlineData("BODY", NntpVerb.Body, NntpVerb.None)] + [InlineData("BODY 1", NntpVerb.Body, NntpVerb.None)] + [InlineData("HEAD ", NntpVerb.Head, NntpVerb.None)] + [InlineData("STAT 300256", NntpVerb.Stat, NntpVerb.None)] + [InlineData("CAPABILITIES", NntpVerb.Capabilities, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("CHECK ", NntpVerb.Check, NntpVerb.None)] + [InlineData("check ", NntpVerb.Check, NntpVerb.None)] + [InlineData("cHeCk ", NntpVerb.Check, NntpVerb.None)] + [InlineData("COMPRESS DEFLATE", NntpVerb.Compress, NntpVerb.None)] + [InlineData("DATE", NntpVerb.Date, NntpVerb.None)] + [InlineData("GROUP misc.test", NntpVerb.Group, NntpVerb.None)] + [InlineData("HDR", NntpVerb.Hdr, NntpVerb.None)] + [InlineData("HEAD", NntpVerb.Head, NntpVerb.None)] + [InlineData("HELP", NntpVerb.Help, NntpVerb.None)] + [InlineData("IHAVE ", NntpVerb.Ihave, NntpVerb.None)] + [InlineData("LAST", NntpVerb.Last, NntpVerb.None)] + [InlineData("LIST", NntpVerb.List, NntpVerb.None)] + [InlineData("LIST ACTIVE", NntpVerb.List, NntpVerb.Active)] + [InlineData("LIST COUNTS", NntpVerb.List, NntpVerb.Counts)] + [InlineData("LIST COUNTS misc.*", NntpVerb.List, NntpVerb.Counts)] + [InlineData("LIST HEADERS", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST HEADERS MSGID", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST HEADERS RANGE", NntpVerb.List, NntpVerb.Headers)] + [InlineData("LIST MOTD", NntpVerb.List, NntpVerb.Motd)] + [InlineData("LIST NEWSGROUPS", NntpVerb.List, NntpVerb.Newsgroups)] + [InlineData("LIST OVERVIEW.FMT", NntpVerb.List, NntpVerb.OverviewFmt)] + [InlineData("LISTGROUP", NntpVerb.ListGroup, NntpVerb.None)] + [InlineData("MODE READER", NntpVerb.Mode, NntpVerb.Reader)] + [InlineData("MODE STREAM", NntpVerb.Mode, NntpVerb.Stream)] + [InlineData("mode reader", NntpVerb.Mode, NntpVerb.Reader)] + [InlineData("mode stream", NntpVerb.Mode, NntpVerb.Stream)] + [InlineData("NEXT", NntpVerb.Next, NntpVerb.None)] + [InlineData("OVER", NntpVerb.Over, NntpVerb.None)] + [InlineData("POST", NntpVerb.Post, NntpVerb.None)] + [InlineData("QUIT", NntpVerb.Quit, NntpVerb.None)] + [InlineData("STARTTLS", NntpVerb.StartTls, NntpVerb.None)] + [InlineData("STAT", NntpVerb.Stat, NntpVerb.None)] + [InlineData("TAKETHIS ", NntpVerb.TakeThis, NntpVerb.None)] + [InlineData("takethis ", NntpVerb.TakeThis, NntpVerb.None)] + [InlineData("AUTHINFO USER alice", NntpVerb.AuthInfo, NntpVerb.User)] + [InlineData("AUTHINFO PASS secret", NntpVerb.AuthInfo, NntpVerb.Pass)] + [InlineData("AUTHINFO SASL PLAIN", NntpVerb.AuthInfo, NntpVerb.Sasl)] + [InlineData("authinfo user alice", NntpVerb.AuthInfo, NntpVerb.User)] + [InlineData("BENCHIT", NntpVerb.BenchIt, NntpVerb.None)] + [InlineData("SPEEDTEST GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("SPEEDTEST usenet-ninja", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("speedtest GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + [InlineData("SpEeDtEsT GIGANEWS", NntpVerb.SpeedTest, NntpVerb.None)] + public void Parse_SupportedCommands_AreValid(string text, NntpVerb verb, NntpVerb qualifier) + { + var (command, _) = NntpCommandTestParse.Parse(text); + Assert.True(command.IsValid); + Assert.Equal(NntpParseStatus.Ok, command.Status); + Assert.Equal(verb, command.Verb); + Assert.Equal(qualifier, command.Qualifier); + } + + /// RFC 3977 §3.1: tokens are separated by one or more SP or TAB. + [Theory] + [InlineData("CHECK\t")] + [InlineData("MODE\tSTREAM")] + [InlineData("AUTHINFO\tUSER\talice")] + [InlineData("QUIT\t")] + [InlineData(" DATE ")] + [InlineData("\tHELP")] + public void Parse_SpAndTabWhitespace_IsAccepted(string text) + { + Assert.True(NntpCommandTestParse.ParseCommand(text).IsValid); + } + + [Theory] + [InlineData("", NntpParseStatus.Empty)] + [InlineData(" ", NntpParseStatus.Empty)] + [InlineData("\t", NntpParseStatus.Empty)] + [InlineData(" ", NntpParseStatus.Empty)] + [InlineData("NOSUCHCMD", NntpParseStatus.UnknownVerb)] + [InlineData("NEWNEWS", NntpParseStatus.UnknownVerb)] + [InlineData("NEWGROUPS", NntpParseStatus.UnknownVerb)] + [InlineData("newnews * 19990624 000000 GMT", NntpParseStatus.UnknownVerb)] + [InlineData("NEWGROUPS 19990624 000000 GMT", NntpParseStatus.UnknownVerb)] + [InlineData("ARTICLEX", NntpParseStatus.UnknownVerb)] + [InlineData("MODE", NntpParseStatus.UnknownQualifier)] + [InlineData("MODE UNKNOWN", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO GENERIC x", NntpParseStatus.UnknownQualifier)] + [InlineData("AUTHINFO USERXYZ", NntpParseStatus.UnknownQualifier)] + [InlineData("ARTICLE foo", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 12a", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE -1", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 10000000000000000", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE <123@example.com", NntpParseStatus.InvalidArgument)] + [InlineData("ARTICLE 123 extra", NntpParseStatus.ExtraArgument)] + [InlineData("HEAD not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("HEAD 123 extra", NntpParseStatus.ExtraArgument)] + [InlineData("BODY 12345678901234567", NntpParseStatus.InvalidArgument)] + [InlineData("BODY extra", NntpParseStatus.ExtraArgument)] + [InlineData("GROUP", NntpParseStatus.MissingArgument)] + [InlineData("GROUP a b", NntpParseStatus.ExtraArgument)] + [InlineData("LIST ACTIVE u[ks].*", NntpParseStatus.InvalidArgument)] + [InlineData("LIST COUNTS u[ks].*", NntpParseStatus.InvalidArgument)] + [InlineData("LIST ACTIVE foo bar", NntpParseStatus.ExtraArgument)] + [InlineData("LIST COUNTS foo bar", NntpParseStatus.ExtraArgument)] + [InlineData("LIST extra", NntpParseStatus.UnknownQualifier)] + [InlineData("LIST SOMETHING-THAT-DOES-NOT-EXIST", NntpParseStatus.UnknownQualifier)] + [InlineData("LIST MOTD extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST OVERVIEW.FMT extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST HEADERS FOO", NntpParseStatus.InvalidArgument)] + [InlineData("LIST HEADERS MSGID extra", NntpParseStatus.ExtraArgument)] + [InlineData("LIST ACTIVE.TIMES", NntpParseStatus.UnknownQualifier)] + [InlineData("CHECK", NntpParseStatus.MissingArgument)] + [InlineData("CHECK ", NntpParseStatus.MissingArgument)] + [InlineData("TAKETHIS", NntpParseStatus.MissingArgument)] + [InlineData("IHAVE", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO USER", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO PASS", NntpParseStatus.MissingArgument)] + [InlineData("AUTHINFO SASL", NntpParseStatus.MissingArgument)] + [InlineData("COMPRESS", NntpParseStatus.MissingArgument)] + [InlineData("CHECK not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK x>", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK x<", NntpParseStatus.InvalidArgument)] + [InlineData("TAKETHIS not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("IHAVE not-an-id", NntpParseStatus.InvalidArgument)] + [InlineData("COMPRESS deflate", NntpParseStatus.InvalidArgument)] + [InlineData("CHECK extra", NntpParseStatus.ExtraArgument)] + [InlineData("TAKETHIS extra", NntpParseStatus.ExtraArgument)] + [InlineData("IHAVE extra", NntpParseStatus.ExtraArgument)] + [InlineData("QUIT extra", NntpParseStatus.ExtraArgument)] + [InlineData("HELP MORE", NntpParseStatus.ExtraArgument)] + [InlineData("COMPRESS DEFLATE EXTRA", NntpParseStatus.ExtraArgument)] + [InlineData("SPEEDTEST", NntpParseStatus.MissingArgument)] + [InlineData("SPEEDTEST ", NntpParseStatus.MissingArgument)] + [InlineData("SPEEDTEST GIGANEWS extra", NntpParseStatus.ExtraArgument)] + [InlineData("SPEEDTEST Usenet Ninja", NntpParseStatus.ExtraArgument)] + public void Parse_InvalidCommands_HaveExpectedStatus(string text, NntpParseStatus status) + { + var command = NntpCommandTestParse.ParseCommand(text); + Assert.False(command.IsValid); + Assert.Equal(status, command.Status); + } + + /// + /// IHAVE without a message-id is parser MissingArgument (501), not the old + /// stub-handler 500. The parser owns required-parameter syntax. + /// + [Fact] + public void Ihave_WithoutMessageId_IsMissingArgument_NotOk() + { + var command = NntpCommandTestParse.ParseCommand("IHAVE"); + Assert.Equal(NntpVerb.Ihave, command.Verb); + Assert.Equal(NntpParseStatus.MissingArgument, command.Status); + Assert.False(command.IsValid); + } + + [Fact] + public void Parse_DoesNotAllocate_OnRepresentativeLines() + { + string[] lines = + [ + "CHECK ", + "TAKETHIS ", + "MODE STREAM", + "MODE READER", + "AUTHINFO USER foo", + "AUTHINFO PASS bar", + "QUIT", + "CHECK not-an-id", + "", + ]; + + foreach (var bytes in lines.Select(static text => Encoding.ASCII.GetBytes(text))) + { + for (var i = 0; i < 8; i++) + { + _ = NntpCommandParser.Parse(bytes); + } + + var before = GC.GetAllocatedBytesForCurrentThread(); + var sink = 0; + for (var i = 0; i < 256; i++) + { + var command = NntpCommandParser.Parse(bytes); + sink += (int)command.Verb + (int)command.Status + command.TokenCount; + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + Assert.NotEqual(int.MinValue, sink); + } + } + + [Theory] + [InlineData("NOSUCHCMD", "500 Unknown command")] + [InlineData("NEWNEWS", "500 Unknown command")] + [InlineData("NEWGROUPS", "500 Unknown command")] + [InlineData("NEWNEWS * 19990624 000000 GMT", "500 Unknown command")] + [InlineData("NEWGROUPS 19990624 000000 GMT", "500 Unknown command")] + [InlineData("MODE UNKNOWN", "501 Unknown command variant")] + [InlineData("LIST DISTRIB.PATS", "501 Unknown command variant")] + [InlineData("ARTICLE foo", "501 Syntax error")] + [InlineData("HEAD 123 extra", "501 Syntax error")] + [InlineData("BODY 12345678901234567", "501 Syntax error")] + [InlineData("STAT extra", "501 Syntax error")] + [InlineData("CHECK", "501 Syntax error")] + [InlineData("CHECK not-an-id", "501 Syntax error")] + [InlineData("CHECK extra", "501 Syntax error")] + [InlineData("IHAVE", "501 Syntax error")] + [InlineData("IHAVE not-an-id", "501 Syntax error")] + [InlineData("AUTHINFO USER", "501 AUTHINFO USER requires a username")] + [InlineData("AUTHINFO PASS", "501 AUTHINFO PASS requires a password")] + [InlineData("AUTHINFO GENERIC x", "501 Unknown command variant")] + [InlineData("COMPRESS", "501 COMPRESS requires a single algorithm argument")] + [InlineData("COMPRESS deflate", "501 Syntactically incorrect compression algorithm")] + [InlineData("QUIT extra", "501 Syntax error")] + [InlineData("HELP MORE", "501 Syntax error")] + [InlineData("SPEEDTEST", "501 Syntax error")] + [InlineData("SPEEDTEST GIGANEWS extra", "501 Syntax error")] + public async Task InvalidCommand_IsRejectedWithProtocolError_AndDoesNotReachHandler(string commandLine, string response) + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, commandLine); + Assert.Equal(response, await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task OverlongCommandLine_Is501_AndTheNextCommandStillRuns() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await dispatcher.DispatchAsync( + session, + NntpCommand.OverlongLine, + ReadOnlyMemory.Empty, + writer, + CancellationToken.None); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task InvalidCheck_DoesNotInvokeHandler_EvenWhenTransitAuthorized() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "CHECK not-an-id"); + var line = await duplex.ReadClientLineAsync(); + Assert.Equal("501 Syntax error", line); + Assert.DoesNotContain("238", line, StringComparison.Ordinal); + } + + [Theory] + [InlineData("")] + [InlineData("NOSUCHCMD")] + [InlineData("MODE")] + [InlineData("AUTHINFO GENERIC x")] + [InlineData("CHECK")] + [InlineData("CHECK not-an-id")] + [InlineData("IHAVE")] + [InlineData("TAKETHIS not-an-id")] + [InlineData("QUIT extra")] + public async Task InvalidCommand_CannotConstructHandlerContext(string commandLine) + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var invalid = NntpCommandTestParse.ParseCommand(commandLine); + Assert.False(invalid.IsValid); + var writer = new NntpResponseWriter(duplex.ServerOutput); + Assert.Throws(() => + new NntpCommandContext(session, invalid, ReadOnlyMemory.Empty, writer)); + } + + [Fact] + public async Task Ihave_ValidMessageId_ReachesHandler() + { + await using var duplex = await ParserDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + + var parsed = NntpCommandTestParse.ParseCommand("IHAVE "); + Assert.True(parsed.IsValid); + var dispatch = NntpCommandTestParse.DispatchAsync(dispatcher, session, writer, "IHAVE ").AsTask(); + Assert.Contains("335 Send article to be transferred", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientAsync(".\r\n"); + await dispatch.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public void MessageId_CommandArgument_UsesCommonValidator_AndFieldLimit() + { + Assert.True(NntpCommandParser.Parse("CHECK "u8).IsValid); + Assert.False(NntpCommandParser.Parse("CHECK "u8).IsValid); + Assert.False(NntpCommandParser.Parse("CHECK <@>"u8).IsValid); + Assert.True(NntpCommandParser.Parse("ARTICLE "u8).IsValid); + + var exactField = BuildMessageId(NntpMessageIdValidation.MaxMessageIdLength); + Assert.True(NntpMessageIdValidation.IsValidMessageId(exactField)); + Assert.True(NntpCommandParser.Parse(BuildCommand("CHECK ", exactField)).IsValid); + + var overField = BuildMessageId(NntpMessageIdValidation.MaxMessageIdLength + 1); + Assert.False(NntpMessageIdValidation.IsValidMessageId(overField)); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse(BuildCommand("CHECK ", overField)).Status); + + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK <>"u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK "u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK "u8).Status); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandParser.Parse("CHECK b@c>"u8).Status); + } + + private static byte[] BuildMessageId(int totalLength) + { + var bytes = new byte[totalLength]; + bytes[0] = (byte)'<'; + bytes[1] = (byte)'a'; + bytes[2] = (byte)'@'; + bytes[^1] = (byte)'>'; + if (totalLength > 4) + { + bytes.AsSpan(3, totalLength - 4).Fill((byte)'b'); + } + + return bytes; + } + + private static byte[] BuildCommand(string prefix, ReadOnlySpan argument) + { + var prefixBytes = Encoding.ASCII.GetBytes(prefix); + var line = new byte[prefixBytes.Length + argument.Length]; + prefixBytes.CopyTo(line); + argument.CopyTo(line.AsSpan(prefixBytes.Length)); + return line; + } + + private sealed class ParserDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public static Task CreateAsync() => Task.FromResult(new ParserDuplex()); + + public NntpSession CreateSession(IArticleIngestionQueue? queue = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTestParse.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTestParse.cs new file mode 100644 index 00000000..f451d678 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTestParse.cs @@ -0,0 +1,28 @@ +using System.Text; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +internal static class NntpCommandTestParse +{ + public static (NntpCommand Command, byte[] Line) Parse(string text) + { + var line = Encoding.ASCII.GetBytes(text); + return (NntpCommandParser.Parse(line), line); + } + + public static NntpCommand ParseCommand(string text) => Parse(text).Command; + + public static ValueTask DispatchAsync( + NntpCommandDispatcher dispatcher, + NntpSession session, + NntpResponseWriter response, + string text, + CancellationToken cancellationToken = default) + { + var (command, line) = Parse(text); + return dispatcher.DispatchAsync(session, command, line, response, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTxStatusLoggingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTxStatusLoggingTests.cs new file mode 100644 index 00000000..0a8f838c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCommandTxStatusLoggingTests.cs @@ -0,0 +1,333 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// Semantic TX status logging: Debug-gated construction, writer isolation, no wire decode. +/// +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class NntpCommandTxStatusLoggingTests +{ + [Fact] + public void Writer_HasNoLoggingInspectionSurface() + { + var type = typeof(NntpResponseWriter); + Assert.Null(type.GetMethod("TrackCommandResponse", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)); + Assert.Null(type.GetField("CommandResponseScope", BindingFlags.Static | BindingFlags.NonPublic)); + Assert.Null(type.GetNestedType("CommandResponseTrackingScope", BindingFlags.NonPublic)); + Assert.DoesNotContain(type.GetFields(BindingFlags.Instance | BindingFlags.Static | BindingFlags.NonPublic), + f => f.FieldType.IsGenericType && f.FieldType.GetGenericTypeDefinition() == typeof(AsyncLocal<>)); + } + + [Fact] + public void FormatCheck_UsesSemanticResultNotWireDecode() + { + var id = ""u8; + Assert.Equal("238 send article to be transferred", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Unseen, id)); + Assert.Equal("438 ", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Seen, id)); + Assert.Equal("431 ", + NntpCommandStatusText.FormatCheck(HistoryLookupResult.Unavailable, id)); + } + + [Fact] + public void FormatTakeThis_UsesSemanticOutcome() + { + var id = ""u8; + Assert.Equal("239 ", NntpCommandStatusText.FormatTakeThisAccepted(id)); + Assert.Equal("439 ", NntpCommandStatusText.FormatTakeThisRejected(id)); + } + + [Fact] + public void StaticStatus_IsNotDecodedFromWireBytes() + { + Assert.Equal("500 Command not implemented", NntpResponseStatus.CommandNotImplemented); + Assert.Equal("412 No newsgroup selected", NntpResponseStatus.NoNewsgroupSelected); + Assert.Equal("420 Current article number is invalid", NntpResponseStatus.CurrentArticleNumberInvalid); + Assert.Equal("423 No article with that number", NntpResponseStatus.NoArticleWithNumber); + Assert.Equal("430 No article with that message-id", NntpResponseStatus.NoArticleWithMessageId); + Assert.Equal("503 Data item not stored", NntpResponseStatus.ListDataItemNotStored); + Assert.Equal("215 Order of fields in overview database.", NntpResponseStatus.ListOverviewFmtFollows); + Assert.Equal("215 headers and metadata items supported:", NntpResponseStatus.ListHeadersFollows); + Assert.Equal("100 Help text follows", NntpResponseStatus.HelpTextFollows); + Assert.Equal("400 Service temporarily unavailable", NntpResponseStatus.ServiceTemporarilyUnavailable); + } + + [Fact] + public async Task WriteCompletion_DisabledLogger_DoesNotFormatClientOrStatus() + { + var logger = new GateLogger(enabled: false); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + + NntpCommandExecution.WriteCompletion( + logger, + session, + "CHECK", + TimeSpan.FromSeconds(0.012), + statusLine: "must-not-be-logged"); + + Assert.Empty(logger.Messages); + Assert.Equal(0, logger.FormatCount); + } + + [Fact] + public async Task RunAsync_DisabledLogger_DoesNotSetStatusLine() + { + var logger = new GateLogger(enabled: false); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("HELP"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + await NntpCommandExecution.RunAsync( + logger, + context, + "HELP", + static (ctx, ct) => NntpCommandReply.WriteAsync( + ctx, + new GateLogger(enabled: false), + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + ct), + CancellationToken.None); + + Assert.Null(context.StatusLine); + Assert.Empty(logger.Messages); + } + + [Fact] + public async Task RunAsync_EnabledLogger_NotesSemanticHelpStatus() + { + var logger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("HELP"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + await NntpCommandExecution.RunAsync( + logger, + context, + "HELP", + static (ctx, ct) => NntpCommandReply.WriteAsync( + ctx, + new GateLogger(enabled: true), + NntpResponses.HelpComplete, + NntpResponseStatus.HelpTextFollows, + ct), + CancellationToken.None); + + Assert.Equal(NntpResponseStatus.HelpTextFollows, context.StatusLine); + var tx = Assert.Single(logger.Messages, m => m.Contains("TX: HELP", StringComparison.Ordinal)); + Assert.Contains("TX: HELP [100 Help text follows] executed in", tx, StringComparison.Ordinal); + Assert.DoesNotContain("ARTICLE [message-id / article-number]", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_HandlerThrowsBeforeResponse_KeepsNoStatusFormat() + { + var logger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + var (command, line) = NntpCommandTestParse.Parse("DATE"); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + var context = new NntpCommandContext(session, command, line, writer); + + var thrown = await Assert.ThrowsAsync(async () => + await NntpCommandExecution.RunAsync( + logger, + context, + "DATE", + static (_, _) => throw new InvalidOperationException("boom"), + CancellationToken.None)); + Assert.Equal("boom", thrown.Message); + + var tx = Assert.Single(logger.Messages, m => m.Contains("TX: DATE", StringComparison.Ordinal)); + Assert.Contains("TX: DATE executed in", tx, StringComparison.Ordinal); + Assert.Contains("[failed]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("[403", tx, StringComparison.Ordinal); + Assert.DoesNotContain("[no response]", tx, StringComparison.Ordinal); + Assert.DoesNotContain("111 ", tx, StringComparison.Ordinal); + Assert.Null(context.StatusLine); + } + + [Fact] + public async Task ConcurrentContexts_KeepIsolatedStatusLines() + { + var firstLogger = new GateLogger(enabled: true); + var secondLogger = new GateLogger(enabled: true); + await using var harness = new StatusHarness(); + var session = harness.CreateSession(); + await using var writer = new NntpResponseWriter(harness.ServerOutput); + + var (checkA, lineA) = NntpCommandTestParse.Parse("CHECK "); + var (checkB, lineB) = NntpCommandTestParse.Parse("CHECK "); + var contextA = new NntpCommandContext(session, checkA, lineA, writer); + var contextB = new NntpCommandContext(session, checkB, lineB, writer); + + await Task.WhenAll( + NntpCommandExecution.RunAsync( + firstLogger, + contextA, + "CHECK", + (ctx, _) => + { + ctx.StatusLine = NntpCommandStatusText.FormatCheck( + HistoryLookupResult.Unseen, + ""u8); + return ValueTask.CompletedTask; + }, + CancellationToken.None).AsTask(), + NntpCommandExecution.RunAsync( + secondLogger, + contextB, + "CHECK", + (ctx, _) => + { + ctx.StatusLine = NntpCommandStatusText.FormatCheck( + HistoryLookupResult.Seen, + ""u8); + return ValueTask.CompletedTask; + }, + CancellationToken.None).AsTask()); + + Assert.Equal("238 send article to be transferred", contextA.StatusLine); + Assert.Equal("438 ", contextB.StatusLine); + Assert.Contains(firstLogger.Messages, m => m.Contains("[238 send article to be transferred]", StringComparison.Ordinal)); + Assert.Contains(secondLogger.Messages, m => m.Contains("[438 ]", StringComparison.Ordinal)); + Assert.DoesNotContain(firstLogger.Messages, m => m.Contains("438", StringComparison.Ordinal)); + Assert.DoesNotContain(secondLogger.Messages, m => m.Contains("238", StringComparison.Ordinal)); + } + + private sealed class GateLogger : ILogger + { + private readonly bool _enabled; + + public GateLogger(bool enabled) => _enabled = enabled; + + public ConcurrentBag Messages { get; } = []; + + public int FormatCount => Messages.Count; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => _enabled && logLevel >= LogLevel.Debug; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + Messages.Add(formatter(state, exception)); + } + } + + private sealed class StatusHarness : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public NntpSession CreateSession() + { + var connection = new PipeConnection(_clientToServer.Reader, _serverToClient.Writer); + return new NntpSession(connection, NullLogger.Instance); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output) + { + Input = input; + Output = output; + ClientIdentity = ConnectionClientIdentity.Direct( + new IPEndPoint(IPAddress.Parse("198.18.0.70"), 49860)); + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpCompressTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCompressTests.cs new file mode 100644 index 00000000..41fdfc85 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpCompressTests.cs @@ -0,0 +1,754 @@ +using System.Collections.Concurrent; +using System.IO.Compression; +using System.Net.Security; +using System.Net.Sockets; +using System.Runtime.InteropServices; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// Session-level COMPRESS DEFLATE (RFC 8054) regressions over the real transport stack. +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpCompressTests +{ + [Fact] + public async Task Compress_Capabilities_AdvertiseThenStopAfterActivation() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + var before = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.Contains("COMPRESS DEFLATE", before); + Assert.Contains("STARTTLS", before); + Assert.DoesNotContain("MODE-READER", before); + Assert.Contains("AUTHINFO USER", before); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + var after = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.True(duplex.Connection.IsCompressed); + Assert.DoesNotContain(after, static c => c.StartsWith("COMPRESS", StringComparison.Ordinal)); + Assert.DoesNotContain("STARTTLS", after); + Assert.DoesNotContain("MODE-READER", after); + Assert.Contains("AUTHINFO", after); + Assert.DoesNotContain("AUTHINFO USER", after); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_Capabilities_AfterRejectedModeReader_StillAdvertiseCompressUntilActive() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + var caps = await ReadCapabilitiesFromDuplexAsync(duplex); + Assert.Contains("COMPRESS DEFLATE", caps); + Assert.DoesNotContain("READER", caps); + Assert.DoesNotContain("MODE-READER", caps); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Theory] + [InlineData("COMPRESS", "501 ")] + [InlineData("COMPRESS DEFLATE EXTRA", "501 ")] + [InlineData("COMPRESS deflate", "501 ")] + [InlineData("COMPRESS Deflate", "501 ")] + [InlineData("COMPRESS FOO", "503 ")] + [InlineData("COMPRESS GZIP", "503 ")] + public async Task Compress_SyntaxAndUnsupportedAlgorithm(string command, string expectedPrefix) + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, command); + Assert.StartsWith(expectedPrefix, await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_Deflate_Exact206Wire_AndIsCompressed() + { + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Equal("206 Compression active", await duplex.ReadClientLineAsync()); + Assert.True(session.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_Deflate_SessionRoundTrip_Repeated_AuthinfoRejected() + { + Assert.True(ProducesRawDeflateNotZlibOrGzip()); + + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var session = new NntpSession( + server, + NullLogger.Instance, + authenticationProvider: provider, + allowCleartextAuth: true); + var sessionTask = session.RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.Equal("206 Compression active", await ReadPlainLineAsync(clientSocket, plain)); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await WriteDeflateLineAsync(deflate, "COMPRESS DEFLATE"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await WriteDeflateLineAsync(deflate, "AUTHINFO USER alice"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + await WriteDeflateLineAsync(deflate, "AUTHINFO PASS secret"); + Assert.StartsWith("502 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + Assert.Equal("AUTHINFO PASS ", NntpCommandLogFormat.RedactRxLine("AUTHINFO PASS secret")); + + await WriteDeflateLineAsync(deflate, "QUIT"); + Assert.StartsWith("205 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_Logging_OwnedByCompress_WithRecordingFactory() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await CompressTestDuplex.CreateAsync(); + var session = new NntpSession( + duplex.Connection, + recording.CreateLogger(), + loggerFactory: recording); + var sessionTask = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.True(duplex.Connection.IsCompressed); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: COMPRESS DEFLATE", StringComparison.Ordinal)); + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: COMPRESS [206 Compression active] executed in", StringComparison.Ordinal)); + Assert.Contains("[DEFLATE active]", tx, StringComparison.Ordinal); + Assert.Contains(recording.Categories, c => c == typeof(Compress).FullName); + } + + [Fact] + public async Task Compress_ClientSendsCompressedBytesImmediatelyAfter206_ConsumedByDeflate() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + await WriteDeflateLineAsync(deflate, "DATE"); + + await WaitForServerCompressedAsync(server); + Assert.False(server.IsCompleted); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_AuthinfoBeforeCompress_ThenCompressSucceeds() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + authenticationProvider: provider, + allowCleartextAuth: true) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("AUTHINFO USER alice\r\n"u8.ToArray()); + Assert.StartsWith("381 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await clientSocket.SendAsync("AUTHINFO PASS secret\r\n"u8.ToArray()); + Assert.StartsWith("281 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_AfterStartTls_LayersDeflateAboveTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var provider = new FixedAuthProvider("alice", "secret"); + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider, + authenticationProvider: provider, + allowCleartextAuth: false) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket, plain); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var ssl = new SslStream(stream, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "AUTHINFO USER alice"); + Assert.StartsWith("381 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + await WriteSslLineAsync(ssl, "AUTHINFO PASS secret"); + Assert.StartsWith("281 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await WriteSslLineAsync(ssl, "COMPRESS DEFLATE"); + Assert.Equal("206 Compression active", await ReadSslLineAsync(ssl)); + await WaitForServerCompressedAsync(server); + Assert.True(server.IsTls); + Assert.True(server.IsCompressed); + + await using var deflate = new NntpDeflateStream(ssl); + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_PipelinedPlaintextAfterCommand_IsDiscarded() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\nDATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await using var stream = WrapAfterPlain(clientSocket, plain); + await using var deflate = new NntpDeflateStream(stream); + + await WriteDeflateLineAsync(deflate, "DATE"); + Assert.StartsWith("111 ", await ReadDeflateLineAsync(deflate), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Compress_ActivationFailureBefore206_Returns403() + { + await using var duplex = await CompressTestDuplex.CreateAsync(pauseReadsFails: true); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "COMPRESS DEFLATE"); + Assert.Contains("403 Unable to activate compression", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.False(session.Connection.IsCompressed); + Assert.False(session.Connection.IsCompleted); + } + + [Fact] + public async Task Compress_UpgradeFailureAfter206_TerminatesWithoutSecondaryStatus() + { + await using var duplex = await CompressTestDuplex.CreateAsync(upgradeFails: true); + var sessionTask = duplex.CreateSession().RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("COMPRESS DEFLATE"); + Assert.StartsWith("206 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(duplex.Connection.IsCompleted); + Assert.False(duplex.Connection.IsCompressed); + } + + [Fact] + public async Task Compress_MalformedCompressedInput_TerminatesConnection() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var plain = new PlainReceiveBuffer(); + + var sessionTask = new NntpSession(server, NullLogger.Instance).RunAsync(); + _ = await ReadPlainLineAsync(clientSocket, plain); + + await clientSocket.SendAsync("COMPRESS DEFLATE\r\n"u8.ToArray()); + Assert.StartsWith("206 ", await ReadPlainLineAsync(clientSocket, plain), StringComparison.Ordinal); + await WaitForServerCompressedAsync(server); + + await clientSocket.SendAsync(new byte[] { 0x00, 0x01, 0x02, 0x03, 0xFF }); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.IsCompleted); + } + + private static bool ProducesRawDeflateNotZlibOrGzip() + { + using var ms = new MemoryStream(); + using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) + { + ds.Write("nntp"u8); + } + + var compressed = ms.ToArray(); + return compressed.Length > 0 + && compressed[0] != 0x78 + && !(compressed.Length >= 2 && compressed[0] == 0x1F && compressed[1] == 0x8B); + } + + private static async Task WaitForServerCompressedAsync(INntpConnection server) + { + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsCompressed) + { + wait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before DEFLATE was published."); + } + + await Task.Delay(10, wait.Token); + } + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + wait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before TLS was published."); + } + + await Task.Delay(10, wait.Token); + } + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static async Task ReadPlainLineAsync(Socket socket, PlainReceiveBuffer? buffer = null) + { + buffer ??= new PlainReceiveBuffer(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + if (buffer.TryReadLine(out var line)) + { + return line; + } + + var scratch = new byte[512]; + var n = await socket.ReceiveAsync(scratch, cts.Token); + Assert.True(n > 0, $"Socket closed while reading plain line (serverCompleted unknown)."); + buffer.Append(scratch.AsSpan(0, n)); + } + } + + /// Preserves leftover octets across plain-line reads before a DEFLATE wrap. + private sealed class PlainReceiveBuffer + { + private readonly List _bytes = []; + + public void Append(ReadOnlySpan data) => _bytes.AddRange(data); + + public ReadOnlyMemory TakeLeftover() + { + if (_bytes.Count == 0) + { + return ReadOnlyMemory.Empty; + } + + var leftover = _bytes.ToArray(); + _bytes.Clear(); + return leftover; + } + + public bool TryReadLine(out string line) + { + line = string.Empty; + var text = Encoding.ASCII.GetString(CollectionsMarshal.AsSpan(_bytes)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx < 0) + { + return false; + } + + line = text[..idx]; + var consumed = idx + 2; + _bytes.RemoveRange(0, consumed); + return true; + } + } + + private static Stream WrapAfterPlain(Socket socket, PlainReceiveBuffer buffer) + { + var network = new NetworkStream(socket, ownsSocket: false); + var leftover = buffer.TakeLeftover(); + return leftover.IsEmpty + ? network + : new PrefixedStream(network, leftover, leaveInnerOpen: false); + } + + private static async Task WriteSslLineAsync(SslStream ssl, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await ssl.WriteAsync(bytes); + await ssl.FlushAsync(); + } + + private static async Task ReadSslLineAsync(SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task WriteDeflateLineAsync(Stream deflate, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await deflate.WriteAsync(bytes); + await deflate.FlushAsync(); + } + + private static async Task ReadDeflateLineAsync(Stream deflate) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await deflate.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task> ReadCapabilitiesFromDuplexAsync(CompressTestDuplex duplex) + { + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = new List(); + while (true) + { + var line = await duplex.ReadClientLineAsync(); + if (line == ".") + { + break; + } + + caps.Add(line); + } + + return caps; + } + + private sealed class FixedAuthProvider : INntpAuthenticationProvider + { + private readonly string _user; + private readonly string _pass; + + public FixedAuthProvider(string user, string pass) + { + _user = user; + _pass = pass; + } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken) + { + if (username == _user && password == _pass) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success( + username, + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false))); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + public ConcurrentBag Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + Categories.Add(categoryName); + return new RecordingLogger(Messages); + } + + public void Dispose() + { + } + } + + private sealed class RecordingLogger : ILogger + { + private readonly ConcurrentBag _messages; + + public RecordingLogger(ConcurrentBag messages) => _messages = messages; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _messages.Add(formatter(state, exception)); + } + } + + private sealed class CompressTestDuplex : IAsyncDisposable + { + private readonly System.IO.Pipelines.Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly System.IO.Pipelines.Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly PipeConnection _connection; + + private CompressTestDuplex(bool pauseReadsFails, bool upgradeFails) + { + _connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + pauseReadsFails, + upgradeFails); + } + + public System.IO.Pipelines.PipeWriter ServerOutput => _serverToClient.Writer; + public INntpConnection Connection => _connection; + + public static Task CreateAsync( + bool pauseReadsFails = false, + bool upgradeFails = false) => + Task.FromResult(new CompressTestDuplex(pauseReadsFails, upgradeFails)); + + public NntpSession CreateSession() => + new(_connection, NullLogger.Instance); + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private readonly bool _pauseReadsFails; + private readonly bool _upgradeFails; + private int _compressed; + + public PipeConnection( + System.IO.Pipelines.PipeReader input, + System.IO.Pipelines.PipeWriter output, + bool pauseReadsFails, + bool upgradeFails) + { + Input = input; + Output = output; + _pauseReadsFails = pauseReadsFails; + _upgradeFails = upgradeFails; + ClientIdentity = VectorNNTPD.TransitService.Networking.Proxy.ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + } + + public System.IO.Pipelines.PipeReader Input { get; } + public System.IO.Pipelines.PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public VectorNNTPD.TransitService.Networking.Proxy.ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + _pauseReadsFails + ? Task.FromException(new InvalidOperationException("simulated pause failure")) + : Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + if (_upgradeFails) + { + return Task.FromException(new InvalidOperationException("simulated upgrade failure")); + } + + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionFoundationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionFoundationTests.cs new file mode 100644 index 00000000..d1a5812d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionFoundationTests.cs @@ -0,0 +1,305 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class NntpSessionFoundationTests +{ + [Fact] + public void Authorization_DefaultsDenyTransitAndPosting() + { + var authz = NntpAuthorization.Unauthenticated; + Assert.False(authz.IsAuthenticated); + Assert.False(authz.AuthorizedReader); + Assert.False(authz.AuthorizedTransit); + Assert.False(authz.PostingPermitted); + Assert.False(authz.StreamingPermitted); + } + + [Fact] + public void CommandParser_SplitsVerbAndArguments() + { + var caps = NntpCommandTestParse.ParseCommand("CAPABILITIES"); + Assert.Equal(NntpVerb.Capabilities, caps.Verb); + Assert.Equal(0, caps.TokenCount); + Assert.True(caps.IsValid); + + var (auth, authLine) = NntpCommandTestParse.Parse("AUTHINFO USER fred"); + Assert.Equal(NntpVerb.AuthInfo, auth.Verb); + Assert.Equal(NntpVerb.User, auth.Qualifier); + Assert.Equal("fred", Encoding.ASCII.GetString(auth.ArgumentSpan(authLine))); + + var empty = NntpCommandParser.Parse(" "u8); + Assert.False(empty.IsValid); + Assert.Equal(NntpParseStatus.Empty, empty.Status); + } + + [Fact] + public async Task Dispatcher_UnknownCommand_Returns500() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "NOSUCHCMD"); + Assert.Contains("500 Unknown command", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_UnknownAuthinfoVariant_Returns501() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "AUTHINFO GENERIC x"); + Assert.Contains("501 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_ReaderCommandOnTransit_Returns502BeforeAuthentication() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "ARTICLE"); + Assert.Contains("502 Command not available on this server", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_TransitCommandWithoutAuth_Returns480() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "CHECK "); + Assert.Contains("480 Authentication required", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_TransitWhenAuthenticatedButNotAuthorized_Returns502() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "IHAVE "); + Assert.Contains("502 Permission denied", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Dispatcher_ModeStreamWhenAuthenticatedWithoutStreaming_Returns502() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: false)); + var response = new NntpResponseWriter(duplex.ServerOutput); + var dispatcher = new NntpCommandDispatcher(); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, "MODE STREAM"); + Assert.Contains("502 Streaming not permitted", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + } + + [Fact] + public async Task Session_GreetingAndQuit_PublicCommands() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var sessionTask = duplex.CreateSession().RunAsync(); + + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + Assert.Equal(18, date.Length); // "111 " + 14 digit stamp + + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + Assert.StartsWith("101 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + var caps = new List(); + while (true) + { + var line = await duplex.ReadClientLineAsync(); + if (line == ".") + { + break; + } + + caps.Add(line); + } + + Assert.Contains("VERSION 2", caps); + Assert.Contains("AUTHINFO USER", caps); + Assert.Contains("COMPRESS DEFLATE", caps); + Assert.Contains("IHAVE", caps); + Assert.Contains("STREAMING", caps); + Assert.Contains("SPEEDTEST", caps); + Assert.DoesNotContain("READER", caps); + Assert.DoesNotContain("MODE-READER", caps); + Assert.DoesNotContain("POST", caps); + Assert.DoesNotContain(caps, static c => c.StartsWith("LIST", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask; + } + + [Fact] + public async Task ModeReader_OnTransit_Returns502_EvenWhenPostingPermitted() + { + await using var duplex = await SessionTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(NntpAuthorization.Unauthenticated.With(postingPermitted: true)); + var sessionTask = session.RunAsync(); + + Assert.StartsWith("200 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await sessionTask; + } + + private sealed class SessionTestDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + private SessionTestDuplex() + { + } + + public PipeWriter ServerOutput => _serverToClient.Writer; + public PipeReader ServerInput => _clientToServer.Reader; + + public static Task CreateAsync() => Task.FromResult(new SessionTestDuplex()); + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + ServerInput, + ServerOutput, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + private int _compressed; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionIdleTimeoutTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionIdleTimeoutTests.cs new file mode 100644 index 00000000..4a71dd4b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionIdleTimeoutTests.cs @@ -0,0 +1,838 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// NNTP command idle timeout: session-owned waiter, ControllableTimeProvider, no 300s sleeps. +/// +public sealed class NntpSessionIdleTimeoutTests +{ + private static readonly TimeSpan Idle = TimeSpan.FromSeconds(5); + private static readonly TimeSpan Safety = TimeSpan.FromSeconds(5); + + private static readonly NntpAuthorization TransitAuth = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Fact] + public void DisconnectReason_IdleTimeout_FormatsForTcpLog() + { + Assert.Equal("IdleTimeout", TcpDisconnectReason.IdleTimeout.ToString()); + } + + [Fact] + public void SessionConstructor_RejectsIdleTimeOutsideValidatedRange() + { + Assert.Throws(() => + new NntpSession( + new StubConnection(), + NullLogger.Instance, + commandIdleTimeout: TimeSpan.Zero)); + Assert.Throws(() => + new NntpSession( + new StubConnection(), + NullLogger.Instance, + commandIdleTimeout: TimeSpan.FromSeconds(NntpdOptions.MaxIdleTime + 1))); + } + + [Fact] + public async Task NewSession_StaysConnectedBeforeIdleTime_ThenDisconnects() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(1)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.True(session.IdleWatchTaskForTests!.IsCompletedSuccessfully); + } + + [Fact] + public async Task CommandActivity_ResetsIdleTimeout() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + // Drain the greeting Delay so the waiter re-samples. T+Idle would have + // disconnected without the DATE reset. + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(2)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task MultipleCommands_TimeoutMeasuredFromMostRecent() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle - TimeSpan.FromSeconds(1)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle - TimeSpan.FromSeconds(2)); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task PartialCommandLine_DoesNotResetIdleTimeout() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + await duplex.WriteClientAsync("DAT"); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task ActiveAuthinfoCommand_LongerThanIdleTime_IsNotKilled() + { + var clock = new ControllableTimeProvider(); + var hold = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var provider = new GatedAuthProvider(hold, entered); + + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: provider); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + await entered.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + clock.Advance(Idle); + await Task.Delay(20); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + + hold.TrySetResult(NntpAuthenticationResult.Failed); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + var afterPass = await duplex.ReadClientLineAsync(); + Assert.StartsWith("481 ", afterPass, StringComparison.Ordinal); + Assert.False(run.IsCompleted); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task ActiveCheckLookup_LongerThanIdleTime_IsNotKilled() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + await history.Started.Task.WaitAsync(Safety); + await WaitUntilAsync(() => session.Pipeline is { Occupied: 1 }, Safety); + Assert.True(session.CommandWorkForTests > 0); + + clock.Advance(Idle); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.Equal(1, session.Pipeline!.Occupied); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.StartsWith("238 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task ActiveTakeThisReceive_LongerThanIdleTime_IsNotKilled() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + + var receiveStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.HoldReceive = hold.Task; + session.TakeThisWindow.AfterReceiveStarted = () => receiveStarted.TrySetResult(); + + await duplex.WriteClientAsync( + "TAKETHIS \r\n" + + CanonicalArticleText.Destuffed("") + + ".\r\n"); + await receiveStarted.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + clock.Advance(Idle); + await Task.Delay(20); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.Equal(1, session.TakeThisWindow.Occupied); + + hold.TrySetResult(); + await WaitUntilAsync(() => session.TakeThisWindow.Occupied == 0, Safety); + Assert.StartsWith("239 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task Quit_CancelsIdleWatchWithoutLeak() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run.WaitAsync(Safety); + + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + Assert.True(session.IdleWatchTaskForTests!.IsCompleted); + Assert.False(session.IdleWatchTaskForTests.IsFaulted); + Assert.False(session.IdleWatchTaskForTests.IsCanceled); + } + + [Fact] + public async Task CommandWorkConcurrentWithExpiry_DoesNotDisconnectAfterReset() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + session.BeginCommandWork(); + Assert.False(session.IdleCloseCommittedForTests); + hold.TrySetResult(); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.False(session.IdleCloseCommittedForTests); + + session.EndCommandWork(); + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task BeginAndEndDuringIdleCommit_StaleIdleDecisionDoesNotClose() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + session.BeginCommandWork(); + session.EndCommandWork(); + Assert.Equal(0, session.CommandWorkForTests); + Assert.False(session.IdleCloseCommittedForTests); + hold.TrySetResult(); + await Task.Yield(); + Assert.False(run.IsCompleted); + Assert.Equal(TcpDisconnectReason.Unspecified, session.CloseReasonForTests); + Assert.False(session.IdleCloseCommittedForTests); + + await WaitUntilAsync(() => clock.HasScheduledTimers, Safety); + clock.Advance(Idle); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + [Fact] + public async Task IdleCommitThenBegin_CommandObservesClosingSession() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var aboutTo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.IdleAboutToCommit = aboutTo; + session.IdleCommitHold = hold.Task; + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(Idle); + await aboutTo.Task.WaitAsync(Safety); + hold.TrySetResult(); + await WaitUntilAsync(() => session.IdleCloseCommittedForTests, Safety); + + session.BeginCommandWork(); + Assert.True(session.IdleCloseCommittedForTests); + session.EndCommandWork(); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + Assert.Equal(0, session.CommandWorkForTests); + } + + [Fact] + public async Task SuccessfulCommand_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task HandlerException_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: new ThrowingAuthProvider()); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + _ = await duplex.ReadClientLineAsync(); + await WaitUntilAsync(() => session.CommandWorkForTests == 0, Safety); + Assert.False(run.IsCompleted); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task CommandCancellation_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var hold = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, authenticationProvider: new GatedAuthProvider(hold, entered)); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("AUTHINFO USER fred"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + await entered.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + Assert.Equal(TcpDisconnectReason.ProtocolClose, session.CloseReasonForTests); + } + + [Fact] + public async Task CheckEmitException_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + await history.Started.Task.WaitAsync(Safety); + await WaitUntilAsync(() => session.Pipeline is { Occupied: 1 }, Safety); + + session.Pipeline!.BeforeEnqueueProbe = static () => + throw new InvalidOperationException("emit-boom"); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await WaitUntilAsync( + () => session.CommandWorkForTests == 0 && session.Pipeline!.Occupied == 0, + Safety); + Assert.False(run.IsCompleted); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task CheckImmediateEmitException_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + session.Pipeline!.BeforeEnqueueProbe = static () => + throw new InvalidOperationException("emit-boom"); + await duplex.WriteClientLineAsync("CHECK "); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + Assert.Equal(0, session.Pipeline.Occupied); + } + + [Fact] + public async Task PipelineShutdown_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await duplex.WriteClientLineAsync("CHECK "); + await history.Started.Task.WaitAsync(Safety); + await WaitUntilAsync(() => session.Pipeline is { Occupied: 1 }, Safety); + Assert.True(session.CommandWorkForTests > 0); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + } + + [Fact] + public async Task TakeThisEmitException_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var detached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + session.TakeThisWindow!.AfterArticleDetached = _ => detached.TrySetResult(); + await duplex.WriteClientAsync("TAKETHIS \r\nSubject: x\r\n\r\nbody\r\n.\r\n"); + await detached.Task.WaitAsync(Safety); + + session.TakeThisWindow.BeforeEnqueueProbe = static () => + throw new InvalidOperationException("emit-boom"); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await WaitUntilAsync( + () => session.CommandWorkForTests == 0 && session.TakeThisWindow.Occupied == 0, + Safety); + Assert.False(run.IsCompleted); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(Safety); + } + + [Fact] + public async Task TakeThisPipelineShutdown_ReleasesCommandWork() + { + var clock = new ControllableTimeProvider(); + var history = new GatedHistoryDb(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var receiveStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, historyDb: history, articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + session.TakeThisWindow!.HoldReceive = hold.Task; + session.TakeThisWindow.AfterReceiveStarted = () => receiveStarted.TrySetResult(); + await duplex.WriteClientAsync("TAKETHIS \r\nSubject: x\r\n\r\nbody\r\n.\r\n"); + await receiveStarted.Task.WaitAsync(Safety); + Assert.True(session.CommandWorkForTests > 0); + + session.RequestClose(); + await run.WaitAsync(Safety); + Assert.Equal(0, session.CommandWorkForTests); + hold.TrySetResult(); + history.Release.TrySetResult(HistoryLookupResult.Unseen); + } + + [Fact] + public async Task ConfiguredIdleTime_IsHonouredBySession() + { + var clock = new ControllableTimeProvider(); + var idle = TimeSpan.FromSeconds(2); + await using var duplex = new IdleDuplex(); + var session = duplex.CreateSession(clock, idle); + var run = session.RunAsync(); + + _ = await duplex.ReadClientLineAsync(); + await session.IdleWatchArmed.WaitAsync(Safety); + await Task.Yield(); + + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + Assert.False(run.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(1)); + await run.WaitAsync(Safety); + Assert.Equal(TcpDisconnectReason.IdleTimeout, session.CloseReasonForTests); + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + while (!condition()) + { + cts.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private sealed class IdleDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession( + ControllableTimeProvider clock, + TimeSpan? idle = null, + IHistoryDb? historyDb = null, + IArticleIngestionQueue? articleIngestion = null, + INntpAuthenticationProvider? authenticationProvider = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: authenticationProvider, + articleIngestion: articleIngestion, + historyDb: historyDb, + commandIdleTimeout: idle ?? Idle, + timeProvider: clock); + } + + public async Task WriteClientLineAsync(string line) => + await WriteClientAsync(line + "\r\n"); + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(Safety); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class StubConnection : INntpConnection + { + public PipeReader Input => throw new NotSupportedException(); + public PipeWriter Output => throw new NotSupportedException(); + public EndPoint? RemoteEndPoint => null; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => CancellationToken.None; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class GatedAuthProvider( + TaskCompletionSource hold, + TaskCompletionSource entered) : INntpAuthenticationProvider + { + public async ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + entered.TrySetResult(); + return await hold.Task.WaitAsync(cancellationToken); + } + } + + private sealed class ThrowingAuthProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) => + throw new InvalidOperationException("handler-boom"); + } + + private sealed class GatedHistoryDb : IHistoryDb + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + WaitAsync(cancellationToken); + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + WaitAsync(cancellationToken); + + public void Remember(ReadOnlyMemory messageId) + { + } + + public bool ContainsLocal(in HistoryDigest digest) => false; + + private async ValueTask WaitAsync(CancellationToken cancellationToken) + { + Started.TrySetResult(); + return await Release.Task.WaitAsync(cancellationToken); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs new file mode 100644 index 00000000..06207404 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpSessionInputReaderLifecycleTests.cs @@ -0,0 +1,192 @@ +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// Session RX owns Connection.Input. Teardown must not complete that reader while +/// NntpContinuousRxReader.ReadUnitAsync can still call ReadAsync. +/// +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpSessionInputReaderLifecycleTests +{ + [Fact] + public async Task RemoteFin_WhileSessionWaitingForCommand_EndsWithoutReaderCompletedException() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task RemoteFin_AfterDateCommand_EndsWithoutReaderCompletedException() + { + await using var host = await TransportTestHost.StartPlainAsync(); + var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + await client.SendAsync("DATE\r\n"u8.ToArray()); + Assert.StartsWith("111 ", await ReadLineAsync(client), StringComparison.Ordinal); + + client.Shutdown(SocketShutdown.Send); + client.Dispose(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task RequestClose_WhileSessionWaitingForCommand_CompletesReaderAfterRxStops() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + Assert.False(server.InputReaderCompletedForTests); + + session.RequestClose(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + [Fact] + public async Task ConnectionCompleteAsync_WhileSessionWaitingForCommand_DoesNotRaceInputReader() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var logger = new CapturingSessionLogger(); + var session = new NntpSession(server, logger); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + + await server.CompleteAsync().WaitAsync(TimeSpan.FromSeconds(5)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + logger.Errors, + static e => e.Contains("Reading is not allowed after reader was completed", StringComparison.Ordinal)); + Assert.True(server.InputReaderCompletedForTests); + } + + [Fact] + public async Task Quit_CompletesInputReaderOnce_AfterCommandLoopStops() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = Assert.IsType(await host.AcceptAsync()); + var session = new NntpSession(server, NullLogger.Instance); + var run = session.RunAsync(); + + Assert.StartsWith("201 ", await ReadLineAsync(client), StringComparison.Ordinal); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await ReadLineAsync(client)); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(server.InputReaderCompletedForTests); + Assert.True(server.IsCompleted); + } + + private static async Task ReadLineAsync(Socket socket) + { + var buffer = new byte[256]; + var stored = new List(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + var n = await socket.ReceiveAsync(buffer, SocketFlags.None, timeout.Token); + Assert.True(n > 0); + for (var i = 0; i < n; i++) + { + stored.Add(buffer[i]); + } + + var span = stored.ToArray().AsSpan(); + var crlf = span.IndexOf("\r\n"u8); + if (crlf >= 0) + { + return Encoding.ASCII.GetString(span[..crlf]); + } + } + } + + private sealed class CapturingSessionLogger : ILogger + { + public List Errors { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel < LogLevel.Error) + { + return; + } + + var message = formatter(state, exception); + if (exception is not null) + { + Errors.Add(exception.ToString()); + } + + if (!string.IsNullOrEmpty(message)) + { + Errors.Add(message); + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/NntpStartTlsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/NntpStartTlsTests.cs new file mode 100644 index 00000000..a7d13c6c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/NntpStartTlsTests.cs @@ -0,0 +1,286 @@ +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// Session-level STARTTLS lifecycle regressions (plain → 382 → TLS). +[Collection(nameof(TransportTestHostCollection))] +public sealed class NntpStartTlsTests +{ + [Fact] + public async Task StartTls_Success_Stress_NoUnconsumedPlaintextFalsePositive() + { + // Expose the historical ClientHello→Input race under normal AuthenticateAsClient timing. + for (var i = 0; i < 30; i++) + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + } + + [Fact] + public async Task StartTls_Success_ThenDateOverTls() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + var greeting = await ReadPlainLineAsync(clientSocket); + Assert.StartsWith("201 ", greeting, StringComparison.Ordinal); + + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + Assert.False(server.IsCompleted); + Assert.False(sessionTask.IsCompleted); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + // Tear down without QUIT to avoid racing CompleteAsync against the last status line. + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_CommandFullyConsumed_NoFalsePositiveDrain() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + Assert.False(server.IsCompleted); + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_PipelinedCommand_IsDiscarded_ThenTlsSucceeds() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\nDATE\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await using var network = new NetworkStream(clientSocket, ownsSocket: true); + await using var ssl = new SslStream(network, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsClientAsync(CreateClientSslOptions()); + await WaitForServerTlsAsync(server); + + await WriteSslLineAsync(ssl, "DATE"); + Assert.StartsWith("111 ", await ReadSslLineAsync(ssl), StringComparison.Ordinal); + + await server.CompleteAsync(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task StartTls_HandshakeFailure_IsTerminal_NoSecondaryNntpResponse() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + await clientSocket.SendAsync(new byte[] { 0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x28 }); + + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.IsCompleted); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.IsTls); + + using var idleCts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + var buffer = new byte[64]; + try + { + var n = await clientSocket.ReceiveAsync(buffer, idleCts.Token); + if (n > 0) + { + var text = Encoding.ASCII.GetString(buffer, 0, n); + Assert.DoesNotContain("403 ", text, StringComparison.Ordinal); + Assert.DoesNotContain("Command failed", text, StringComparison.Ordinal); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (SocketException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + [Fact] + public async Task StartTls_ClientClosesAfter382_SessionEndsCleanly() + { + var pfx = TransportTestShared.CreatePfx("nntpd01.usenet.ninja"); + await using var host = await TransportTestHost.StartPlainWithCertificateAsync(pfx); + using var clientSocket = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + + var sessionTask = new NntpSession( + server, + NullLogger.Instance, + certificateProvider: host.CertificateProvider) + .RunAsync(); + + _ = await ReadPlainLineAsync(clientSocket); + await clientSocket.SendAsync("STARTTLS\r\n"u8.ToArray()); + Assert.StartsWith("382 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + + clientSocket.Dispose(); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(server.ConnectionClosed.IsCancellationRequested); + Assert.False(server.IsTls); + } + + private static async Task WaitForServerTlsAsync(INntpConnection server) + { + using var tlsWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!server.IsTls) + { + tlsWait.Token.ThrowIfCancellationRequested(); + if (server.IsCompleted || server.ConnectionClosed.IsCancellationRequested) + { + throw new InvalidOperationException("Server closed before TLS mode was published."); + } + + await Task.Delay(10, tlsWait.Token); + } + } + + private static SslClientAuthenticationOptions CreateClientSslOptions() => + new() + { + TargetHost = "nntpd01.usenet.ninja", + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + RemoteCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static async Task ReadPlainLineAsync(Socket socket) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private static async Task WriteSslLineAsync(SslStream ssl, string command) + { + var bytes = Encoding.ASCII.GetBytes(command + "\r\n"); + await ssl.WriteAsync(bytes); + await ssl.FlushAsync(); + } + + private static async Task ReadSslLineAsync(SslStream ssl) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await ssl.ReadAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/PostArticleValidationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/PostArticleValidationTests.cs new file mode 100644 index 00000000..41546a81 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/PostArticleValidationTests.cs @@ -0,0 +1,489 @@ +using System.Net; +using System.Text; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class PostArticleValidationTests +{ + private static readonly DateTimeOffset Now = + new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public void MissingDate_IsSynthesized() + { + var parsed = Parse(Build(date: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.True(parsed.DateSynthesized); + Assert.Equal(Now, parsed.AuthorDate); + } + + [Fact] + public void OmittedDate_NormalizeWritesGeneratedDateOnce() + { + var parsed = Parse(Build(date: null, extraHeaders: "Path: client.path\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + var generated = PostRfcDate.Format(Now); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + generated + "\r\n", text, StringComparison.Ordinal); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("client.path", text, StringComparison.Ordinal); + } + + [Fact] + public void MissingFrom_IsRejected() + { + AssertRejected(Build(from: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingNewsgroups_IsRejected() + { + AssertRejected(Build(newsgroups: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingSubject_IsRejected() + { + AssertRejected(Build(subject: null), PostingFailureCategory.MissingRequiredHeader); + } + + [Fact] + public void MissingMessageId_IsSynthesized() + { + var parsed = Parse(Build(messageId: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.True(parsed.MessageIdSynthesized); + Assert.StartsWith("<", parsed.MessageId, StringComparison.Ordinal); + Assert.EndsWith("@usenet.ninja>", parsed.MessageId, StringComparison.Ordinal); + Assert.Equal(47, parsed.MessageId!.Length); + } + + [Fact] + public void SynthesizedMessageId_UsesConfiguredDomainAndMd5Length() + { + var parsed = Parse(Build(messageId: null)); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Matches(@"^<[0-9a-f]{32}@usenet\.ninja>$", parsed.MessageId); + } + + [Fact] + public void MalformedMessageId_IsRejected() + { + AssertRejected(Build(messageId: "not-an-id"), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: "b>"), PostingFailureCategory.InvalidMessageId); + } + + [Fact] + public void Rfc5536MessageId_IsAccepted() + { + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: "<\"quoted\"@example.com>"), PostingFailureCategory.InvalidMessageId); + AssertRejected(Build(messageId: ""), PostingFailureCategory.InvalidMessageId); + var parsed = Parse(Build(messageId: "")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal("", parsed.MessageId); + + parsed = Parse(Build(messageId: "")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal("", parsed.MessageId); + } + + [Fact] + public void MalformedDate_IsRejected() + { + AssertRejected(Build(date: "yesterday"), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void FutureDateBeyondPolicy_IsRejected() + { + AssertRejected(Build(date: PostRfcDate.Format(Now.AddHours(25))), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void ExcessivelyOldDate_IsRejected() + { + AssertRejected(Build(date: PostRfcDate.Format(Now.AddDays(-15))), PostingFailureCategory.InvalidDate); + } + + [Fact] + public void GmtDate_IsAccepted() + { + var parsed = Parse(Build(date: "25 Sep 2026 12:00:00 GMT")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal(Now, parsed.AuthorDate); + Assert.False(parsed.DateSynthesized); + } + + [Fact] + public void DuplicateSingleton_IsRejected() + { + var extra = "Subject: first\r\nSubject: second\r\n"; + AssertRejected(Build(extraHeaders: extra, subject: null), PostingFailureCategory.DuplicateHeader); + } + + [Fact] + public void MalformedNewsgroup_IsRejected() + { + AssertRejected(Build(newsgroups: "misc..test"), PostingFailureCategory.InvalidNewsgroups); + } + + [Fact] + public void DuplicateNewsgroup_IsRejected() + { + AssertRejected(Build(newsgroups: "misc.test,misc.test"), PostingFailureCategory.InvalidNewsgroups); + } + + [Fact] + public void MultipleValidNewsgroups_AreAccepted() + { + var parsed = Parse(Build(newsgroups: "misc.test, alt.test")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + Assert.Equal(["misc.test", "alt.test"], parsed.Newsgroups); + } + + [Fact] + public void FollowupToPoster_IsAccepted() + { + var parsed = Parse(Build(extraHeaders: "Followup-To: poster\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + } + + [Fact] + public void MalformedFollowupTo_IsRejected() + { + AssertRejected(Build(extraHeaders: "Followup-To: ..bad\r\n"), PostingFailureCategory.InvalidFollowupTo); + } + + [Fact] + public void InReplyToNotInReferences_IsRejected() + { + AssertRejected( + Build(extraHeaders: "References: \r\nIn-Reply-To: \r\n"), + PostingFailureCategory.InvalidReferences); + } + + [Fact] + public void ControlHeader_IsRejected() + { + AssertRejected( + Build(extraHeaders: "Control: cancel \r\n"), + PostingFailureCategory.InvalidControl); + } + + [Fact] + public void ControlHeader_AuthorizedCancel_IsAccepted() + { + var parsed = Parse(Build( + messageId: "", + extraHeaders: "Control: cancel \r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _, + controlCancelPermitted: true)); + Assert.Equal("", parsed.MessageId); + } + + [Fact] + public void ControlHeader_Authorized_RejectsOtherVerb() + { + var parsed = Parse(Build(extraHeaders: "Control: newgroup misc.test\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void ControlHeader_Authorized_RejectsMalformedCancel() + { + var parsed = Parse(Build(extraHeaders: "Control: cancel not-an-id\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void ControlHeader_Authorized_RejectsCancelWithExtraArgument() + { + var parsed = Parse(Build(extraHeaders: "Control: cancel extra\r\n")); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure, + controlCancelPermitted: true)); + Assert.Equal(PostingFailureCategory.InvalidControl, failure.Category); + } + + [Fact] + public void MalformedApproved_IsRejected() + { + AssertRejected(Build(extraHeaders: "Approved: not-a-mailbox\r\n"), PostingFailureCategory.InvalidApproved); + } + + [Fact] + public void MailRoutingHeader_IsPolicyRejected() + { + AssertRejected(Build(extraHeaders: "Received: from nowhere\r\n"), PostingFailureCategory.PolicyRejected); + } + + [Fact] + public void LeadingFold_IsRejected() + { + Assert.False(PostHeaderParser.TryParse( + Encoding.ASCII.GetBytes(" Subject: folded\r\n\r\nbody\r\n"), + out _, + out var failure)); + Assert.Equal(PostingFailureCategory.MalformedHeader, failure.Category); + } + + [Fact] + public void EmbeddedNul_IsRejected() + { + var bytes = Encoding.ASCII.GetBytes(Build()); + bytes[10] = 0; + Assert.False(PostHeaderParser.TryParse(bytes, out _, out var failure)); + Assert.Equal(PostingFailureCategory.MalformedHeader, failure.Category); + } + + [Fact] + public void NewsgroupPolicy_ReportsCatalogUnavailable() + { + var evaluation = SyntaxOnlyNewsgroupPostingPolicy.Instance.Evaluate(["misc.test"]); + Assert.Equal(NewsgroupCatalogStatus.CatalogUnavailable, evaluation.Status); + Assert.Null(evaluation.Failure); + } + + [Fact] + public void Normalize_ReplacesServerOwnedHeaders_AndPreservesDate() + { + var date = PostRfcDate.Format(Now); + var parsed = Parse(Build( + date: date, + extraHeaders: + "Path: client.path\r\n" + + "Injection-Date: forged\r\n" + + "Injection-Info: forged\r\n" + + "NNTP-Posting-Date: forged\r\n" + + "NNTP-Posting-Host: 1.2.3.4\r\n" + + "X-Trace: forged\r\n" + + "Xref: nntpd01.usenet.ninja misc.test:1\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var peer = IPAddress.Parse("192.0.2.10"); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(peer, 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + Assert.Contains("Date: " + date, text, StringComparison.Ordinal); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date: " + date, text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Date", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + Assert.DoesNotContain("192.0.2.10", text, StringComparison.Ordinal); + Assert.DoesNotContain("1.2.3.4", text, StringComparison.Ordinal); + Assert.DoesNotContain("posting-host", text, StringComparison.Ordinal); + Assert.Contains( + "Injection-Info: nntpd01.usenet.ninja; logging-data=\"\"; mail-complaints-to=\"" + + NntpdOptions.DefaultMailComplaintsTo + "\"", + text, + StringComparison.Ordinal); + var xtraceStart = text.IndexOf("X-Trace: ", StringComparison.Ordinal); + Assert.True(xtraceStart >= 0); + var xtrace = text[(xtraceStart + "X-Trace: ".Length)..text.IndexOf("\r\n", xtraceStart, StringComparison.Ordinal)]; + Assert.StartsWith(AesGcmPostingTraceProtector.TokenPrefix, xtrace, StringComparison.Ordinal); + Assert.NotEqual(Convert.ToBase64String(peer.GetAddressBytes()), xtrace); + Assert.True(protector.TryUnprotect(xtrace, out var recovered)); + Assert.Equal(peer, recovered.Address); + Assert.Equal(119, recovered.Port); + Assert.Equal(Now, recovered.InjectedAtUtc); + Assert.DoesNotContain("Path: client.path", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + Assert.DoesNotContain("forged", text, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(text, "Path:")); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Equal(1, CountOccurrences(text, "Injection-Date:")); + Assert.Equal(1, CountOccurrences(text, "X-Trace:")); + } + + [Fact] + public void Normalize_PreservesClientXPgpSig_AndDoesNotTreatItAsServerOwned() + { + var date = PostRfcDate.Format(Now); + var parsed = Parse(Build( + date: date, + extraHeaders: + "Control: cancel \r\n" + + "X-PGP-Sig: BouncyCastle Subject,Control,Message-ID,Date,From,Sender\r\n" + + "\tiQCVAwUBNhiHnsJdOtO4janBAQGFFwP/SCsq1Isgw8DXHDaRkr0cdkZidVH41N6d\r\n")); + Assert.True(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out _, + controlCancelPermitted: true)); + var protector = AesGcmPostingTraceProtector.Create( + new NntpdOptions { XTraceKey = TestHostFactory.TestXTraceKey }); + var normalized = PostHeaderNormalizer.Normalize( + parsed, + Now, + "nntpd01.usenet.ninja", + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Parse("192.0.2.10"), 119)), + NntpdOptions.DefaultMailComplaintsTo, + protector); + var text = Encoding.ASCII.GetString(normalized.Span); + Assert.Contains("X-PGP-Sig: BouncyCastle Subject,Control,Message-ID,Date,From,Sender\r\n", text, StringComparison.Ordinal); + Assert.Contains("\tiQCVAwUBNhiHnsJdOtO4janBAQGFFwP/SCsq1Isgw8DXHDaRkr0cdkZidVH41N6d\r\n", text, StringComparison.Ordinal); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("X-Trace: ", text, StringComparison.Ordinal); + } + + private static void AssertRejected(string article, PostingFailureCategory category) + { + var parsed = Parse(article); + Assert.False(PostArticleValidator.TryValidate( + parsed, + Now, + SyntaxOnlyNewsgroupPostingPolicy.Instance, + out var failure)); + Assert.Equal(category, failure.Category); + } + + private static ParsedPostArticle Parse(string article) + { + Assert.True(PostHeaderParser.TryParse(Encoding.ASCII.GetBytes(article), out var parsed, out var failure), failure.Detail); + Assert.NotNull(parsed); + return parsed!; + } + + private static string Build( + string? date = "25 Sep 2026 12:00:00 +0000", + string? from = "poster@example.com", + string? newsgroups = "misc.test", + string? subject = "test", + string? messageId = "", + string extraHeaders = "", + string body = "body\r\n") + { + var sb = new StringBuilder(); + if (date is not null) + { + sb.Append("Date: ").Append(date).Append("\r\n"); + } + + if (from is not null) + { + sb.Append("From: ").Append(from).Append("\r\n"); + } + + if (newsgroups is not null) + { + sb.Append("Newsgroups: ").Append(newsgroups).Append("\r\n"); + } + + if (subject is not null) + { + sb.Append("Subject: ").Append(subject).Append("\r\n"); + } + + if (messageId is not null) + { + sb.Append("Message-ID: ").Append(messageId).Append("\r\n"); + } + + sb.Append(extraHeaders); + sb.Append("\r\n"); + sb.Append(body); + return sb.ToString(); + } + + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private static int CountHeader(string text, string header) + { + var count = text.StartsWith(header, StringComparison.Ordinal) ? 1 : 0; + return count + CountOccurrences(text, "\r\n" + header); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/PostingTraceProtectorTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/PostingTraceProtectorTests.cs new file mode 100644 index 00000000..d041328e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/PostingTraceProtectorTests.cs @@ -0,0 +1,224 @@ +using System.Net; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Logging; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Tests.Fixtures; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class PostingTraceProtectorTests +{ + private static readonly byte[] CurrentKey = Convert.FromHexString(TestHostFactory.TestXTraceKey); + private static readonly byte[] PreviousKey = Convert.FromHexString( + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + [Fact] + public void Protect_RoundTripsTransportIdentity() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("198.18.0.123"), + 1199, + new DateTimeOffset(2026, 9, 25, 12, 0, 0, TimeSpan.Zero), + Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")); + + var token = protector.Protect(payload); + Assert.StartsWith(AesGcmPostingTraceProtector.TokenPrefix, token, StringComparison.Ordinal); + Assert.DoesNotContain("198.18.0.123", token, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToBase64String(payload.Address.GetAddressBytes()), token, StringComparison.Ordinal); + Assert.DoesNotContain(Convert.ToBase64String(Encoding.ASCII.GetBytes("198.18.0.123")), token, StringComparison.Ordinal); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.InjectedAtUtc, recovered.InjectedAtUtc); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void Protect_RoundTripsAuthenticatedUsername() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("198.18.0.123"), + 1199, + new DateTimeOffset(2026, 9, 25, 12, 0, 0, TimeSpan.Zero), + Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"), + "poster+tag/name"); + + var token = protector.Protect(payload); + Assert.DoesNotContain("poster+tag/name", token, StringComparison.Ordinal); + Assert.DoesNotContain("198.18.0.123", token, StringComparison.Ordinal); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.InjectedAtUtc, recovered.InjectedAtUtc); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Equal("poster+tag/name", recovered.AuthenticatedUsername); + } + + [Fact] + public void Protect_EmptyUsername_IsAbsent() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var token = protector.Protect(new PostingTracePayload( + IPAddress.Loopback, + 119, + DateTimeOffset.UnixEpoch, + Guid.Empty, + " ")); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void LegacyV1Payload_DecryptsWithoutUsername() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload( + IPAddress.Parse("192.0.2.10"), + 119, + DateTimeOffset.UnixEpoch, + Guid.Parse("11111111-2222-3333-4444-555555555555")); + var token = protector.ProtectLegacyV1(payload); + Assert.True(protector.TryUnprotect(token, out var recovered)); + Assert.Equal(payload.Address, recovered.Address); + Assert.Equal(payload.Port, recovered.Port); + Assert.Equal(payload.TraceId, recovered.TraceId); + Assert.Null(recovered.AuthenticatedUsername); + } + + [Fact] + public void PreviousKey_CanDecryptUsernamePayload() + { + var oldProtector = new AesGcmPostingTraceProtector(PreviousKey); + var token = oldProtector.Protect(new PostingTracePayload( + IPAddress.Parse("192.0.2.55"), + 563, + DateTimeOffset.UnixEpoch, + Guid.NewGuid(), + "newsmaster")); + var rotated = new AesGcmPostingTraceProtector(CurrentKey, PreviousKey); + Assert.True(rotated.TryUnprotect(token, out var recovered)); + Assert.Equal("newsmaster", recovered.AuthenticatedUsername); + Assert.False(new AesGcmPostingTraceProtector(CurrentKey).TryUnprotect(token, out _)); + } + + [Fact] + public void InvalidToken_IsRejected() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + Assert.False(protector.TryUnprotect("not-a-token", out _)); + Assert.False(protector.TryUnprotect("v1.$$$$", out _)); + Assert.False(protector.TryUnprotect("v2.AAAA", out _)); + } + + [Fact] + public void MalformedInnerPayloads_DoNotDecrypt() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(LegacyWithExtraByte()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(UnknownVersion()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2LengthMismatch()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2InvalidUtf8()), out _)); + Assert.False(protector.TryUnprotect(protector.ProtectRawForTests(Version2EmbeddedNul()), out _)); + } + + private static byte[] FixedIPv4Prefix(byte version) + { + var buffer = new byte[32]; + buffer[0] = version; + buffer[1] = 4; + buffer[2] = 127; + buffer[3] = 0; + buffer[4] = 0; + buffer[5] = 1; + buffer[7] = 119; + return buffer; + } + + private static byte[] LegacyWithExtraByte() + { + var prefix = FixedIPv4Prefix(1); + var extra = new byte[prefix.Length + 1]; + prefix.CopyTo(extra, 0); + extra[^1] = 1; + return extra; + } + + private static byte[] UnknownVersion() => FixedIPv4Prefix(99); + + private static byte[] Version2LengthMismatch() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 3]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 5; + raw[34] = (byte)'x'; + return raw; + } + + private static byte[] Version2InvalidUtf8() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 3]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 1; + raw[34] = 0xFF; + return raw; + } + + private static byte[] Version2EmbeddedNul() + { + var prefix = FixedIPv4Prefix(2); + var raw = new byte[prefix.Length + 5]; + prefix.CopyTo(raw, 0); + raw[32] = 0; + raw[33] = 3; + raw[34] = (byte)'a'; + raw[35] = 0; + raw[36] = (byte)'b'; + return raw; + } + + [Fact] + public void Protect_DoesNotReuseToken() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var payload = new PostingTracePayload(IPAddress.Loopback, 119, DateTimeOffset.UnixEpoch, Guid.Empty); + Assert.NotEqual(protector.Protect(payload), protector.Protect(payload)); + } + + [Fact] + public void TamperedToken_IsRejected() + { + var protector = new AesGcmPostingTraceProtector(CurrentKey); + var token = protector.Protect(new PostingTracePayload(IPAddress.Loopback, 119, DateTimeOffset.UnixEpoch, Guid.NewGuid())); + var chars = token.ToCharArray(); + chars[^1] = chars[^1] == 'A' ? 'B' : 'A'; + Assert.False(protector.TryUnprotect(new string(chars), out _)); + } + + [Fact] + public void PreviousKey_CanDecryptRotatedToken() + { + var oldProtector = new AesGcmPostingTraceProtector(PreviousKey); + var token = oldProtector.Protect(new PostingTracePayload(IPAddress.Parse("192.0.2.55"), 563, DateTimeOffset.UnixEpoch, Guid.NewGuid())); + var rotated = new AesGcmPostingTraceProtector(CurrentKey, PreviousKey); + Assert.True(rotated.TryUnprotect(token, out var recovered)); + Assert.Equal(IPAddress.Parse("192.0.2.55"), recovered.Address); + Assert.False(new AesGcmPostingTraceProtector(CurrentKey).TryUnprotect(token, out _)); + } + + [Fact] + public void Protector_DoesNotAcceptLogger() + { + var ctors = typeof(AesGcmPostingTraceProtector).GetConstructors(BindingFlags.Public | BindingFlags.Instance); + Assert.DoesNotContain(ctors, static ctor => ctor.GetParameters().Any(static p => + typeof(ILogger).IsAssignableFrom(p.ParameterType))); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/QuitCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/QuitCommandTests.cs new file mode 100644 index 00000000..6a46db4d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/QuitCommandTests.cs @@ -0,0 +1,591 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Networking.Transport; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// QUIT command — RFC 3977 §5.4 and peer-disappears-during-205 race. +[Collection(nameof(TransportTestHostCollection))] +public sealed class QuitCommandTests +{ + [Fact] + public async Task Quit_Returns205_AndEndsSession() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(session.Connection.IsCompleted); + } + + [Fact] + public async Task Quit_WithTrailingArgs_Returns501_AndSessionRemainsUsable() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT NOW"); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("DATE"); + var date = await duplex.ReadClientLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Quit_Logging_RxCentralized_TxOwnedByQuit_NoDuplicate() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Contains(recording.Messages, m => m.Contains("RX: QUIT", StringComparison.Ordinal)); + Assert.Equal( + 1, + recording.Messages.Count(m => + m.Contains("TX: QUIT [205 Connection closing] executed in", StringComparison.Ordinal))); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && m.Contains("[failed]", StringComparison.Ordinal)); + Assert.Contains(recording.Categories, c => c == typeof(Quit).FullName); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task Quit_NoFurtherCommandsProcessed() + { + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("DATE"); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + // DATE after terminal QUIT must not produce a response. + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + await Assert.ThrowsAnyAsync( + async () => await duplex.ReadClientLineAsync(cts.Token)); + } + + [Fact] + public async Task Quit_PeerDisconnectDuringWrite_IsQuietAndTerminal() + { + var recording = new RecordingLoggerFactory(); + await using var duplex = await QuitDuplex.CreateAsync(); + var session = duplex.CreateSession(recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + // Complete the server→client pipe so FlushAsync of 205 observes a completed writer (peer gone). + await duplex.CompleteServerToClientAsync(); + + await duplex.WriteClientLineAsync("QUIT"); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(session.Connection.IsCompleted); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + // May be clean success (enqueue before complete) or peer-disconnected detail. + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + Assert.DoesNotContain("[failed]", tx, StringComparison.Ordinal); + } + + [Fact] + public async Task Quit_Socket_NormalCloseAfter205() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var recording = new RecordingLoggerFactory(); + var session = new NntpSession( + server, + recording.CreateLogger(), + loggerFactory: recording); + var run = session.RunAsync(); + + _ = await ReadSocketLineAsync(client); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.Equal("205 Connection closing", await ReadSocketLineAsync(client)); + client.Shutdown(SocketShutdown.Both); + client.Close(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && !m.Contains("[failed]", StringComparison.Ordinal)); + } + + [Fact] + public async Task Quit_Socket_ImmediateClientCloseAfterQuit_IsNotApplicationError() + { + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var recording = new RecordingLoggerFactory(); + var session = new NntpSession( + server, + recording.CreateLogger(), + loggerFactory: recording); + var run = session.RunAsync(); + + _ = await ReadSocketLineAsync(client); + await client.SendAsync("QUIT\r\n"u8.ToArray()); + // Do not wait for 205 — close immediately (common client race). + client.LingerState = new LingerOption(true, 0); + client.Close(); + + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + recording.Messages, + m => m.Contains("NNTP session ended with an error", StringComparison.OrdinalIgnoreCase) + && recording.Messages.Any(x => x.Contains("QUIT failed", StringComparison.Ordinal))); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + var tx = Assert.Single( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal)); + // Immediate RST can complete the connection after a successful 205 write; the + // existing RunAsync IsCompleted heuristic may then append [failed]. That is + // not an application error when 205 was selected for TX. + Assert.True( + tx.Contains("[205 Connection closing]", StringComparison.Ordinal) + || !tx.Contains("[failed]", StringComparison.Ordinal), + tx); + } + + [Fact] + public async Task Quit_GenuineNonPeerFailure_IsNotSwallowed() + { + var recording = new RecordingLoggerFactory(); + NntpCommandLoggers.Configure(recording); + + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + await using var connection = new FailOnWaitConnection( + identity, + new IOException("simulated non-peer transport failure")); + var session = new NntpSession( + connection, + recording.CreateLogger(), + loggerFactory: recording); + var response = new NntpResponseWriter(connection.Output); + var (command, line) = NntpCommandTestParse.Parse("QUIT"); + var context = new NntpCommandContext(session, command, line, response); + + try + { + await Quit.HandleAsync(context, CancellationToken.None); + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // Expected — non-peer failure must propagate. + } + + Assert.Contains( + recording.Messages, + m => m.Contains("QUIT failed", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + recording.Messages, + m => m.Contains("TX: QUIT", StringComparison.Ordinal) + && m.Contains("executed in", StringComparison.Ordinal) + && m.Contains("[failed]", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(SocketError.ConnectionReset)] + [InlineData(SocketError.ConnectionAborted)] + [InlineData((SocketError)32)] + public void PeerDisconnect_ClassifiesResetLikeSocketErrors(SocketError error) + { + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + using var cts = new CancellationTokenSource(); + var connection = new StaticConnection(identity, cts.Token, completed: false); + var ex = new IOException("send failed", new SocketException((int)error)); + Assert.True(NntpPeerDisconnect.IsPeerDisconnect(ex, connection)); + } + + [Fact] + public void PeerDisconnect_DoesNotClassifyArbitraryIOException() + { + var identity = ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119)); + using var cts = new CancellationTokenSource(); + var connection = new StaticConnection(identity, cts.Token, completed: false); + Assert.False(NntpPeerDisconnect.IsPeerDisconnect(new IOException("disk full"), connection)); + } + + private static async Task ReadSocketLineAsync(Socket socket) + { + var buffer = new byte[512]; + var total = 0; + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total)); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer.AsSpan(0, total)); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("line too long"); + } + + private sealed class QuitDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new QuitDuplex()); + + public NntpSession CreateSession(ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return loggerFactory is null + ? new NntpSession(connection, NullLogger.Instance) + : new NntpSession( + connection, + loggerFactory.CreateLogger(), + loggerFactory: loggerFactory); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync(CancellationToken cancellationToken = default) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (!cancellationToken.CanBeCanceled) + { + cts.CancelAfter(TimeSpan.FromSeconds(5)); + } + + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task CompleteServerToClientAsync() + { + await _serverToClient.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + private int _completed; + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => Volatile.Read(ref _completed) == 1; + public long OutboundIdleVersion => 0; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + if (Interlocked.Exchange(ref _completed, 1) == 0) + { + _cts.Cancel(); + } + + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + /// + /// Connection that accepts the 205 write but fails + /// with a non-peer IOException. + /// + private sealed class FailOnWaitConnection : INntpConnection + { + private readonly Pipe _pipe = new(NntpPipeOptions.Create()); + private readonly CancellationTokenSource _cts = new(); + private readonly Exception _waitFailure; + + public FailOnWaitConnection(ConnectionClientIdentity identity, Exception waitFailure) + { + ClientIdentity = identity; + _waitFailure = waitFailure; + Input = _pipe.Reader; + Output = _pipe.Writer; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => false; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.FromException(_waitFailure); + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.FromException(_waitFailure); + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public async ValueTask DisposeAsync() + { + await _pipe.Writer.CompleteAsync(); + await _pipe.Reader.CompleteAsync(); + _cts.Dispose(); + } + } + + private sealed class StaticConnection : INntpConnection + { + private readonly Pipe _pipe = new(NntpPipeOptions.Create()); + + public StaticConnection(ConnectionClientIdentity identity, CancellationToken closed, bool completed) + { + ClientIdentity = identity; + ConnectionClosed = closed; + IsCompleted = completed; + Input = _pipe.Reader; + Output = _pipe.Writer; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed { get; } + public bool IsCompleted { get; } + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + private readonly object _gate = new(); + public List Messages { get; } = []; + public List Categories { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) + { + lock (_gate) + { + Categories.Add(categoryName); + } + + return new RecordingLogger(categoryName, this); + } + + public void Dispose() + { + } + + public void Add(string message) + { + lock (_gate) + { + Messages.Add(message); + } + } + + private sealed class RecordingLogger(string category, RecordingLoggerFactory owner) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + owner.Add(formatter(state, exception)); + if (exception is not null) + { + owner.Add($"{category}: {exception.GetType().Name}: {exception.Message}"); + } + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCommandTests.cs new file mode 100644 index 00000000..7e300609 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCommandTests.cs @@ -0,0 +1,694 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Tests.Transit; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session.SpeedTest; + +/// +/// SPEEDTEST parser-adjacent dispatch, Transit binding, limits, and measurement contracts. +/// +public sealed class SpeedTestCommandTests +{ + [Fact] + public void Parser_RejectsControlBytesAndOverlongPeer() + { + var tooLong = "SPEEDTEST " + new string('A', TransitPeersOptionsValidator.MaxPeerIdentifierLength + 1); + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandTestParse.ParseCommand(tooLong).Status); + + var control = "SPEEDTEST " + ((char)0x01) + "peer"; + Assert.Equal(NntpParseStatus.InvalidArgument, NntpCommandTestParse.ParseCommand(control).Status); + } + + [Fact] + public async Task Unauthenticated_Returns480() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task AuthenticatedWithoutTransit_Returns502() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false)); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task UnknownPeer_AndArbitraryHostOrIp_Return502() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var session = duplex.CreateAuthorizedSession(); + + await DispatchAsync(session, duplex, "SPEEDTEST NOSUCH"); + Assert.Equal("502 UNKNOWN SPEEDTEST PEER", await duplex.ReadClientLineAsync()); + + await DispatchAsync(session, duplex, "SPEEDTEST news.example.com"); + Assert.Equal("502 UNKNOWN SPEEDTEST PEER", await duplex.ReadClientLineAsync()); + + await DispatchAsync(session, duplex, "SPEEDTEST 192.0.2.1"); + Assert.Equal("502 UNKNOWN SPEEDTEST PEER", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task IdentifierUsenetNinja_Succeeds_AndReportsPeerName() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + duplex.Store.Replace( + TransitTestPeers.Snapshot( + "usenet-ninja", + TransitTestPeers.Peer(allowFrom: ["127.0.0.1"], peerName: "Usenet Ninja"))); + var session = duplex.CreateAuthorizedSession("usenet-ninja"); + + var read = duplex.ReadSpeedTestAsync(); + await DispatchAsync(session, duplex, "SPEEDTEST usenet-ninja"); + var result = await read; + + Assert.Equal("290 SPEEDTEST usenet-ninja TX", result.Ready); + Assert.Equal("291 SPEEDTEST COMPLETE", result.Complete); + Assert.Contains("PEER=usenet-ninja", result.Fields); + Assert.Contains("PEERNAME=Usenet Ninja", result.Fields); + Assert.DoesNotContain("PEER=Usenet Ninja", result.Fields); + } + + [Fact] + public void DisplayNameTwoTokens_IsExtraArgument() + { + var parsed = NntpCommandTestParse.ParseCommand("SPEEDTEST Usenet Ninja"); + Assert.Equal(NntpParseStatus.ExtraArgument, parsed.Status); + Assert.False(parsed.IsValid); + } + + [Fact] + public async Task PeerNameIsNotACommandArgument() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + duplex.Store.Replace( + TransitTestPeers.Snapshot( + "usenet-ninja", + TransitTestPeers.Peer(allowFrom: ["127.0.0.1"], peerName: "Usenet Ninja"))); + var session = duplex.CreateAuthorizedSession("usenet-ninja"); + await DispatchAsync(session, duplex, "SPEEDTEST Usenet"); + Assert.Equal("502 UNKNOWN SPEEDTEST PEER", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task SessionPeerMismatch_Returns502() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var store = duplex.Store; + store.Replace( + TransitConfigurationSnapshot.Create( + new TransitPeersOptions + { + ["GIGANEWS"] = TransitTestPeers.Peer(allowFrom: ["127.0.0.1"]), + ["INVISION"] = TransitTestPeers.Peer(allowFrom: ["127.0.0.2"]), + })); + var session = duplex.CreateAuthorizedSession("GIGANEWS"); + await DispatchAsync(session, duplex, "SPEEDTEST INVISION"); + Assert.Equal("502 SPEEDTEST PEER MISMATCH", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task CapabilitiesAndHelp_AdvertiseSpeedTest() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var session = duplex.CreateSession(); + var caps = duplex.ReadMultilineAsync(); + await DispatchAsync(session, duplex, "CAPABILITIES"); + var capBody = await caps; + Assert.Contains("SPEEDTEST", capBody); + + var help = duplex.ReadMultilineAsync(); + await DispatchAsync(session, duplex, "HELP"); + var helpBody = await help; + Assert.Contains("SPEEDTEST ", helpBody); + } + + [Fact] + public async Task MissingCoordinator_Returns503() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var session = duplex.CreateAuthorizedSession(includeCoordinator: false); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + Assert.Equal("503 SPEEDTEST NOT SUPPORTED", await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task SuccessfulTx_ReportsMeasuredFields_AndDoesNotTouchHistoryOrIngestion() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var history = new CountingHistoryDb(); + var queue = new CountingIngestionQueue(); + var session = duplex.CreateAuthorizedSession(historyDb: history, ingestion: queue); + + var read = duplex.ReadSpeedTestAsync(); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + var result = await read; + + Assert.Equal("290 SPEEDTEST GIGANEWS TX", result.Ready); + Assert.Equal(4096, result.PayloadBytes); + Assert.True(result.PayloadBytes % SpeedTestPayload.LineBytes == 0); + Assert.Equal('#', (char)result.Payload[0]); + Assert.Equal("291 SPEEDTEST COMPLETE", result.Complete); + Assert.Contains("PEER=GIGANEWS", result.Fields); + Assert.Contains("PEERNAME=Giganews, Inc.", result.Fields); + Assert.Contains("IMPLEMENTATION=VectorNNTP", result.Fields); + Assert.Contains("DIRECTION=TX", result.Fields); + Assert.Contains("BYTES=4096", result.Fields); + Assert.Contains("RTT_US=NOT-MEASURED", result.Fields); + Assert.Contains("RX=NOT-MEASURED", result.Fields); + Assert.Contains("OUTBOUND=NOT-AVAILABLE", result.Fields); + Assert.Contains("COMPLETE", result.Fields); + Assert.Contains("PATH=DIRECT-PIPE", result.Fields); + Assert.Contains(result.Fields, static f => f.StartsWith("DIRECT_PIPE_FLUSHES=", StringComparison.Ordinal)); + Assert.DoesNotContain("THROUGHPUT_MBPS=NOT-MEASURED", result.Fields); + Assert.DoesNotContain("THROUGHPUT_GBIT=NOT-MEASURED", result.Fields); + Assert.Equal(0, history.Calls); + Assert.Equal(0, queue.Calls); + Assert.Equal(0, duplex.Coordinator.GlobalInFlight); + } + + [Fact] + public async Task ConcurrencyLimit_Returns400_ThenSucceedsAfterRelease() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + Assert.True(duplex.Coordinator.TryAcquire("GIGANEWS", out var held)); + var session = duplex.CreateAuthorizedSession(); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + Assert.Equal("400 SPEEDTEST BUSY", await duplex.ReadClientLineAsync()); + + held!.Dispose(); + var read = duplex.ReadSpeedTestAsync(); + await DispatchAsync(session, duplex, "SPEEDTEST GIGANEWS"); + var result = await read; + Assert.StartsWith("290 SPEEDTEST GIGANEWS TX", result.Ready, StringComparison.Ordinal); + Assert.Equal(0, duplex.Coordinator.GlobalInFlight); + } + + [Fact] + public async Task Cancellation_DoesNotReportComplete_AndReleasesLease() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(maxBytes: SpeedTestPayload.ChunkBytes * 8); + var session = duplex.CreateAuthorizedSession(); + using var cts = new CancellationTokenSource(); + + var dispatch = NntpCommandTestParse.DispatchAsync( + new NntpCommandDispatcher(), + session, + new NntpResponseWriter(duplex.ServerOutput), + "SPEEDTEST GIGANEWS", + cts.Token).AsTask(); + + var ready = await duplex.ReadClientLineAsync(); + Assert.Equal("290 SPEEDTEST GIGANEWS TX", ready); + cts.Cancel(); + + await dispatch; + Assert.Equal(0, duplex.Coordinator.GlobalInFlight); + Assert.False(await duplex.TryReadCompleteAsync()); + } + + [Fact] + public async Task WritePayloadAsync_StopsOnDurationBeforeByteCeiling() + { + var pipe = new Pipe(NntpPipeOptions.Create()); + await using var writer = new NntpResponseWriter(pipe.Writer); + var drain = Task.Run(async () => + { + while (true) + { + var result = await pipe.Reader.ReadAsync().ConfigureAwait(false); + pipe.Reader.AdvanceTo(result.Buffer.End); + if (result.IsCompleted) + { + break; + } + } + }); + + var limits = new SpeedTestLimits(TimeSpan.FromMilliseconds(20), 1_000_000_000, 1, 1); + var beforeEnqueue = writer.ChannelEnqueueCount; + var (bytes, elapsed, flushes) = await VectorNNTPD.TransitService.Session.Commands.SpeedTest + .WritePayloadAsync(writer, limits, CancellationToken.None); + Assert.Equal(beforeEnqueue, writer.ChannelEnqueueCount); + Assert.True(flushes > 0); + Assert.False(writer.DirectPipeExclusive); + await pipe.Writer.CompleteAsync(); + await drain; + + Assert.True(bytes < 1_000_000_000); + Assert.True(elapsed < TimeSpan.FromSeconds(1)); + Assert.True(elapsed >= TimeSpan.Zero); + } + + [Fact] + public void FormatResult_UsesNotMeasured_WhenDurationOrBytesMissing() + { + var text = Encoding.UTF8.GetString( + VectorNNTPD.TransitService.Session.Commands.SpeedTest.FormatResult("GIGANEWS", "Giganews, Inc.", 0, TimeSpan.FromSeconds(1))); + Assert.Contains("PEER=GIGANEWS", text, StringComparison.Ordinal); + Assert.Contains("PEERNAME=Giganews, Inc.", text, StringComparison.Ordinal); + Assert.Contains("THROUGHPUT_MBPS=NOT-MEASURED", text, StringComparison.Ordinal); + Assert.Contains("THROUGHPUT_GBIT=NOT-MEASURED", text, StringComparison.Ordinal); + Assert.Contains("BYTES=0", text, StringComparison.Ordinal); + + var measured = Encoding.UTF8.GetString( + VectorNNTPD.TransitService.Session.Commands.SpeedTest.FormatResult("GIGANEWS", "Giganews, Inc.", 1_000_000, TimeSpan.FromSeconds(1))); + Assert.Contains("THROUGHPUT_MBPS=8.000", measured, StringComparison.Ordinal); + Assert.Contains("THROUGHPUT_GBIT=0.008000", measured, StringComparison.Ordinal); + } + + [Fact] + public void FormatResult_ThroughputFormula_IsBitsOverSeconds() + { + var text = Encoding.UTF8.GetString( + VectorNNTPD.TransitService.Session.Commands.SpeedTest.FormatResult("GIGANEWS", "Giganews, Inc.", 125_000_000, TimeSpan.FromSeconds(1))); + Assert.Contains("THROUGHPUT_MBPS=1000.000", text, StringComparison.Ordinal); + Assert.Contains("THROUGHPUT_GBIT=1.000000", text, StringComparison.Ordinal); + } + + [Fact] + public async Task DirectPayload_UsesExistingSessionOutput_AndDoesNotEnqueueChunks() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(); + var writer = new NntpResponseWriter(duplex.ServerOutput); + var session = duplex.CreateAuthorizedSession(); + Assert.Same(duplex.ServerOutput, session.Connection.Output); + Assert.Same(session.Connection.Output, writer.UnderlyingOutput); + + var before = writer.ChannelEnqueueCount; + var drain = duplex.ReadClientLineAsync(); + await using (var lease = await writer.AcquireDirectTxPipeAsync()) + { + Assert.True(writer.DirectPipeExclusive); + Assert.Same(session.Connection.Output, lease.PipeWriter); + var slice = SpeedTestPayload.Take(SpeedTestPayload.LineBytes); + await lease.WriteAndFlushAsync(slice); + Assert.Equal(before, writer.ChannelEnqueueCount); + Assert.True(lease.FlushCount > 0); + } + + Assert.StartsWith("#", await drain, StringComparison.Ordinal); + Assert.False(writer.DirectPipeExclusive); + } + + [Fact] + public async Task NormalWriteLine_StillUsesResponseWriterChannel() + { + var pipe = new Pipe(NntpPipeOptions.Create()); + await using var writer = new NntpResponseWriter(pipe.Writer); + var drain = Task.Run(async () => + { + while (true) + { + var result = await pipe.Reader.ReadAsync().ConfigureAwait(false); + pipe.Reader.AdvanceTo(result.Buffer.End); + if (result.IsCompleted) + { + break; + } + } + }); + var before = writer.ChannelEnqueueCount; + await writer.WriteLineAsync(200, "Hello"); + Assert.Equal(before + 1, writer.ChannelEnqueueCount); + Assert.False(writer.DirectPipeExclusive); + Assert.Equal(0, writer.DirectPipeFlushCount); + await pipe.Writer.CompleteAsync(); + await drain; + } + + [Fact] + public async Task ExclusiveLease_RejectsConcurrentChannelWrite_AndSecondAcquire() + { + var pipe = new Pipe(NntpPipeOptions.Create()); + await using var writer = new NntpResponseWriter(pipe.Writer); + var drain = Task.Run(async () => + { + while (true) + { + var result = await pipe.Reader.ReadAsync().ConfigureAwait(false); + pipe.Reader.AdvanceTo(result.Buffer.End); + if (result.IsCompleted) + { + break; + } + } + }); + await using var lease = await writer.AcquireDirectTxPipeAsync(); + await Assert.ThrowsAsync(() => writer.WriteLineAsync(200, "nope").AsTask()); + await Assert.ThrowsAsync(async () => await writer.AcquireDirectTxPipeAsync()); + await lease.DisposeAsync(); + await writer.WriteLineAsync(200, "ok"); + await pipe.Writer.CompleteAsync(); + await drain; + } + + [Fact] + public async Task DirectCommand_ChannelEnqueueIsStatusOnly_SamePipe() + { + await using var duplex = await SpeedTestDuplex.CreateAsync(maxBytes: SpeedTestPayload.ChunkBytes); + var writer = new NntpResponseWriter(duplex.ServerOutput); + var session = duplex.CreateAuthorizedSession(); + var before = writer.ChannelEnqueueCount; + var read = duplex.ReadSpeedTestAsync(); + await NntpCommandTestParse.DispatchAsync( + new NntpCommandDispatcher(), + session, + writer, + "SPEEDTEST GIGANEWS"); + var result = await read; + Assert.Equal("290 SPEEDTEST GIGANEWS TX", result.Ready); + Assert.Equal(SpeedTestPayload.ChunkBytes, result.PayloadBytes); + Assert.Equal("291 SPEEDTEST COMPLETE", result.Complete); + Assert.Contains("PATH=DIRECT-PIPE", result.Fields); + Assert.Equal(before + 2, writer.ChannelEnqueueCount); + Assert.True(writer.DirectPipeFlushCount > 0); + Assert.False(writer.DirectPipeExclusive); + Assert.Same(session.Connection.Output, writer.UnderlyingOutput); + } + + private static ValueTask DispatchAsync(NntpSession session, SpeedTestDuplex duplex, string command) => + NntpCommandTestParse.DispatchAsync(new NntpCommandDispatcher(), session, new NntpResponseWriter(duplex.ServerOutput), command); + + private sealed class SpeedTestDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + public TransitConfigurationStore Store { get; } = new(); + public SpeedTestCoordinator Coordinator { get; private set; } = null!; + + public static Task CreateAsync(long maxBytes = 4096, int maxDurationSeconds = 10) + { + var duplex = new SpeedTestDuplex(); + duplex.Store.Replace( + TransitTestPeers.Snapshot( + "GIGANEWS", + TransitTestPeers.Peer(allowFrom: ["127.0.0.1"], peerName: "Giganews, Inc."))); + duplex.Coordinator = SpeedTestCoordinator.Create( + duplex.Store, + new NntpdOptions + { + SpeedTest = new SpeedTestOptions + { + MaxBytes = maxBytes, + MaxDurationSeconds = maxDurationSeconds, + MaxConcurrent = 2, + MaxConcurrentPerPeer = 1, + }, + }); + return Task.FromResult(duplex); + } + + public NntpSession CreateSession( + ISpeedTestCoordinator? coordinator = null, + IHistoryDb? historyDb = null, + IArticleIngestionQueue? ingestion = null) => + new( + new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))), + NullLogger.Instance, + historyDb: historyDb, + articleIngestion: ingestion, + speedTest: coordinator); + + public NntpSession CreateAuthorizedSession( + string peerName = "GIGANEWS", + bool includeCoordinator = true, + IHistoryDb? historyDb = null, + IArticleIngestionQueue? ingestion = null) + { + var session = CreateSession( + includeCoordinator ? Coordinator : null, + historyDb, + ingestion); + Assert.True(Coordinator.TryResolvePeer(Encoding.ASCII.GetBytes(peerName), out var peer)); + session.SetAuthorization(NntpAuthorization.ForTransitPeer(peer!)); + return session; + } + + public async Task> ReadMultilineAsync() + { + var status = await ReadClientLineAsync().ConfigureAwait(false); + var lines = new List { status }; + while (true) + { + var line = await ReadClientLineAsync().ConfigureAwait(false); + if (line == ".") + { + break; + } + + lines.Add(line); + } + + return lines; + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task TryReadCompleteAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + try + { + while (true) + { + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + if (line is null) + { + return false; + } + + if (line.StartsWith("291 ", StringComparison.Ordinal)) + { + return true; + } + } + } + catch (OperationCanceledException) + { + return false; + } + } + + public async Task ReadSpeedTestAsync() + { + var ready = await ReadClientLineAsync().ConfigureAwait(false); + using var payload = new MemoryStream(); + while (true) + { + var line = await ReadClientLineAsync().ConfigureAwait(false); + if (line == ".") + { + break; + } + + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + payload.Write(bytes); + } + + var complete = await ReadClientLineAsync().ConfigureAwait(false); + var fields = new List(); + while (true) + { + var line = await ReadClientLineAsync().ConfigureAwait(false); + if (line == ".") + { + break; + } + + fields.Add(line); + } + + return new SpeedTestRead(ready, payload.ToArray(), complete, fields); + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed record SpeedTestRead( + string Ready, + byte[] Payload, + string Complete, + List Fields) + { + public int PayloadBytes => Payload.Length; + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + public PipeWriter Output { get; } + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + + private sealed class CountingHistoryDb : IHistoryDb + { + public int Calls; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Calls); + return new ValueTask(HistoryLookupResult.Unseen); + } + + public ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Calls); + return new ValueTask(HistoryLookupResult.Unseen); + } + + public void Remember(ReadOnlyMemory messageId) => Interlocked.Increment(ref Calls); + + public bool ContainsLocal(in HistoryDigest digest) + { + Interlocked.Increment(ref Calls); + return false; + } + } + + private sealed class CountingIngestionQueue : IArticleIngestionQueue + { + public int Calls; + public long MemoryLimitBytes => 1; + public long QueuedBytes => 0; + public long PeakQueuedBytes => 0; + public int MaxArticleBytes => 1024; + public int Count => 0; + public int PeakCount => 0; + public bool IsAccepting => true; + + public ValueTask EnqueueAsync( + InboundArticle article, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref Calls); + return new ValueTask(ArticleEnqueueResult.Accepted); + } + + public bool TryProbeCapacity() + { + Interlocked.Increment(ref Calls); + return true; + } + + public ArticleEnqueueResult TryAdmit(InboundArticle article) + { + Interlocked.Increment(ref Calls); + return ArticleEnqueueResult.Accepted; + } + + public bool TryEnqueue(InboundArticle article) + { + Interlocked.Increment(ref Calls); + return true; + } + + public void Complete() => Interlocked.Increment(ref Calls); + + public ValueTask DequeueAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref Calls); + return ValueTask.FromResult(null); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCoordinatorTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCoordinatorTests.cs new file mode 100644 index 00000000..5c2d937a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestCoordinatorTests.cs @@ -0,0 +1,97 @@ +using System.Text; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session.SpeedTest; +using VectorNNTPD.TransitService.Tests.Transit; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session.SpeedTest; + +public sealed class SpeedTestCoordinatorTests +{ + [Fact] + public void TryResolvePeer_ExactOrdinalMatch_Only() + { + var store = StoreWithPeer("GIGANEWS"); + var coordinator = SpeedTestCoordinator.Create(store, Limits()); + + Assert.True(coordinator.TryResolvePeer("GIGANEWS"u8, out var peer)); + Assert.Equal("GIGANEWS", peer!.Identifier); + Assert.Equal(TransitTestPeers.DefaultPeerDisplayName, peer.PeerName); + Assert.False(coordinator.TryResolvePeer("giganews"u8, out _)); + Assert.False(coordinator.TryResolvePeer("news.example.com"u8, out _)); + Assert.False(coordinator.TryResolvePeer("192.0.2.1"u8, out _)); + Assert.False(coordinator.TryResolvePeer("GIGANEWS:119"u8, out _)); + } + + [Fact] + public void TryResolvePeer_UsesCurrentSnapshot_NotAStaleReference() + { + var store = StoreWithPeer("GIGANEWS"); + var coordinator = SpeedTestCoordinator.Create(store, Limits()); + Assert.True(coordinator.TryResolvePeer("GIGANEWS"u8, out _)); + + store.Replace(TransitTestPeers.Snapshot("INVISION", TransitTestPeers.Peer(allowFrom: ["127.0.0.1"]))); + Assert.False(coordinator.TryResolvePeer("GIGANEWS"u8, out _)); + Assert.True(coordinator.TryResolvePeer("INVISION"u8, out var peer)); + Assert.Equal("INVISION", peer!.Identifier); + } + + [Fact] + public void TryAcquire_EnforcesGlobalAndPerPeerLimits_AndReleases() + { + var store = StoreWithPeer("GIGANEWS"); + store.Replace( + TransitConfigurationSnapshot.Create( + new TransitPeersOptions + { + ["GIGANEWS"] = TransitTestPeers.Peer(allowFrom: ["127.0.0.1"]), + ["INVISION"] = TransitTestPeers.Peer(allowFrom: ["127.0.0.2"]), + })); + var coordinator = SpeedTestCoordinator.Create( + store, + Limits(maxConcurrent: 2, maxPerPeer: 1)); + + Assert.True(coordinator.TryAcquire("GIGANEWS", out var first)); + Assert.False(coordinator.TryAcquire("GIGANEWS", out _)); + Assert.True(coordinator.TryAcquire("INVISION", out var second)); + Assert.False(coordinator.TryAcquire("INVISION", out _)); + Assert.Equal(2, coordinator.GlobalInFlight); + + first!.Dispose(); + first.Dispose(); + Assert.Equal(1, coordinator.GlobalInFlight); + Assert.Equal(0, coordinator.InFlightFor("GIGANEWS")); + + Assert.True(coordinator.TryAcquire("GIGANEWS", out var again)); + second!.Dispose(); + again!.Dispose(); + Assert.Equal(0, coordinator.GlobalInFlight); + } + + [Fact] + public void NameEquals_IsExactOrdinal() + { + Assert.True(SpeedTestCoordinator.NameEquals("GIGANEWS", "GIGANEWS"u8)); + Assert.False(SpeedTestCoordinator.NameEquals("GIGANEWS", "giganews"u8)); + Assert.False(SpeedTestCoordinator.NameEquals("GIGANEWS", "GIGANEW"u8)); + } + + private static TransitConfigurationStore StoreWithPeer(string name) + { + var store = new TransitConfigurationStore(); + store.Replace(TransitTestPeers.Snapshot(name, TransitTestPeers.Peer(allowFrom: ["127.0.0.1"]))); + return store; + } + + private static NntpdOptions Limits(int maxConcurrent = 2, int maxPerPeer = 1) => + new() + { + SpeedTest = new SpeedTestOptions + { + MaxBytes = 4096, + MaxDurationSeconds = 10, + MaxConcurrent = maxConcurrent, + MaxConcurrentPerPeer = maxPerPeer, + }, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestPayloadTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestPayloadTests.cs new file mode 100644 index 00000000..85264144 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/SpeedTest/SpeedTestPayloadTests.cs @@ -0,0 +1,40 @@ +using System.Runtime.InteropServices; +using VectorNNTPD.TransitService.Session.SpeedTest; + +namespace VectorNNTPD.TransitService.Tests.Session.SpeedTest; + +public sealed class SpeedTestPayloadTests +{ + [Fact] + public void Chunk_IsBoundedAndReusable() + { + Assert.Equal(64 * 1024, SpeedTestPayload.ChunkBytes); + Assert.Equal(SpeedTestPayload.ChunkBytes, SpeedTestPayload.Chunk.Length); + Assert.True(MemoryMarshal.TryGetArray(SpeedTestPayload.Chunk, out var first)); + Assert.True(MemoryMarshal.TryGetArray(SpeedTestPayload.Chunk, out var second)); + Assert.Same(first.Array, second.Array); + } + + [Fact] + public void Lines_NeverStartWithDot() + { + var span = SpeedTestPayload.Chunk.Span; + for (var offset = 0; offset < span.Length; offset += SpeedTestPayload.LineBytes) + { + Assert.NotEqual((byte)'.', span[offset]); + Assert.Equal((byte)'#', span[offset]); + Assert.Equal((byte)'\r', span[offset + SpeedTestPayload.LineBytes - 2]); + Assert.Equal((byte)'\n', span[offset + SpeedTestPayload.LineBytes - 1]); + } + } + + [Fact] + public void Take_IsCrlfAligned_AndEmptyBelowOneLine() + { + Assert.True(SpeedTestPayload.Take(SpeedTestPayload.LineBytes - 1).IsEmpty); + Assert.Equal(SpeedTestPayload.LineBytes, SpeedTestPayload.Take(SpeedTestPayload.LineBytes).Length); + Assert.Equal(SpeedTestPayload.LineBytes, SpeedTestPayload.Take(SpeedTestPayload.LineBytes + 100).Length); + Assert.Equal(SpeedTestPayload.ChunkBytes, SpeedTestPayload.Take(SpeedTestPayload.ChunkBytes).Length); + Assert.Equal(SpeedTestPayload.ChunkBytes, SpeedTestPayload.Take(SpeedTestPayload.ChunkBytes + 1).Length); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/StreamingPostArticleReaderTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/StreamingPostArticleReaderTests.cs new file mode 100644 index 00000000..fc45b115 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/StreamingPostArticleReaderTests.cs @@ -0,0 +1,546 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Text; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Session.Commands.Posting; +using VectorNNTPD.TransitService.Session.Framing; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class StreamingPostArticleReaderTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 25, 12, 0, 0, TimeSpan.Zero); + private static readonly ConnectionClientIdentity Client = + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119)); + + [Fact] + public async Task Streaming_MatchesLegacyMaterialize_ForNormalAndDotBodies() + { + foreach (var stuffed in new[] { "hello\r\n", "..dot\r\n", "...triple\r\n", "....quad\r\n", "a\r\n..\r\n...\r\n" } + .Select(static body => ClientArticle(body: body))) + { + var streaming = await ReadStreamingAsync(stuffed); + var legacy = await LegacyPostMaterialize.ProcessAsync(stuffed, CreateOptions()); + Assert.Equal(StreamingPostReadStatus.Completed, streaming.Status); + Assert.True(streaming.Wire.Span.SequenceEqual(legacy.Span)); + } + } + + [Fact] + public async Task FoldedHeader_IsPreservedOnWire() + { + var stuffed = ClientArticle(extraHeaders: "User-Agent: foo\r\n\tbar\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Contains("User-Agent: foo\r\n\tbar\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task DuplicateSingleton_IsRejected() + { + var stuffed = ClientArticle(extraHeaders: "Subject: again\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.DuplicateHeader, result.Failure.Category); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task MissingRequiredHeaders_AreRejected() + { + var stuffed = "From: poster@example.com\r\nSubject: test\r\n\r\nbody\r\n.\r\n"; + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MissingRequiredHeader, result.Failure.Category); + Assert.Equal("Newsgroups", result.Failure.Detail); + } + + [Theory] + [InlineData("From")] + [InlineData("Newsgroups")] + [InlineData("Subject")] + public async Task MissingMandatoryHeader_IsRejected(string header) + { + var stuffed = header switch + { + "From" => "Newsgroups: misc.test\r\nSubject: test\r\n\r\nbody\r\n.\r\n", + "Newsgroups" => "From: poster@example.com\r\nSubject: test\r\n\r\nbody\r\n.\r\n", + _ => "From: poster@example.com\r\nNewsgroups: misc.test\r\n\r\nbody\r\n.\r\n", + }; + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MissingRequiredHeader, result.Failure.Category); + Assert.Equal(header, result.Failure.Detail); + } + + [Fact] + public async Task MissingDate_IsGeneratedOnce() + { + var result = await ReadStreamingAsync(ClientArticle(date: null)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + PostRfcDate.Format(Now) + "\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task SuppliedDate_IsPreservedOnce() + { + const string date = "25 Sep 2026 11:00:00 +0000"; + var result = await ReadStreamingAsync(ClientArticle(date: date)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Date: ")); + Assert.Contains("Date: " + date + "\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MalformedDate_IsRejected() + { + var result = await ReadStreamingAsync(ClientArticle(date: "yesterday")); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidDate, result.Failure.Category); + } + + [Theory] + [InlineData(25, 0)] + [InlineData(0, -15)] + public async Task DateOutsidePolicy_IsRejected(int hours, int days) + { + var date = PostRfcDate.Format(Now.AddHours(hours).AddDays(days)); + var result = await ReadStreamingAsync(ClientArticle(date: date)); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidDate, result.Failure.Category); + } + + [Fact] + public async Task MalformedMessageId_IsRejected() + { + var result = await ReadStreamingAsync(ClientArticle(messageId: "not-an-id")); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.InvalidMessageId, result.Failure.Category); + } + + [Fact] + public async Task AbsentPath_IsServerGeneratedOnce() + { + var result = await ReadStreamingAsync(ClientArticle()); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + } + + [Fact] + public async Task SuppliedPath_IsDiscardedAndReplacedOnce() + { + var result = await ReadStreamingAsync(ClientArticle(extraHeaders: "Path: evil.path\r\n")); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountHeader(text, "Path: ")); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("evil.path", text, StringComparison.Ordinal); + } + + [Fact] + public async Task ExistingMessageId_IsPreservedOnce() + { + var stuffed = ClientArticle(messageId: ""); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal("", result.MessageId); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountOccurrences(text, "Message-ID:")); + Assert.Contains("Message-ID: ", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MissingMessageId_IsGeneratedOnce() + { + var stuffed = ClientArticle(messageId: null); + var result = await ReadStreamingAsync(stuffed); + Assert.Matches(@"^<[0-9a-f]{32}@usenet\.ninja>$", result.MessageId); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Equal(1, CountOccurrences(text, "Message-ID:")); + Assert.Contains("Message-ID: " + result.MessageId, text, StringComparison.Ordinal); + Assert.Contains("logging-data=\"" + result.MessageId + "\"", text, StringComparison.Ordinal); + } + + [Fact] + public async Task ServerOwnedHeaders_AreReplaced_AndClientValuesDiscarded() + { + var stuffed = ClientArticle( + extraHeaders: + "Path: evil.path\r\n" + + "Injection-Date: forged\r\n" + + "Injection-Info: forged\r\n" + + "NNTP-Posting-Date: forged\r\n" + + "NNTP-Posting-Host: 203.0.113.9\r\n" + + "X-Trace: forged\r\n" + + "Xref: other 1\r\n"); + var result = await ReadStreamingAsync(stuffed); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.Contains("Injection-Date: " + PostRfcDate.Format(Now), text, StringComparison.Ordinal); + Assert.Contains( + "Injection-Info: nntpd01.usenet.ninja; logging-data=\"\"; mail-complaints-to=\"abuse@usenet.ninja\"", + text, + StringComparison.Ordinal); + Assert.Contains("X-Trace: v1.test-token\r\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Date", text, StringComparison.Ordinal); + Assert.DoesNotContain("NNTP-Posting-Host", text, StringComparison.Ordinal); + Assert.DoesNotContain("evil.path", text, StringComparison.Ordinal); + Assert.DoesNotContain("forged", text, StringComparison.Ordinal); + Assert.DoesNotContain("Xref:", text, StringComparison.Ordinal); + Assert.DoesNotContain("203.0.113.9", text, StringComparison.Ordinal); + } + + [Fact] + public async Task MailComplaintsTo_IsWrittenOnInjectionInfo() + { + var options = CreateOptions(); + var custom = new StreamingPostReadOptions + { + MaxArticleSize = options.MaxArticleSize, + Time = options.Time, + NewsgroupPolicy = options.NewsgroupPolicy, + InjectionIdentity = options.InjectionIdentity, + ClientIdentity = options.ClientIdentity, + MailComplaintsTo = "ops@usenet.ninja", + TraceProtector = options.TraceProtector, + }; + var result = await ReadStreamingAsync(ClientArticle(), custom); + Assert.Contains( + "mail-complaints-to=\"ops@usenet.ninja\"", + Encoding.ASCII.GetString(result.Wire.Span), + StringComparison.Ordinal); + } + + [Fact] + public async Task BodyDotVariants_AreStoredDestuffed() + { + var stuffed = ClientArticle(body: "plain\r\n..\r\n...\r\n....\r\n"); + var result = await ReadStreamingAsync(stuffed); + var text = Encoding.ASCII.GetString(result.Wire.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal("plain\r\n.\r\n..\r\n...\r\n", body); + } + + [Theory] + [InlineData("hello\r\n", "hello\r\n")] + [InlineData("..hidden\r\n", ".hidden\r\n")] + [InlineData("...hidden\r\n", "..hidden\r\n")] + [InlineData("....hidden\r\n", "...hidden\r\n")] + public async Task Body_RemovesOneStuffingLayer(string stuffedBody, string destuffedBody) + { + var result = await ReadStreamingAsync(ClientArticle(body: stuffedBody)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + var text = Encoding.ASCII.GetString(result.Wire.Span); + var body = text[(text.IndexOf("\r\n\r\n", StringComparison.Ordinal) + 4)..]; + Assert.Equal(destuffedBody, body); + Assert.Contains("Path: .POSTED\r\n", text, StringComparison.Ordinal); + Assert.False(result.Wire.Span.EndsWith(".\r\n"u8)); + } + + [Fact] + public async Task CrlfAndDotSplitAcrossSegments_ProduceIdenticalWire() + { + var stuffed = ClientArticle(body: "..hello\r\n...more\r\n"); + var expected = await ReadStreamingAsync(stuffed); + foreach (var chunkSize in new[] { 1, 2, 3, 7 }) + { + var segmented = await ReadStreamingAsync(stuffed, CreateOptions(), chunkSize); + Assert.Equal(StreamingPostReadStatus.Completed, segmented.Status); + Assert.True(expected.Wire.Span.SequenceEqual(segmented.Wire.Span), "chunk " + chunkSize); + } + } + + [Fact] + public async Task ExactMaxArticleSize_IsAccepted() + { + var destuffed = PaddedDestuffed(512); + Assert.Equal(512, destuffed.Length); + var stuffed = destuffed + ".\r\n"; + var result = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: 512)); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + Assert.Equal(512, result.DestuffedSize); + } + + [Fact] + public async Task MaxArticleSizePlusOne_IsRejected() + { + var destuffed = PaddedDestuffed(513); + var result = await ReadStreamingAsync(destuffed + ".\r\n", CreateOptions(maxArticleSize: 512)); + Assert.Equal(StreamingPostReadStatus.TooLarge, result.Status); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task DotStuffing_DoesNotCountTowardMaxArticleSize() + { + var prefix = ClientArticle(body: string.Empty, includeTerminator: false); + var destuffedPrefix = Encoding.ASCII.GetByteCount(prefix); + var destuffedBody = Encoding.ASCII.GetByteCount(".hello\r\n"); + var limit = destuffedPrefix + destuffedBody; + var stuffed = prefix + "..hello\r\n.\r\n"; + Assert.True(Encoding.ASCII.GetByteCount(stuffed) - 3 > limit); + + var accepted = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: limit)); + Assert.Equal(StreamingPostReadStatus.Completed, accepted.Status); + Assert.Equal(limit, accepted.DestuffedSize); + Assert.Contains(".hello\r\n", Encoding.ASCII.GetString(accepted.Wire.Span), StringComparison.Ordinal); + Assert.DoesNotContain("..hello\r\n", Encoding.ASCII.GetString(accepted.Wire.Span), StringComparison.Ordinal); + + var rejected = await ReadStreamingAsync(stuffed, CreateOptions(maxArticleSize: limit - 1)); + Assert.Equal(StreamingPostReadStatus.TooLarge, rejected.Status); + } + + [Fact] + public async Task Cancellation_Throws_AndDoesNotCompleteWire() + { + var pipe = new Pipe(); + using var cts = new CancellationTokenSource(); + var read = StreamingPostArticleReader.ReadAsync(pipe.Reader, CreateOptions(), cts.Token); + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => await read); + } + + [Fact] + public async Task EofBeforeTerminator_IsIncomplete() + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(ClientArticle(includeTerminator: false))); + await pipe.Writer.CompleteAsync(); + var result = await StreamingPostArticleReader.ReadAsync(pipe.Reader, CreateOptions(), CancellationToken.None); + Assert.Equal(StreamingPostReadStatus.Incomplete, result.Status); + Assert.True(result.Wire.IsEmpty); + } + + [Fact] + public async Task Completed_ExposesParsedMessageIdAndNewsgroups() + { + // Finish()'s Completed path requires a materialized ParsedPostArticle; regressions that + // leave _parsed null must not surface as empty identity fields. + var result = await ReadStreamingAsync(ClientArticle()); + Assert.Equal(StreamingPostReadStatus.Completed, result.Status); + Assert.False(string.IsNullOrEmpty(result.MessageId)); + Assert.Contains("misc.test", result.Newsgroups); + Assert.False(result.Wire.IsEmpty); + } + + [Fact] + public async Task MalformedArticle_IsRejected() + { + var result = await ReadStreamingAsync("not a header\r\n.\r\n"); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.MalformedHeader, result.Failure.Category); + } + + [Fact] + public async Task MissingTraceProtector_IsRejectedAfterHeaders() + { + var options = new StreamingPostReadOptions + { + MaxArticleSize = 64 * 1024, + Time = new FixedTimeProvider(Now), + NewsgroupPolicy = SyntaxOnlyNewsgroupPostingPolicy.Instance, + InjectionIdentity = "nntpd01.usenet.ninja", + ClientIdentity = Client, + MailComplaintsTo = NntpdOptions.DefaultMailComplaintsTo, + TraceProtector = null, + }; + var result = await ReadStreamingAsync(ClientArticle(), options); + Assert.Equal(StreamingPostReadStatus.Rejected, result.Status); + Assert.Equal(PostingFailureCategory.PersistenceFailure, result.Failure.Category); + } + + private static async Task ReadStreamingAsync( + string stuffedWithTerminator, + StreamingPostReadOptions? options = null, + int? chunkSize = null) + { + options ??= CreateOptions(); + var bytes = Encoding.ASCII.GetBytes(stuffedWithTerminator); + if (chunkSize is int size && size > 0) + { + var sequence = SegmentedSequence.FromChunks(bytes, size); + var reader = PipeReader.Create(sequence); + return await StreamingPostArticleReader.ReadAsync(reader, options, CancellationToken.None); + } + + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(bytes); + await pipe.Writer.CompleteAsync(); + return await StreamingPostArticleReader.ReadAsync(pipe.Reader, options, CancellationToken.None); + } + + private static StreamingPostReadOptions CreateOptions(int maxArticleSize = 64 * 1024) => + new() + { + MaxArticleSize = maxArticleSize, + Time = new FixedTimeProvider(Now), + NewsgroupPolicy = SyntaxOnlyNewsgroupPostingPolicy.Instance, + InjectionIdentity = "nntpd01.usenet.ninja", + ClientIdentity = Client, + MailComplaintsTo = NntpdOptions.DefaultMailComplaintsTo, + TraceProtector = DeterministicTraceProtector.Instance, + }; + + private static string ClientArticle( + string? messageId = "", + string extraHeaders = "", + string body = "body\r\n", + bool includeTerminator = true, + string? date = "") + { + var sb = new StringBuilder(); + if (date is not null) + { + sb.Append("Date: ").Append(date.Length == 0 ? PostRfcDate.Format(Now) : date).Append("\r\n"); + } + + sb.Append("From: poster@example.com\r\n"); + sb.Append("Newsgroups: misc.test\r\n"); + sb.Append("Subject: test\r\n"); + if (messageId is not null) + { + sb.Append("Message-ID: ").Append(messageId).Append("\r\n"); + } + + sb.Append(extraHeaders); + sb.Append("\r\n"); + sb.Append(body); + if (includeTerminator) + { + sb.Append(".\r\n"); + } + + return sb.ToString(); + } + + private static string PaddedDestuffed(int destuffedBytes) + { + var prefix = ClientArticle(body: string.Empty, includeTerminator: false); + var needed = destuffedBytes - Encoding.ASCII.GetByteCount(prefix); + Assert.True(needed >= 2); + return prefix + new string('Z', needed - 2) + "\r\n"; + } + + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var index = 0; + while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } + + private static int CountHeader(string text, string header) + { + var count = text.StartsWith(header, StringComparison.Ordinal) ? 1 : 0; + return count + CountOccurrences(text, "\r\n" + header); + } + + private sealed class FixedTimeProvider(DateTimeOffset utc) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => utc; + } + + internal sealed class DeterministicTraceProtector : IPostingTraceProtector + { + public static DeterministicTraceProtector Instance { get; } = new(); + + public string Protect(PostingTracePayload payload) => "v1.test-token"; + + public bool TryUnprotect(ReadOnlySpan token, out PostingTracePayload payload) + { + payload = default; + return false; + } + } +} + +internal static class LegacyPostMaterialize +{ + public static async Task> ProcessAsync( + string stuffedWithTerminator, + StreamingPostReadOptions options) + { + var pipe = new Pipe(); + await pipe.Writer.WriteAsync(Encoding.ASCII.GetBytes(stuffedWithTerminator)); + await pipe.Writer.CompleteAsync(); + var destuffed = await NntpMultilineDataReader.ReadArticleAsync( + pipe.Reader, + options.MaxArticleSize, + CancellationToken.None); + Assert.Equal(NntpMultilineReadStatus.Completed, destuffed.Status); + Assert.True(PostHeaderParser.TryParse(destuffed.Payload, out var parsed, out var failure), failure.Detail); + Assert.True( + PostArticleValidator.TryValidate(parsed!, options.Time.GetUtcNow(), options.NewsgroupPolicy, out failure), + failure.Detail); + var normalized = PostHeaderNormalizer.Normalize( + parsed!, + options.Time.GetUtcNow(), + options.InjectionIdentity, + options.ClientIdentity, + options.MailComplaintsTo, + options.TraceProtector!); + return normalized; + } +} + +internal static class SegmentedSequence +{ + public static ReadOnlySequence FromChunks(ReadOnlySpan source, int chunkSize) + { + ArgumentOutOfRangeException.ThrowIfLessThan(chunkSize, 1); + if (source.IsEmpty) + { + return ReadOnlySequence.Empty; + } + + BufferSegment? first = null; + BufferSegment? last = null; + var offset = 0; + while (offset < source.Length) + { + var take = Math.Min(chunkSize, source.Length - offset); + var memory = source.Slice(offset, take).ToArray(); + var segment = new BufferSegment(memory); + if (first is null) + { + first = segment; + } + else + { + last!.Append(segment); + } + + last = segment; + offset += take; + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(ReadOnlyMemory memory) + { + Memory = memory; + } + + public void Append(BufferSegment next) + { + next.RunningIndex = RunningIndex + Memory.Length; + Next = next; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisCommandTests.cs new file mode 100644 index 00000000..a8ab25b2 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisCommandTests.cs @@ -0,0 +1,1084 @@ +using System.IO.Pipelines; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Newsgroups; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Networking.Transport; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// RFC 4644 TAKETHIS streaming / pipelining / ingestion contracts. +[Collection(nameof(TransportTestHostCollection))] +public sealed class TakeThisCommandTests +{ + private static NntpAuthorization TransitAuth { get; } = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Fact] + public async Task TakeThis_ValidArticle_Returns239_AndEnqueues() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "body\r\n"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.NotNull(article); + Assert.Equal(id, article!.MessageId); + Assert.Equal(InboundArticleProducer.TakeThis, article.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + Assert.True(article.Payload.Equals(article.Record.ArtData)); + Assert.Equal(article.Record.ArtData.Length, article.Record.ArtSize); + Assert.Equal(1, article.Record.ArtLines); + Assert.NotEqual(ArticleType.None, article.Record.ArtType); + Assert.True(article.Record.MessageId.SequenceEqual(Encoding.ASCII.GetBytes(id))); + Assert.True(article.Record.Fields.MessageId.IsPresent); + Assert.True(article.Record.Fields.Newsgroups.IsPresent); + Assert.True(article.Record.Fields.Date.IsPresent); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task WantTrashFalse_UnknownGroup_Returns439_AndWritesRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession( + queue, + newsLog: news, + transit: new TransitOptions { WantTrash = false, LogTrash = false }, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)]))); + session.SetAuthorization(TransitAuth); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync( + BuildTakeThis(id, CanonicalArticleText.Destuffed(id, newsgroups: "unknown.un.carried"))); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(439, evt.ResponseCode); + Assert.Equal( + IngressNewsReasons.WithGroups(IngressNewsReasons.NewsgroupNotCarried, ["unknown.un.carried"]), + Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_TextOnlyCapability_RejectsYEncoded() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + session.ApplySuccessfulAuthentication( + "poster", + TransitAuth, + new NntpAccountPolicy("poster", 0, 0, 0, 0, "c", ArticleTypeCapabilities.TextOnly)); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + var decoded = new byte[] { 0x41 }; + var crc = VectorNNTPD.Common.Articles.YEnc.YEncCrc32.Compute(decoded); + var encoded = unchecked((byte)(decoded[0] + 42)); + var body = $"=ybegin line=128 size=1 name=t.bin\r\n{(char)encoded}\r\n=yend size=1 crc32={crc:x8}\r\n"; + var article = CanonicalArticleText.Destuffed(id, body); + await duplex.WriteClientAsync(BuildTakeThis(id, article)); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(IngressNewsReasons.ArticleTypeNotPermitted, Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_ValidArticle_DoesNotEvaluatePostFilter() + { + var filter = new CountingPostFilter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, postFilter: filter); + session.SetAuthorization(TransitAuth); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "body\r\n"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, filter.EvaluateCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task StreamTakeThis_InvalidArticle_Returns439_AndDoesNotEnqueue() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, "..foo\r\nbar\r\n")); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task StreamTakeThis_BenchHeaderSet_Returns439_AndDoesNotEnqueue() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync( + BuildTakeThis( + id, + "From: benchmark@vectornntp.local\r\n" + + "Subject: TAKETHIS benchmark\r\n" + + "Message-ID: \r\n" + + "\r\n" + + "1234567890abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnopqrstuvwxyz12345678\r\n")); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task StreamTakeThis_ValidDottedBody_QueuesDestuffedArtData() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "..foo\r\nbar\r\n").Replace("..foo\r\n", "...foo\r\n", StringComparison.Ordinal))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(ArticleParseStatus.CanonicalV1, article!.Record.ParseStatus); + var text = Encoding.ASCII.GetString(article.Record.ArtData.Span); + Assert.Contains("\r\n\r\n...foo\r\nbar\r\n", text, StringComparison.Ordinal); + Assert.True(article.Payload.Equals(article.Record.ArtData)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_PipelinedThree_ResponsesOrdered_AllQueued() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var ids = new[] { "", "", "" }; + var payload = new StringBuilder(); + foreach (var id in ids) + { + payload.Append(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "x\r\n"))); + } + + await duplex.WriteClientAsync(payload.ToString()); + + foreach (var id in ids) + { + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + } + + for (var i = 0; i < ids.Length; i++) + { + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(ids[i], article!.MessageId); + } + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_TwoPipelinedInOneWrite_IdleFlushPeekLeavesSecondArticleReadable() + { + // Session idle-flush peeks Connection.Input after each TAKETHIS. That peek must + // AdvanceTo(Start, Start). AdvanceTo(Start, End) marks leftover examined, so the + // next STREAM ReadAsync waits even though the second TAKETHIS is already buffered. + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string first = ""; + const string second = ""; + await duplex.WriteClientAsync( + BuildTakeThis(first, CanonicalArticleText.Destuffed(first, "a\r\n")) + + BuildTakeThis(second, CanonicalArticleText.Destuffed(second, "b\r\n"))); + + Assert.Equal($"239 {first}", await duplex.ReadClientLineAsync()); + Assert.Equal($"239 {second}", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task TakeThis_SlowOutboundFlush_DoesNotBlockArticleReceive() + { + // Pause after ~1 byte so the first 239 flush blocks until the client reads. + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(pauseWriterThreshold: 1); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var ids = new[] { "", "", "" }; + var payload = new StringBuilder(); + foreach (var id in ids) + { + payload.Append(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "y\r\n"))); + } + + await duplex.WriteClientAsync(payload.ToString()); + + // Articles must be accepted into the queue without the client reading any 239 yet. + using var waitCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (queue.Count < 3 && !waitCts.IsCancellationRequested) + { + await Task.Delay(5, waitCts.Token); + } + + Assert.Equal(3, queue.Count); + + foreach (var id in ids) + { + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + } + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_SlowSpoolWriter_DoesNotBlockEnqueue() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = new ArticleIngestionOptions() }), + NullLogger.Instance); + + await writer.StartAsync(CancellationToken.None); + + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var ids = new[] { "", "", "" }; + foreach (var id in ids) + { + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "z\r\n"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + } + + queue.Complete(); + using var drainCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await writer.StopAsync(drainCts.Token); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_QueueFull_AppliesBackpressureThenAccepts() + { + const string firstId = ""; + const string secondId = ""; + var firstPayload = CanonicalArticleText.Destuffed(firstId, "a\r\n"); + var firstRecord = ArticleRecordIngress.TryCreateFromDestuffed( + new VectorNNTPD.Common.Articles.Parsing.NntpArticleParser("nntpd01.usenet.ninja"), + Encoding.ASCII.GetBytes(firstPayload)); + Assert.True(firstRecord.IsAccepted); + var queue = new ArticleIngestionQueue( + new ArticleIngestionOptions(), + firstRecord.Record.ArtSize); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis(firstId, firstPayload)); + Assert.Equal($"239 {firstId}", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + // Second response must wait until capacity frees (enqueue backpressure). + var secondResponse = duplex.ReadClientLineAsync(); + await duplex.WriteClientAsync(BuildTakeThis(secondId, CanonicalArticleText.Destuffed(secondId, "b\r\n"))); + await Task.Delay(80); + Assert.False(secondResponse.IsCompleted); + + var first = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(firstId, first!.MessageId); + + Assert.Equal($"239 {secondId}", await secondResponse.WaitAsync(TimeSpan.FromSeconds(5))); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_DisconnectMidArticle_DoesNotEnqueue() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync("TAKETHIS \r\nSubject: x\r\n\r\nno-terminator"); + await duplex.CompleteClientInputAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TakeThis_InvalidMessageId_Returns501_AfterConsumingArticle() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("not-a-msgid", "Subject: x\r\n\r\ny\r\n")); + Assert.Equal("501 Syntax error", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_WithoutTransitAuth_Returns480() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_QueueUnavailable_Returns400AndCloses() + { + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(DisabledArticleIngestionQueue.Instance); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", "Subject: t\r\n\r\nz\r\n")); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task TakeThis_TooLarge_Returns439() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, "Subject: oversized-payload-here\r\n\r\nbody\r\n")); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_TooLarge_WritesRejectedNews() + { + var news = new RecordingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, "Subject: oversized-payload-here\r\n\r\nbody\r\n")); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + var evt = Assert.Single(news.Events); + Assert.Equal(NewsLogDisposition.Rejected, evt.Disposition); + Assert.Equal(439, evt.ResponseCode); + Assert.Equal(IngressNewsReasons.ArticleTooLarge, Encoding.ASCII.GetString(evt.Reason.Span)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_NewsWriterFailure_On439_DoesNotChangeRejectionResponse() + { + var news = new ThrowingNewsLogWriter(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 16, + }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue, newsLog: news); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, "Subject: oversized-payload-here\r\n\r\nbody\r\n")); + Assert.Equal($"439 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Equal(1, news.WriteCalls); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_MixedAcceptReject_ResponseMessageIdsCorrelate() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 256, + }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var ok = ""; + var big = ""; + var payload = BuildTakeThis(ok, CanonicalArticleText.Destuffed(ok, "x\r\n")) + + BuildTakeThis(big, "Subject: this-is-definitely-too-large\r\n\r\n" + new string('Z', 300) + "\r\n"); + await duplex.WriteClientAsync(payload); + + Assert.Equal($"239 {ok}", await duplex.ReadClientLineAsync()); + Assert.Equal($"439 {big}", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DefaultUnspecified_UsesStreamDataPlaneRx() + { + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + } + + [Fact] + public async Task ModeStream_KeepsStreamDataPlaneRx_WithoutSettingMode() + { + var source = System.Net.IPAddress.Parse("192.0.2.10"); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession( + new ArticleIngestionQueue(new ArticleIngestionOptions()), + TransitTestPeers.ForAllowFrom(source), + source); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + + var id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "body\r\n"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ModeReader_OnTransit_Returns502_AndSessionStaysOnStreamDataPlane() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + // The Transit role never enters reader mode: MODE READER is unavailable and RX stays on the stream data plane. + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + + // TAKETHIS after the rejected MODE READER is still pre-consumed by the stream data plane. + session.SetAuthorization(TransitAuth); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "reader\r\n"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(id, article!.MessageId); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + Assert.Contains("reader\r\n", Encoding.ASCII.GetString(article.Record.ArtData.Span), StringComparison.Ordinal); + + const string stuffedId = ""; + await duplex.WriteClientAsync(BuildTakeThis(stuffedId, CanonicalArticleText.Destuffed(stuffedId, ".foo\r\nbar\r\n").Replace(".foo\r\n", "..foo\r\n", StringComparison.Ordinal))); + Assert.Equal($"239 {stuffedId}", await duplex.ReadClientLineAsync()); + var destuffed = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(ArticleParseStatus.CanonicalV1, destuffed!.Record.ParseStatus); + Assert.Contains("\r\n\r\n..foo\r\nbar\r\n", Encoding.ASCII.GetString(destuffed.Record.ArtData.Span), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task ModeReader_OnTransit_Returns502_AndTakeThisWithoutTransit_DoesNotConsumeFollowingQuit() + { + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run; + } + + [Fact] + public async Task StreamRx_CheckThenTakeThis_CommandsInsideArticleRemainPayload() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + Assert.Equal(NntpReceiveStrategy.StreamDataPlane, session.ReceiveStrategy); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var stored = "QUIT\r\nCHECK x\r\nTAKETHIS y\r\n"; + await duplex.WriteClientAsync( + "CHECK \r\n" + BuildTakeThis("", CanonicalArticleText.Destuffed("", stored)) + "QUIT\r\n"); + + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Contains(stored, Encoding.ASCII.GetString(article!.Record.ArtData.Span), StringComparison.Ordinal); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run; + } + + [Fact] + public async Task ModeStream_Returns203_WithoutChangingMode() + { + var source = System.Net.IPAddress.Parse("192.0.2.10"); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession( + new ArticleIngestionQueue(new ArticleIngestionOptions()), + TransitTestPeers.ForAllowFrom(source), + source); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task Capabilities_AdvertisesStreaming() + { + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(new ArticleIngestionQueue(new ArticleIngestionOptions())); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CAPABILITIES"); + var lines = new List(); + string line; + do + { + line = await duplex.ReadClientLineAsync(); + lines.Add(line); + } + while (line != "."); + + Assert.Contains(lines, static l => l.Equals("STREAMING", StringComparison.Ordinal)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task SpoolWriter_DiscardsQueuedArticlesWithoutPersisting() + { + var options = new ArticleIngestionOptions { QueueCapacity = 4 }; + var queue = new ArticleIngestionQueue(options); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = options }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + + queue.Complete(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await writer.StopAsync(cts.Token); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task SpoolWriter_ShutdownDrainsAlreadyQueuedArticles() + { + var options = new ArticleIngestionOptions { QueueCapacity = 8 }; + var queue = new ArticleIngestionQueue(options); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions { ArticleIngestion = options }), + NullLogger.Instance); + await writer.StartAsync(CancellationToken.None); + + for (var i = 0; i < 3; i++) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + Assert.True(queue.TryEnqueue(article)); + } + + var stop = writer.StopAsync(CancellationToken.None); + await stop.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task TakeThis_RealTcp_PipelinedWithoutWaitingForResponses() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 16 }); + await using var host = await TransportTestHost.StartPlainAsync(); + using var client = await host.ConnectPlainClientAsync(); + await using var server = await host.AcceptAsync(); + var session = new NntpSession( + server, + NullLogger.Instance, + articleIngestion: queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + + _ = await ReadSocketLineAsync(client); + + var ids = new[] { "", "", "" }; + var batch = new StringBuilder(); + foreach (var id in ids) + { + batch.Append(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "body\r\n"))); + } + + var bytes = Encoding.ASCII.GetBytes(batch.ToString()); + await client.SendAsync(bytes); + + foreach (var id in ids) + { + Assert.Equal($"239 {id}", await ReadSocketLineAsync(client)); + } + + for (var i = 0; i < ids.Length; i++) + { + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(ids[i], article!.MessageId); + } + + await client.SendAsync("QUIT\r\n"u8.ToArray()); + _ = await ReadSocketLineAsync(client); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task MultilineReader_Unit_UnstuffsAndTerminates() + { + var pipe = new Pipe(); + var wire = Encoding.ASCII.GetBytes("..leading\r\nplain\r\n.\r\n"); + await pipe.Writer.WriteAsync(wire); + await pipe.Writer.FlushAsync(); + await pipe.Writer.CompleteAsync(); + + var result = await NntpMultilineDataReader + .ReadArticleAsync(pipe.Reader, 1024, CancellationToken.None); + + Assert.Equal(NntpMultilineReadStatus.Completed, result.Status); + Assert.Equal(".leading\r\nplain\r\n", Encoding.ASCII.GetString(result.Payload.Span)); + } + + [Fact] + public async Task TakeThis_UnknownNewsgroup_WantTrash_Returns239() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync( + BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "body\r\n", "unknown.un.carried"))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("unknown.un.carried", Encoding.ASCII.GetString(article!.Record.Newsgroups)); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TakeThis_NewsWriterFailure_DoesNotEmitSecondResponseOrRequeue() + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 8 }); + var throwing = new ThrowingNewsLogWriter(); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(new NntpdOptions + { + ArticleIngestion = new ArticleIngestionOptions(), + Transit = new TransitOptions { WantTrash = true, LogTrash = true }, + }), + NullLogger.Instance, + newsLog: throwing, + catalogue: new StaticNewsgroupCatalogue( + NewsgroupSnapshot.Create( + [new NewsgroupDefinition("alt.test", string.Empty, 2, 1, NewsgroupPostingStatus.Allowed)]))); + await writer.StartAsync(CancellationToken.None); + + await using var duplex = await TakeThisDuplex.CreateAsync(); + var session = duplex.CreateSession(queue); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id))); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + queue.Complete(); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await writer.StopAsync(wait.Token); + + Assert.Equal(1, throwing.WriteCalls); + Assert.Equal(0, queue.Count); + } + + private static string BuildTakeThis(string messageId, string articleWithoutTerminator) => + $"TAKETHIS {messageId}\r\n{articleWithoutTerminator}.\r\n"; + + private static async Task ReadSocketLineAsync(Socket socket) + { + var buffer = new byte[1]; + var line = new StringBuilder(); + while (true) + { + var n = await socket.ReceiveAsync(buffer, SocketFlags.None); + if (n == 0) + { + break; + } + + line.Append((char)buffer[0]); + if (line.Length >= 2 && line[^2] == '\r' && line[^1] == '\n') + { + return line.ToString(0, line.Length - 2); + } + } + + return line.ToString(); + } + + private sealed class TakeThisDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer; + private readonly Pipe _serverToClient; + + private TakeThisDuplex(PipeOptions? outputOptions) + { + _clientToServer = new Pipe(); + _serverToClient = new Pipe(outputOptions ?? PipeOptions.Default); + } + + public static Task CreateAsync(long? pauseWriterThreshold = null) + { + PipeOptions? outputOptions = null; + if (pauseWriterThreshold is { } pause) + { + outputOptions = new PipeOptions( + pauseWriterThreshold: pause, + resumeWriterThreshold: pause, + useSynchronizationContext: false); + } + + return Task.FromResult(new TakeThisDuplex(outputOptions)); + } + + public NntpSession CreateSession( + IArticleIngestionQueue queue, + ITransitPeerAuthorization? transitPeers = null, + System.Net.IPAddress? clientAddress = null, + VectorNNTPD.TransitService.History.IHistoryDb? historyDb = null, + IPostFilter? postFilter = null, + INewsLogWriter? newsLog = null, + TransitOptions? transit = null, + INewsgroupCatalogue? catalogue = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct( + new System.Net.IPEndPoint(clientAddress ?? System.Net.IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: transitPeers, + historyDb: historyDb, + postFilter: postFilter, + newsgroupCatalogue: catalogue, + newsLog: newsLog, + transit: transit); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task CompleteClientInputAsync() + { + await _clientToServer.Writer.CompleteAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + private int _compressed; + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public System.Net.EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisPipelineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisPipelineTests.cs new file mode 100644 index 00000000..7c2fd0c8 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisPipelineTests.cs @@ -0,0 +1,1363 @@ +using System.Collections.Concurrent; +using System.IO.Pipelines; +using System.Net; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Session.Framing; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// TAKETHIS bounded pipeline: RX frames and detaches owned articles; Peek/enqueue/239 +/// complete off the RX stack; one PipeReader consumer; ordered immediate TX flush. +/// +public sealed class TakeThisPipelineTests +{ + private static NntpAuthorization TransitAuth { get; } = new( + isAuthenticated: true, + authorizedReader: false, + authorizedTransit: true, + postingPermitted: false, + streamingPermitted: true); + + [Fact] + public void PipelineDepth_MatchesCheckWindow() + { + Assert.Equal(CheckPipeline.Depth, TakeThisPipeline.Depth); + Assert.Equal(16, TakeThisPipeline.Depth); + } + + [Fact] + public async Task MessageIdAvailable_StartsHistoryPeek_BeforeArticleCompletes() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var peekHook = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterPeekStarted = () => peekHook.TrySetResult(); + + const string id = ""; + await duplex.WriteClientAsync($"TAKETHIS {id}\r\n{CanonicalArticleText.Destuffed(id, "body-without-terminator\r\n")}"); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await peekHook.Task.WaitAsync(safety.Token); + await history.FirstPeek.Task.WaitAsync(safety.Token); + + Assert.Equal(1, history.PeekStarted); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await Task.Yield(); + Assert.Null(duplex.TryReadClientLine()); + + await duplex.WriteClientAsync(".\r\n"); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(id, article!.MessageId); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TwoThirtyNine_WaitsForCompleteArticle_EvenWhenHistoryAlreadyReady() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var peekHook = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterPeekStarted = () => peekHook.TrySetResult(); + + const string id = ""; + await duplex.WriteClientAsync($"TAKETHIS {id}\r\n{CanonicalArticleText.Destuffed(id, string.Empty)}"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await peekHook.Task.WaitAsync(safety.Token); + Assert.Equal(1, history.PeekStarted); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientAsync("body\r\n.\r\n"); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TwoThirtyNine_WaitsForHistory_AfterArticleReceived() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, "x\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(() => session.TakeThisWindow is { Occupied: 1 } && history.PeekStarted == 1, safety.Token); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal($"239 {id}", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + Assert.Contains(id, history.Remembered); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ArticleReceiveAndHistory_Overlap_SecondArticleReceivedWhileFirstPeekPending() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n"))); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(() => history.PeekStarted == 1 && session.TakeThisWindow!.Occupied == 1, safety.Token); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + await WaitUntilAsync(() => history.PeekStarted == 2 && session.TakeThisWindow!.Occupied == 2, safety.Token); + + Assert.True(session.TakeThisWindow!.Occupied <= TakeThisPipeline.Depth); + Assert.True(session.TakeThisWindow.InFlightCompletions <= TakeThisPipeline.Depth); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(2, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task HoldReceive_PreventsNextTakeThisFromBeingParsed() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receiveA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var peekB = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receivesInFlight = 0; + var maxReceivesInFlight = 0; + session.TakeThisWindow!.HoldReceive = hold.Task; + session.TakeThisWindow.AfterPeekStarted = () => + { + if (history.PeekStarted >= 2) + { + peekB.TrySetResult(); + } + }; + session.TakeThisWindow.AfterReceiveStarted = () => + { + var n = Interlocked.Increment(ref receivesInFlight); + if (n > maxReceivesInFlight) + { + maxReceivesInFlight = n; + } + + receiveA.TrySetResult(); + }; + session.TakeThisWindow.AfterReceiveCompleted = () => Interlocked.Decrement(ref receivesInFlight); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await receiveA.Task.WaitAsync(safety.Token); + Assert.Equal(1, history.PeekStarted); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Equal(1, Volatile.Read(ref receivesInFlight)); + Assert.False(peekB.Task.IsCompleted); + + hold.TrySetResult(); + await peekB.Task.WaitAsync(safety.Token); + Assert.Equal(2, history.PeekStarted); + Assert.True(Volatile.Read(ref maxReceivesInFlight) == 1); + + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, maxReceivesInFlight); + Assert.Equal(0, Volatile.Read(ref receivesInFlight)); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task OpenArticle_ConsumesFollowingTakeThisCommandAsPayload() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var receiveA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var peekB = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterReceiveStarted = () => receiveA.TrySetResult(); + session.TakeThisWindow.AfterPeekStarted = () => + { + if (history.PeekStarted >= 2) + { + peekB.TrySetResult(); + } + }; + + await duplex.WriteClientAsync("TAKETHIS \r\nSubject: a\r\n\r\npartial\r\n"); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await receiveA.Task.WaitAsync(safety.Token); + await history.FirstPeek.Task.WaitAsync(safety.Token); + Assert.Equal(1, history.PeekStarted); + Assert.Equal(1, session.TakeThisWindow.Occupied); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + Assert.False(peekB.Task.IsCompleted); + Assert.Equal(1, history.PeekStarted); + + Assert.Equal("439 ", await duplex.ReadClientLineAsync()); + Assert.False(peekB.Task.IsCompleted); + Assert.Equal(1, history.PeekStarted); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TwoOccupiedSlots_NeverHaveTwoReceivesInFlight() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var receiveStarted = 0; + var receiveCompleted = 0; + var inFlight = 0; + var maxInFlight = 0; + var firstReceiveDone = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondReceiveStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterReceiveStarted = () => + { + var n = Interlocked.Increment(ref inFlight); + if (n > maxInFlight) + { + maxInFlight = n; + } + + var started = Interlocked.Increment(ref receiveStarted); + if (started == 2) + { + secondReceiveStarted.TrySetResult(); + } + }; + session.TakeThisWindow.AfterReceiveCompleted = () => + { + Interlocked.Decrement(ref inFlight); + if (Interlocked.Increment(ref receiveCompleted) == 1) + { + firstReceiveDone.TrySetResult(); + } + }; + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n"))); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await firstReceiveDone.Task.WaitAsync(safety.Token); + Assert.Equal(1, receiveStarted); + Assert.Equal(0, Volatile.Read(ref inFlight)); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Equal(1, history.PeekStarted); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + await secondReceiveStarted.Task.WaitAsync(safety.Token); + await WaitUntilAsync( + () => receiveCompleted >= 2 && session.TakeThisWindow.Occupied == 2, + safety.Token); + + Assert.Equal(1, maxInFlight); + Assert.Equal(2, history.PeekStarted); + Assert.Equal(2, session.TakeThisWindow.Occupied); + Assert.Equal(0, Volatile.Read(ref inFlight)); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DetachedArticle_AllowsNextReceive_WhilePeekRemainsGated() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var detached = new List(); + var receiveADone = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receiveBStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterArticleDetached = payload => + { + detached.Add(payload.ToArray()); + if (detached.Count >= 1) + { + receiveADone.TrySetResult(); + } + }; + session.TakeThisWindow.AfterReceiveStarted = () => + { + if (history.PeekStarted >= 2) + { + receiveBStarted.TrySetResult(); + } + }; + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n"))); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await receiveADone.Task.WaitAsync(safety.Token); + Assert.Single(detached); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Equal(0, session.TakeThisWindow.ActiveArticleReads); + Assert.Null(duplex.TryReadClientLine()); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + await receiveBStarted.Task.WaitAsync(safety.Token); + await WaitUntilAsync( + () => detached.Count == 2 && session.TakeThisWindow.Occupied == 2, + safety.Token); + + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + Assert.Equal(2, history.PeekStarted); + Assert.Equal(2, session.TakeThisWindow.InFlightCompletions); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task FastPeek_DoesNotSerializeRx_BehindHeldEmit() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var holdEmit = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var emitStarted = 0; + var receiveBStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var peekB = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var noTwoThirtyNineOnDetachA = false; + session.TakeThisWindow!.BeforeEnqueueProbe = () => + { + if (Interlocked.Increment(ref emitStarted) == 1) + { + return new ValueTask(holdEmit.Task); + } + + return ValueTask.CompletedTask; + }; + session.TakeThisWindow.AfterArticleDetached = _ => + { + if (history.PeekStarted == 1) + { + noTwoThirtyNineOnDetachA = duplex.TryReadClientLine() is null; + } + }; + session.TakeThisWindow.AfterPeekStarted = () => + { + if (history.PeekStarted >= 2) + { + peekB.TrySetResult(); + } + }; + session.TakeThisWindow.AfterReceiveStarted = () => + { + if (history.PeekStarted >= 2) + { + receiveBStarted.TrySetResult(); + } + }; + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await peekB.Task.WaitAsync(safety.Token); + await receiveBStarted.Task.WaitAsync(safety.Token); + Assert.True(noTwoThirtyNineOnDetachA); + Assert.True(holdEmit.Task.IsCompleted is false); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + + holdEmit.TrySetResult(); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ThreeDetachedArticles_HeldCompletion_RetainOwnedBuffers() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var detached = new List>(); + session.TakeThisWindow!.AfterArticleDetached = payload => detached.Add(payload); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "A\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "B\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "C\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync( + () => detached.Count == 3 && session.TakeThisWindow.Occupied == 3, + safety.Token); + + Assert.Equal(3, history.PeekStarted); + Assert.Equal(0, session.TakeThisWindow.ActiveArticleReads); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(CanonicalArticleText.Destuffed("", "A\r\n"), Encoding.ASCII.GetString(detached[0].Span)); + Assert.Equal(CanonicalArticleText.Destuffed("", "B\r\n"), Encoding.ASCII.GetString(detached[1].Span)); + Assert.Equal(CanonicalArticleText.Destuffed("", "C\r\n"), Encoding.ASCII.GetString(detached[2].Span)); + Assert.True(MemoryMarshal.TryGetArray(detached[0], out var aSeg)); + Assert.True(MemoryMarshal.TryGetArray(detached[1], out var bSeg)); + Assert.True(MemoryMarshal.TryGetArray(detached[2], out var cSeg)); + Assert.Equal(IHaveArticleReader.InitialCapacity, aSeg.Array!.Length); + Assert.NotSame(aSeg.Array, bSeg.Array); + Assert.NotSame(bSeg.Array, cSeg.Array); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task DepthFull_DoesNotAdmitSeventeenthUntilSlotReleases() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var detached = 0; + session.TakeThisWindow!.AfterArticleDetached = _ => Interlocked.Increment(ref detached); + + var extra = TakeThisPipeline.Depth + 1; + var batch = new StringBuilder(); + for (var i = 0; i < extra; i++) + { + batch.Append(BuildTakeThis($"", CanonicalArticleText.Destuffed($"", "x\r\n"))); + } + + await duplex.WriteClientAsync(batch.ToString()); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync( + () => Volatile.Read(ref detached) == TakeThisPipeline.Depth + && session.TakeThisWindow.Occupied == TakeThisPipeline.Depth, + safety.Token); + + Assert.Equal(TakeThisPipeline.Depth, history.PeekStarted); + Assert.Equal(0, session.TakeThisWindow.ActiveArticleReads); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + Assert.Null(duplex.TryReadClientLine()); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + await WaitUntilAsync(() => history.PeekStarted == extra, safety.Token); + for (var i = 0; i < extra; i++) + { + Assert.Equal($"239 ", await duplex.ReadClientLineAsync()); + } + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TakeThisInFlight_CheckDoesNotLookupUntilDrain() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n")) + + "CHECK \r\n"); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync( + () => history.PeekStarted == 2 && session.TakeThisWindow is { Occupied: 2 }, + safety.Token); + Assert.True(session.Pipeline is null || session.Pipeline.Occupied == 0); + Assert.Null(duplex.TryReadClientLine()); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.Equal(3, history.PeekStarted); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TakeThisInFlight_QuitWaitsForOrderedTwoThirtyNine() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var detached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterArticleDetached = _ => detached.TrySetResult(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + "QUIT\r\n"); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await detached.Task.WaitAsync(safety.Token); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Null(duplex.TryReadClientLine()); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task DetachedPayload_SurvivesSubsequentArticleReceive() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + ReadOnlyMemory first = default; + var firstDetached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondDetached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.AfterArticleDetached = payload => + { + if (first.IsEmpty) + { + first = payload; + firstDetached.TrySetResult(); + return; + } + + secondDetached.TrySetResult(); + }; + + var wireA = CanonicalArticleText.Destuffed("", ".dot\r\n") + .Replace(".dot\r\n", "..dot\r\n", StringComparison.Ordinal); + await duplex.WriteClientAsync(BuildTakeThis("", wireA)); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await firstDetached.Task.WaitAsync(safety.Token); + var snapshot = first.ToArray(); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + await secondDetached.Task.WaitAsync(safety.Token); + Assert.Equal(wireA, Encoding.ASCII.GetString(first.Span)); + Assert.Equal(snapshot, first.ToArray()); + Assert.True(MemoryMarshal.TryGetArray(first, out var segment)); + Assert.Equal(IHaveArticleReader.InitialCapacity, segment.Array!.Length); + + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task TooLarge_ThenNextTakeThis_IsNotConsumedAsPayload() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 8, + MaxArticleBytes = 256, + }); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + BuildTakeThis("", "Subject: oversized\r\n\r\n" + new string('Z', 300) + "\r\n") + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "ok\r\n"))); + + Assert.Equal("439 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal("", article!.MessageId); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task SinglePipeReader_MaxActiveArticleReadsIsOne() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "3\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync( + () => session.TakeThisWindow is { Occupied: 3, ActiveArticleReads: 0 }, + safety.Token); + Assert.Equal(1, session.TakeThisWindow!.MaxActiveArticleReads); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task SlowHeadProcess_DoesNotBlockLaterDestuff_ResponsesStayOrdered() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var holdN = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new ConcurrentBag(); + var completed = new ConcurrentBag(); + session.TakeThisWindow!.HoldProcess = id => + { + var text = Encoding.ASCII.GetString(id.Span); + return text == "" ? holdN.Task : slowEntered.Task; + }; + session.TakeThisWindow.AfterProcessStarted = id => + { + var text = Encoding.ASCII.GetString(id.Span); + started.Add(text); + if (text == "") + { + slowEntered.TrySetResult(); + } + }; + session.TakeThisWindow.AfterProcessCompleted = id => completed.Add(Encoding.ASCII.GetString(id.Span)); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "N\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "N1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "N2\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await WaitUntilAsync( + () => completed.Contains("") && completed.Contains(""), + safety.Token); + + Assert.Contains("", started); + Assert.DoesNotContain("", completed); + Assert.True(session.TakeThisWindow.MaxActiveArticleProcessing >= 2); + Assert.Equal(3, session.TakeThisWindow.Occupied); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + Assert.Empty(history.Remembered); + + holdN.TrySetResult(); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Contains("", completed); + Assert.Equal(3, queue.Count); + Assert.Equal(3, history.Remembered.Count); + Assert.Contains("", history.Remembered); + Assert.Contains("", history.Remembered); + Assert.Contains("", history.Remembered); + Assert.Equal(1, session.TakeThisWindow.MaxActiveArticleReads); + Assert.Equal(0, session.TakeThisWindow.Occupied); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task HistorySeen_SkipsDestuff() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Seen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var processStarted = 0; + session.TakeThisWindow!.AfterProcessStarted = _ => Interlocked.Increment(ref processStarted); + + await duplex.WriteClientAsync(BuildTakeThis("", CanonicalArticleText.Destuffed("", "x\r\n"))); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(0, Volatile.Read(ref processStarted)); + Assert.Equal(0, session.TakeThisWindow.MaxActiveArticleProcessing); + Assert.Equal(0, queue.Count); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Shutdown_CancelsHeldProcess_ReleasesSlotWithoutResponse() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var hold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + session.TakeThisWindow!.HoldProcess = _ => hold.Task; + session.TakeThisWindow.AfterProcessStarted = _ => started.TrySetResult(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "x\r\n"))); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await started.Task.WaitAsync(safety.Token); + Assert.Equal(1, session.TakeThisWindow.Occupied); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + + await session.TakeThisWindow.ShutdownAsync().AsTask().WaitAsync(safety.Token); + Assert.Equal(0, session.TakeThisWindow.Occupied); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(0, queue.Count); + Assert.Empty(history.Remembered); + + await session.Connection.CompleteAsync(); + try + { + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + // codeql[cs/catch-of-all-exceptions] Best-effort test cleanup must tolerate any filesystem or disposal failure. + catch (Exception ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + // RX may observe connection close after the pipeline already shut down. + } + } + + [Fact] + public async Task HistorySeen_Returns239_DoesNotEnqueue() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Seen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", "Subject: d\r\n\r\nz\r\n")); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + Assert.Empty(history.Remembered); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task RememberedId_TakeThis_RemainsAcceptedDuplicate() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + history.Remember(""u8.ToArray()); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", "Subject: d\r\n\r\nz\r\n")); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(0, queue.Count); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("438 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task HistoryMiss_Returns239_EnqueuesAndRemembers() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, ".stuffed\r\nplain\r\n").Replace(".stuffed\r\n", "..stuffed\r\n", StringComparison.Ordinal))); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(id, article!.MessageId); + Assert.Equal(InboundArticleProducer.TakeThis, article.Producer); + Assert.Equal(ArticleParseStatus.CanonicalV1, article.Record.ParseStatus); + Assert.True(article.Payload.Equals(article.Record.ArtData)); + Assert.Contains("\r\n\r\n..stuffed\r\nplain\r\n", Encoding.ASCII.GetString(article.Record.ArtData.Span), StringComparison.Ordinal); + Assert.Contains("", history.Remembered); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task HistoryUnavailable_Returns400AndCloses() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unavailable); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", "Subject: x\r\n\r\ny\r\n")); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task HistoryThrows_Returns400AndCloses() + { + var history = new GatedHistoryDb { ThrowOnPeek = true }; + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync(BuildTakeThis("", "Subject: x\r\n\r\ny\r\n")); + Assert.Equal("400 Service temporarily unavailable", await duplex.ReadClientLineAsync()); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, queue.Count); + } + + [Fact] + public async Task QueuedPayload_IsSingleOwnedWireBuffer_NotToArrayCopy() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + const string id = ""; + await duplex.WriteClientAsync(BuildTakeThis(id, CanonicalArticleText.Destuffed(id, ".dot\r\n").Replace(".dot\r\n", "..dot\r\n", StringComparison.Ordinal))); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + var article = await queue.DequeueAsync(CancellationToken.None); + Assert.Equal(ArticleParseStatus.CanonicalV1, article!.Record.ParseStatus); + Assert.True(article.Payload.Equals(article.Record.ArtData)); + Assert.True(MemoryMarshal.TryGetArray(article.Payload, out var segment)); + Assert.NotNull(segment.Array); + Assert.Equal(article.Record.ArtSize, segment.Array.Length); + Assert.Equal(article.Record.ArtSize, article.Payload.Length); + Assert.Contains("\r\n\r\n..dot\r\n", Encoding.ASCII.GetString(article.Record.ArtData.Span), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task Window_IsBounded_AtCheckDepth() + { + var history = new GatedHistoryDb(); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var extra = TakeThisPipeline.Depth + 1; + var batch = new StringBuilder(); + for (var i = 0; i < extra; i++) + { + batch.Append(BuildTakeThis($"", CanonicalArticleText.Destuffed($"", "x\r\n"))); + } + + await duplex.WriteClientAsync(batch.ToString()); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync( + () => history.PeekStarted == TakeThisPipeline.Depth && session.TakeThisWindow!.Occupied == TakeThisPipeline.Depth, + safety.Token); + + Assert.Equal(TakeThisPipeline.Depth, history.PeekStarted); + Assert.Equal(TakeThisPipeline.Depth, session.TakeThisWindow!.Occupied); + Assert.True(session.TakeThisWindow.PeakOccupied <= TakeThisPipeline.Depth); + Assert.True(session.TakeThisWindow.InFlightCompletions <= TakeThisPipeline.Depth); + Assert.Null(duplex.TryReadClientLine()); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + for (var i = 0; i < extra; i++) + { + Assert.Equal($"239 ", await duplex.ReadClientLineAsync()); + } + + Assert.Equal(extra, queue.Count); + await QuitAsync(duplex, run); + } + + [Fact] + public async Task FasterSecondHistory_DoesNotReorderResponses() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + BuildTakeThis("", CanonicalArticleText.Destuffed("", "2\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(() => session.TakeThisWindow is { Occupied: 2 }, safety.Token); + Assert.Null(duplex.TryReadClientLine()); + + history.Release.TrySetResult(HistoryLookupResult.Unseen); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ImmediateFlush_EmitsFirst239_WhileNextArticleStillOpen() + { + var history = new GatedHistoryDb(); + history.Force("", HistoryLookupResult.Unseen); + history.Force("", HistoryLookupResult.Unseen); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + BuildTakeThis("", CanonicalArticleText.Destuffed("", "1\r\n")) + + $"TAKETHIS \r\n{CanonicalArticleText.Destuffed("", "no-term\r\n")}"); + + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Null(duplex.TryReadClientLine()); + Assert.Equal(1, queue.Count); + + await duplex.WriteClientAsync(".\r\n"); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task CheckThenTakeThis_DrainsCheckBeforeTakeThisPeek() + { + var redis = new FakeRedisService(); + var checkId = ""u8.ToArray(); + var block = BlockKey(redis, checkId); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var queue = NewQueue(); + await using var duplex = new TakeThisPipelineDuplex(); + var session = duplex.CreateSession(queue, history); + session.SetAuthorization(TransitAuth); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientAsync( + "CHECK \r\n" + BuildTakeThis("", CanonicalArticleText.Destuffed("", "z\r\n"))); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await WaitUntilAsync(() => session.Pipeline is { Occupied: >= 1 }, safety.Token); + Assert.True(session.TakeThisWindow is null || session.TakeThisWindow.Occupied == 0); + Assert.Equal(1, redis.Database.ExistsStartedCount); + Assert.Null(duplex.TryReadClientLine()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(checkId))); + + block.TrySetResult(); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(checkId))); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(1, queue.Count); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + private static ArticleIngestionQueue NewQueue() => + new(new ArticleIngestionOptions { QueueCapacity = 32 }); + + private static string BuildTakeThis(string messageId, string? articleWithoutTerminator = null) => + $"TAKETHIS {messageId}\r\n{(articleWithoutTerminator ?? CanonicalArticleText.Destuffed(messageId))}.\r\n"; + + private static async Task QuitAsync(TakeThisPipelineDuplex duplex, Task run) + { + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static async Task WaitUntilAsync(Func condition, CancellationToken cancellationToken) + { + while (!condition()) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Yield(); + } + } + + private static TaskCompletionSource BlockKey(FakeRedisService redis, ReadOnlySpan messageId) + { + var key = HistoryRedisKeys.Create(HistoryDigest.FromMessageId(messageId)); + var block = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + redis.Database.BlockExistsByKey[Convert.ToHexString(key)] = block; + return block; + } + + private sealed class GatedHistoryDb : IHistoryDb + { + private readonly ConcurrentDictionary _forced = new(StringComparer.Ordinal); + private readonly HashSet _local = []; + private readonly object _gate = new(); + + public TaskCompletionSource Release { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource FirstPeek { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ConcurrentBag Remembered { get; } = []; + + public int PeekStarted; + + public bool ThrowOnPeek { get; set; } + + public void Force(string messageId, HistoryLookupResult result) => + _forced[messageId] = result; + + public ValueTask LookupAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) => + PeekAsync(messageId, cancellationToken); + + public async ValueTask PeekAsync( + ReadOnlyMemory messageId, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref PeekStarted); + FirstPeek.TrySetResult(); + if (ThrowOnPeek) + { + throw new InvalidOperationException("HistoryDB test failure."); + } + + var id = Encoding.ASCII.GetString(messageId.Span); + if (_forced.TryGetValue(id, out var forced)) + { + return forced; + } + + return await Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + public void Remember(ReadOnlyMemory messageId) + { + var text = Encoding.ASCII.GetString(messageId.Span); + Remembered.Add(text); + lock (_gate) + { + _local.Add(HistoryDigest.FromMessageId(messageId.Span)); + } + } + + public bool ContainsLocal(in HistoryDigest digest) + { + lock (_gate) + { + return _local.Contains(digest); + } + } + } + + private sealed class TakeThisPipelineDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public NntpSession CreateSession(IArticleIngestionQueue queue, IHistoryDb history) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(IPAddress.Loopback, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + historyDb: history); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public string? TryReadClientLine() + { + if (!_serverToClient.Reader.TryRead(out var result)) + { + return null; + } + + var buffer = result.Buffer; + if (!NntpDelimiterSearch.TryReadLine(ref buffer, out var lineBytes)) + { + _serverToClient.Reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + return null; + } + + var text = Encoding.ASCII.GetString(System.Buffers.BuffersExtensions.ToArray(in lineBytes)); + _serverToClient.Reader.AdvanceTo(buffer.Start, buffer.Start); + return text; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + + public PipeWriter Output { get; } = output; + + public ConnectionClientIdentity ClientIdentity { get; } = identity; + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public bool IsCompleted => _closed.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisStageProbeTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisStageProbeTests.cs new file mode 100644 index 00000000..cc2aea33 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/TakeThisStageProbeTests.cs @@ -0,0 +1,87 @@ +using System.Diagnostics; +using VectorNNTPD.TransitService.Session; + +namespace VectorNNTPD.TransitService.Tests.Session; + +public sealed class TakeThisStageProbeTests +{ + [Fact] + public void Default_IsDisabled() + { + Assert.False(TakeThisStageProbe.IsEnabled); + Assert.Null(TakeThisStageProbe.CreateSessionLog()); + } + + [Fact] + public void Percentile_IsMonotonic() + { + var values = new double[] { 10, 20, 30, 40, 50 }; + Assert.Equal(10, TakeThisStageReport.Percentile(values, 0)); + Assert.Equal(30, TakeThisStageReport.Percentile(values, 0.5)); + Assert.Equal(50, TakeThisStageReport.Percentile(values, 1)); + Assert.True(TakeThisStageReport.Percentile(values, 0.95) >= TakeThisStageReport.Percentile(values, 0.50)); + } + + [Fact] + public void Intervals_ClassifyReceiveAsDominant_WhenReceiveIsLargest() + { + var origin = Stopwatch.GetTimestamp(); + var freq = Stopwatch.Frequency; + long AtMs(double ms) => origin + (long)(ms / 1000.0 * freq); + + var sample = new TakeThisStageSample( + CommandParsedTs: origin, + PeekStartTs: origin, + PeekCompleteTs: AtMs(0.2), + ReceiveStartTs: origin, + ReceiveCompleteTs: AtMs(1.0), + EmitStartTs: AtMs(1.05), + EnqueueStartTs: AtMs(1.06), + EnqueueAcceptedTs: AtMs(1.08), + RememberStartTs: AtMs(1.08), + RememberCompleteTs: AtMs(1.09), + ReplyEnqueueStartTs: AtMs(1.09), + ReplyEnqueueCompleteTs: AtMs(1.12), + OccupiedAtEmit: 3, + PeakOccupied: 8, + ArticleBytes: 1000, + Outcome: "unseen"); + + var intervals = TakeThisStageReport.Intervals(sample); + Assert.Equal("article receive", intervals.Dominant); + Assert.True(intervals.ReceiveUs > intervals.PeekUs); + Assert.True(intervals.OverlapUs >= intervals.ReceiveUs - 1); + } + + [Fact] + public void Report_IncludesRequestedIntervals() + { + var origin = Stopwatch.GetTimestamp(); + var sample = new TakeThisStageSample( + origin, + origin, + origin + Stopwatch.Frequency / 1000, + origin, + origin + Stopwatch.Frequency / 500, + origin + Stopwatch.Frequency / 400, + origin + Stopwatch.Frequency / 400, + origin + Stopwatch.Frequency / 350, + origin + Stopwatch.Frequency / 350, + origin + Stopwatch.Frequency / 340, + origin + Stopwatch.Frequency / 340, + origin + Stopwatch.Frequency / 300, + OccupiedAtEmit: 2, + PeakOccupied: 4, + ArticleBytes: 64, + Outcome: "unseen"); + + var text = TakeThisStageReport.Format([sample], peakOccupied: 4, "test", csvPath: null); + Assert.Contains("HistoryDB Peek", text, StringComparison.Ordinal); + Assert.Contains("article receive", text, StringComparison.Ordinal); + Assert.Contains("queue EnqueueAsync", text, StringComparison.Ordinal); + Assert.Contains("HistoryDB Remember", text, StringComparison.Ordinal); + Assert.Contains("239 Channel enqueue", text, StringComparison.Ordinal); + Assert.Contains("ready → emit (order)", text, StringComparison.Ordinal); + Assert.Contains("Peak occupied", text, StringComparison.Ordinal); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/TransitPeerAuthorizationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/TransitPeerAuthorizationTests.cs new file mode 100644 index 00000000..f2b89f83 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/TransitPeerAuthorizationTests.cs @@ -0,0 +1,398 @@ +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Networking.Transport; +using VectorNNTPD.TransitService.Tests.Transit; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// Connection-time transit/streaming peer ACL (top-level named Transit peers). +[Collection(nameof(TransportTestHostCollection))] +public sealed class TransitPeerAuthorizationTests +{ + private static readonly IPAddress AllowedPeer = IPAddress.Parse("198.18.0.70"); + private static readonly IPAddress OtherPeer = IPAddress.Parse("198.18.0.71"); + + [Fact] + public async Task DefaultAcl_ModeStream_Returns480() + { + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(TransitPeerAuthorization.Disabled); + AssertPeerUnauthorized(session); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowedPeer_ReceivesTransitStreamingPrivileges_NotAuthenticated() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(peers); + + AssertNamedTransitPeer(session); + Assert.False(session.Authentication.IsAuthenticated); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowedPeer_ModeStream_Returns203_WithoutChangingMode() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + Assert.False(session.Authorization.IsAuthenticated); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowedPeer_CheckTakeThisIhave_NotBlockedByAuthentication() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(peers, queue); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientBytesAsync(Encoding.ASCII.GetBytes(CanonicalArticleText.Stuffed("") + ".\r\n")); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + await duplex.WriteClientBytesAsync(Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("") + ".\r\n")); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(2, queue.Count); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowedPeer_PostAndReaderCommands_AreUnavailableOnTransit() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("LIST"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("GROUP alt.test"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("ARTICLE"); + Assert.Equal("502 Command not available on this server", await duplex.ReadClientLineAsync()); + + Assert.False(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.PostingPermitted); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task DifferentSourceAddress_DoesNotReceiveTransitPrivileges() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var duplex = await PeerDuplex.CreateAsync(OtherPeer); + var session = duplex.CreateSession(peers); + AssertPeerUnauthorized(session); + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AllowedPeer_FailedAuthinfo_RestoresPeerPrivileges_NotAuthenticated() + { + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var duplex = await PeerDuplex.CreateAsync(AllowedPeer); + var session = duplex.CreateSession(peers); + AssertNamedTransitPeer(session); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER nobody"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + + Assert.False(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedTransit); + Assert.True(session.Authorization.StreamingPermitted); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void Resolve_MatchesIpv6AndMappedIpv4() + { + var v6 = IPAddress.Parse("2001:db8::1"); + var peers = TransitTestPeers.ForAllowFrom([v6, AllowedPeer]); + AssertTransitPeer(peers.Resolve(v6), TransitTestPeers.DefaultPeerName); + AssertTransitPeer(peers.Resolve(AllowedPeer), TransitTestPeers.DefaultPeerName); + AssertTransitPeer(peers.Resolve(IPAddress.Parse("::ffff:198.18.0.70")), TransitTestPeers.DefaultPeerName); + Assert.Equal(NntpAuthorization.Unauthenticated, peers.Resolve(OtherPeer)); + } + + [Fact] + public async Task TcpProxyEffectiveClient_AllowedPeer_GrantsTransit() + { + var trusted = new TrustedProxyHosts([IPAddress.Loopback]); + var peers = TransitTestPeers.ForAllowFrom(AllowedPeer); + await using var host = await TransportTestHost.StartPlainWithProxyAsync(trusted); + + using var clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + await clientSocket.ConnectAsync(host.EndPoint); + await clientSocket.SendAsync(Encoding.ASCII.GetBytes( + "PROXY TCP4 198.18.0.70 127.0.0.1 40000 119\r\n")); + + await using var server = await host.AcceptAsync(); + Assert.Equal(AllowedPeer, server.ClientIdentity.ClientAddress); + + var session = new NntpSession( + server, + NullLogger.Instance, + transitPeerAuthorization: peers); + AssertNamedTransitPeer(session); + + var sessionTask = session.RunAsync(); + var greeting = await ReadPlainLineAsync(clientSocket); + Assert.StartsWith("20", greeting, StringComparison.Ordinal); + + await clientSocket.SendAsync("MODE STREAM\r\n"u8.ToArray()); + Assert.Equal("203 Streaming permitted", await ReadPlainLineAsync(clientSocket)); + + await clientSocket.SendAsync("QUIT\r\n"u8.ToArray()); + Assert.StartsWith("205 ", await ReadPlainLineAsync(clientSocket), StringComparison.Ordinal); + await sessionTask.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static void AssertNamedTransitPeer(NntpSession session) + { + AssertTransitPeer(session.Authorization, TransitTestPeers.DefaultPeerName); + Assert.Equal(AllowedPeer, session.ClientAddress); + } + + private static void AssertTransitPeer(NntpAuthorization authorization, string peerName) + { + Assert.False(authorization.IsAuthenticated); + Assert.True(authorization.AuthorizedTransit); + Assert.True(authorization.StreamingPermitted); + Assert.False(authorization.AuthorizedReader); + Assert.False(authorization.PostingPermitted); + Assert.Equal(peerName, authorization.TransitPeerName); + Assert.NotNull(authorization.TransitPeerPolicy); + } + + private static void AssertPeerUnauthorized(NntpSession session) + { + Assert.False(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.False(session.Authorization.StreamingPermitted); + Assert.False(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.PostingPermitted); + Assert.Null(session.Authorization.TransitPeerName); + } + + private static async Task ReadPlainLineAsync(Socket socket) + { + var buffer = new byte[512]; + var total = 0; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (total < buffer.Length) + { + var n = await socket.ReceiveAsync(buffer.AsMemory(total), cts.Token); + Assert.True(n > 0); + total += n; + var text = Encoding.ASCII.GetString(buffer, 0, total); + var idx = text.IndexOf("\r\n", StringComparison.Ordinal); + if (idx >= 0) + { + return text[..idx]; + } + } + + throw new InvalidOperationException("Line too long."); + } + + private sealed class PeerDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly IPAddress _clientAddress; + + private PeerDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress clientAddress) => + Task.FromResult(new PeerDuplex(clientAddress)); + + public NntpSession CreateSession( + ITransitPeerAuthorization peers, + IArticleIngestionQueue? queue = null) + { + var identity = ConnectionClientIdentity.Direct(new IPEndPoint(_clientAddress, 40000)); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + identity); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientBytesAsync(ReadOnlyMemory bytes) + { + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + private int _compressed; + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => Volatile.Read(ref _compressed) == 1; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) + { + Volatile.Write(ref _compressed, 1); + return Task.CompletedTask; + } + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Session/TransitReaderCommandIsolationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Session/TransitReaderCommandIsolationTests.cs new file mode 100644 index 00000000..3bcc8622 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Session/TransitReaderCommandIsolationTests.cs @@ -0,0 +1,256 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; + +namespace VectorNNTPD.TransitService.Tests.Session; + +/// +/// The Transit role does not host reader commands. Every reader-only verb (article retrieval, group +/// selection, LIST, OVER/HDR, POST) is answered 502 Command not available on this server before +/// authentication, authorization, or syntax-specific handling, and never changes session state. +/// The reader command behavior itself is covered by the ReaderService test project. +/// +public sealed class TransitReaderCommandIsolationTests +{ + private const string CommandNotAvailable = "502 Command not available on this server"; + + [Theory] + [InlineData("ARTICLE")] + [InlineData("ARTICLE 1")] + [InlineData("ARTICLE ")] + [InlineData("HEAD")] + [InlineData("HEAD 1")] + [InlineData("HEAD ")] + [InlineData("BODY")] + [InlineData("BODY 1")] + [InlineData("BODY ")] + [InlineData("STAT")] + [InlineData("STAT 1")] + [InlineData("STAT ")] + [InlineData("GROUP misc.test")] + [InlineData("LISTGROUP")] + [InlineData("LISTGROUP misc.test")] + [InlineData("LAST")] + [InlineData("NEXT")] + [InlineData("OVER 1-2")] + [InlineData("HDR Subject 1")] + [InlineData("LIST")] + [InlineData("LIST ACTIVE")] + [InlineData("LIST ACTIVE misc.*")] + [InlineData("LIST COUNTS")] + [InlineData("LIST NEWSGROUPS")] + [InlineData("LIST OVERVIEW.FMT")] + [InlineData("LIST HEADERS")] + [InlineData("LIST MOTD")] + [InlineData("POST")] + public async Task ReaderCommand_Unauthenticated_Returns502(string commandLine) + { + await using var duplex = new IsolationDuplex(); + var session = duplex.CreateSession(); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + } + + [Theory] + [InlineData("ARTICLE 1")] + [InlineData("ARTICLE ")] + [InlineData("HEAD ")] + [InlineData("BODY ")] + [InlineData("STAT ")] + [InlineData("GROUP misc.test")] + [InlineData("LISTGROUP misc.test")] + [InlineData("LAST")] + [InlineData("NEXT")] + [InlineData("OVER 1-2")] + [InlineData("HDR Subject 1")] + [InlineData("LIST")] + [InlineData("LIST ACTIVE")] + [InlineData("LIST COUNTS")] + [InlineData("LIST NEWSGROUPS")] + [InlineData("LIST OVERVIEW.FMT")] + [InlineData("LIST HEADERS")] + [InlineData("LIST MOTD")] + [InlineData("POST")] + public async Task ReaderCommand_FullyAuthorized_StillReturns502(string commandLine) + { + await using var duplex = new IsolationDuplex(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: true, + postingPermitted: true, + streamingPermitted: true)); + var dispatcher = new NntpCommandDispatcher(); + var response = new NntpResponseWriter(duplex.ServerOutput); + + await NntpCommandTestParse.DispatchAsync(dispatcher, session, response, commandLine); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + } + + [Fact] + public async Task Post_OnTransit_Returns502_AndDoesNotEnterArticleInput() + { + await using var duplex = new IsolationDuplex(); + var session = duplex.CreateSession(); + session.SetAuthorization(new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false)); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("POST"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + + // If POST had opened article input, DATE would be consumed as article text instead of answered. + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run; + } + + [Fact] + public async Task ReaderCommands_OnTransit_DoNotSelectGroupOrChangeSessionMode() + { + await using var duplex = new IsolationDuplex(); + var session = duplex.CreateSession(); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("GROUP misc.test"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("LIST"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("MODE READER"); + Assert.Equal(CommandNotAvailable, await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + await duplex.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await run; + } + + private sealed class IsolationDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public PipeWriter ServerOutput => _serverToClient.Writer; + + public NntpSession CreateSession() + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 119))); + return new NntpSession(connection, NullLogger.Instance); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public System.Net.EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public System.Net.EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementConnectionTests.cs b/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementConnectionTests.cs new file mode 100644 index 00000000..574e7d38 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementConnectionTests.cs @@ -0,0 +1,764 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Transport.Vatp; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Tests.Storage; + +public sealed class ArticlePlacementConnectionTests +{ + [Fact] + public async Task Several_articles_reuse_one_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + for (var i = 0; i < 3; i++) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal(ArticlePlacementKind.Accepted, result.Kind); + } + + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + Assert.Equal(3, server.Results); + } + + [Fact] + public async Task Two_targets_do_not_share_a_connection() + { + await using var left = StoreServer.Start(); + await using var right = StoreServer.Start(); + await using var client = Client(maxOutstanding: 1); + var a = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var b = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(a.Record, Entry(left.Port, "cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(b.Record, Entry(right.Port, "cache02.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(2, client.ConnectCount); + Assert.Equal(1, left.AcceptedConnections); + Assert.Equal(1, right.AcceptedConnections); + } + + [Fact] + public async Task Fqdn_case_uses_one_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 1); + var lower = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var upper = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(lower.Record, Entry(server.Port, "cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(upper.Record, Entry(server.Port, "Cache01.usenet.ninja"), CancellationToken.None)).Kind); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + } + + [Fact] + public async Task Limit_one_does_not_overlap_stores() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await Task.Delay(50); + Assert.Equal(1, server.PeakOutstanding); + server.ReleaseResults(); + Assert.Equal(ArticlePlacementKind.Accepted, (await firstTask).Kind); + Assert.Equal(ArticlePlacementKind.Accepted, (await secondTask).Kind); + Assert.Equal(1, server.PeakOutstanding); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Two_stores_complete_with_the_result_for_their_stream() + { + await using var server = StoreServer.Start(holdResult: true, duplicateSecond: true); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(2); + Assert.Equal(2, server.PeakOutstanding); + server.ReleaseResults(); + var results = await Task.WhenAll(firstTask, secondTask); + Assert.Contains(results, static result => result.Kind == ArticlePlacementKind.Accepted); + Assert.Contains(results, static result => result.Kind == ArticlePlacementKind.Duplicate); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(0, server.ParseFailures); + } + + [Fact] + public async Task Dead_connection_is_replaced_once() + { + await using var server = StoreServer.Start(closeAfterAccepted: 1); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(first.Record, target, CancellationToken.None)).Kind); + var replaced = ArticlePlacementKind.TransportFailure; + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline && replaced != ArticlePlacementKind.Accepted) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + replaced = (await client.PlaceAsync(article.Record, target, CancellationToken.None)).Kind; + } + + Assert.Equal(ArticlePlacementKind.Accepted, replaced); + Assert.Equal(2, client.ConnectCount); + Assert.Equal(2, server.AcceptedConnections); + } + + [Fact] + public async Task Closed_port_does_not_dial_again_during_backoff() + { + await using var client = new ArticlePlacementClient( + NullLogger.Instance, + new VatpClientOptions(), + ArticleTransferLimits.Default, + maxOutstandingStores: 1) + { + TestTcpConnectHost = "127.0.0.1", + }; + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var target = Entry(port); + var first = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal(ArticlePlacementKind.TransportFailure, first.Kind); + Assert.StartsWith("SocketException:", first.Failure, StringComparison.Ordinal); + var second = await client.PlaceAsync(article.Record, target, CancellationToken.None); + Assert.Equal("backoff", second.Failure); + Assert.Equal(1, client.DialAttempts); + } + + [Fact] + public async Task Drop_after_end_is_not_success() + { + await using var server = StoreServer.Start(dropAfterEnd: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Remote_rejection_is_not_success() + { + await using var server = StoreServer.Start(reject: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.RejectedInvalid, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Cancel_while_waiting_for_a_slot_sends_nothing() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var target = Entry(server.Port); + var held = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var heldTask = client.PlaceAsync(held.Record, target, CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + var queued = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(queued.Record, target, cts.Token); + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(1, server.Stores); + server.ReleaseResults(); + Assert.Equal(ArticlePlacementKind.Accepted, (await heldTask).Kind); + } + + [Fact] + public async Task Shutdown_cancels_an_outstanding_store_and_rejects_new_calls() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None).AsTask(); + await server.WaitForStoresAsync(1); + await client.DisposeAsync(); + var result = await pending; + Assert.NotEqual(ArticlePlacementKind.Accepted, result.Kind); + var later = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.TransportFailure, later.Kind); + Assert.Equal("shutdown", later.Failure); + } + + [Fact] + public async Task Stream_ids_are_not_reused_on_a_live_connection() + { + await using var server = StoreServer.Start(); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + for (var i = 0; i < 3; i++) + { + var article = CanonicalArticleText.CreateQueued($"", InboundArticleProducer.TakeThis); + Assert.Equal(ArticlePlacementKind.Accepted, (await client.PlaceAsync(article.Record, target, CancellationToken.None)).Kind); + } + + var ids = server.StreamIds; + Assert.Equal(new uint[] { 1, 2, 3 }, ids); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Cancel_during_dial_does_not_open_a_second_connection() + { + await using var server = StoreServer.Start(handshakeDelay: TimeSpan.FromMilliseconds(400)); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + using var cts = new CancellationTokenSource(); + var firstArticle = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var secondArticle = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var first = client.PlaceAsync(firstArticle.Record, target, cts.Token).AsTask(); + var started = DateTime.UtcNow; + while (client.DialAttempts < 1) + { + if (DateTime.UtcNow - started > TimeSpan.FromSeconds(2)) + { + throw new TimeoutException("Dial did not start."); + } + + await Task.Delay(5); + } + + var second = client.PlaceAsync(secondArticle.Record, target, CancellationToken.None).AsTask(); + await cts.CancelAsync(); + var firstResult = await first; + var secondResult = await second; + Assert.Equal(ArticlePlacementKind.Cancelled, firstResult.Kind); + Assert.Equal(ArticlePlacementKind.Accepted, secondResult.Kind); + Assert.Equal(1, client.ConnectCount); + Assert.Equal(1, server.AcceptedConnections); + } + + [Fact] + public async Task Two_streams_keep_independent_window_credit() + { + var limits = new ArticleTransferLimits { InitialStreamWindowBytes = 32 }; + await using var server = StoreServer.Start(initialCredit: 32); + await using var client = Client(maxOutstanding: 2, limits: limits); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('a', 200) + "\r\n"); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('b', 200) + "\r\n"); + var results = await Task.WhenAll( + client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(), + client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask()); + Assert.All(results, static result => Assert.Equal(ArticlePlacementKind.Accepted, result.Kind)); + Assert.Equal(0, server.ParseFailures); + Assert.Equal(2, server.PeakOutstanding); + Assert.Equal(1, client.ConnectCount); + } + + [Fact] + public async Task Connection_loss_after_end_is_ambiguous_for_every_outstanding_store() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 2); + var target = Entry(server.Port); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var firstTask = client.PlaceAsync(first.Record, target, CancellationToken.None).AsTask(); + var secondTask = client.PlaceAsync(second.Record, target, CancellationToken.None).AsTask(); + await server.WaitForEndsAsync(2); + server.DropConnections(); + var results = await Task.WhenAll(firstTask, secondTask); + Assert.All(results, static result => Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind)); + Assert.All(results, static result => Assert.False(result.Succeeded)); + } + + [Fact] + public async Task Cancel_while_blocked_on_credit_does_not_report_success() + { + var limits = new ArticleTransferLimits { InitialStreamWindowBytes = 16 }; + await using var server = StoreServer.Start(grantWindows: false, initialCredit: 16); + await using var client = Client(maxOutstanding: 1, limits: limits, ioTimeout: TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis, body: new string('c', 400) + "\r\n"); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), cts.Token).AsTask(); + await server.WaitForDataAsync(1); + await cts.CancelAsync(); + var result = await pending; + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(0, server.Ends); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Client_does_not_send_the_body_before_window_credit() + { + await using var server = StoreServer.Start(initialCredit: 0); + await using var client = Client(maxOutstanding: 1, ioTimeout: TimeSpan.FromSeconds(5)); + using var cts = new CancellationTokenSource(); + var article = CanonicalArticleText.CreateQueued( + "", + InboundArticleProducer.TakeThis, + body: new string('d', 400) + "\r\n"); + var pending = client.PlaceAsync(article.Record, Entry(server.Port), cts.Token).AsTask(); + await server.WaitForStoresAsync(1); + Assert.Equal(0, server.DataFrames); + await cts.CancelAsync(); + var result = await pending; + Assert.Equal(ArticlePlacementKind.Cancelled, result.Kind); + Assert.Equal(0, server.DataFrames); + } + + [Fact] + public async Task Timeout_after_end_is_ambiguous() + { + await using var server = StoreServer.Start(holdResult: true); + await using var client = Client(maxOutstanding: 1, ioTimeout: TimeSpan.FromMilliseconds(300)); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry(server.Port), CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, result.Kind); + Assert.False(result.Succeeded); + } + + [Fact] + public async Task Timeout_before_handshake_is_not_success() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var accept = listener.AcceptTcpClientAsync(); + await using var client = new ArticlePlacementClient( + NullLogger.Instance, + new VatpClientOptions + { + ConnectTimeout = TimeSpan.FromMilliseconds(200), + TlsHandshakeTimeout = TimeSpan.FromMilliseconds(200), + IoTimeout = TimeSpan.FromMilliseconds(200), + }, + ArticleTransferLimits.Default, + maxOutstandingStores: 1) + { + TestTcpConnectHost = "127.0.0.1", + ServerCertificateValidationCallback = static (_, _, _, _) => true, + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var place = client.PlaceAsync(article.Record, Entry(port), CancellationToken.None).AsTask(); + using var accepted = await accept; + var result = await place; + listener.Stop(); + Assert.Equal(ArticlePlacementKind.TransportFailure, result.Kind); + Assert.Equal("timeout", result.Failure); + Assert.False(result.Succeeded); + } + + [Fact] + public void Production_placement_stays_disabled() + { + var property = typeof(VectorNNTPD.TransitService.ArticleIngestion.IncomingSpoolWriterService) + .GetProperty("ArticlePlacementEnabled", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + Assert.NotNull(property); + Assert.False((bool)property.GetValue(null)!); + } + + [Fact] + public void Outstanding_limit_rejects_values_other_than_one_or_two() + { + Assert.Throws(() => + new ArticlePlacementClient(NullLogger.Instance, 64)); + } + + private static ArticlePlacementClient Client( + int maxOutstanding, + ArticleTransferLimits? limits = null, + TimeSpan? ioTimeout = null) => + new( + NullLogger.Instance, + new VatpClientOptions + { + ConnectTimeout = TimeSpan.FromSeconds(2), + TlsHandshakeTimeout = TimeSpan.FromSeconds(2), + IoTimeout = ioTimeout ?? TimeSpan.FromSeconds(2), + }, + limits ?? ArticleTransferLimits.Default, + maxOutstanding) + { + TestTcpConnectHost = "127.0.0.1", + ServerCertificateValidationCallback = static (_, _, _, _) => true, + }; + + private static StorageServerFleetEntry Entry(int port, string fqdn = "cache01.usenet.ninja") => + new(1, fqdn, 10_000, 0, 10_000, DateTimeOffset.UtcNow, port); + + private sealed class StoreServer : IAsyncDisposable + { + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _accept; + private readonly ConcurrentDictionary _streams = new(); + private readonly ConcurrentQueue _streamIds = new(); + private int _outstanding; + private int _peak; + private int _stores; + private int _ends; + private int _dataFrames; + private int _results; + private int _acceptedConnections; + private int _parseFailures; + + private StoreServer( + TcpListener listener, + bool holdResult, + bool duplicateSecond, + bool dropAfterEnd, + bool reject, + int closeAfterAccepted, + bool grantWindows, + int initialCredit, + TimeSpan handshakeDelay) + { + _listener = listener; + HoldResult = holdResult; + DuplicateSecond = duplicateSecond; + DropAfterEnd = dropAfterEnd; + Reject = reject; + CloseAfterAccepted = closeAfterAccepted; + GrantWindows = grantWindows; + InitialCredit = initialCredit; + HandshakeDelay = handshakeDelay; + Port = ((IPEndPoint)listener.LocalEndpoint).Port; + _accept = Task.Run(AcceptLoop); + } + + public int Port { get; } + + public bool HoldResult { get; } + + public bool DuplicateSecond { get; } + + public bool DropAfterEnd { get; } + + public bool Reject { get; } + + public int CloseAfterAccepted { get; } + + public bool GrantWindows { get; } + + public int InitialCredit { get; } + + public TimeSpan HandshakeDelay { get; } + + public int PeakOutstanding => Volatile.Read(ref _peak); + + public int Stores => Volatile.Read(ref _stores); + + public int Ends => Volatile.Read(ref _ends); + + public int DataFrames => Volatile.Read(ref _dataFrames); + + public uint[] StreamIds => _streamIds.ToArray(); + + public int Results => Volatile.Read(ref _results); + + public int AcceptedConnections => Volatile.Read(ref _acceptedConnections); + + public int ParseFailures => Volatile.Read(ref _parseFailures); + + private TaskCompletionSource ResultsReleased { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public static StoreServer Start( + bool holdResult = false, + bool duplicateSecond = false, + bool dropAfterEnd = false, + bool reject = false, + int closeAfterAccepted = 0, + bool grantWindows = true, + int initialCredit = 1024 * 1024, + TimeSpan handshakeDelay = default) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return new StoreServer( + listener, + holdResult, + duplicateSecond, + dropAfterEnd, + reject, + closeAfterAccepted, + grantWindows, + initialCredit, + handshakeDelay); + } + + public void DropConnections() + { + foreach (var stream in _streams.Keys) + { + stream.Close(); + } + } + + public Task WaitForStoresAsync(int count) => WaitForAsync(() => Stores, count, "STORE frames did not arrive."); + + public Task WaitForEndsAsync(int count) => WaitForAsync(() => Ends, count, "END frames did not arrive."); + + public Task WaitForDataAsync(int count) => WaitForAsync(() => DataFrames, count, "DATA frames did not arrive."); + + private static async Task WaitForAsync(Func read, int count, string message) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (read() < count) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException(message); + } + + await Task.Delay(10); + } + } + + public void ReleaseResults() => ResultsReleased.TrySetResult(); + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _listener.Stop(); + try + { + await _accept.WaitAsync(TimeSpan.FromSeconds(2)); + } + catch (Exception ex) when (ex is OperationCanceledException or TimeoutException or SocketException or ObjectDisposedException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private async Task AcceptLoop() + { + while (!_stop.IsCancellationRequested) + { + TcpClient tcp; + try + { + tcp = await _listener.AcceptTcpClientAsync(_stop.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + return; + } + + Interlocked.Increment(ref _acceptedConnections); + _ = HandleAsync(tcp); + } + } + + private async Task HandleAsync(TcpClient tcp) + { + var certificate = VatpTestCertificates.CreateServerCertificate(); + using var tcpDisposable = tcp; + // codeql[cs/local-not-disposed] SslStream ownership transfers to the VATP/TLS session fixture for connection lifetime. + var ssl = new SslStream(tcp.GetStream(), leaveInnerStreamOpen: false); + _streams.TryAdd(ssl, 0); + try + { + if (HandshakeDelay > TimeSpan.Zero) + { + await Task.Delay(HandshakeDelay, _stop.Token); + } + + await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions + { + ServerCertificate = certificate, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }); + var buffer = new byte[64 * 1024]; + var held = new List(); + var credit = new Dictionary(); + var order = new List(); + var pendingResults = new List(); + Task? readTask = null; + while (!_stop.IsCancellationRequested) + { + if (HoldResult && ResultsReleased.Task.IsCompleted) + { + await WritePendingResultsAsync(ssl, pendingResults, order); + } + + var parsed = VatpFrameParser.ParseOneFrame(held.ToArray(), VatpProtocol.DefaultMaxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + Interlocked.Increment(ref _parseFailures); + return; + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } frame) + { + readTask ??= ssl.ReadAsync(buffer, _stop.Token).AsTask(); + if (HoldResult && !ResultsReleased.Task.IsCompleted) + { + await Task.WhenAny(readTask, ResultsReleased.Task); + if (!readTask.IsCompleted) + { + continue; + } + } + + var read = await readTask; + readTask = null; + if (read == 0) + { + return; + } + + for (var i = 0; i < read; i++) + { + held.Add(buffer[i]); + } + + continue; + } + + held.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + if (frame.Header.Type == VatpFrameType.Hello) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeHello(VatpProtocol.DefaultMaxFramePayload)); + continue; + } + + if (frame.Header.Type == VatpFrameType.Store) + { + var now = Interlocked.Increment(ref _outstanding); + int previous; + do + { + previous = Volatile.Read(ref _peak); + if (now <= previous) + { + break; + } + } + while (Interlocked.CompareExchange(ref _peak, now, previous) != previous); + + Interlocked.Increment(ref _stores); + credit[frame.Header.StreamId] = InitialCredit; + _streamIds.Enqueue(frame.Header.StreamId); + order.Add(frame.Header.StreamId); + if (InitialCredit > 0) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)InitialCredit)); + } + + continue; + } + + if (frame.Header.Type == VatpFrameType.Data) + { + var length = frame.Payload.Length; + if (!credit.TryGetValue(frame.Header.StreamId, out var available) || length > available) + { + Interlocked.Increment(ref _parseFailures); + return; + } + + credit[frame.Header.StreamId] = available - length; + Interlocked.Increment(ref _dataFrames); + if (length > 0 && GrantWindows) + { + await WriteAsync(ssl, VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)length)); + credit[frame.Header.StreamId] += length; + } + + continue; + } + + if (frame.Header.Type != VatpFrameType.End) + { + continue; + } + + Interlocked.Increment(ref _ends); + if (DropAfterEnd) + { + return; + } + + if (HoldResult && !ResultsReleased.Task.IsCompleted) + { + pendingResults.Add(frame.Header.StreamId); + continue; + } + + if (await WriteResultAsync(ssl, frame.Header.StreamId, order)) + { + return; + } + } + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or ObjectDisposedException) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + _streams.TryRemove(ssl, out _); + await ssl.DisposeAsync(); + } + } + + private async Task WritePendingResultsAsync(SslStream ssl, List pendingResults, List order) + { + while (pendingResults.Count > 0) + { + var streamId = pendingResults[0]; + pendingResults.RemoveAt(0); + if (await WriteResultAsync(ssl, streamId, order)) + { + return; + } + } + } + + private async Task WriteResultAsync(SslStream ssl, uint streamId, List order) + { + var index = order.IndexOf(streamId); + var outcome = Reject + ? (byte)ArticlePlacementKind.RejectedInvalid + : DuplicateSecond && index == 1 + ? (byte)ArticlePlacementKind.Duplicate + : (byte)ArticlePlacementKind.Accepted; + await WriteAsync(ssl, VatpFrameEncoder.EncodeResult(streamId, outcome)); + Interlocked.Decrement(ref _outstanding); + var sent = Interlocked.Increment(ref _results); + return CloseAfterAccepted > 0 && sent >= CloseAfterAccepted; + } + + private static async Task WriteAsync(Stream stream, VatpFrameEncoder.EncodedFrame frame) + { + if (!frame.Header.IsEmpty) + { + await stream.WriteAsync(frame.Header); + } + + if (!frame.Payload.IsEmpty) + { + await stream.WriteAsync(frame.Payload); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementTests.cs b/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementTests.cs new file mode 100644 index 00000000..7e927920 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Storage/ArticlePlacementTests.cs @@ -0,0 +1,1183 @@ +using System.Buffers; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Reflection; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Tests.Storage; + +public sealed class ArticlePlacementTests +{ + private static readonly DateTimeOffset Now = DateTimeOffset.Parse("2026-09-30T00:00:00Z"); + + [Fact] + public void Selector_PrefersHighestAvailableBytes_ThenServerId_ThenFqdn() + { + var entries = new StorageServerFleetEntry[] + { + Entry("cache02.example", 2, available: 10, port: 563), + Entry("cache09.example", 9, available: 50, port: 563), + Entry("cache03.example", 3, available: 50, port: 563), + }; + Assert.True(StorageServerPlacementSelector.TrySelect(entries, out var selected)); + Assert.Equal(3, selected.ServerId); + + var tiedId = new StorageServerFleetEntry[] + { + Entry("m.example", 4, available: 20, port: 563), + Entry("a.example", 4, available: 20, port: 563), + }; + Assert.True(StorageServerPlacementSelector.TrySelect(tiedId, out var fqdn)); + Assert.Equal("a.example", fqdn.Fqdn); + } + + [Fact] + public void Selector_ExcludesMissingPort_AndEmptySetDoesNotSelect() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 100, port: null), Now); + registry.ApplyAdvertisement(Advertisement("stale.example", 2, 500, port: 563), Now.AddSeconds(-10)); + var active = registry.GetActive(Now); + Assert.Contains(active, static entry => entry.Fqdn == "cache01.example"); + Assert.DoesNotContain(active, static entry => entry.Fqdn == "stale.example"); + Assert.False(StorageServerPlacementSelector.TrySelect(active, out _)); + Assert.False(StorageServerPlacementSelector.TrySelect([], out _)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + public async Task LocalProducer_PlacesOnceAfterPersist(InboundArticleProducer producer) + { + var client = new RecordingPlacement(); + var registry = RegistryWithTarget(); + var article = CanonicalArticleText.CreateQueued("", producer); + await RunAsync(article, client, registry); + Assert.Equal(0, client.Calls); + Assert.False(client.SawPersist); + } + + [Fact] + public async Task BackFiller_StoresOnceOnTheLowestServerId() + { + var client = new RecordingPlacement(); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Fact] + public async Task NoEligibleServer_DoesNotDial() + { + var client = new RecordingPlacement { FailIfCalled = true }; + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 100, port: null), Now); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, registry, time: Now); + Assert.Equal(0, client.Calls); + } + + [Fact] + public async Task PlacementFailure_DoesNotRequeue() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Conflict), + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunAsync(article, client, RegistryWithTarget()); + Assert.Equal(0, client.Calls); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + public async Task LocalProducer_PlacesOnceAndDoesNotSelectASecondTarget(InboundArticleProducer producer) + { + var client = new RecordingPlacement(); + var article = CanonicalArticleText.CreateQueued("", producer); + var logs = new ListLogger(); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Fact] + public async Task BackFiller_Duplicate_IsOneStoreAndDoesNotReplicate() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Duplicate), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2621, logs.EventIds); + } + + [Fact] + public async Task BackFiller_Conflict_LogsAndDoesNotStoreAgain() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Conflict), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2622, logs.EventIds); + } + + [Theory] + [InlineData(ArticlePlacementKind.RejectedCapacity, 2623)] + [InlineData(ArticlePlacementKind.RejectedInvalid, 2625)] + public async Task BackFiller_Rejection_DoesNotTryAnotherServer(ArticlePlacementKind kind, int eventId) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(kind), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(eventId, logs.EventIds); + } + + [Fact] + public async Task BackFiller_TransportFailure_DoesNotTryAnotherServer() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.TransportFailure, "IOException"), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunObservedAsync(article, client, DivergentTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2627, logs.EventIds); + } + + [Fact] + public async Task PeerProducer_PlacesOnceOnTheHighestFreeSpace() + { + var client = new RecordingPlacement(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunObservedAsync(article, client, DivergentTargets()); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + } + + [Fact] + public async Task BackFiller_ConcurrentArticles_DoNotShareStoreState() + { + var client = new ConcurrentPlacement(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + await RunManyAsync([first, second], client, DivergentTargets(), workers: 2); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.Empty(client.Records); + Assert.NotEqual(first.Record.ArtId, second.Record.ArtId); + } + + [Fact] + public async Task BackFiller_SameArticleConcurrently_StoresTwice() + { + var client = new ConcurrentPlacement(); + var first = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + var second = CanonicalArticleText.CreateQueued("", InboundArticleProducer.BackFiller); + Assert.Equal(first.Record.ArtId, second.Record.ArtId); + await RunManyAsync([first, second], client, DivergentTargets(), workers: 2); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Records); + Assert.Empty(client.Targets); + } + + [Fact] + public async Task NoSecondEligible_LeavesSingleCopy() + { + var client = new RecordingPlacement(); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, RegistryWithTarget(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2620, logs.EventIds); + } + + [Theory] + [InlineData(ArticlePlacementKind.Conflict, 2622)] + [InlineData(ArticlePlacementKind.RejectedCapacity, 2623)] + [InlineData(ArticlePlacementKind.RejectedInvalid, 2625)] + [InlineData(ArticlePlacementKind.TransportFailure, 2627)] + [InlineData(ArticlePlacementKind.AcknowledgementNotObserved, 2629)] + public async Task FirstFailure_DoesNotAttemptReplica(ArticlePlacementKind kind, int eventId) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(kind, kind == ArticlePlacementKind.TransportFailure ? "IOException" : null), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.Post); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(eventId, logs.EventIds); + } + + [Fact] + public async Task CancellationBeforeFirstAcceptance_DoesNotAttemptReplica() + { + using var cts = new CancellationTokenSource(); + var client = new RecordingPlacement + { + CancelAt = CancelPoint.FirstCallReturnsCancelled, + Cancel = cts, + }; + var logs = new ListLogger(); + await PlaceDirectAsync( + CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis), + client, + TwoTargets(), + cts.Token, + logs); + Assert.Equal(1, client.Calls); + Assert.Equal(ArticlePlacementKind.Cancelled, client.Outcomes[0].Kind); + Assert.Equal("cache01.example", client.Targets[0].Fqdn); + Assert.Contains(2628, logs.EventIds); + Assert.Single(client.Targets); + Assert.DoesNotContain(logs.EventIds, static id => id >= 2630); + } + + [Fact] + public async Task DuplicatePrimary_CompletesAfterOnePlacement() + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.Duplicate), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + await RunAsync(article, client, TwoTargets(), logs); + Assert.Equal(0, client.Calls); + Assert.Empty(client.Targets); + Assert.DoesNotContain(2621, logs.EventIds); + } + + [Fact] + public async Task CancellationAfterFirstAcceptance_DoesNotStartReplica() + { + using var cts = new CancellationTokenSource(); + var client = new RecordingPlacement + { + CancelAt = CancelPoint.AfterFirstAccepted, + Cancel = cts, + }; + var logs = new ListLogger(); + await PlaceDirectAsync( + CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis), + client, + TwoTargets(), + cts.Token, + logs); + Assert.Equal(1, client.Calls); + Assert.Equal(ArticlePlacementKind.Accepted, client.Outcomes[0].Kind); + Assert.Equal("cache01.example", client.Targets[0].Fqdn); + Assert.Contains(2620, logs.EventIds); + Assert.Single(client.Targets); + Assert.DoesNotContain(logs.EventIds, static id => id >= 2630); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.IHave)] + [InlineData(InboundArticleProducer.Post)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_RetriesOnceOnTheSameTargetAndPayload(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(TimeSpan.FromSeconds(5), delay.Requested); + Assert.Equal(1, delay.Calls); + AssertSamePlacement(article, client, producer); + Assert.Equal(ArticlePlacementKind.Accepted, client.Outcomes[1].Kind); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=Accepted"); + Assert.Equal(2, logs.EventIds.Count(static id => id is 2624 or 2620)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_SecondDuplicate_SettlesOnTheSameTarget(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.Duplicate)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(1, delay.Calls); + Assert.Equal(TimeSpan.FromSeconds(5), delay.Requested); + AssertSamePlacement(article, client, producer); + Assert.Equal(ArticlePlacementKind.Duplicate, client.Outcomes[1].Kind); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=Duplicate"); + Assert.Contains(2621, logs.EventIds); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task RejectedPressure_SecondPressure_IsTerminal(InboundArticleProducer producer) + { + var delay = new ImmediatePressureDelay(); + var client = SequenceClient( + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure)); + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(2, client.Calls); + Assert.Equal(1, delay.Calls); + AssertSamePlacement(article, client, producer); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + AssertAttempt(logs.Messages, 2, "outcome=RejectedPressure"); + Assert.Equal(2, logs.Messages.Count(static message => message.Contains("outcome=RejectedPressure", StringComparison.Ordinal))); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=3", StringComparison.Ordinal)); + } + + [Theory] + [MemberData(nameof(TerminalPlacementOutcomes))] + public async Task TerminalOutcomes_AreNotRetried(InboundArticleProducer producer, ArticlePlacementKind kind) + { + var delay = new ImmediatePressureDelay(); + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult( + kind, + kind == ArticlePlacementKind.TransportFailure ? "IOException" : null), + }; + var logs = new ListLogger(); + var article = CanonicalArticleText.CreateQueued($"", producer); + await InvokePlacementAsync(article, client, delay.Delay, CancellationToken.None, logs); + + Assert.Equal(1, client.Calls); + Assert.Equal(0, delay.Calls); + AssertOriginalTarget(producer, client.Targets[0]); + AssertSameArtData(article.Record, client.Records[0]); + Assert.Contains(logs.Messages, message => message.Contains("attempt=1", StringComparison.Ordinal)); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=2", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(InboundArticleProducer.TakeThis)] + [InlineData(InboundArticleProducer.BackFiller)] + public async Task CancellationDuringPressureDelay_DoesNotStartAttemptTwo(InboundArticleProducer producer) + { + var client = new RecordingPlacement + { + Result = new ArticlePlacementResult(ArticlePlacementKind.RejectedPressure), + }; + var logs = new ListLogger(); + using var cts = new CancellationTokenSource(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var article = CanonicalArticleText.CreateQueued($"", producer); + var pending = InvokePlacementAsync( + article, + client, + async (delay, token) => + { + Assert.Equal(TimeSpan.FromSeconds(5), delay); + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token).ConfigureAwait(false); + }, + cts.Token, + logs); + + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + await pending.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(1, client.Calls); + AssertOriginalTarget(producer, client.Targets[0]); + AssertSameArtData(article.Record, client.Records[0]); + AssertAttempt(logs.Messages, 1, "outcome=RejectedPressure"); + Assert.Contains( + logs.Messages, + static message => message.Contains("placement cancelled", StringComparison.Ordinal) + && message.Contains("attempt=1", StringComparison.Ordinal)); + Assert.DoesNotContain(logs.Messages, static message => message.Contains("attempt=2", StringComparison.Ordinal)); + } + + public static IEnumerable TerminalPlacementOutcomes() + { + var producers = new[] { InboundArticleProducer.TakeThis, InboundArticleProducer.BackFiller }; + var kinds = new[] + { + ArticlePlacementKind.RejectedCapacity, + ArticlePlacementKind.Conflict, + ArticlePlacementKind.RejectedInvalid, + ArticlePlacementKind.TransportFailure, + ArticlePlacementKind.AcknowledgementNotObserved, + ArticlePlacementKind.Cancelled, + }; + foreach (var producer in producers) + { + foreach (var kind in kinds) + { + yield return [producer, kind]; + } + } + } + + [Fact] + public void Placement_DoesNotReferenceLookupOrReadPool() + { + var placementFields = typeof(ArticlePlacementClient).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(placementFields, static field => field.FieldType.Name.Contains("Lookup", StringComparison.Ordinal)); + var writerFields = typeof(IncomingSpoolWriterService).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(writerFields, static field => field.FieldType.Name.Contains("Lookup", StringComparison.Ordinal)); + Assert.DoesNotContain( + typeof(StorageArticleLookupService).GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic), + static method => method.Name.Contains("Replica", StringComparison.Ordinal) + || method.Name.Contains("Primary", StringComparison.Ordinal)); + } + + [Fact] + public async Task PlacementClient_SocketException_ReportsErrorCodeHResultAndMessage() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + + var client = new ArticlePlacementClient(NullLogger.Instance) + { + TestTcpConnectHost = "127.0.0.1", + }; + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis); + var result = await client.PlaceAsync(article.Record, Entry("cache01.usenet.ninja", 1, 100, port), CancellationToken.None); + + Assert.Equal(ArticlePlacementKind.TransportFailure, result.Kind); + Assert.NotNull(result.Failure); + Assert.StartsWith("SocketException: SocketErrorCode=", result.Failure, StringComparison.Ordinal); + Assert.Contains("SocketErrorCode=ConnectionRefused", result.Failure, StringComparison.Ordinal); + Assert.Contains(" HResult=", result.Failure, StringComparison.Ordinal); + Assert.Contains(" Message=", result.Failure, StringComparison.Ordinal); + } + + [Fact] + public void PlacementClient_DoesNotUseReadConnectionPool() + { + var fields = typeof(ArticlePlacementClient).GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.DoesNotContain(fields, static field => field.FieldType == typeof(VatpConnectionPool)); + Assert.False(typeof(VatpArticleClient).IsAssignableTo(typeof(IArticlePlacementClient))); + } + + [Fact] + public async Task Transfer_ResultAccepted_AndWindowPacing() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + var limits = new ArticleTransferLimits + { + InitialStreamWindowBytes = 32, + MaxStreamCreditBytes = 1024 * 1024, + DefaultMaxFramePayload = 64, + }; + Assert.True(article.ArtSize > 32); + await using var link = new Link(); + var server = Task.Run(() => ScriptedServer.RunAsync(link.Server, result: (byte)ArticlePlacementKind.Accepted)); + var result = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + limits, + TimeSpan.FromSeconds(5), + CancellationToken.None); + Assert.Equal(ArticlePlacementKind.Accepted, result.Kind); + Assert.True(await server.WaitAsync(TimeSpan.FromSeconds(5))); + } + + [Fact] + public async Task Transfer_DropAfterEnd_IsAcknowledgementNotObserved() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.IHave).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + await using var link = new Link(); + var server = Task.Run(() => ScriptedServer.RunAsync(link.Server, result: null)); + var placed = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + ArticleTransferLimits.Default, + TimeSpan.FromSeconds(5), + CancellationToken.None); + Assert.Equal(ArticlePlacementKind.AcknowledgementNotObserved, placed.Kind); + Assert.False(placed.Succeeded); + await server.WaitAsync(TimeSpan.FromSeconds(5)); + } + + [Fact] + public async Task Transfer_CancelBeforeSend_DoesNotReportSuccess() + { + var article = CanonicalArticleText.CreateQueued("", InboundArticleProducer.TakeThis).Record; + Assert.True(ArticlePlacementMeta.TryFromRecord(article, out var meta)); + await using var link = new Link(); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + var placed = await VatpPlacementTransfer.PlaceAsync( + link.Client, + article, + meta, + ArticleTransferLimits.Default, + TimeSpan.FromSeconds(5), + cts.Token); + Assert.Equal(ArticlePlacementKind.Cancelled, placed.Kind); + Assert.False(placed.Succeeded); + } + + private static Task RunAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs = null, + DateTimeOffset? time = null) => + RunCoreAsync(article, client, registry, logs, time); + + private static async Task PlaceDirectAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + CancellationToken cancellationToken, + ListLogger? logs = null) + { + var writer = CreateWriter(client, registry, logs, time: null); + var place = typeof(IncomingSpoolWriterService).GetMethod( + "PlaceAfterPersistAsync", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(place); + try + { + var pending = (Task)place.Invoke(writer, [article, cancellationToken])!; + await pending.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static IncomingSpoolWriterService CreateWriter( + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs, + DateTimeOffset? time) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + return new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(time ?? Now), + placement: client, + placementRegistry: registry); + } + + private static async Task RunObservedAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs = null) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task RunManyAsync( + InboundArticle[] articles, + ConcurrentPlacement client, + StorageServerRegistry registry, + int workers) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions(workers)), + NullLogger.Instance, + timeProvider: new FixedTime(Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + foreach (var article in articles) + { + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + } + + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task RunCoreAsync( + InboundArticle article, + RecordingPlacement client, + StorageServerRegistry registry, + ListLogger? logs, + DateTimeOffset? time) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + var writer = new IncomingSpoolWriterService( + queue, + Options.Create(PlacementOptions()), + (ILogger?)logs ?? NullLogger.Instance, + timeProvider: new FixedTime(time ?? Now), + placement: client, + placementRegistry: registry); + await writer.StartAsync(CancellationToken.None); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(article, CancellationToken.None)); + queue.Complete(); + await writer.StopAsync(CancellationToken.None); + } + + private static async Task InvokePlacementAsync( + InboundArticle article, + RecordingPlacement client, + Func delay, + CancellationToken cancellationToken, + ListLogger? logs = null) + { + var writer = CreateWriter( + client, + TargetsFor(article.Producer), + logs, + time: null); + writer.PressureRetryDelay = delay; + var methodName = article.Producer == InboundArticleProducer.BackFiller + ? "StoreBackFillerAsync" + : "PlaceAfterPersistAsync"; + var place = typeof(IncomingSpoolWriterService).GetMethod( + methodName, + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(place); + try + { + var pending = (Task)place.Invoke(writer, [article, cancellationToken])!; + await pending; + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static RecordingPlacement SequenceClient(params ArticlePlacementResult[] sequence) => + new() + { + Sequence = sequence, + ExpectedCalls = sequence.Length, + }; + + private static StorageServerRegistry TargetsFor(InboundArticleProducer producer) => + producer == InboundArticleProducer.BackFiller ? DivergentTargets() : TwoTargets(); + + private static void AssertOriginalTarget(InboundArticleProducer producer, StorageServerFleetEntry target) + { + if (producer == InboundArticleProducer.BackFiller) + { + Assert.Equal(2, target.ServerId); + Assert.Equal("lowid.example", target.Fqdn); + return; + } + + Assert.Equal(1, target.ServerId); + Assert.Equal("cache01.example", target.Fqdn); + } + + private static void AssertSamePlacement( + InboundArticle article, + RecordingPlacement client, + InboundArticleProducer producer) + { + Assert.Equal(2, client.Calls); + Assert.Equal(2, client.Targets.Count); + Assert.Equal(2, client.Records.Count); + AssertOriginalTarget(producer, client.Targets[0]); + Assert.Equal(client.Targets[0].ServerId, client.Targets[1].ServerId); + Assert.Equal(client.Targets[0].Fqdn, client.Targets[1].Fqdn); + Assert.Equal(client.Targets[0].VatpPort, client.Targets[1].VatpPort); + Assert.Equal(article.Record.ArtId, client.Records[0].ArtId); + Assert.Equal(article.Record.ArtId, client.Records[1].ArtId); + AssertSameArtData(article.Record, client.Records[0]); + AssertSameArtData(article.Record, client.Records[1]); + AssertSameArtData(client.Records[0], client.Records[1]); + } + + private static void AssertAttempt(IReadOnlyList messages, int attempt, string outcome) => + Assert.Contains( + messages, + message => message.Contains($"attempt={attempt}", StringComparison.Ordinal) + && message.Contains(outcome, StringComparison.Ordinal)); + + private static void AssertSameArtData(ArticleRecord expected, ArticleRecord actual) + { + Assert.True(MemoryMarshal.TryGetArray(expected.ArtData, out var expectedArray)); + Assert.True(MemoryMarshal.TryGetArray(actual.ArtData, out var actualArray)); + Assert.Same(expectedArray.Array, actualArray.Array); + Assert.Equal(expectedArray.Offset, actualArray.Offset); + Assert.Equal(expectedArray.Count, actualArray.Count); + } + + private static NntpdOptions PlacementOptions(int workers = 1) => + new() + { + ArticleIngestion = new ArticleIngestionOptions + { + MinWorkers = workers, + MaxWorkers = workers, + ScaleIntervalSeconds = 3600, + OverviewDbMinPublisherWorkers = 1, + OverviewDbMaxPublisherWorkers = 1, + }, + }; + + private static StorageServerRegistry TwoTargets() + { + var registry = RegistryWithTarget(); + registry.ApplyAdvertisement(Advertisement("cache02.example", 2, 100, 564), Now); + return registry; + } + + private static StorageServerRegistry DivergentTargets() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("lowid.example", 2, 10, 564), Now); + registry.ApplyAdvertisement(Advertisement("highspace.example", 9, 5000, 563), Now); + return registry; + } + + private static StorageServerRegistry RegistryWithTarget() + { + var registry = new StorageServerRegistry(); + registry.ApplyAdvertisement(Advertisement("cache01.example", 1, 1000, 563), Now); + return registry; + } + + private static StorageServerAdvertisement Advertisement(string fqdn, int serverId, long available, int? port) => + new(1, serverId, fqdn, 10_000, 10_000 - available, available, Now, port); + + private static StorageServerFleetEntry Entry(string fqdn, int serverId, long available, int? port) => + new(serverId, fqdn, 10_000, 10_000 - available, available, Now, port); + + private sealed class FixedTime(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } + + private enum CancelPoint + { + None, + FirstCallReturnsCancelled, + AfterFirstAccepted, + } + + private sealed class ConcurrentPlacement : IArticlePlacementClient + { + private readonly Barrier _barrier = new(2); + private readonly List _targets = []; + private readonly List _records = []; + private readonly object _gate = new(); + + public int Calls { get; private set; } + + public IReadOnlyList Targets + { + get + { + lock (_gate) + { + return [.. _targets]; + } + } + } + + public IReadOnlyList Records + { + get + { + lock (_gate) + { + return [.. _records]; + } + } + } + + public TaskCompletionSource BothStored { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + lock (_gate) + { + Calls++; + _targets.Add(target); + _records.Add(record); + } + + _barrier.SignalAndWait(cancellationToken); + BothStored.TrySetResult(); + return ValueTask.FromResult(new ArticlePlacementResult(ArticlePlacementKind.Accepted)); + } + } + + private sealed class RecordingPlacement : IArticlePlacementClient + { + private readonly List _targets = []; + private readonly List _records = []; + private readonly List _outcomes = []; + + public int Calls { get; private set; } + + public bool SawPersist { get; private set; } + + public bool FailIfCalled { get; init; } + + public ArticleRecord Last { get; private set; } + + public ArticlePlacementResult Result { get; init; } = new(ArticlePlacementKind.Accepted); + + public ArticlePlacementResult[]? Sequence { get; init; } + + public int ExpectedCalls { get; init; } = 1; + + public CancelPoint CancelAt { get; init; } + + public CancellationTokenSource? Cancel { get; init; } + + public Action? BeforeStore { get; init; } + + public IReadOnlyList Targets => _targets; + + public IReadOnlyList Records => _records; + + public IReadOnlyList Outcomes => _outcomes; + + public TaskCompletionSource Finished { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask PlaceAsync( + ArticleRecord record, + StorageServerFleetEntry target, + CancellationToken cancellationToken) + { + if (FailIfCalled) + { + throw new InvalidOperationException("Placement dialed without an eligible server."); + } + + var call = ++Calls; + SawPersist = true; + Last = record; + _targets.Add(target); + _records.Add(record); + BeforeStore?.Invoke(call, target); + + ArticlePlacementResult outcome; + if (CancelAt == CancelPoint.FirstCallReturnsCancelled && call == 1) + { + Cancel!.Cancel(); + outcome = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + else if (CancelAt == CancelPoint.AfterFirstAccepted && call == 1) + { + outcome = new ArticlePlacementResult(ArticlePlacementKind.Accepted); + Cancel!.Cancel(); + } + else if (cancellationToken.IsCancellationRequested) + { + outcome = new ArticlePlacementResult(ArticlePlacementKind.Cancelled); + } + else + { + outcome = Sequence is { } sequence && call - 1 < sequence.Length + ? sequence[call - 1] + : Result; + } + + _outcomes.Add(outcome); + if (call >= ExpectedCalls) + { + Finished.TrySetResult(); + } + + return ValueTask.FromResult(outcome); + } + } + + private sealed class ImmediatePressureDelay + { + public int Calls { get; private set; } + + public TimeSpan Requested { get; private set; } + + public Task Delay(TimeSpan delay, CancellationToken cancellationToken) + { + Calls++; + Requested = delay; + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + } + + private sealed class ListLogger : ILogger + { + public List EventIds { get; } = []; + + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + EventIds.Add(eventId.Id); + Messages.Add(formatter(state, exception)); + } + } + + private sealed class Link : IAsyncDisposable + { + private readonly Pipe _toServer = new(); + private readonly Pipe _toClient = new(); + + public Link() + { + Client = new PipeStream(_toClient.Reader, _toServer.Writer); + Server = new PipeStream(_toServer.Reader, _toClient.Writer); + } + + public Stream Client { get; } + + public Stream Server { get; } + + public async ValueTask DisposeAsync() + { + await Client.DisposeAsync(); + await Server.DisposeAsync(); + } + } + + private sealed class PipeStream(PipeReader reader, PipeWriter writer) : Stream + { + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + if (result.Buffer.IsEmpty && result.IsCompleted) + { + return 0; + } + + var length = (int)Math.Min(buffer.Length, result.Buffer.Length); + var written = 0; + foreach (var segment in result.Buffer.Slice(0, length)) + { + segment.Span.CopyTo(buffer.Span[written..]); + written += segment.Length; + } + + reader.AdvanceTo(result.Buffer.GetPosition(length)); + return length; + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + var copied = buffer.ToArray(); + await writer.WriteAsync(copied, cancellationToken).ConfigureAwait(false); + var flushed = await writer.FlushAsync(cancellationToken).ConfigureAwait(false); + if (flushed.IsCompleted) + { + throw new IOException("peer closed"); + } + } + + public override async ValueTask DisposeAsync() + { + try + { + await writer.CompleteAsync().ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + + try + { + await reader.CompleteAsync().ConfigureAwait(false); + } + catch (InvalidOperationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + } + + private static class ScriptedServer + { + public static async Task RunAsync(Stream stream, byte? result) + { + var pending = new List(); + var buffer = new byte[8192]; + var windows = 0; + while (true) + { + var read = await stream.ReadAsync(buffer); + if (read == 0) + { + return windows > 0 || result is null; + } + + pending.AddRange(buffer.AsSpan(0, read).ToArray()); + while (pending.Count > 0) + { + var parsed = VatpFrameParser.ParseOneFrame(pending.ToArray(), VatpProtocol.DefaultMaxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status != VatpFrameParseStatus.Success || parsed.Frame is not { } frame) + { + return false; + } + + pending.RemoveRange(0, checked((int)parsed.ConsumedBytes)); + if (frame.Header.Type == VatpFrameType.Hello) + { + var hello = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeHello(VatpProtocol.DefaultMaxFramePayload)); + await stream.WriteAsync(hello); + } + else if (frame.Header.Type == VatpFrameType.Meta + && TryDecodeMeta(frame.Payload, out var meta) + && meta.ArtSize > 0) + { + var grant = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, (uint)meta.ArtSize)); + await stream.WriteAsync(grant); + } + else if (frame.Header.Type == VatpFrameType.Data && frame.Payload.Length > 0) + { + windows++; + var window = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeWindow(frame.Header.StreamId, checked((uint)frame.Payload.Length))); + await stream.WriteAsync(window); + } + else if (frame.Header.Type == VatpFrameType.End) + { + if (result is byte outcome) + { + var encoded = VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeResult(frame.Header.StreamId, outcome)); + await stream.WriteAsync(encoded); + } + else + { + await stream.DisposeAsync(); + } + + return true; + } + } + } + } + + private static bool TryDecodeMeta(ReadOnlySequence payload, out ArticleCanonicalTransferMeta meta) + { + var bytes = new byte[payload.Length]; + payload.CopyTo(bytes); + return VatpMetaCodec.TryDecode(bytes, out meta, out _); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Storage/StorageArticleLookupServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageArticleLookupServiceTests.cs new file mode 100644 index 00000000..24c842f2 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageArticleLookupServiceTests.cs @@ -0,0 +1,498 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Storage; + +public sealed class StorageArticleLookupServiceTests +{ + private static readonly ArticleId ArticleX = ArticleId.FromMessageId(""u8); + + [Fact] + public async Task MultipleOwners_FirstPositiveCompletes_AndDuplicateIsIgnored() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, publication.Exchange); + Assert.Equal(string.Empty, publication.RoutingKey); + Assert.Equal(CacheFleetTopology.LookupExpirationMilliseconds, publication.Expiration); + Assert.NotNull(service.CurrentReplyTo); + Assert.Equal(service.CurrentReplyTo, publication.ReplyTo); + + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + Assert.NotNull(request); + + var first = CreateResponse(request!, 2, "cache02.usenet.ninja"); + var second = CreateResponse(request!, 3, "cache03.usenet.ninja"); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(second)); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.Equal("cache02.usenet.ninja", result.Fqdn); + Assert.Equal(2, result.ServerId); + Assert.Equal(1, service.OutstandingCorrelations); + Assert.NotNull(result.Alternates); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 4, "cache04.usenet.ninja"))); + + var alternate = await result.Alternates!.WaitForAlternateAsync(CancellationToken.None); + Assert.True(alternate.HasValue); + var alternateCopy = alternate.GetValueOrDefault(); + Assert.Equal(3, alternateCopy.ServerId); + Assert.Equal("cache03.usenet.ninja", alternateCopy.Fqdn); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 4, "cache04.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task TwoPresentCopies_AreRetainedByExistingFanout() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + Assert.NotNull(request); + + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 2, "cache02.usenet.ninja"))); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 3, "cache03.usenet.ninja"))); + + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.Equal(2, result.ServerId); + Assert.Equal("cache02.usenet.ninja", result.Fqdn); + Assert.NotNull(result.Alternates); + var alternate = await result.Alternates!.WaitForAlternateAsync(CancellationToken.None); + Assert.True(alternate.HasValue); + var alternateCopy = alternate.GetValueOrDefault(); + Assert.Equal(3, alternateCopy.ServerId); + Assert.Equal("cache03.usenet.ninja", alternateCopy.Fqdn); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task NobodyOwns_TimesOutCleanly() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + _ = await WaitForPublicationAsync(factory); + for (var attempt = 0; attempt < 3 && !lookupTask.IsCompleted; attempt++) + { + time.Advance(CacheFleetTopology.LookupTimeout); + await Task.Yield(); + } + + var result = await lookupTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task LateResponse_AfterDeadline_IsIgnored() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 1, "cache01.usenet.ninja"))); + var result = await lookupTask; + Assert.Equal(StorageArticleLookupOutcome.Found, result.Outcome); + Assert.NotNull(result.Alternates); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates!.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 2, "cache02.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task AlternateWait_CompletesWhenOriginalWindowClosesWithoutSecondCandidate() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync( + publication.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1(CreateResponse(request!, 1, "cache01.usenet.ninja"))); + var result = await lookupTask; + Assert.NotNull(result.Alternates); + + var alternateTask = result.Alternates!.WaitForAlternateAsync(CancellationToken.None).AsTask(); + Assert.False(alternateTask.IsCompleted); + time.Advance(CacheFleetTopology.LookupTimeout); + var alternate = await alternateTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Null(alternate); + Assert.Equal(0, service.OutstandingCorrelations); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 2, "cache02.usenet.ninja"))); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task ConcurrentLookups_CorrelateIndependently() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance); + var articleA = ArticleId.FromMessageId(""u8); + var articleB = ArticleId.FromMessageId(""u8); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupA = service.LookupAsync(articleA, CancellationToken.None); + var lookupB = service.LookupAsync(articleB, CancellationToken.None); + var publications = await WaitForPublicationCountAsync(factory, 2); + var requestA = publications.Single(p => + StorageArticleLookupWireProtocol.TryParseRequestV1(p.Body, out var r, out _) + && r!.ArticleId == articleA); + var requestB = publications.Single(p => + StorageArticleLookupWireProtocol.TryParseRequestV1(p.Body, out var r, out _) + && r!.ArticleId == articleB); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(requestA.Body, out var parsedA, out _)); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(requestB.Body, out var parsedB, out _)); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + + await consume.DeliverAsync( + requestB.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + CreateResponse(parsedB!, 3, "cache03.usenet.ninja"))); + await consume.DeliverAsync( + requestA.CorrelationId, + StorageArticleLookupWireProtocol.SerializeResponseV1( + CreateResponse(parsedA!, 1, "cache01.usenet.ninja"))); + + var resultA = await lookupA; + var resultB = await lookupB; + Assert.Equal("cache01.usenet.ninja", resultA.Fqdn); + Assert.Equal("cache03.usenet.ninja", resultB.Fqdn); + Assert.NotEqual(resultA.RequestId, resultB.RequestId); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task DuplicateServerId_AndSameEndpoint_AreNotSecondCandidates() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time); + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var first = CreateResponse(request!, 2, "cache02.usenet.ninja"); + var consume = factory.LastConnection!.RpcChannels.First(static c => c.ConsumedQueue is not null); + await consume.DeliverAsync(publication.CorrelationId, StorageArticleLookupWireProtocol.SerializeResponseV1(first)); + var result = await lookupTask; + + var duplicateServer = new StorageArticleLookupResponse( + 1, + request!.RequestId, + 2, + "cache02.usenet.ninja", + request.ArticleId, + 1192); + var sameEndpoint = new StorageArticleLookupResponse( + 1, + request.RequestId, + 9, + first.Fqdn, + request.ArticleId, + first.VatpPort); + Assert.False(service.TryDispatchResponse(publication.CorrelationId, duplicateServer)); + Assert.False(service.TryDispatchResponse(publication.CorrelationId, sameEndpoint)); + + time.Advance(CacheFleetTopology.LookupTimeout); + await result.Alternates!.WhenClosed.WaitAsync(TimeSpan.FromSeconds(2)); + var alternate = await result.Alternates.WaitForAlternateAsync(CancellationToken.None); + Assert.Null(alternate); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task InvalidCandidateResponses_AreIgnoredUntilAValidOneArrives() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + new FakeTimeProvider()); + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.True(StorageArticleLookupWireProtocol.TryParseRequestV1(publication.Body, out var request, out _)); + var otherArticle = ArticleId.FromMessageId(""u8); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 0, "cache00.usenet.ninja"))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request!, 256, "cache256.usenet.ninja"))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + Guid.NewGuid(), + 4, + "cache04.usenet.ninja", + request!.ArticleId, + 1191))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + request.RequestId, + 5, + "cache05.usenet.ninja", + otherArticle, + 1191))); + Assert.False(service.TryDispatchResponse( + publication.CorrelationId, + new StorageArticleLookupResponse( + 1, + request.RequestId, + 6, + "cache06.usenet.ninja", + request.ArticleId, + 0))); + Assert.False(lookupTask.IsCompleted); + + Assert.True(service.TryDispatchResponse( + publication.CorrelationId, + CreateResponse(request, 7, "cache07.usenet.ninja"))); + var result = await lookupTask; + Assert.Equal(7, result.ServerId); + Assert.Equal(request.RequestId, result.RequestId); + + await service.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task EmptyRegistry_DoesNotPublish_AndReturnsExistingMiss() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + storageServers: new StorageServerRegistry()); + + var result = await service.LookupAsync(ArticleX, CancellationToken.None); + + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Equal("Storage article lookup timed out with no positive response.", result.Error); + Assert.Null(factory.LastConnection); + Assert.Null(service.CurrentReplyTo); + } + + [Fact] + public async Task RegisteredStorageServer_StillPublishesFleetLookup() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var time = new FakeTimeProvider(); + var registry = new StorageServerRegistry(); + var now = DateTimeOffset.Parse("2026-10-04T12:00:00Z"); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 1000, 100, 900, now, 1191), + now); + var service = new StorageArticleLookupService( + rabbit, + Options.Create(CreateOptions()), + NullLogger.Instance, + time, + registry); + + await rabbit.StartAsync(CancellationToken.None); + await service.StartAsync(CancellationToken.None); + + var lookupTask = service.LookupAsync(ArticleX, CancellationToken.None); + var publication = await WaitForPublicationAsync(factory); + Assert.Equal(CacheFleetTopology.RequestsExchangeName, publication.Exchange); + Assert.Equal(string.Empty, publication.RoutingKey); + + for (var attempt = 0; attempt < 3 && !lookupTask.IsCompleted; attempt++) + { + time.Advance(CacheFleetTopology.LookupTimeout); + await Task.Yield(); + } + + var result = await lookupTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(StorageArticleLookupOutcome.NotFound, result.Outcome); + Assert.Equal("Storage article lookup timed out with no positive response.", result.Error); + await service.StopAsync(CancellationToken.None); + } + + private static StorageArticleLookupResponse CreateResponse( + StorageArticleLookupRequest request, + int serverId, + string fqdn) => + new( + 1, + request.RequestId, + serverId, + fqdn, + request.ArticleId, + 1191); + + private static RabbitMqService CreateRabbitMq(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static NntpdOptions CreateOptions() + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = 1; + return options; + } + + private static async Task WaitForPublicationAsync(FakeRabbitMqConnectionFactory factory) + { + var publications = await WaitForPublicationCountAsync(factory, 1); + return publications[0]; + } + + private static async Task> WaitForPublicationCountAsync( + FakeRabbitMqConnectionFactory factory, + int count) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2); + while (DateTime.UtcNow < deadline) + { + var connection = factory.LastConnection; + if (connection is not null) + { + var publications = connection.RpcChannels + .SelectMany(static channel => channel.Publications) + .Where(static p => p.Exchange == CacheFleetTopology.RequestsExchangeName) + .ToList(); + if (publications.Count >= count) + { + return publications; + } + } + + await Task.Delay(10); + } + + throw new TimeoutException($"Expected {count} cache.requests publications."); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs new file mode 100644 index 00000000..0d4190ac --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerFleetConsumerServiceTests.cs @@ -0,0 +1,200 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.Storage; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.RabbitMq; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Storage; + +public sealed class StorageServerFleetConsumerServiceTests +{ + [Fact] + public async Task StartAsync_DeclaresEphemeralQueue_BoundToBroadcast() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var options = Options.Create(CreateNntpdOptions(1)); + var consumer = new StorageServerFleetConsumerService( + rabbit, + options, + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + + var connection = factory.LastConnection!; + var channel = Assert.Single(connection.ManualAckChannels); + Assert.Equal("cache.nntpd01.usenet.ninja", consumer.CurrentQueueName); + Assert.Equal("cache.nntpd01.usenet.ninja", channel.ConsumedQueue); + Assert.Contains( + channel.Exchanges, + static e => e.Name == CacheBroadcastTopology.ExchangeName && e.Type == "fanout" && e.Durable); + var queue = Assert.Single(channel.Queues); + Assert.Equal("cache.nntpd01.usenet.ninja", queue.Name); + Assert.False(queue.Durable); + Assert.True(queue.Exclusive); + Assert.True(queue.AutoDelete); + var binding = Assert.Single(channel.Bindings); + Assert.Equal(queue.Name, binding.Queue); + Assert.Equal(CacheBroadcastTopology.ExchangeName, binding.Exchange); + + await consumer.StopAsync(CancellationToken.None); + Assert.Equal(1, channel.DisposeCount); + Assert.Null(consumer.CurrentQueueName); + } + + [Fact] + public async Task IndependentNntpdInstances_UseDistinctQueues() + { + var factoryA = new FakeRabbitMqConnectionFactory(); + var factoryB = new FakeRabbitMqConnectionFactory(); + await using var rabbitA = CreateRabbitMq(factoryA); + await using var rabbitB = CreateRabbitMq(factoryB); + var consumerA = new StorageServerFleetConsumerService( + rabbitA, + Options.Create(CreateNntpdOptions(1)), + new StorageServerRegistry(), + NullLogger.Instance); + var consumerB = new StorageServerFleetConsumerService( + rabbitB, + Options.Create(CreateNntpdOptions(2)), + new StorageServerRegistry(), + NullLogger.Instance); + + await rabbitA.StartAsync(CancellationToken.None); + await rabbitB.StartAsync(CancellationToken.None); + await consumerA.StartAsync(CancellationToken.None); + await consumerB.StartAsync(CancellationToken.None); + + Assert.Equal("cache.nntpd01.usenet.ninja", consumerA.CurrentQueueName); + Assert.Equal("cache.nntpd02.usenet.ninja", consumerB.CurrentQueueName); + Assert.NotEqual(consumerA.CurrentQueueName, consumerB.CurrentQueueName); + + await consumerA.StopAsync(CancellationToken.None); + await consumerB.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task Delivery_UpdatesRegistry_AndAcks() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var consumer = new StorageServerFleetConsumerService( + rabbit, + Options.Create(CreateNntpdOptions(1)), + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + + var channel = Assert.Single(factory.LastConnection!.ManualAckChannels); + var body = StorageServerAdvertisementWireProtocol.SerializeV1( + new StorageServerAdvertisement( + 1, + 7, + "cache07.usenet.ninja", + 1_000_000, + 250_000, + 750_000, + DateTimeOffset.Parse("2026-09-29T12:00:00Z"))); + await channel.DeliverAsync(42, body); + + Assert.Contains(42UL, channel.Acks); + Assert.True(registry.TryGet("cache07.usenet.ninja", out var entry)); + Assert.Equal(7, entry.ServerId); + Assert.Equal(1_000_000, entry.TotalBytes); + Assert.Equal(750_000, entry.AvailableBytes); + + await consumer.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task LifecycleDelivery_UpdatesSelection() + { + var factory = new FakeRabbitMqConnectionFactory(); + await using var rabbit = CreateRabbitMq(factory); + var registry = new StorageServerRegistry(); + var consumer = new StorageServerFleetConsumerService( + rabbit, + Options.Create(CreateNntpdOptions(1)), + registry, + NullLogger.Instance); + + await rabbit.StartAsync(CancellationToken.None); + await consumer.StartAsync(CancellationToken.None); + var channel = Assert.Single(factory.LastConnection!.ManualAckChannels); + var advertisement = StorageServerAdvertisementWireProtocol.SerializeV1( + new StorageServerAdvertisement( + 1, + 7, + "cache07.usenet.ninja", + 1_000, + 100, + 900, + DateTimeOffset.Parse("2026-09-29T12:00:00Z"), + 1191)); + await channel.DeliverAsync(1, advertisement); + var selected = Assert.Single(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:01Z"))); + Assert.Equal(900, selected.AvailableBytes); + Assert.Equal(1191, selected.VatpPort); + Assert.False(selected.IsDraining); + + var draining = StorageServerLifecycleWireProtocol.SerializeV1( + new StorageServerLifecycleAnnouncement( + 1, + 7, + "cache07.usenet.ninja", + StorageServerLifecycleState.Draining, + DateTimeOffset.Parse("2026-09-29T12:00:02Z"), + 1191)); + await channel.DeliverAsync(2, draining); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:02Z"))); + Assert.True(registry.TryGet("cache07.usenet.ninja", out var drained)); + Assert.True(drained.IsDraining); + Assert.Equal(900, drained.AvailableBytes); + Assert.Equal(1191, drained.VatpPort); + + var ready = """{"version":1,"serverId":7,"fqdn":"cache07.usenet.ninja","state":"Ready","timestamp":"2026-09-29T12:00:04.0000000Z","vatpPort":1191}"""u8.ToArray(); + await channel.DeliverAsync(3, ready); + Assert.Contains(3UL, channel.Acks); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:04Z"))); + Assert.True(registry.TryGet("cache07.usenet.ninja", out drained)); + Assert.True(drained.IsDraining); + Assert.Equal(900, drained.AvailableBytes); + + var invalid = """{"version":1,"serverId":7,"fqdn":"cache07.usenet.ninja","state":"Offline","timestamp":"2026-09-29T12:00:03.0000000Z","vatpPort":1191}"""u8.ToArray(); + await channel.DeliverAsync(4, invalid); + Assert.Contains(4UL, channel.Acks); + Assert.Empty(registry.GetActive(DateTimeOffset.Parse("2026-09-29T12:00:03Z"))); + + await consumer.StopAsync(CancellationToken.None); + } + + private static RabbitMqService CreateRabbitMq(FakeRabbitMqConnectionFactory factory) + { + var options = RabbitMqOptionsTests.CreateValid(); + options.PoolReconnectBaseDelayMs = 50; + options.PoolReconnectMaxDelayMs = 50; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.TransitService:nntpd01.usenet.ninja"), + NullLogger.Instance); + } + + private static NntpdOptions CreateNntpdOptions(int serverId) + { + var options = TestHostFactory.CreateValidOptions(); + options.ServerId = serverId; + return options; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerRegistryTests.cs b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerRegistryTests.cs new file mode 100644 index 00000000..00daa776 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Storage/StorageServerRegistryTests.cs @@ -0,0 +1,279 @@ +using System.Reflection; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.Storage; + +namespace VectorNNTPD.TransitService.Tests.Storage; + +public sealed class StorageServerRegistryTests +{ + [Fact] + public void ApplyAdvertisement_CreatesAndUpdatesEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + var first = new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 1000, 100, 900, t0); + registry.ApplyAdvertisement(first, t0); + + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.Equal(1, entry.ServerId); + Assert.Equal(1000, entry.TotalBytes); + Assert.Equal(100, entry.UsedBytes); + Assert.Equal(900, entry.AvailableBytes); + Assert.Equal(t0, entry.LastSeen); + Assert.True(entry.IsActive(t0)); + + var t1 = t0.AddSeconds(1); + var second = first with { UsedBytes = 200, AvailableBytes = 800, Timestamp = t1 }; + registry.ApplyAdvertisement(second, t1); + + Assert.True(registry.TryGet("cache01.usenet.ninja", out entry)); + Assert.Equal(200, entry.UsedBytes); + Assert.Equal(800, entry.AvailableBytes); + Assert.Equal(t1, entry.LastSeen); + Assert.Equal(StorageServerFleetState.Active, entry.GetState(t1)); + } + + [Fact] + public void GetActive_ExcludesStaleServers_AndKeepsMultipleActive() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 1, "cache01.usenet.ninja", 100, 10, 90, t0), + t0); + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 2, "cache02.usenet.ninja", 200, 20, 180, t0), + t0); + + var activeAtStart = registry.GetActive(t0); + Assert.Equal(2, activeAtStart.Count); + + var staleNow = t0 + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(staleNow)); + Assert.Equal(2, registry.Snapshot().Count); + Assert.Equal(StorageServerFleetState.Stale, registry.Snapshot()[0].GetState(staleNow)); + + registry.ApplyAdvertisement( + new StorageServerAdvertisement(1, 2, "cache02.usenet.ninja", 200, 50, 150, staleNow), + staleNow); + var active = Assert.Single(registry.GetActive(staleNow)); + Assert.Equal("cache02.usenet.ninja", active.Fqdn); + Assert.Equal(50, active.UsedBytes); + } + + [Fact] + public void FirstAdvertisement_MakesTheServerSelectable_WithoutASecondRegistryEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + + var selected = Assert.Single(registry.GetActive(t0)); + Assert.Equal(1, selected.ServerId); + Assert.Equal(1191, selected.VatpPort); + Assert.False(selected.IsDraining); + Assert.True(StorageServerPlacementSelector.TrySelect(registry.GetActive(t0), out var target)); + Assert.Equal("cache01.usenet.ninja", target.Fqdn); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 900 }, + t0.AddSeconds(1)); + var updated = Assert.Single(registry.Snapshot()); + Assert.Equal(900, updated.AvailableBytes); + Assert.False(updated.IsDraining); + Assert.Equal("cache01.usenet.ninja", Assert.Single(registry.GetActive(t0.AddSeconds(1))).Fqdn); + } + + [Fact] + public void Draining_RemovesTheServerFromNewSelection_Immediately() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyAdvertisement(Advertisement("cache02.usenet.ninja", 2, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddTicks(1)), t0); + + var selected = Assert.Single(registry.GetActive(t0)); + Assert.Equal("cache02.usenet.ninja", selected.Fqdn); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var drained)); + Assert.True(drained.IsDraining); + Assert.Equal(500, drained.AvailableBytes); + Assert.True(drained.IsActive(t0)); + } + + [Fact] + public void DelayedAdvertisement_DoesNotClearDraining() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + var drainAt = t0.AddSeconds(2); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 1 }, + drainAt); + + Assert.Empty(registry.GetActive(drainAt)); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(drainAt, entry.LastSeen); + } + + [Fact] + public void CrashWithoutDraining_ExpiresThroughTheLivenessWindow() + { + var registry = new StorageServerRegistry(); + var seen = DateTimeOffset.Parse("2026-09-29T12:00:01Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, seen), seen); + + Assert.Single(registry.GetActive(seen)); + var expired = seen + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(expired)); + Assert.Single(registry.Snapshot()); + Assert.False(registry.Snapshot()[0].IsDraining); + } + + [Fact] + public void Restart_BecomesSelectableFromANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-09-29T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), t0.AddSeconds(1)); + Assert.Empty(registry.GetActive(t0.AddSeconds(1))); + + var restarted = t0.AddSeconds(3); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, restarted) with { AvailableBytes = 42 }, + restarted); + var selected = Assert.Single(registry.GetActive(restarted)); + Assert.False(selected.IsDraining); + Assert.Equal(42, selected.AvailableBytes); + + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), restarted); + Assert.Single(registry.GetActive(restarted)); + Assert.Equal(42, Assert.Single(registry.Snapshot()).AvailableBytes); + + var drainedAgain = restarted.AddSeconds(1); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainedAgain), drainedAgain); + Assert.Empty(registry.GetActive(drainedAgain)); + Assert.Equal(42, Assert.Single(registry.Snapshot()).AvailableBytes); + } + + [Fact] + public void DelayedAdvertisement_AfterRestart_DoesNotReplaceANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var drainAt = t0.AddSeconds(10); + var restartedAt = drainAt.AddSeconds(1); + var received = restartedAt.AddSeconds(1); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, restartedAt) with { AvailableBytes = 42 }, + received); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(1)) with { AvailableBytes = 1 }, + received.AddSeconds(1)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), received.AddSeconds(1)); + + var entry = Assert.Single(registry.GetActive(received.AddSeconds(1))); + Assert.False(entry.IsDraining); + Assert.Equal(received, entry.LastSeen); + Assert.Equal(42, entry.AvailableBytes); + Assert.Single(registry.Snapshot()); + } + + [Fact] + public void AdvertisementNotNewerThanDraining_StaysUnselectable() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var drainAt = t0.AddSeconds(10); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, drainAt), drainAt); + + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, drainAt) with { AvailableBytes = 7 }, + drainAt.AddSeconds(1)); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0.AddSeconds(9)) with { AvailableBytes = 8 }, + drainAt.AddSeconds(2)); + + Assert.Empty(registry.GetActive(drainAt.AddSeconds(2))); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(drainAt, entry.LastSeen); + } + + [Fact] + public void OlderAdvertisement_DoesNotReplaceANewerAdvertisement() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + var newer = t0.AddSeconds(2); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, newer) with { AvailableBytes = 80 }, newer); + registry.ApplyAdvertisement( + Advertisement("cache01.usenet.ninja", 1, t0) with { AvailableBytes = 1 }, + newer.AddSeconds(1)); + + var entry = Assert.Single(registry.GetActive(newer.AddSeconds(1))); + Assert.Equal(80, entry.AvailableBytes); + Assert.Equal(newer, entry.LastSeen); + Assert.False(entry.IsDraining); + } + + [Fact] + public void RepeatedDraining_StaysOneEntry() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(2)), t0.AddSeconds(2)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(3)), t0.AddSeconds(3)); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(2)), t0.AddSeconds(4)); + + Assert.Empty(registry.GetActive(t0.AddSeconds(4))); + var entry = Assert.Single(registry.Snapshot()); + Assert.True(entry.IsDraining); + Assert.Equal(500, entry.AvailableBytes); + Assert.Equal(t0.AddSeconds(3), entry.NewestMessageTimestamp); + } + + [Fact] + public void LivenessExpiry_DoesNotConvertDrainingToSelectable() + { + var registry = new StorageServerRegistry(); + var t0 = DateTimeOffset.Parse("2026-10-01T12:00:00Z"); + registry.ApplyAdvertisement(Advertisement("cache01.usenet.ninja", 1, t0), t0); + registry.ApplyLifecycle(Draining("cache01.usenet.ninja", 1, t0.AddSeconds(1)), t0.AddSeconds(1)); + + Assert.Empty(registry.GetActive(t0.AddSeconds(1))); + var expired = t0.AddSeconds(1) + CacheFleetTopology.LivenessWindow + TimeSpan.FromMilliseconds(1); + Assert.Empty(registry.GetActive(expired)); + Assert.True(registry.TryGet("cache01.usenet.ninja", out var entry)); + Assert.True(entry.IsDraining); + } + + [Fact] + public void Registry_HasNoReadyEpochState() + { + const BindingFlags publicStatic = BindingFlags.Public | BindingFlags.Static; + Assert.Null(typeof(StorageServerFleetEntry).GetProperty("LifecycleOriginTimestamp")); + Assert.Null(typeof(StorageServerFleetEntry).GetProperty("SupersededDrainingTimestamp")); + Assert.Null(typeof(StorageServerLifecycleState).GetField("Ready", publicStatic)); + Assert.NotNull(typeof(StorageServerLifecycleState).GetField(nameof(StorageServerLifecycleState.Draining), publicStatic)); + Assert.Equal(1, StorageServerLifecycleWireProtocol.CurrentVersion); + } + + private static StorageServerAdvertisement Advertisement(string fqdn, int serverId, DateTimeOffset timestamp) => + new(1, serverId, fqdn, 1000, 500, 500, timestamp, 1191); + + private static StorageServerLifecycleAnnouncement Draining(string fqdn, int serverId, DateTimeOffset timestamp) => + new(1, serverId, fqdn, StorageServerLifecycleState.Draining, timestamp, 1191); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Telemetry/ApplicationTelemetryTests.cs b/tests/VectorNNTPD.TransitService.Tests/Telemetry/ApplicationTelemetryTests.cs new file mode 100644 index 00000000..14ebb1e2 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Telemetry/ApplicationTelemetryTests.cs @@ -0,0 +1,1049 @@ +using System.Diagnostics; +using System.IO.Pipelines; +using System.Net; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Diagnostics; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Telemetry; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Tests.Transit; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Telemetry; + +public sealed class ApplicationTelemetryTests +{ + [Fact] + public void Period_IsSixtySeconds() + { + Assert.Equal(TimeSpan.FromMinutes(1), ApplicationTelemetryService.Period); + } + + [Fact] + public async Task FirstSnapshot_EmitsOnlyAfterFirstInterval_AtInformation() + { + var interval = new ControllableInterval(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, interval); + + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => interval.WaitCount >= 1); + Assert.Equal(0, service.EmitCount); + Assert.Empty(logger.Entries); + + interval.ReleaseTick(); + await WaitUntilAsync(() => service.EmitCount == 1); + + Assert.Equal(3, logger.Entries.Count); + Assert.All(logger.Entries, static e => Assert.Equal(LogLevel.Information, e.Level)); + Assert.Equal(2400, logger.Entries[0].EventId.Id); + Assert.Equal(2401, logger.Entries[1].EventId.Id); + Assert.Equal(2402, logger.Entries[2].EventId.Id); + Assert.Contains("HistoryDb lookups=", logger.Entries[0].Message); + Assert.Contains("TransitIngressQueue articles=", logger.Entries[1].Message); + Assert.Contains("ActiveSessions active=", logger.Entries[2].Message); + } + + [Fact] + public async Task Cadence_EmitsOncePerTick_WithoutOverlap() + { + var interval = new ControllableInterval(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, interval); + + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => interval.WaitCount >= 1); + + interval.ReleaseTick(); + await WaitUntilAsync(() => service.EmitCount == 1); + Assert.False(service.IsEmitBusy); + interval.ReleaseTick(); + await WaitUntilAsync(() => service.EmitCount == 2); + Assert.False(service.IsEmitBusy); + + Assert.Equal(6, logger.Entries.Count); + Assert.Equal(2, logger.Entries.Count(static e => e.EventId.Id == 2400)); + Assert.Equal(2, logger.Entries.Count(static e => e.EventId.Id == 2401)); + Assert.Equal(2, logger.Entries.Count(static e => e.EventId.Id == 2402)); + } + + [Fact] + public async Task DisabledFeedDiagnostics_DoesNotDisableTelemetry() + { + Assert.False(NullFeedDiagnostics.Instance.IsEnabled); + + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), 64); + await using var diagnostics = new FeedDiagnosticsService( + NullFeedDiagnostics.Instance, + queue, + new TransitConfigurationStore(), + Options.Create(new NntpdOptions()), + NullLogger.Instance); + await diagnostics.StartAsync(CancellationToken.None); + Assert.Null(diagnostics.Execution); + + var interval = new ControllableInterval(); + var logger = new RecordingLogger(); + var census = new NntpSessionCensus(); + await using var telemetry = new ApplicationTelemetryService( + queue, + census, + logger, + history: null, + timeProvider: null, + interval, + ApplicationTelemetryService.Period); + + await telemetry.StartAsync(CancellationToken.None); + interval.ReleaseTick(); + await WaitUntilAsync(() => telemetry.EmitCount == 1); + Assert.Equal(3, logger.Entries.Count); + Assert.All(logger.Entries, static e => Assert.Equal(LogLevel.Information, e.Level)); + } + + [Fact] + public async Task HistoryDb_ValuesComeFromAuthoritativeHistoryMetrics() + { + var redis = new FakeRedisService(); + var history = new HistoryDb( + redis, + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + Assert.Equal(HistoryLookupResult.Unseen, await history.LookupAsync(""u8.ToArray())); + Assert.Equal(HistoryLookupResult.Seen, await history.LookupAsync(""u8.ToArray())); + redis.IsUnavailable = true; + Assert.Equal(HistoryLookupResult.Unavailable, await history.LookupAsync(""u8.ToArray())); + var captured = history.Capture(); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, history: history); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2400); + Assert.Equal(captured.Lookups, GetInt64(row, "Lookups")); + Assert.Equal(captured.Hits, GetInt64(row, "Hits")); + Assert.Equal(captured.Misses, GetInt64(row, "Misses")); + Assert.Equal(captured.Errors, GetInt64(row, "Errors")); + + logger.Entries.Clear(); + service.Emit(); + var second = Assert.Single(logger.Entries, static e => e.EventId.Id == 2400); + Assert.Equal(0L, GetInt64(second, "Lookups")); + Assert.Equal(0L, GetInt64(second, "Hits")); + Assert.Equal(0L, GetInt64(second, "Misses")); + Assert.Equal(0L, GetInt64(second, "Errors")); + } + + [Fact] + public async Task TransitIngressQueue_ValuesComeFromAuthoritativeQueue() + { + var a = Article(""); + var b = Article(""); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), a.Payload.Length + b.Payload.Length); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(a, CancellationToken.None)); + Assert.Equal(ArticleEnqueueResult.Accepted, await queue.EnqueueAsync(b, CancellationToken.None)); + Assert.Equal( + ArticleEnqueueResult.Rejected, + queue.TryAdmit(Article( + "", + body: string.Concat(Enumerable.Repeat(new string('Z', 80) + "\r\n", 20))))); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, queue: queue); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2401); + Assert.Equal(queue.Count, GetInt32(row, "Articles")); + Assert.Equal(queue.QueuedBytes, GetInt64(row, "Bytes")); + Assert.Equal(queue.PeakCount, GetInt32(row, "PeakArticles")); + Assert.Equal(queue.PeakQueuedBytes, GetInt64(row, "PeakBytes")); + Assert.Equal(queue.WaitingProducerCount, GetInt32(row, "Waiting")); + Assert.Equal(queue.MemoryLimitBytes, GetInt64(row, "Limit")); + Assert.Equal(queue.AdmissionFailureCount, GetInt64(row, "AdmissionFailures")); + Assert.Equal(2, GetInt32(row, "Articles")); + Assert.Equal(a.Payload.Length + b.Payload.Length, GetInt64(row, "Bytes")); + Assert.Equal(1L, GetInt64(row, "AdmissionFailures")); + } + + [Fact] + public async Task ActiveSessions_CountsCompleteServerPopulation() + { + var census = new NntpSessionCensus(); + var idle = CreateSession(); + var receiving = CreateSession(); + var waitingHistory = CreateSession(); + receiving.SetActivityState(FeedSessionState.Receiving); + waitingHistory.SetActivityState(FeedSessionState.WaitingHistory); + census.Register(idle); + census.Register(receiving); + census.Register(waitingHistory); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2402); + Assert.Equal(3, GetInt32(row, "Active")); + Assert.Equal(3, GetInt32(row, "Established")); + Assert.Equal(1, GetInt32(row, "Idle")); + Assert.Equal(1, GetInt32(row, "Receiving")); + Assert.Equal(1, GetInt32(row, "WaitingHistory")); + Assert.Equal(0, GetInt32(row, "WaitingQueue")); + Assert.Equal(0, GetInt32(row, "WaitingWindow")); + Assert.Equal(0, GetInt32(row, "Completing")); + } + + [Fact] + public async Task ActiveSessions_IncludesReaderAndStreamerSessions() + { + var census = new NntpSessionCensus(); + var reader = CreateSession(); + var streamer = CreateSession(); + reader.SetMode(NntpSessionMode.Reader); + streamer.SetMode(NntpSessionMode.Stream); + census.Register(reader); + census.Register(streamer); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2402); + Assert.Equal(2, GetInt32(row, "Active")); + Assert.Equal(2, GetInt32(row, "Established")); + Assert.Equal(NntpSessionMode.Reader, reader.Mode); + Assert.Equal(NntpSessionMode.Stream, streamer.Mode); + } + + [Fact] + public async Task ActiveSessions_DoesNotUseFeedDiagnosticsOrTransitOnlyPopulation() + { + var hub = new FeedDiagnosticsHub(); + var transit = CreateSession(); + Assert.NotNull(hub.OnAccepted(transit)); + var feed = hub.CaptureSnapshot(DisabledArticleIngestionQueue.Instance, new TransitConfigurationStore()); + Assert.Equal(1, feed.ActiveSessions); + + var census = new NntpSessionCensus(); + var reader = CreateSession(); + var streamer = CreateSession(); + reader.SetMode(NntpSessionMode.Reader); + streamer.SetMode(NntpSessionMode.Stream); + census.Register(reader); + census.Register(streamer); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2402); + Assert.Equal(2, GetInt32(row, "Active")); + Assert.NotEqual(feed.ActiveSessions, GetInt32(row, "Active")); + } + + [Fact] + public async Task RunAsync_RegistersReaderAndStreamer_ThenUnregistersOnExit() + { + var census = new NntpSessionCensus(); + var reader = CreateSession(census); + var streamer = CreateSession(census); + reader.SetMode(NntpSessionMode.Reader); + streamer.SetMode(NntpSessionMode.Stream); + + using var cts = new CancellationTokenSource(); + var readerRun = reader.RunAsync(cts.Token); + var streamerRun = streamer.RunAsync(cts.Token); + await WaitUntilAsync(() => census.Capture().Active == 2); + var live = census.Capture(); + Assert.Equal(2, live.Active); + Assert.Equal(2, live.Established); + + await cts.CancelAsync(); + await readerRun; + await streamerRun; + Assert.Equal(0, census.Capture().Active); + } + + [Fact] + public async Task Stop_CancelsLoopWithoutFurtherEmits() + { + var interval = new ControllableInterval(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, interval); + + await service.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => interval.WaitCount >= 1); + await service.StopAsync(CancellationToken.None); + Assert.NotNull(service.Execution); + await service.Execution.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, service.EmitCount); + interval.ReleaseTick(); + await Task.Yield(); + Assert.Equal(0, service.EmitCount); + Assert.Empty(logger.Entries); + } + + [Fact] + public async Task Emit_IsSingleFlight() + { + var census = new BlockingCensus(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census); + + var first = Task.Run(service.Emit, CancellationToken.None); + await census.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var second = Task.Run(service.Emit, CancellationToken.None); + await Task.Yield(); + Assert.Equal(1, census.Concurrent); + census.Release.TrySetResult(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(2, service.EmitCount); + Assert.Equal(1, census.MaxConcurrent); + Assert.False(service.IsEmitBusy); + } + + [Fact] + public async Task ConfiguredPeers_EachEmitsOneInformationRecord() + { + var store = CreatePeerStore(("blueworld-hosting", 10), ("giganews", 100), ("usenet-ninja", 10)); + var metrics = new TransitPeerMetrics(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + + var peers = logger.Entries.Where(static e => e.EventId.Id == 2403).ToArray(); + Assert.Equal(3, peers.Length); + Assert.Equal("blueworld-hosting", peers[0].Properties["PeerName"]); + Assert.Equal("giganews", peers[1].Properties["PeerName"]); + Assert.Equal("usenet-ninja", peers[2].Properties["PeerName"]); + Assert.All(peers, static e => Assert.Equal(LogLevel.Information, e.Level)); + Assert.All(peers, static e => Assert.True(e.Properties.ContainsKey("AverageMbps"))); + Assert.All(peers, static e => Assert.Contains("avg_mbps=", e.Message, StringComparison.Ordinal)); + Assert.Equal(6, logger.Entries.Count); + } + + [Fact] + public async Task UnconfiguredPeer_ProducesNoTelemetry() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + metrics.RecordAccepted("not-a-peer"); + metrics.RecordArticleReceived("not-a-peer", 100); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + + var peers = logger.Entries.Where(static e => e.EventId.Id == 2403).ToArray(); + Assert.Single(peers); + Assert.Equal("giganews", peers[0].Properties["PeerName"]); + Assert.DoesNotContain(logger.Entries, static e => e.Message.StartsWith("not-a-peer ", StringComparison.Ordinal)); + } + + [Fact] + public async Task Giganews_RendersCanonicalIdentifierAndMaxIncoming() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + for (var i = 0; i < 60; i++) + { + metrics.RecordAccepted("giganews"); + metrics.RecordSessionRegistered("giganews"); + } + + const int articles = 1619; + const long bytes = 1_213_768_568L; + var bytesPerArticle = (int)(bytes / articles); + var remainder = (int)(bytes - ((long)bytesPerArticle * articles)); + for (var i = 0; i < articles; i++) + { + var size = i == 0 ? bytesPerArticle + remainder : bytesPerArticle; + metrics.RecordArticleReceived("giganews", size); + } + + for (var i = 0; i < 1840; i++) + { + metrics.RecordCheck("giganews"); + } + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(LogLevel.Information, row.Level); + var expectedMbps = ApplicationTelemetryService.AverageMbps(bytes, ApplicationTelemetryService.Period); + Assert.Equal( + string.Create( + System.Globalization.CultureInfo.InvariantCulture, + $"giganews active=60/100 peak=60 accepted=60 transmitted=0 rejected=0 articles=1619 bytes=1213768568 avg_article={bytes / articles} avg_mbps={expectedMbps:F2} checks=1840"), + row.Message); + Assert.Equal(60, GetInt32(row, "Active")); + Assert.Equal(100, GetInt32(row, "MaxIncomingConnections")); + Assert.Equal(60, GetInt32(row, "Peak")); + Assert.Equal(60L, GetInt64(row, "Accepted")); + Assert.Equal(0L, GetInt64(row, "Transmitted")); + Assert.Equal(0L, GetInt64(row, "Rejected")); + Assert.Equal(1619L, GetInt64(row, "Articles")); + Assert.Equal(1_213_768_568L, GetInt64(row, "Bytes")); + Assert.Equal(1_213_768_568L / 1619L, GetInt64(row, "AverageArticleBytes")); + Assert.Equal(1840L, GetInt64(row, "Checks")); + Assert.Equal(expectedMbps, GetDouble(row, "AverageMbps"), precision: 6); + } + + [Fact] + public async Task PeerActive_IsCurrentEstablishedSessionsFromCensus() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + var census = new NntpSessionCensus(metrics); + var first = CreatePeerSession("giganews", census, metrics); + var second = CreatePeerSession("giganews", census, metrics); + census.Register(first); + census.Register(second); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census, peerMetrics: metrics, transit: store); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(2, GetInt32(row, "Active")); + Assert.Equal(2, GetInt32(row, "Peak")); + Assert.Equal(2, census.Capture().Active); + } + + [Fact] + public async Task PeerAcceptedAndRejected_AreIntervalDeltas() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + metrics.RecordAccepted("giganews"); + metrics.RecordAccepted("giganews"); + metrics.RecordRejected("giganews"); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + var first = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(2L, GetInt64(first, "Accepted")); + Assert.Equal(1L, GetInt64(first, "Rejected")); + + logger.Entries.Clear(); + metrics.RecordAccepted("giganews"); + metrics.RecordRejected("giganews"); + metrics.RecordRejected("giganews"); + service.Emit(); + var second = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(1L, GetInt64(second, "Accepted")); + Assert.Equal(2L, GetInt64(second, "Rejected")); + } + + [Fact] + public async Task PeerArticlesBytesChecks_AreIntervalValues_AndAverageUsesSameInterval() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + metrics.RecordArticleReceived("giganews", 100); + metrics.RecordArticleReceived("giganews", 50); + metrics.RecordCheck("giganews"); + metrics.RecordCheck("giganews"); + metrics.RecordCheck("giganews"); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + var first = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(2L, GetInt64(first, "Articles")); + Assert.Equal(150L, GetInt64(first, "Bytes")); + Assert.Equal(75L, GetInt64(first, "AverageArticleBytes")); + Assert.Equal(3L, GetInt64(first, "Checks")); + + logger.Entries.Clear(); + metrics.RecordArticleReceived("giganews", 40); + service.Emit(); + var second = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(1L, GetInt64(second, "Articles")); + Assert.Equal(40L, GetInt64(second, "Bytes")); + Assert.Equal(40L, GetInt64(second, "AverageArticleBytes")); + Assert.Equal(0L, GetInt64(second, "Checks")); + } + + [Fact] + public async Task Transmitted_IsZeroUntilOutboundAccounting_AndIsNotDerivedFromQueue() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + var queued = Article(""); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), queued.Payload.Length); + Assert.Equal( + ArticleEnqueueResult.Accepted, + await queue.EnqueueAsync(queued, CancellationToken.None)); + metrics.RecordArticleReceived("giganews", queued.Payload.Length); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, queue: queue, peerMetrics: metrics, transit: store); + service.Emit(); + + var first = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(0L, GetInt64(first, "Transmitted")); + Assert.Equal(1L, GetInt64(first, "Articles")); + Assert.Contains(logger.Entries, static e => e.EventId.Id == 2401 && GetInt32(e, "Articles") == 1); + + logger.Entries.Clear(); + metrics.RecordTransmitted("giganews"); + service.Emit(); + var second = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(1L, GetInt64(second, "Transmitted")); + } + + [Fact] + public async Task DisabledFeedDiagnostics_DoesNotDisablePeerTelemetry() + { + Assert.False(NullFeedDiagnostics.Instance.IsEnabled); + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + metrics.RecordAccepted("giganews"); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions(), 64); + await using var diagnostics = new FeedDiagnosticsService( + NullFeedDiagnostics.Instance, + queue, + store, + Options.Create(new NntpdOptions()), + NullLogger.Instance); + await diagnostics.StartAsync(CancellationToken.None); + Assert.Null(diagnostics.Execution); + + var logger = new RecordingLogger(); + await using var telemetry = CreateService(logger, queue: queue, peerMetrics: metrics, transit: store); + telemetry.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(LogLevel.Information, row.Level); + Assert.Equal(1L, GetInt64(row, "Accepted")); + } + + [Fact] + public async Task ReaderAuthFromAllowFromRange_DoesNotIncrementTransitPeerTelemetry() + { + var store = CreatePeerStore(("usenet-ninja", 10)); + var metrics = new TransitPeerMetrics(); + var census = new NntpSessionCensus(metrics); + var session = CreatePeerSession("usenet-ninja", census, metrics, maxIncoming: 10); + Assert.Equal("usenet-ninja", session.Authorization.TransitPeerName); + + session.RecordPeerAccepted(); + census.Register(session); + Assert.Equal(1, metrics.Capture("usenet-ninja").Active); + Assert.Equal(1, metrics.Capture("usenet-ninja").Accepted); + + session.ApplySuccessfulAuthentication("a", MySqlNntpCredentialValidator.ReaderPrivileges); + Assert.Null(session.Authorization.TransitPeerName); + Assert.True(session.Authorization.AuthorizedReader); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census, peerMetrics: metrics, transit: store); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal("usenet-ninja", row.Properties["PeerName"]); + Assert.Equal(0, GetInt32(row, "Active")); + Assert.Equal(10, GetInt32(row, "MaxIncomingConnections")); + Assert.Equal(0L, GetInt64(row, "Accepted")); + Assert.Equal(0, metrics.Capture("usenet-ninja").Active); + Assert.Equal(0, metrics.Capture("usenet-ninja").Accepted); + Assert.Equal(1, census.Capture().Active); + } + + [Fact] + public async Task TransitConnection_IncrementsTransitPeerTelemetry() + { + var store = CreatePeerStore(("usenet-ninja", 10)); + var metrics = new TransitPeerMetrics(); + var census = new NntpSessionCensus(metrics); + var session = CreatePeerSession("usenet-ninja", census, metrics, maxIncoming: 10); + + session.RecordPeerAccepted(); + census.Register(session); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census, peerMetrics: metrics, transit: store); + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal("usenet-ninja", row.Properties["PeerName"]); + Assert.Equal(1, GetInt32(row, "Active")); + Assert.Equal(10, GetInt32(row, "MaxIncomingConnections")); + Assert.Equal(1, GetInt32(row, "Peak")); + Assert.Equal(1L, GetInt64(row, "Accepted")); + Assert.Equal("usenet-ninja", session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedReader); + } + + [Fact] + public async Task PeerSessionCounts_AreIndependent() + { + var store = CreatePeerStore(("blueworld-hosting", 10), ("giganews", 100)); + var metrics = new TransitPeerMetrics(); + var census = new NntpSessionCensus(metrics); + var giga1 = CreatePeerSession("giganews", census, metrics, maxIncoming: 100); + var giga2 = CreatePeerSession("giganews", census, metrics, maxIncoming: 100); + var blue = CreatePeerSession( + "blueworld-hosting", + census, + metrics, + maxIncoming: 10, + address: IPAddress.Parse("192.0.2.20")); + census.Register(giga1); + census.Register(giga2); + census.Register(blue); + metrics.RecordAccepted("giganews"); + metrics.RecordAccepted("giganews"); + metrics.RecordAccepted("blueworld-hosting"); + metrics.RecordArticleReceived("giganews", 80); + metrics.RecordCheck("blueworld-hosting"); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, census: census, peerMetrics: metrics, transit: store); + service.Emit(); + + var peers = logger.Entries.Where(static e => e.EventId.Id == 2403).ToArray(); + Assert.Equal(2, peers.Length); + var blueRow = Assert.Single(peers, static e => (string)e.Properties["PeerName"]! == "blueworld-hosting"); + var gigaRow = Assert.Single(peers, static e => (string)e.Properties["PeerName"]! == "giganews"); + Assert.Equal(1, GetInt32(blueRow, "Active")); + Assert.Equal(1L, GetInt64(blueRow, "Accepted")); + Assert.Equal(0L, GetInt64(blueRow, "Articles")); + Assert.Equal(1L, GetInt64(blueRow, "Checks")); + Assert.Equal(2, GetInt32(gigaRow, "Active")); + Assert.Equal(2L, GetInt64(gigaRow, "Accepted")); + Assert.Equal(1L, GetInt64(gigaRow, "Articles")); + Assert.Equal(0L, GetInt64(gigaRow, "Checks")); + } + + [Fact] + public async Task PeerSnapshots_DoNotDuplicateWithinOneEmit() + { + var store = CreatePeerStore(("giganews", 100), ("usenet-ninja", 10)); + var metrics = new TransitPeerMetrics(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + + var names = logger.Entries + .Where(static e => e.EventId.Id == 2403) + .Select(static e => (string)e.Properties["PeerName"]!) + .ToArray(); + Assert.Equal(["giganews", "usenet-ninja"], names); + Assert.Equal(names.Length, names.Distinct(StringComparer.Ordinal).Count()); + } + + [Fact] + public async Task AvgMbps_KnownSixtySecondInterval_IsDecimalMegabits() + { + const long bytes = 3_227_779_887L; + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + RecordReceivedBytes(metrics, "giganews", bytes); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(bytes, GetInt64(row, "Bytes")); + Assert.Equal(430.37, GetDouble(row, "AverageMbps"), precision: 2); + Assert.Contains("avg_mbps=430.37", row.Message, StringComparison.Ordinal); + Assert.Contains($"bytes={bytes}", row.Message, StringComparison.Ordinal); + + var mebibytePerSecond = bytes / 60d / (1024d * 1024d); + var megabytePerSecond = bytes / 60d / 1_000_000d; + Assert.InRange(GetDouble(row, "AverageMbps"), 430, 431); + Assert.True(GetDouble(row, "AverageMbps") > mebibytePerSecond * 2); + Assert.True(GetDouble(row, "AverageMbps") > megabytePerSecond * 2); + } + + [Fact] + public async Task AvgMbps_ZeroBytes_IsZero() + { + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + var logger = new RecordingLogger(); + await using var service = CreateService(logger, peerMetrics: metrics, transit: store); + + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(0L, GetInt64(row, "Bytes")); + Assert.Equal(0d, GetDouble(row, "AverageMbps")); + Assert.Contains("avg_mbps=0.00", row.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task AvgMbps_UsesSameIntervalBytes_AndScalesWithPeriod() + { + const long bytes = 3_227_779_887L; + var store = CreatePeerStore(("giganews", 100)); + var metrics = new TransitPeerMetrics(); + RecordReceivedBytes(metrics, "giganews", bytes); + var logger = new RecordingLogger(); + await using var service = CreateService( + logger, + peerMetrics: metrics, + transit: store, + period: TimeSpan.FromSeconds(30)); + + service.Emit(); + + var row = Assert.Single(logger.Entries, static e => e.EventId.Id == 2403); + Assert.Equal(bytes, GetInt64(row, "Bytes")); + var expected = ApplicationTelemetryService.AverageMbps(bytes, TimeSpan.FromSeconds(30)); + Assert.Equal(expected, GetDouble(row, "AverageMbps"), precision: 6); + Assert.Equal(860.74, GetDouble(row, "AverageMbps"), precision: 2); + Assert.Equal( + ApplicationTelemetryService.AverageMbps(GetInt64(row, "Bytes"), TimeSpan.FromSeconds(30)), + GetDouble(row, "AverageMbps"), + precision: 6); + } + + [Fact] + public async Task PipelineMetrics_EmitAdditionalInformationRows() + { + var pipeline = new IngestionPipelineMetrics(); + RecordAgo(pipeline.RecordWorkerItem, TimeSpan.FromMilliseconds(8)); + pipeline.AddBusyTicks((long)(0.008 * Stopwatch.Frequency)); + RecordAgo(pipeline.RecordPublishAndConfirm, TimeSpan.FromMilliseconds(4)); + RecordAgo(pipeline.RecordHandoff, TimeSpan.FromMilliseconds(5)); + RecordAgo(pipeline.RecordTakeThisEnqueue, TimeSpan.FromMilliseconds(12)); + pipeline.RecordConfirmFailure(); + pipeline.RecordOccupiedFullWait(Stopwatch.GetTimestamp() - (long)(0.003 * Stopwatch.Frequency)); + + var logger = new RecordingLogger(); + await using var service = CreateService(logger, pipelineMetrics: pipeline); + service.Emit(); + + Assert.Equal(8, logger.Entries.Count); + Assert.All(logger.Entries, static e => Assert.Equal(LogLevel.Information, e.Level)); + var worker = Assert.Single(logger.Entries, static e => e.EventId.Id == 2404); + var handoff = Assert.Single(logger.Entries, static e => e.EventId.Id == 2405); + var takeThis = Assert.Single(logger.Entries, static e => e.EventId.Id == 2406); + var pool = Assert.Single(logger.Entries, static e => e.EventId.Id == 2407); + var overviewQueue = Assert.Single(logger.Entries, static e => e.EventId.Id == 2408); + Assert.Equal(1L, GetInt64(worker, "Items")); + Assert.True(GetDouble(worker, "ArticlesPerSec") > 0); + Assert.Equal(1L, GetInt64(handoff, "ConfirmFailures")); + Assert.Equal(0L, GetInt64(handoff, "ConfirmTimeouts")); + Assert.Equal(1L, GetInt64(takeThis, "OccupiedFullWaits")); + Assert.Contains("IngestionWorker items=", worker.Message, StringComparison.Ordinal); + Assert.Contains("OverviewDbHandoff gate_p95_ms=", handoff.Message, StringComparison.Ordinal); + Assert.Contains("TakeThisPipeline occupied=", takeThis.Message, StringComparison.Ordinal); + Assert.Contains("IngestionWorkerPool workers=", pool.Message, StringComparison.Ordinal); + Assert.Contains("OverviewDbWorkQueue articles=", overviewQueue.Message, StringComparison.Ordinal); + + logger.Entries.Clear(); + service.Emit(); + var secondWorker = Assert.Single(logger.Entries, static e => e.EventId.Id == 2404); + Assert.Equal(0L, GetInt64(secondWorker, "Items")); + } + + private static void RecordAgo(Action record, TimeSpan ago) + { + var start = Stopwatch.GetTimestamp() - (long)(ago.TotalSeconds * Stopwatch.Frequency); + record(start); + } + + private static ApplicationTelemetryService CreateService( + RecordingLogger logger, + IAsyncInterval? interval = null, + IArticleIngestionQueue? queue = null, + INntpSessionCensus? census = null, + IHistoryLookupMetrics? history = null, + ITransitPeerMetrics? peerMetrics = null, + TransitConfigurationStore? transit = null, + TimeSpan? period = null, + IngestionPipelineMetrics? pipelineMetrics = null) => + new( + queue ?? new ArticleIngestionQueue(new ArticleIngestionOptions(), 64), + census ?? new NntpSessionCensus(), + logger, + history, + timeProvider: null, + interval, + period ?? ApplicationTelemetryService.Period, + transit, + peerMetrics, + pipelineMetrics); + + private static void RecordReceivedBytes(TransitPeerMetrics metrics, string peerId, long bytes) + { + var remaining = bytes; + while (remaining > 0) + { + var chunk = remaining > int.MaxValue ? int.MaxValue : (int)remaining; + metrics.RecordArticleReceived(peerId, chunk); + remaining -= chunk; + } + } + + private static NntpSession CreateSession(INntpSessionCensus? census = null) + { + var input = new Pipe(); + var output = new Pipe(); + var connection = new HoldingConnection(input.Reader, output.Writer); + return new NntpSession( + connection, + NullLogger.Instance, + sessionCensus: census); + } + + private static NntpSession CreatePeerSession( + string peerId, + INntpSessionCensus? census = null, + ITransitPeerMetrics? metrics = null, + int maxIncoming = 100, + IPAddress? address = null) + { + address ??= IPAddress.Parse("192.0.2.10"); + var input = new Pipe(); + var output = new Pipe(); + var connection = new HoldingConnection(input.Reader, output.Writer, address); + var auth = TransitTestPeers.ForAllowFrom(address, name: peerId, maxIncoming: maxIncoming); + return new NntpSession( + connection, + NullLogger.Instance, + transitPeerAuthorization: auth, + sessionCensus: census, + peerMetrics: metrics); + } + + private static TransitConfigurationStore CreatePeerStore(params (string Id, int MaxIncoming)[] peers) + { + var options = new TransitPeersOptions(); + foreach (var (id, maxIncoming) in peers) + { + options[id] = TransitTestPeers.Peer(maxIncoming: maxIncoming, allowFrom: ["192.0.2.0/24"]); + } + + var store = new TransitConfigurationStore(); + store.Replace(TransitConfigurationSnapshot.Create(options)); + return store; + } + + private static InboundArticle Article(string messageId, string body = "body\r\n") => + CanonicalArticleText.CreateQueued(messageId, InboundArticleProducer.TakeThis, body); + + private static long GetInt64(LogRow row, string name) => + Convert.ToInt64(row.Properties[name], System.Globalization.CultureInfo.InvariantCulture); + + private static int GetInt32(LogRow row, string name) => + Convert.ToInt32(row.Properties[name], System.Globalization.CultureInfo.InvariantCulture); + + private static double GetDouble(LogRow row, string name) => + Convert.ToDouble(row.Properties[name], System.Globalization.CultureInfo.InvariantCulture); + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(1, cts.Token); + } + } + + private sealed class ControllableInterval : IAsyncInterval + { + private readonly Channel _ticks = Channel.CreateUnbounded(); + + public int WaitCount; + + public ValueTask WaitNextAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref WaitCount); + return _ticks.Reader.ReadAsync(cancellationToken); + } + + public void ReleaseTick() => _ticks.Writer.TryWrite(true); + + public ValueTask DisposeAsync() + { + _ticks.Writer.TryComplete(); + return ValueTask.CompletedTask; + } + } + + private sealed class BlockingCensus : INntpSessionCensus + { + public readonly TaskCompletionSource Entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly TaskCompletionSource Release = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Concurrent; + public int MaxConcurrent; + + public void Register(NntpSession session) + { + } + + public void Unregister(NntpSession session) + { + } + + public void ReleasePeerAttribution(NntpSession session) + { + } + + public NntpSessionCensusSnapshot Capture() + { + var current = Interlocked.Increment(ref Concurrent); + UpdateMax(current); + Entered.TrySetResult(); + Release.Task.GetAwaiter().GetResult(); + Interlocked.Decrement(ref Concurrent); + return new NntpSessionCensusSnapshot(0, 0, 0, 0, 0, 0, 0, 0); + } + + private void UpdateMax(int current) + { + while (true) + { + var observed = Volatile.Read(ref MaxConcurrent); + if (current <= observed + || Interlocked.CompareExchange(ref MaxConcurrent, current, observed) == observed) + { + return; + } + } + } + } + + private sealed class RecordingLogger : ILogger + { + public List Entries { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var properties = new Dictionary(StringComparer.Ordinal); + if (state is IEnumerable> pairs) + { + foreach (var pair in pairs) + { + properties[pair.Key] = pair.Value; + } + } + + var row = new LogRow(logLevel, eventId, formatter(state, exception), properties); + lock (Entries) + { + Entries.Add(row); + } + } + } + + private sealed record LogRow( + LogLevel Level, + EventId EventId, + string Message, + Dictionary Properties); + + private sealed class NullScope : IDisposable + { + public static NullScope Instance { get; } = new(); + + public void Dispose() + { + } + } + + private sealed class HoldingConnection : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public HoldingConnection(PipeReader input, PipeWriter output, IPAddress? address = null) + { + Input = input; + Output = output; + ClientIdentity = ConnectionClientIdentity.Direct( + new IPEndPoint(address ?? IPAddress.Parse("192.0.2.10"), 40000)); + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public ConnectionClientIdentity ClientIdentity { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public bool IsCompleted => _closed.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/CountingPostFilter.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/CountingPostFilter.cs new file mode 100644 index 00000000..d31128a9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/CountingPostFilter.cs @@ -0,0 +1,19 @@ +using VectorNNTPD.TransitService.PostFilter; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Counts calls. Accepts without reserving. +internal sealed class CountingPostFilter : IPostFilter +{ + public int EvaluateCalls { get; private set; } + + public ValueTask EvaluateAsync( + PostFilterRequest request, + CancellationToken cancellationToken = default) + { + _ = request; + cancellationToken.ThrowIfCancellationRequested(); + EvaluateCalls++; + return ValueTask.FromResult(PostFilterResult.AcceptedDisabled()); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeApplicationService.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeApplicationService.cs new file mode 100644 index 00000000..3ff47d9e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeApplicationService.cs @@ -0,0 +1,118 @@ +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +internal sealed class FakeApplicationService : IApplicationService +{ + private readonly Func? _onStart; + private readonly Func? _onStop; + private readonly TaskCompletionSource? _executionTcs; + private int _startCount; + private int _stopCount; + private int _stopAttemptCount; + + public FakeApplicationService( + string name, + Func? onStart = null, + Func? onStop = null, + bool withExecution = false) + { + Name = name; + _onStart = onStart; + _onStop = onStop; + if (withExecution) + { + _executionTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + public string Name { get; } + + public Task? Execution => _executionTcs?.Task; + + public int StartCount => Volatile.Read(ref _startCount); + + public int StopCount => Volatile.Read(ref _stopCount); + + /// Number of times was entered, including canceled/aborted attempts. + public int StopAttemptCount => Volatile.Read(ref _stopAttemptCount); + + public List SharedStartOrder { get; set; } = []; + + public List SharedStopOrder { get; set; } = []; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StopGate { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource StopEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool HoldStopUntilReleased { get; set; } + + /// + /// When set with , stop waits on without + /// observing the cancellation token (non-cooperative stop). + /// + public bool IgnoreStopCancellation { get; set; } + + public void CompleteExecution(Exception? exception = null) + { + if (_executionTcs is null) + { + throw new InvalidOperationException("Execution was not enabled for this fake."); + } + + if (exception is null) + { + _executionTcs.TrySetResult(); + } + else + { + _executionTcs.TrySetException(exception); + } + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (_onStart is not null) + { + await _onStart(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _startCount); + SharedStartOrder.Add(Name); + Started.TrySetResult(); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _stopAttemptCount); + StopEntered.TrySetResult(); + + if (HoldStopUntilReleased) + { + if (IgnoreStopCancellation) + { + await StopGate.Task.ConfigureAwait(false); + } + else + { + await StopGate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (_onStop is not null) + { + await _onStop(cancellationToken).ConfigureAwait(false); + } + + if (!IgnoreStopCancellation) + { + cancellationToken.ThrowIfCancellationRequested(); + } + + Interlocked.Increment(ref _stopCount); + SharedStopOrder.Add(Name); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeCloudflareDnsClient.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeCloudflareDnsClient.cs new file mode 100644 index 00000000..fbd2665a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeCloudflareDnsClient.cs @@ -0,0 +1,343 @@ +using VectorNNTPD.Common.Cloudflare; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// +/// In-memory Cloudflare DNS client for offline tests. Never performs network I/O. +/// +internal sealed class FakeCloudflareDnsClient : ICloudflareDnsClient +{ + private readonly object _sync = new(); + private readonly List _records = []; + private int _nextId = 1; + + public int ListCallCount { get; private set; } + public int ListAllCallCount { get; private set; } + public int CreateCallCount { get; private set; } + public int UpdateCallCount { get; private set; } + public int DeleteCallCount { get; private set; } + + public Func? OnMutate { get; set; } + public Exception? ListException { get; set; } + public Exception? ListAllException { get; set; } + public Exception? CreateException { get; set; } + public Exception? UpdateException { get; set; } + public Exception? DeleteException { get; set; } + public Exception? FailListAfterMutationsException { get; set; } + public int MutationsBeforeListFailure { get; set; } = int.MaxValue; + + /// + /// How many delete calls should fail when is set. + /// Use -1 (default) for unlimited failures; a positive count fails that many times then succeeds. + /// + public int DeleteFailRemaining { get; set; } = -1; + + /// + /// When true, a failing delete still removes the record then throws (timeout after apply). + /// + public bool ApplyDeleteThenFailUncertain { get; set; } + + /// When set, create fails only for this DNS record type (e.g. AAAA). + public string? CreateFailType { get; set; } + + /// When set with , throws this instead of . + public Exception? CreateFailTypeException { get; set; } + + /// + /// How many matching creates should fail before succeeding. + /// Use for permanent failure. + /// + public int CreateFailTypeRemaining { get; set; } = int.MaxValue; + + /// + /// When true, a failing create for still mutates the store, then throws + /// (simulates timeout after Cloudflare applied the change). + /// + public bool ApplyCreateThenFailUncertain { get; set; } + + public int SuccessfulCreateCount { get; private set; } + public int SuccessfulDeleteCount { get; private set; } + + /// Ordered mutation labels such as Create:A or Delete for successful mutations. + public List MutationOrder { get; } = []; + + /// + /// Optional transform applied to typed list results (e.g. inject an external record during verification). + /// Arguments: record type, listed records, current list call count. + /// + public Func, int, IReadOnlyList>? + TransformListedRecords + { get; set; } + + /// + /// Optional transform applied to results. + /// Arguments: listed records, current list-all call count. + /// + public Func, int, IReadOnlyList>? + TransformListedAllRecords + { get; set; } + + public IReadOnlyList Snapshot() + { + lock (_sync) + { + return _records.Select(Clone).ToArray(); + } + } + + public void Seed(params CloudflareDnsRecord[] records) + { + lock (_sync) + { + _records.Clear(); + foreach (var record in records) + { + _records.Add(Clone(record)); + } + } + } + + /// Adds a record without clearing existing ones (simulates an external DNS actor). + public void Inject(CloudflareDnsRecord record) + { + lock (_sync) + { + _records.Add(Clone(record)); + } + } + + public Task> ListRecordsAsync( + string zoneId, + string fqdn, + string type, + CancellationToken cancellationToken) + { + ListCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + + if (ListException is not null) + { + throw ListException; + } + + if (CreateCallCount + UpdateCallCount + DeleteCallCount >= MutationsBeforeListFailure + && FailListAfterMutationsException is not null) + { + throw FailListAfterMutationsException; + } + + IReadOnlyList matches; + lock (_sync) + { + var normalized = fqdn.Trim().TrimEnd('.').ToLowerInvariant(); + matches = _records + .Where(r => + string.Equals(r.Type, type, StringComparison.OrdinalIgnoreCase) + && string.Equals(r.Name.Trim().TrimEnd('.'), normalized, StringComparison.OrdinalIgnoreCase)) + .Select(Clone) + .ToArray(); + } + + if (TransformListedRecords is not null) + { + matches = TransformListedRecords(type, matches, ListCallCount); + } + + return Task.FromResult(matches); + } + + public Task> ListAllRecordsForNameAsync( + string zoneId, + string fqdn, + CancellationToken cancellationToken) + { + ListAllCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + + if (ListAllException is not null) + { + throw ListAllException; + } + + if (ListException is not null) + { + throw ListException; + } + + if (CreateCallCount + UpdateCallCount + DeleteCallCount >= MutationsBeforeListFailure + && FailListAfterMutationsException is not null) + { + throw FailListAfterMutationsException; + } + + IReadOnlyList matches; + lock (_sync) + { + var normalized = fqdn.Trim().TrimEnd('.').ToLowerInvariant(); + matches = _records + .Where(r => + string.Equals(r.Name.Trim().TrimEnd('.'), normalized, StringComparison.OrdinalIgnoreCase)) + .Select(Clone) + .ToArray(); + } + + if (TransformListedAllRecords is not null) + { + matches = TransformListedAllRecords(matches, ListAllCallCount); + } + + return Task.FromResult(matches); + } + + public async Task CreateRecordAsync( + string zoneId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + CreateCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + var failForThisType = CreateFailType is not null + && CreateFailTypeRemaining > 0 + && string.Equals(request.Type, CreateFailType, StringComparison.OrdinalIgnoreCase); + var failEx = failForThisType + ? CreateFailTypeException ?? new CloudflareDnsException($"create {request.Type} failed") + : CreateFailType is null + ? CreateException + : null; + + if (failEx is not null) + { + if (failForThisType) + { + CreateFailTypeRemaining--; + } + + if (ApplyCreateThenFailUncertain && failForThisType) + { + lock (_sync) + { + _records.Add(new CloudflareDnsRecord + { + Id = $"rec-{_nextId++}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + Ttl = request.Ttl, + Proxied = request.Proxied, + }); + SuccessfulCreateCount++; + MutationOrder.Add($"Create:{request.Type}"); + } + } + + throw failEx; + } + + lock (_sync) + { + var record = new CloudflareDnsRecord + { + Id = $"rec-{_nextId++}", + Type = request.Type, + Name = request.Name, + Content = request.Content, + Ttl = request.Ttl, + Proxied = request.Proxied, + }; + _records.Add(record); + SuccessfulCreateCount++; + MutationOrder.Add($"Create:{request.Type}"); + return Clone(record); + } + } + + public async Task UpdateRecordAsync( + string zoneId, + string recordId, + CloudflareDnsRecordWriteRequest request, + CancellationToken cancellationToken) + { + UpdateCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + if (UpdateException is not null) + { + throw UpdateException; + } + + lock (_sync) + { + var existing = _records.FirstOrDefault(r => r.Id == recordId) + ?? throw new CloudflareDnsException($"Record '{recordId}' was not found."); + existing.Type = request.Type; + existing.Name = request.Name; + existing.Content = request.Content; + existing.Ttl = request.Ttl; + existing.Proxied = request.Proxied; + MutationOrder.Add($"Update:{request.Type}"); + return Clone(existing); + } + } + + public async Task DeleteRecordAsync( + string zoneId, + string recordId, + CancellationToken cancellationToken) + { + DeleteCallCount++; + cancellationToken.ThrowIfCancellationRequested(); + if (OnMutate is not null) + { + await OnMutate(cancellationToken).ConfigureAwait(false); + } + + if (DeleteException is not null && DeleteFailRemaining != 0) + { + if (DeleteFailRemaining > 0) + { + DeleteFailRemaining--; + } + + if (ApplyDeleteThenFailUncertain) + { + lock (_sync) + { + _records.RemoveAll(r => r.Id == recordId); + SuccessfulDeleteCount++; + MutationOrder.Add("Delete"); + } + } + + throw DeleteException; + } + + lock (_sync) + { + _records.RemoveAll(r => r.Id == recordId); + SuccessfulDeleteCount++; + MutationOrder.Add("Delete"); + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + private static CloudflareDnsRecord Clone(CloudflareDnsRecord record) => + new() + { + Id = record.Id, + Type = record.Type, + Name = record.Name, + Content = record.Content, + Ttl = record.Ttl, + Proxied = record.Proxied, + }; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs new file mode 100644 index 00000000..8e624a92 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeLocalIpAddressAssignee.cs @@ -0,0 +1,40 @@ +using System.Net; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.Common.Configuration; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// +/// Deterministic local-address probe for configuration tests. +/// +internal sealed class FakeLocalIpAddressAssignee : ILocalIpAddressAssignee +{ + private readonly HashSet _assigned; + private readonly bool _assignAll; + private readonly IReadOnlyList _unicast; + + public FakeLocalIpAddressAssignee(bool assignAll) + { + _assignAll = assignAll; + _assigned = new HashSet(); + _unicast = assignAll + ? [TestHostFactory.TestIpv4, TestHostFactory.TestIpv6] + : Array.Empty(); + } + + public FakeLocalIpAddressAssignee(params IPAddress[] assigned) + { + _assignAll = false; + _assigned = new HashSet(assigned); + _unicast = assigned; + } + + public bool IsLocallyAssigned(IPAddress address) + { + ArgumentNullException.ThrowIfNull(address); + return _assignAll || _assigned.Contains(address); + } + + public IReadOnlyList GetAssignedUnicastAddresses() => _unicast; +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeNntpDb.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeNntpDb.cs new file mode 100644 index 00000000..a1c162db --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeNntpDb.cs @@ -0,0 +1,447 @@ +using MySqlConnector; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Moderation; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.PostFilter; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// In-memory MySQL connection factory for offline NntpDB tests. +internal sealed class FakeNntpDbConnectionFactory : INntpDbConnectionFactory +{ + private readonly object _sync = new(); + private readonly List _connections = []; + + public int OpenCount { get; private set; } + + public int OpenAttemptCount { get; private set; } + + public Exception? OpenException { get; set; } + + public int RemainingOpenFailures { get; set; } + + public TaskCompletionSource? BlockOpen { get; set; } + + public TaskCompletionSource? OpenStarted { get; set; } + + public TaskCompletionSource? BlockSelectOne { get; set; } + + public TaskCompletionSource? SelectOneStarted { get; set; } + + public int NextHealthCheckResult { get; set; } = 1; + + public Exception? NextHealthCheckException { get; set; } + + public IReadOnlyList Newsgroups { get; set; } = []; + + public Exception? QueryNewsgroupsException { get; set; } + + public Dictionary Users { get; } = new(StringComparer.Ordinal); + + public Exception? QueryUserException { get; set; } + + public Exception? ConsumeAccountBytesException { get; set; } + + public IReadOnlyList Moderators { get; set; } = []; + + public Exception? QueryModeratorsException { get; set; } + + public PostFilterPolicyRecord? PostFilterPolicy { get; set; } = + new(1, DateTimeOffset.UtcNow, new PostFilterOptions()); + + public Dictionary PostFilterRevisions { get; } = []; + + public long? PostFilterCurrentRevision { get; set; } + + public Exception? QueryPostFilterPolicyException { get; set; } + + public Exception? InsertPostFilterRejectionException { get; set; } + + public TaskCompletionSource? BlockQueryNewsgroups { get; set; } + + public TaskCompletionSource? QueryNewsgroupsStarted { get; set; } + + public IReadOnlyList? SharedConfigurationRows { get; set; } + + public TransitCatalogueRecord? TransitCatalogue { get; set; } = TransitCatalogueRecord.Empty; + + public Exception? SharedConfigurationException { get; set; } + + public IReadOnlyList Connections + { + get + { + lock (_sync) + { + return _connections.ToArray(); + } + } + } + + public int QueryNewsgroupsCount + { + get + { + lock (_sync) + { + var total = 0; + foreach (var connection in _connections) + { + total += connection.QueryNewsgroupsCount; + } + + return total; + } + } + } + + public async Task OpenAsync(string connectionString, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + OpenAttemptCount++; + OpenStarted?.TrySetResult(); + if (BlockOpen is not null) + { + await BlockOpen.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (RemainingOpenFailures > 0) + { + RemainingOpenFailures--; + throw new NntpDbUnavailableException("down"); + } + + if (OpenException is not null) + { + throw OpenException; + } + + var connection = new FakeNntpDbConnection + { + HealthCheckResult = NextHealthCheckResult, + HealthCheckException = NextHealthCheckException, + BlockSelectOne = BlockSelectOne, + SelectOneStarted = SelectOneStarted, + Newsgroups = Newsgroups, + QueryNewsgroupsException = QueryNewsgroupsException, + Users = Users, + QueryUserException = QueryUserException, + ConsumeAccountBytesException = ConsumeAccountBytesException, + Moderators = Moderators, + QueryModeratorsException = QueryModeratorsException, + PostFilterPolicy = PostFilterPolicy, + PostFilterRevisions = PostFilterRevisions, + PostFilterCurrentRevision = PostFilterCurrentRevision, + QueryPostFilterPolicyException = QueryPostFilterPolicyException, + InsertPostFilterRejectionException = InsertPostFilterRejectionException, + BlockQueryNewsgroups = BlockQueryNewsgroups, + QueryNewsgroupsStarted = QueryNewsgroupsStarted, + SharedConfigurationRows = SharedConfigurationRows, + SharedConfigurationException = SharedConfigurationException, + TransitCatalogue = TransitCatalogue, + }; + lock (_sync) + { + OpenCount++; + _connections.Add(connection); + } + + return connection; + } +} + +/// In-memory logical MySQL connection with failure injection. +internal sealed class FakeNntpDbConnection : INntpDbConnection, INntpSharedConfigurationRowSource +{ + public int SelectOneCount { get; private set; } + + public int DisposeCount { get; private set; } + + public int HealthCheckResult { get; set; } = 1; + + public Exception? HealthCheckException { get; set; } + + public TaskCompletionSource? BlockSelectOne { get; set; } + + public TaskCompletionSource? SelectOneStarted { get; set; } + + public IReadOnlyList Newsgroups { get; set; } = []; + + public Exception? QueryNewsgroupsException { get; set; } + + public Dictionary Users { get; set; } = new(StringComparer.Ordinal); + + public Exception? QueryUserException { get; set; } + + public Exception? ConsumeAccountBytesException { get; set; } + + public int ConsumeAccountBytesCount { get; private set; } + + public int QueryAccountByteRemainingCount { get; private set; } + + public IReadOnlyList Moderators { get; set; } = []; + + public Exception? QueryModeratorsException { get; set; } + + public int QueryModeratorsCount { get; private set; } + + public PostFilterPolicyRecord? PostFilterPolicy { get; set; } + + public Dictionary PostFilterRevisions { get; set; } = []; + + public long? PostFilterCurrentRevision { get; set; } + + public Exception? QueryPostFilterPolicyException { get; set; } + + public int QueryPostFilterPolicyCount { get; private set; } + + public TaskCompletionSource? BlockQueryNewsgroups { get; set; } + + public TaskCompletionSource? QueryNewsgroupsStarted { get; set; } + + public int QueryNewsgroupsCount { get; private set; } + + public int QueryUserCount { get; private set; } + + public async ValueTask SelectOneAsync(CancellationToken cancellationToken) + { + SelectOneStarted?.TrySetResult(); + if (BlockSelectOne is not null) + { + await BlockSelectOne.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + SelectOneCount++; + if (HealthCheckException is not null) + { + throw HealthCheckException; + } + + return HealthCheckResult; + } + + public async ValueTask> QueryNewsgroupsAsync(CancellationToken cancellationToken) + { + QueryNewsgroupsStarted?.TrySetResult(); + if (BlockQueryNewsgroups is not null) + { + await BlockQueryNewsgroups.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + QueryNewsgroupsCount++; + if (QueryNewsgroupsException is not null) + { + throw QueryNewsgroupsException; + } + + return Newsgroups; + } + + public ValueTask QueryUserAccountAsync(string accountName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + QueryUserCount++; + if (QueryUserException is not null) + { + throw QueryUserException; + } + + return Users.TryGetValue(accountName, out var record) + ? ValueTask.FromResult(record) + : ValueTask.FromResult(null); + } + + public ValueTask ConsumeAccountBytesAsync( + string accountName, + long bytes, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + ArgumentOutOfRangeException.ThrowIfNegative(bytes); + cancellationToken.ThrowIfCancellationRequested(); + ConsumeAccountBytesCount++; + if (ConsumeAccountBytesException is not null) + { + throw ConsumeAccountBytesException; + } + + lock (Users) + { + if (!Users.TryGetValue(accountName, out var record)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var current = record.ByteLimit < 0 ? 0 : record.ByteLimit; + var consumed = bytes > current ? current : bytes; + var remaining = current - consumed; + Users[accountName] = WithByteLimit(record, remaining); + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, consumed)); + } + } + + public ValueTask QueryAccountByteRemainingAsync( + string accountName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + QueryAccountByteRemainingCount++; + if (QueryUserException is not null) + { + throw QueryUserException; + } + + if (!Users.TryGetValue(accountName, out var record)) + { + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.AccountNotFound, 0, 0)); + } + + var remaining = record.ByteLimit < 0 ? 0 : record.ByteLimit; + return ValueTask.FromResult( + new AccountByteConsumeResult(AccountByteConsumeStatus.Consumed, remaining, 0)); + } + + public ValueTask> QueryEnabledModeratorsAsync( + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryModeratorsCount++; + if (QueryModeratorsException is not null) + { + throw QueryModeratorsException; + } + + return ValueTask.FromResult(Moderators); + } + + public ValueTask QueryPostFilterPolicyAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryPostFilterPolicyCount++; + if (QueryPostFilterPolicyException is not null) + { + throw QueryPostFilterPolicyException; + } + + if (PostFilterRevisions.Count > 0) + { + if (PostFilterCurrentRevision is not { } revision) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult( + PostFilterRevisions.TryGetValue(revision, out var versioned) ? versioned : null); + } + + return ValueTask.FromResult(PostFilterPolicy); + } + + public TransitCatalogueRecord? TransitCatalogue { get; set; } + + public Exception? QueryTransitCatalogueException { get; set; } + + public ValueTask QueryTransitCatalogueAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueryTransitCatalogueException is not null) + { + throw QueryTransitCatalogueException; + } + + return ValueTask.FromResult(TransitCatalogue); + } + + public List Rejections { get; } = []; + + public Exception? InsertPostFilterRejectionException { get; set; } + + public int InsertPostFilterRejectionCount { get; private set; } + + public ValueTask InsertPostFilterRejectionAsync( + PostFilterRejectionEvidence evidence, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(evidence); + cancellationToken.ThrowIfCancellationRequested(); + InsertPostFilterRejectionCount++; + if (InsertPostFilterRejectionException is not null) + { + throw InsertPostFilterRejectionException; + } + + Rejections.Add(evidence); + return ValueTask.CompletedTask; + } + + public IReadOnlyList? SharedConfigurationRows { get; set; } + + public Exception? SharedConfigurationException { get; set; } + + public MySqlCommand CreateCommand() => + throw new NotSupportedException("Fake NntpDB connections do not create provider commands."); + + public ValueTask BeginTransactionAsync(CancellationToken cancellationToken) => + throw new NotSupportedException("Fake NntpDB connections do not begin transactions."); + + public ValueTask BeginTransactionAsync( + System.Data.IsolationLevel isolationLevel, + CancellationToken cancellationToken) => + throw new NotSupportedException("Fake NntpDB connections do not begin transactions."); + + public ValueTask> ReadCandidatesAsync( + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (SharedConfigurationException is not null) + { + throw SharedConfigurationException; + } + + IReadOnlyList rows = SharedConfigurationRows + ?? [new NntpSharedConfigurationCandidate( + 5 * 1024 * 1024, + "news.usenet.ninja", + null, + "https://acme-v02.api.letsencrypt.org/directory", + 14, + "5811a29d39a0732afb5f160c9b137c3d", + "usenet.ninja")]; + return ValueTask.FromResult(rows); + } + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + + private static NntpUserRecord WithByteLimit(NntpUserRecord record, long byteLimit) => + new( + record.AccountName, + record.AccountPassword, + record.AllowAuthPlain, + record.AllowAuthScram256, + record.ScramSalt, + record.ScramIterations, + record.ScramStoredKey, + record.ScramServerKey, + record.RateLimitBps, + byteLimit, + record.SessionLimit, + record.SrcIpLimit, + record.IsEnabled, + record.CustomerId, + record.AllowedArtTypes); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRabbitMq.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRabbitMq.cs new file mode 100644 index 00000000..234573d5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRabbitMq.cs @@ -0,0 +1,1216 @@ +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using VectorNNTPD.Common.Messaging.Cache; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.RabbitMq; +using VectorNNTPD.TransitService.RabbitMq.ArticleWork; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// In-memory RabbitMQ connection factory for offline tests. +internal sealed class FakeRabbitMqConnectionFactory : IRabbitMqConnectionFactory +{ + private int _connectCount; + private int _attemptCount; + private readonly SemaphoreSlim _attemptPulse = new(0, int.MaxValue); + + /// Gets the number of successful connects. + public int ConnectCount => Volatile.Read(ref _connectCount); + + /// Gets the number of connect attempts, including failures. + public int AttemptCount => Volatile.Read(ref _attemptCount); + + /// Gets the most recently created connection. + public FakeRabbitMqConnection? LastConnection { get; private set; } + + /// Gets every connection created by this factory. + public List Connections { get; } = []; + + /// When set, every connect attempt throws this exception. + public Exception? ConnectException { get; set; } + + /// + /// Remaining connect attempts that throw or a default broker failure. + /// After the budget is exhausted, connects succeed unless is set. + /// + public int RemainingConnectFailures { get; set; } + + /// Signaled when a connect attempt begins, before . + public TaskCompletionSource? ConnectStarted { get; set; } + + /// When set, connect waits on this source before completing. + public TaskCompletionSource? BlockConnect { get; set; } + + /// Signaled after each successful connect with the new . + public TaskCompletionSource? Connected { get; set; } + + /// When , created connections report IsOpen == false. + public bool ReturnUnusableConnection { get; set; } + + /// Copied onto each created connection as . + public Exception? QueueDeclareException { get; set; } + + /// + /// Copied onto each created connection as . + /// + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// Waits until at least connect attempts have started. + public async Task WaitForAttemptsAsync(int count, CancellationToken cancellationToken) + { + while (AttemptCount < count) + { + await _attemptPulse.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + public async Task ConnectAsync( + RabbitMqOptions options, + string connectionName, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(connectionName); + + Interlocked.Increment(ref _attemptCount); + _attemptPulse.Release(); + ConnectStarted?.TrySetResult(); + if (BlockConnect is not null) + { + await BlockConnect.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (RemainingConnectFailures > 0) + { + RemainingConnectFailures--; + throw ConnectException ?? new InvalidOperationException("broker down"); + } + + if (ConnectException is not null) + { + throw ConnectException; + } + + var count = Interlocked.Increment(ref _connectCount); + var hosts = options.Hosts ?? []; + var connection = new FakeRabbitMqConnection( + hosts.Length > 0 ? hosts[0] : "127.0.0.1", + options.Port ?? 5672, + string.IsNullOrWhiteSpace(options.VirtualHost) ? "/" : options.VirtualHost.Trim(), + connectionName) + { + IsOpen = !ReturnUnusableConnection, + QueueDeclareException = QueueDeclareException, + }; + foreach (var pair in PassiveConsumerCounts) + { + connection.PassiveConsumerCounts[pair.Key] = pair.Value; + } + + LastConnection = connection; + Connections.Add(connection); + Connected?.TrySetResult(count); + return connection; + } +} + +/// In-memory RabbitMQ connection for lifecycle tests. +internal sealed class FakeRabbitMqConnection : IRabbitMqConnection +{ + /// Initializes a new fake connection. + public FakeRabbitMqConnection(string host, int port, string virtualHost, string clientProvidedName) + { + Host = host; + Port = port; + VirtualHost = virtualHost; + ClientProvidedName = clientProvidedName; + } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// + public string Host { get; } + + /// + public int Port { get; } + + /// + public string VirtualHost { get; } + + /// + public string ClientProvidedName { get; } + + /// + public event EventHandler? ConnectionBlocked; + + /// + public event EventHandler? ConnectionUnblocked; + + /// Gets how many times the connection was disposed. + public int DisposeCount { get; private set; } + + /// Signaled when begins, before . + public TaskCompletionSource? DisposeStarted { get; set; } + + /// When set, waits on this source before completing. + public TaskCompletionSource? BlockDispose { get; set; } + + /// + public event EventHandler? ConnectionLost; + + /// Gets every topology channel created on this connection. + public List TopologyChannels { get; } = []; + + /// Gets every RPC channel created on this connection. + public List RpcChannels { get; } = []; + + /// Gets every confirm-enabled publish channel created on this connection. + public List PublishChannels { get; } = []; + + /// Gets every async-confirm publish channel created on this connection. + public List AsyncConfirmPublishChannels { get; } = []; + + /// Gets every manual-ack channel created on this connection. + public List ManualAckChannels { get; } = []; + + /// When set, throws this exception. + public Exception? CreateRpcChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreateManualAckChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreatePublishChannelException { get; set; } + + /// When set, throws this exception. + public Exception? CreateAsyncConfirmPublishChannelException { get; set; } + + /// Optional configurator invoked for every newly created publish channel. + public Action? ConfigurePublishChannel { get; set; } + + /// Optional configurator invoked for every newly created async-confirm channel. + public Action? ConfigureAsyncConfirmPublishChannel { get; set; } + + /// When set, throws this exception. + public Exception? CreateTopologyChannelException { get; set; } + + /// When set, the next created topology channel throws this exception from queue declare. + public Exception? QueueDeclareException { get; set; } + + /// When set, the next created topology channel throws this exception from exchange declare. + public Exception? ExchangeDeclareException { get; set; } + + /// When set, the next created topology channel throws this exception from queue bind. + public Exception? QueueBindException { get; set; } + + /// + /// Passive-declare consumer counts copied onto each new topology channel (ArticleWork tests). + /// + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// When set, throws on new topology channels. + public Exception? QueueDeclarePassiveException { get; set; } + + /// + /// Queue names for which passive declare throws NOT_FOUND on each new topology channel + /// (simulates missing provider queues under BackFiller-owned topology). + /// + public HashSet PassiveDeclareNotFoundQueues { get; } = new(StringComparer.Ordinal); + + /// + public Task CreateTopologyChannelAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateTopologyChannelException is not null) + { + throw CreateTopologyChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for topology declaration."); + } + + var channel = new FakeRabbitMqTopologyChannel + { + ExchangeDeclareException = ExchangeDeclareException, + QueueDeclareException = QueueDeclareException, + QueueBindException = QueueBindException, + QueueDeclarePassiveException = QueueDeclarePassiveException, + }; + foreach (var pair in PassiveConsumerCounts) + { + channel.PassiveConsumerCounts[pair.Key] = pair.Value; + } + + foreach (var queue in PassiveDeclareNotFoundQueues) + { + channel.PassiveDeclareNotFoundQueues.Add(queue); + } + + TopologyChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateRpcChannelAsync(long generation, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateRpcChannelException is not null) + { + throw CreateRpcChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for article-work RPC."); + } + + var channel = new FakeRabbitMqRpcChannel(generation); + RpcChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateManualAckChannelAsync( + long generation, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateManualAckChannelException is not null) + { + throw CreateManualAckChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for channel creation."); + } + + var channel = new FakeRabbitMqManualAckChannel(generation); + ManualAckChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreatePublishChannelAsync(long generation, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreatePublishChannelException is not null) + { + throw CreatePublishChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for publication."); + } + + var channel = new FakeRabbitMqPublishChannel(generation); + ConfigurePublishChannel?.Invoke(channel); + PublishChannels.Add(channel); + return Task.FromResult(channel); + } + + /// + public Task CreateAsyncConfirmPublishChannelAsync( + long generation, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (CreateAsyncConfirmPublishChannelException is not null) + { + throw CreateAsyncConfirmPublishChannelException; + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ connection is not open for publication."); + } + + var channel = new FakeRabbitMqAsyncConfirmPublishChannel(generation); + ConfigureAsyncConfirmPublishChannel?.Invoke(channel); + AsyncConfirmPublishChannels.Add(channel); + return Task.FromResult(channel); + } + + /// Raises as a peer-initiated disconnect. + public void SimulateLost(ushort replyCode = 320, string replyText = "CONNECTION FORCED") + { + IsOpen = false; + ConnectionLost?.Invoke(this, new RabbitMqConnectionLostEventArgs(replyCode, replyText, "Peer")); + } + + /// + public async ValueTask DisposeAsync() + { + IsOpen = false; + DisposeStarted?.TrySetResult(); + if (BlockDispose is not null) + { + await BlockDispose.Task.ConfigureAwait(false); + } + + DisposeCount++; + } + + /// Gets every exchange declared on any topology channel of this connection. + public IReadOnlyList ExchangeDeclarations => + TopologyChannels.SelectMany(static channel => channel.Exchanges).ToList(); + + /// Gets every queue declared on any topology channel of this connection. + public IReadOnlyList QueueDeclarations => + TopologyChannels.SelectMany(static channel => channel.Queues).ToList(); + + /// Gets every binding declared on any topology channel of this connection. + public IReadOnlyList BindingDeclarations => + TopologyChannels.SelectMany(static channel => channel.Bindings).ToList(); +} + +/// In-memory declare-only RabbitMQ channel for topology tests. +internal sealed class FakeRabbitMqTopologyChannel : IRabbitMqTopologyChannel +{ + /// Gets recorded exchange declarations in call order. + public List Exchanges { get; } = []; + + /// Gets recorded queue declarations in call order. + public List Queues { get; } = []; + + /// Gets recorded queue bindings in call order. + public List Bindings { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? ExchangeDeclareException { get; set; } + + /// When set, throws this exception. + public Exception? QueueDeclareException { get; set; } + + /// When set, throws this exception. + public Exception? QueueBindException { get; set; } + + /// Passive-declare consumer counts keyed by queue name (tests). + public Dictionary PassiveConsumerCounts { get; } = new(StringComparer.Ordinal); + + /// When set, throws this exception. + public Exception? QueueDeclarePassiveException { get; set; } + + /// Queues that throw a broker-style NOT_FOUND on passive declare. + public HashSet PassiveDeclareNotFoundQueues { get; } = new(StringComparer.Ordinal); + + /// Number of passive queue declares performed on this channel. + public int PassiveDeclareCount { get; private set; } + + /// Passive-declare call targets in order. + public List PassiveDeclareQueues { get; } = []; + + /// + public Task ExchangeDeclareAsync( + string exchange, + string type, + bool durable, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (ExchangeDeclareException is not null) + { + throw ExchangeDeclareException; + } + + Exchanges.Add(new FakeRabbitMqExchangeDeclaration( + exchange, + type, + durable, + autoDelete, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueueDeclareException is not null) + { + throw QueueDeclareException; + } + + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclarePassiveAsync(string queue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + PassiveDeclareCount++; + PassiveDeclareQueues.Add(queue); + if (QueueDeclarePassiveException is not null) + { + throw QueueDeclarePassiveException; + } + + if (PassiveDeclareNotFoundQueues.Contains(queue)) + { + throw new InvalidOperationException($"NOT_FOUND - no queue '{queue}' in vhost '/'"); + } + + PassiveConsumerCounts.TryGetValue(queue, out var consumers); + return Task.FromResult(new RabbitMqQueueStats(MessageCount: 0, ConsumerCount: consumers)); + } + + /// + public Task QueueBindAsync( + string queue, + string exchange, + string routingKey, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (QueueBindException is not null) + { + throw QueueBindException; + } + + Bindings.Add(new FakeRabbitMqBindingDeclaration( + queue, + exchange, + routingKey, + CopyArguments(arguments))); + return Task.CompletedTask; + } + + /// + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + + private static IReadOnlyDictionary? CopyArguments( + IReadOnlyDictionary? arguments) + { + if (arguments is null) + { + return null; + } + + return new Dictionary(arguments, StringComparer.Ordinal); + } +} + +/// In-memory RabbitMQ channel for article-work RPC tests. +internal sealed class FakeRabbitMqRpcChannel : IRabbitMqRpcChannel +{ + /// Initializes a new fake RPC channel. + public FakeRabbitMqRpcChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// Gets recorded queue declarations. + public List Queues { get; } = []; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// Gets the consume handler when a consumer has started. + public Func? DeliveryHandler { get; private set; } + + /// Gets the most recent delivery passed to the consumer, when any. + public RabbitMqRpcDelivery? LastDelivery { get; private set; } + + /// Gets the consumed queue name. + public string? ConsumedQueue { get; private set; } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task PublishAsync( + string exchange, + string routingKey, + string correlationId, + string requestId, + string replyTo, + string contentType, + string expiration, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqRpcChannel)); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqRpcPublication( + exchange, + routingKey, + correlationId, + requestId, + replyTo, + contentType, + expiration, + body.ToArray())); + return Task.CompletedTask; + } + + /// + public Task ConsumeAsync( + string queue, + Func onDelivery, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConsumedQueue = queue; + DeliveryHandler = onDelivery; + return Task.FromResult("fake-consumer"); + } + + /// Delivers one message to the registered consumer. + public Task DeliverAsync( + string? correlationId, + ReadOnlyMemory body, + string? contentType = "application/json", + string? requestId = null, + string? expiration = ArticleWorkRpcAmqp.ExpirationMilliseconds) + { + if (DeliveryHandler is null) + { + throw new InvalidOperationException("No RPC consumer is registered."); + } + + var delivery = new RabbitMqRpcDelivery( + correlationId, + requestId, + contentType, + expiration, + body.ToArray(), + Generation); + LastDelivery = delivery; + return DeliveryHandler(delivery); + } + + /// + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// In-memory confirm-enabled publish channel for OverviewDB handoff tests. +internal sealed class FakeRabbitMqPublishChannel : IRabbitMqPublishChannel +{ + /// Initializes a new fake publish channel. + public FakeRabbitMqPublishChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// + /// Remaining publications that throw or a default nack. + /// After the budget is exhausted, publications succeed unless stays set + /// and is zero with a sticky exception. + /// + public int RemainingPublishFailures { get; set; } + + /// + /// When , waits until cancellationToken is cancelled + /// instead of completing a confirm. + /// + public bool HoldUntilCancelled { get; set; } + + /// + /// When , marks the channel closed at the start of the next + /// publish so the not-open failure path is exercised after the channel was obtained. + /// + public bool CloseBeforePublish { get; set; } + + /// + /// Optional await injected after open checks and before recording a successful confirm. + /// Used by concurrency tests to observe in-flight publishes. + /// + public Func? BeforeConfirmAsync { get; set; } + + /// + public Task PublishConfirmedAsync( + string exchange, + string routingKey, + string messageId, + string appId, + string expiration, + ReadOnlyMemory body, + CancellationToken cancellationToken) => + PublishConfirmedAsync( + new RabbitMqConfirmedPublication( + exchange, + routingKey, + messageId, + appId, + CorrelationId: null, + ContentType: null, + RequestIdHeader: null, + expiration, + Persistent: true, + Mandatory: true, + body), + cancellationToken); + + /// + public async Task PublishConfirmedAsync( + RabbitMqConfirmedPublication publication, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(publication); + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqPublishChannel)); + } + + if (CloseBeforePublish) + { + IsOpen = false; + } + + if (HoldUntilCancelled) + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ publish channel is not open."); + } + + if (BeforeConfirmAsync is not null) + { + await BeforeConfirmAsync().ConfigureAwait(false); + } + + if (RemainingPublishFailures > 0) + { + RemainingPublishFailures--; + throw PublishException ?? new InvalidOperationException("RabbitMQ negatively acknowledged the publication."); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqConfirmedPublication( + publication.Exchange, + publication.RoutingKey, + publication.MessageId, + publication.AppId, + publication.CorrelationId, + publication.ContentType, + publication.RequestIdHeader, + publication.ExpirationMilliseconds, + publication.Persistent, + publication.Mandatory, + publication.Body.ToArray())); + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// +/// In-memory async-confirm publish channel. returns after the write; +/// auto-acks by default so outstanding windows drain without a real broker. +/// +internal sealed class FakeRabbitMqAsyncConfirmPublishChannel : IRabbitMqAsyncConfirmPublishChannel +{ + private long _nextSequence = 1; + private AsyncEventHandler? _acks; + private AsyncEventHandler? _nacks; + private AsyncEventHandler? _returns; + + public FakeRabbitMqAsyncConfirmPublishChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded publications. + public List Publications { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// When true (default), raises BasicAck after each successful publish. + public bool AutoAck { get; set; } = true; + + /// Delay before auto-ack (simulates quorum confirm latency). + public TimeSpan AutoAckDelay { get; set; } + + /// When set, throws this exception. + public Exception? PublishException { get; set; } + + /// Remaining publications that throw before succeeding. + public int RemainingPublishFailures { get; set; } + + /// Optional await before completing a publish write. + public Func? BeforePublishAsync { get; set; } + + /// + public event AsyncEventHandler BasicAcksAsync + { + add => _acks += value; + remove => _acks -= value; + } + + /// + public event AsyncEventHandler BasicNacksAsync + { + add => _nacks += value; + remove => _nacks -= value; + } + + /// + public event AsyncEventHandler BasicReturnAsync + { + add => _returns += value; + remove => _returns -= value; + } + + /// + public ValueTask GetNextPublishSequenceNumberAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var seq = (ulong)Interlocked.Increment(ref _nextSequence) - 1; + return ValueTask.FromResult(seq); + } + + /// + public async Task PublishAsync( + string exchange, + string routingKey, + string messageId, + string appId, + string expiration, + ulong publishSequenceNumber, + ReadOnlyMemory body, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (DisposeCount > 0) + { + throw new ObjectDisposedException(nameof(FakeRabbitMqAsyncConfirmPublishChannel)); + } + + if (!IsOpen) + { + throw new InvalidOperationException("RabbitMQ publish channel is not open."); + } + + if (BeforePublishAsync is not null) + { + await BeforePublishAsync().ConfigureAwait(false); + } + + if (RemainingPublishFailures > 0) + { + RemainingPublishFailures--; + throw PublishException + ?? new InvalidOperationException("RabbitMQ publish write failed."); + } + + if (PublishException is not null) + { + throw PublishException; + } + + Publications.Add(new FakeRabbitMqConfirmedPublication( + exchange, + routingKey, + messageId, + appId, + CorrelationId: null, + ContentType: null, + RequestIdHeader: null, + expiration, + Persistent: true, + Mandatory: OverviewDbTopology.Mandatory, + body.ToArray())); + + if (AutoAck) + { + var delay = AutoAckDelay; + var seq = publishSequenceNumber; + _ = Task.Run(async () => + { + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay).ConfigureAwait(false); + } + + await RaiseAckAsync(seq, multiple: false).ConfigureAwait(false); + }); + } + } + + /// Raises a Basic.Ack for tests. + public Task RaiseAckAsync(ulong deliveryTag, bool multiple) => + _acks?.Invoke(this, new BasicAckEventArgs(deliveryTag, multiple, CancellationToken.None)) + ?? Task.CompletedTask; + + /// Raises a Basic.Nack for tests. + public Task RaiseNackAsync(ulong deliveryTag, bool multiple) => + _nacks?.Invoke(this, new BasicNackEventArgs(deliveryTag, multiple, requeue: false, CancellationToken.None)) + ?? Task.CompletedTask; + + /// Raises a Basic.Return for tests. + public Task RaiseReturnAsync(ulong publishSequenceNumber, ReadOnlyMemory body) + { + var properties = new BasicProperties + { + Headers = new Dictionary + { + // Mirror production AMQP table representation (invariant decimal string). + [Constants.PublishSequenceNumberHeader] = + RabbitMqClientAsyncConfirmPublishChannel.FormatPublishSequenceNumberHeader( + publishSequenceNumber), + }, + }; + return _returns?.Invoke( + this, + new BasicReturnEventArgs( + 312, + "NO_ROUTE", + OverviewDbTopology.DefaultExchange, + OverviewDbTopology.RoutingKey, + properties, + body, + CancellationToken.None)) + ?? Task.CompletedTask; + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// Recorded article-work RPC publication. +internal sealed record FakeRabbitMqRpcPublication( + string Exchange, + string RoutingKey, + string CorrelationId, + string RequestId, + string ReplyTo, + string ContentType, + string Expiration, + byte[] Body); + +/// In-memory manual-ack RabbitMQ channel for consume/settle tests. +internal sealed class FakeRabbitMqManualAckChannel : IRabbitMqManualAckChannel +{ + /// Initializes a new fake manual-ack channel. + public FakeRabbitMqManualAckChannel(long generation) + { + Generation = generation; + } + + /// + public long Generation { get; } + + /// + public bool IsOpen { get; set; } = true; + + /// + public event EventHandler? ChannelFaulted; + + /// Gets recorded exchange declarations. + public List Exchanges { get; } = []; + + /// Gets recorded queue declarations. + public List Queues { get; } = []; + + /// Gets recorded bindings. + public List Bindings { get; } = []; + + /// Gets recorded acknowledgements. + public List Acks { get; } = []; + + /// Gets recorded negative acknowledgements. + public List<(ulong DeliveryTag, bool Requeue)> Nacks { get; } = []; + + /// Gets how many times the channel was disposed. + public int DisposeCount { get; private set; } + + /// Gets the consume handler when a consumer has started. + public Func? DeliveryHandler { get; private set; } + + /// Gets the consumed queue name. + public string? ConsumedQueue { get; private set; } + + /// Gets the configured prefetch. + public ushort PrefetchCount { get; private set; } + + /// + public Task ExchangeDeclareAsync( + string exchange, + string type, + bool durable, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Exchanges.Add(new FakeRabbitMqExchangeDeclaration( + exchange, + type, + durable, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task QueueDeclareAsync( + string queue, + bool durable, + bool exclusive, + bool autoDelete, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Queues.Add(new FakeRabbitMqQueueDeclaration( + queue, + durable, + exclusive, + autoDelete, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task QueueBindAsync( + string queue, + string exchange, + string routingKey, + IReadOnlyDictionary? arguments, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Bindings.Add(new FakeRabbitMqBindingDeclaration( + queue, + exchange, + routingKey, + arguments is null ? null : new Dictionary(arguments, StringComparer.Ordinal))); + return Task.CompletedTask; + } + + /// + public Task BasicConsumeAsync( + string queue, + ushort prefetchCount, + Func onDelivery, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConsumedQueue = queue; + PrefetchCount = prefetchCount; + DeliveryHandler = onDelivery; + return Task.FromResult("fake-manual-ack-consumer"); + } + + /// + public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + + /// + public Task BasicAckAsync(ulong deliveryTag, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Acks.Add(deliveryTag); + return Task.CompletedTask; + } + + /// + public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Nacks.Add((deliveryTag, requeue)); + return Task.CompletedTask; + } + + /// + public Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Nacks.Add((deliveryTag, requeue)); + return Task.CompletedTask; + } + + /// Delivers one message to the registered manual-ack consumer. + public Task DeliverAsync( + ulong deliveryTag, + ReadOnlyMemory body, + string? contentType = StorageServerAdvertisementWireProtocol.JsonContentType, + string? exchange = CacheFleetTopology.BroadcastExchangeName, + string? routingKey = "", + string? correlationId = null, + string? replyTo = null) + { + if (DeliveryHandler is null) + { + throw new InvalidOperationException("No manual-ack consumer is registered."); + } + + var delivery = new RabbitMqManualAckDelivery( + deliveryTag, + body.ToArray(), + correlationId, + replyTo, + contentType, + RequestIdHeader: null, + Redelivered: false, + routingKey, + exchange, + ConsumerTag: "fake-manual-ack-consumer", + Generation); + return DeliveryHandler(delivery); + } + + /// + public ValueTask DisposeAsync() + { + IsOpen = false; + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// Recorded confirm-enabled publication. +internal sealed record FakeRabbitMqConfirmedPublication( + string Exchange, + string RoutingKey, + string MessageId, + string? AppId, + string? CorrelationId, + string? ContentType, + string? RequestIdHeader, + string Expiration, + bool Persistent, + bool Mandatory, + byte[] Body); + +/// Recorded exchange declaration. +internal sealed record FakeRabbitMqExchangeDeclaration( + string Name, + string Type, + bool Durable, + bool AutoDelete, + IReadOnlyDictionary? Arguments); + +/// Recorded queue declaration. +internal sealed record FakeRabbitMqQueueDeclaration( + string Name, + bool Durable, + bool Exclusive, + bool AutoDelete, + IReadOnlyDictionary? Arguments); + +/// Recorded queue binding. +internal sealed record FakeRabbitMqBindingDeclaration( + string Queue, + string Exchange, + string RoutingKey, + IReadOnlyDictionary? Arguments); diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRedis.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRedis.cs new file mode 100644 index 00000000..f6c74fda --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeRedis.cs @@ -0,0 +1,536 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Text; +using VectorNNTPD.TransitService.SessionState.BytesAccounting; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.PostFilter.Quota; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Redis; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// In-memory Redis connection factory for offline tests. +internal sealed class FakeRedisConnectionFactory : IRedisConnectionFactory +{ + public int ConnectCount { get; private set; } + + public Exception? ConnectException { get; set; } + + public FakeRedisConnection? LastConnection { get; private set; } + + public Task ConnectAsync(RedisOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + if (ConnectException is not null) + { + throw ConnectException; + } + + ConnectCount++; + LastConnection = new FakeRedisConnection(); + return Task.FromResult(LastConnection); + } +} + +/// In-memory Redis connection wrapping one . +internal sealed class FakeRedisConnection : IRedisConnection +{ + public FakeRedisDatabase Database { get; } = new(); + + public int DisposeCount { get; private set; } + + public IRedisDatabase GetDatabase() => Database; + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } +} + +/// In-memory Redis database with optional failure injection. +internal sealed class FakeRedisDatabase : IRedisDatabase +{ + private readonly ConcurrentDictionary _keys = + new(StringComparer.Ordinal); + + public int PingCount { get; set; } + + public int KeyExistsCount { get; set; } + + private int _existsStartedCount; + + public int ExistsStartedCount => Volatile.Read(ref _existsStartedCount); + + public ConcurrentDictionary BlockExistsByKey { get; } = new(StringComparer.Ordinal); + + public int SetCount { get; set; } + + public int GetCount { get; set; } + + public Exception? ExistsException { get; set; } + + public Exception? GetException { get; set; } + + public Exception? SetException { get; set; } + + public TimeSpan LastSetExpiry { get; private set; } + + public TaskCompletionSource? BlockSet { get; set; } + + public TaskCompletionSource? BlockExists { get; set; } + + public TaskCompletionSource? ExistsStarted { get; set; } + + public int NotifyExistsStartedAt { get; set; } + + public TaskCompletionSource? ExistsReached { get; set; } + + public int ScriptEvaluateCount { get; set; } + + public Exception? ScriptException { get; set; } + + public TaskCompletionSource? BlockScript { get; set; } + + public TaskCompletionSource? ScriptStarted { get; set; } + + internal SessionStateEngine SessionStateEngine { get; } = new(); + + internal TransitPeerStateEngine TransitPeerStateEngine { get; } = new(); + + internal AccountByteEngine AccountByteEngine { get; } = new(); + + internal PostFilterQuotaEngine PostFilterQuotaEngine { get; } = new(); + + public Task PingAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + PingCount++; + return Task.FromResult(TimeSpan.FromMilliseconds(1)); + } + + public async ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + var started = Interlocked.Increment(ref _existsStartedCount); + ExistsStarted?.TrySetResult(); + if (NotifyExistsStartedAt > 0 && started >= NotifyExistsStartedAt) + { + ExistsReached?.TrySetResult(); + } + if (BlockExistsByKey.TryGetValue(ToKey(key), out var keyedBlock)) + { + await keyedBlock.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + else if (BlockExists is not null) + { + await BlockExists.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + KeyExistsCount++; + if (ExistsException is not null) + { + throw ExistsException; + } + + if (!_keys.TryGetValue(ToKey(key), out var entry)) + { + return false; + } + + if (entry.Expiry <= DateTimeOffset.UtcNow) + { + _keys.TryRemove(ToKey(key), out _); + return false; + } + + return true; + } + + public ValueTask GetAsync( + ReadOnlyMemory key, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + GetCount++; + if (GetException is not null) + { + return ValueTask.FromException(GetException); + } + + var name = ToKey(key); + if (!_keys.TryGetValue(name, out var entry)) + { + return ValueTask.FromResult(null); + } + + if (entry.Expiry <= DateTimeOffset.UtcNow) + { + _keys.TryRemove(name, out _); + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(entry.Value); + } + + public async ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) + { + if (BlockSet is not null) + { + await BlockSet.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (SetException is not null) + { + SetCount++; + throw SetException; + } + + LastSetExpiry = expiry; + _keys[ToKey(key)] = (value.ToArray(), DateTimeOffset.UtcNow.Add(expiry)); + SetCount++; + } + + public async ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(script); + ArgumentNullException.ThrowIfNull(keys); + ArgumentNullException.ThrowIfNull(values); + ScriptStarted?.TrySetResult(); + if (BlockScript is not null) + { + await BlockScript.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + ScriptEvaluateCount++; + if (ScriptException is not null) + { + throw ScriptException; + } + + if (keys.Length == 1) + { + var key = Encoding.UTF8.GetString(keys[0].Span); + if (script == AccountByteScripts.Apply) + { + return AccountByteEngine.Apply(key, Utf8(values[0]), ParseLong(values[1]), ParseLong(values[2])); + } + + if (script == AccountByteScripts.Observe) + { + return AccountByteEngine.Observe(key); + } + + if (script == AccountByteScripts.Delete) + { + return AccountByteEngine.Delete(key); + } + + return EvaluateTransit(script, key, values); + } + + if (keys.Length >= 3 && script == SessionStateScripts.RenewAndApply) + { + return EvaluateRenewAndApply(keys, values); + } + + if (keys.Length == 2 && TryEvaluateQuota(script, keys, values, out var quotaResult)) + { + return quotaResult; + } + + if (keys.Length < 2) + { + throw new InvalidOperationException("Admission EVAL requires KEYS[1] source and KEYS[2] session."); + } + + var sourceKey = Encoding.UTF8.GetString(keys[0].Span); + var sessionKey = Encoding.UTF8.GetString(keys[1].Span); + if (script == SessionStateScripts.TryAdmit) + { + return SessionStateEngine.TryAdmit( + sourceKey, + sessionKey, + Utf8(values[0]), + Utf8(values[1]), + ParseInt(values[2]), + ParseInt(values[3]), + ParseLong(values[4]), + ParseLong(values[5]), + ParseLong(values[6]), + ParseLong(values[7]), + trackSessions: values.Length > 8 && ParseInt(values[8]) == 1); + } + + if (script == SessionStateScripts.Release) + { + return SessionStateEngine.Release( + sourceKey, + sessionKey, + Utf8(values[0]), + Utf8(values[1]), + ParseLong(values[2]), + ParseLong(values[3])); + } + + if (script == SessionStateScripts.Renew) + { + var ipCount = ParseInt(values[4]); + var sources = new (string Ip, long Generation)[ipCount]; + for (var i = 0; i < ipCount; i++) + { + sources[i] = (Utf8(values[5 + (i * 2)]), ParseLong(values[6 + (i * 2)])); + } + + return SessionStateEngine.Renew( + sourceKey, + sessionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + sources); + } + + if (script == SessionStateScripts.ReleaseOwner) + { + return SessionStateEngine.ReleaseOwner(sourceKey, sessionKey, Utf8(values[0])); + } + + throw new NotSupportedException("FakeRedis only evaluates cluster admission scripts."); + } + + private bool TryEvaluateQuota( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + out long result) + { + result = 0; + var quotaKey = Encoding.UTF8.GetString(keys[0].Span); + var multipostKey = Encoding.UTF8.GetString(keys[1].Span); + if (script == PostFilterQuotaScripts.Reserve) + { + var reservation = ParseReservation(Utf8(values[2]), ParseLong(values[3])); + result = PostFilterQuotaEngine.Reserve( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0]), + ParseLong(values[1]), + new PostFilterQuotaWindows(ParseLong(values[8]), ParseLong(values[9])), + new PostFilterQuotaCeilings( + ParseLong(values[10]), + ParseLong(values[11]), + ParseLong(values[12]), + ParseLong(values[13]), + ParseLong(values[14]), + ParseLong(values[15])), + ParseLong(values[4]), + ParseLong(values[5]), + ParseInt(values[6]), + Utf8(values[7])); + return true; + } + + if (script == PostFilterQuotaScripts.Commit) + { + var reservation = ParseReservation(Utf8(values[1]), ParseLong(values[2])); + result = PostFilterQuotaEngine.Commit( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0]), + new PostFilterQuotaWindows(ParseLong(values[3]), ParseLong(values[4])), + new PostFilterQuotaCeilings( + ParseLong(values[5]), + ParseLong(values[6]), + ParseLong(values[7]), + ParseLong(values[8]), + ParseLong(values[9]), + ParseLong(values[10]))); + return true; + } + + if (script == PostFilterQuotaScripts.Release) + { + var reservation = ParseReservation(Utf8(values[1]), ParseLong(values[2])); + result = PostFilterQuotaEngine.Release( + quotaKey, + multipostKey, + reservation, + ParseLong(values[0])); + return true; + } + + return false; + } + + private static PostFilterReservationId ParseReservation(string token, long generation) + { + var last = token.LastIndexOf(':'); + var mid = token.LastIndexOf(':', last - 1); + return new PostFilterReservationId(token[..mid], token[(mid + 1)..last], generation); + } + + private long EvaluateRenewAndApply(ReadOnlyMemory[] keys, ReadOnlyMemory[] values) + { + var sourceKey = Encoding.UTF8.GetString(keys[0].Span); + var sessionKey = Encoding.UTF8.GetString(keys[1].Span); + var bytesKey = Encoding.UTF8.GetString(keys[2].Span); + var ipCount = ParseInt(values[4]); + var sources = new (string Ip, long Generation)[ipCount]; + for (var i = 0; i < ipCount; i++) + { + sources[i] = (Utf8(values[5 + (i * 2)]), ParseLong(values[6 + (i * 2)])); + } + + var renewed = SessionStateEngine.Renew( + sourceKey, + sessionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + sources); + var remaining = AccountByteEngine.Apply( + bytesKey, + Utf8(values[5 + (ipCount * 2)]), + ParseLong(values[6 + (ipCount * 2)]), + ParseLong(values[7 + (ipCount * 2)])); + return SessionStateBytePack.Encode(renewed == 1, remaining); + } + + private long EvaluateTransit(string script, string connectionKey, ReadOnlyMemory[] values) + { + if (script == TransitPeerStateScripts.TryAdmit) + { + return TransitPeerStateEngine.TryAdmit( + connectionKey, + Utf8(values[0]), + ParseInt(values[1]), + ParseLong(values[2]), + ParseLong(values[3]), + ParseLong(values[4])); + } + + if (script == TransitPeerStateScripts.Release) + { + return TransitPeerStateEngine.Release(connectionKey, Utf8(values[0]), ParseLong(values[1])); + } + + if (script == TransitPeerStateScripts.Renew) + { + return TransitPeerStateEngine.Renew( + connectionKey, + Utf8(values[0]), + ParseLong(values[1]), + ParseLong(values[2]), + ParseLong(values[3])); + } + + if (script == TransitPeerStateScripts.ReleaseOwner) + { + return TransitPeerStateEngine.ReleaseOwner(connectionKey, Utf8(values[0])); + } + + throw new NotSupportedException("FakeRedis only evaluates cluster admission scripts."); + } + + public void Seed(ReadOnlyMemory key, TimeSpan? expiry = null) => + Seed(key, new byte[] { 1 }, expiry); + + public void Seed(ReadOnlyMemory key, ReadOnlyMemory value, TimeSpan? expiry = null) => + _keys[ToKey(key)] = (value.ToArray(), DateTimeOffset.UtcNow.Add(expiry ?? TimeSpan.FromHours(2))); + + public bool Contains(ReadOnlyMemory key) => _keys.ContainsKey(ToKey(key)); + + public bool TryGetStored(ReadOnlyMemory key, out byte[] value, out DateTimeOffset expiry) + { + if (_keys.TryGetValue(ToKey(key), out var entry)) + { + value = entry.Value; + expiry = entry.Expiry; + return true; + } + + value = []; + expiry = default; + return false; + } + + public void Expire(ReadOnlyMemory key) => _keys.TryRemove(ToKey(key), out _); + + private static string ToKey(ReadOnlyMemory key) => Convert.ToHexString(key.Span); + + private static string Utf8(ReadOnlyMemory value) => Encoding.UTF8.GetString(value.Span); + + private static int ParseInt(ReadOnlyMemory value) => + int.Parse(Utf8(value), CultureInfo.InvariantCulture); + + private static long ParseLong(ReadOnlyMemory value) => + long.Parse(Utf8(value), CultureInfo.InvariantCulture); +} + +/// Direct double for HistoryDB tests. +internal sealed class FakeRedisService : IRedisService +{ + public FakeRedisDatabase Database { get; } = new(); + + IRedisDatabase IRedisService.Database => Database; + + public bool IsUnavailable { get; set; } + + public int BeginRejectCount { get; private set; } + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + if (IsUnavailable) + { + BeginRejectCount++; + return false; + } + + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + if (!succeeded) + { + IsUnavailable = true; + } + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public void Recover() => IsUnavailable = false; + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.KeyExistsAsync(key, cancellationToken); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + Database.GetAsync(key, cancellationToken); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + Database.SetAsync(key, value, expiry, cancellationToken); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeSmtpServer.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeSmtpServer.cs new file mode 100644 index 00000000..636891cb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/FakeSmtpServer.cs @@ -0,0 +1,594 @@ +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +public sealed class FakeSmtpServerOptions +{ + public bool ImplicitTls { get; set; } + + public bool AdvertiseStartTls { get; set; } + + public bool AdvertiseAuthPlain { get; set; } + + public bool AdvertiseAuthLogin { get; set; } + + public bool RequireTlsForAuthentication { get; set; } = true; + + public string? ExpectedUsername { get; set; } + + public string? ExpectedPassword { get; set; } + + public long? Size { get; set; } + + public int MailFromCode { get; set; } = 250; + + public int DefaultRcptCode { get; set; } = 250; + + public Dictionary RcptCodes { get; } = new(StringComparer.OrdinalIgnoreCase); + + public int DataIntroCode { get; set; } = 354; + + public int DataDoneCode { get; set; } = 250; + + public int TransientDataFailures { get; set; } + + public bool CloseAfterGreeting { get; set; } + + public bool CloseAfterEhlo { get; set; } + + public string Greeting { get; set; } = "220 test.example ESMTP"; + + public X509Certificate2? Certificate { get; set; } +} + +public sealed class FakeSmtpServer : IAsyncDisposable +{ + private readonly FakeSmtpServerOptions _options; + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly List _commands = []; + private readonly List _rawLines = []; + private readonly Task? _run; + private int _dataAttempts; + private bool _sawPlaintextBeforeTls; + + public FakeSmtpServer(FakeSmtpServerOptions? options = null) + { + _options = options ?? new FakeSmtpServerOptions(); + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _run = Task.Run(() => AcceptLoopAsync(_cts.Token)); + } + + public int Port { get; } + + public IReadOnlyList Commands + { + get + { + lock (_commands) + { + return [.. _commands]; + } + } + } + + public IReadOnlyList RawLines + { + get + { + lock (_rawLines) + { + return [.. _rawLines]; + } + } + } + + public byte[]? LastMessage { get; private set; } + + public bool Authenticated { get; private set; } + + public bool StartedTls { get; private set; } + + public bool SawPlaintextSmtpBeforeTls => _sawPlaintextBeforeTls; + + public X509Certificate2? Certificate => _options.Certificate; + + public SmtpOptions ClientOptions( + SmtpSecurityMode security, + string? username = null, + string? password = null) => + new() + { + Host = "127.0.0.1", + Port = Port, + Security = security, + Username = username ?? string.Empty, + Password = password ?? string.Empty, + RequireTlsForAuthentication = security != SmtpSecurityMode.None, + ConnectTimeout = TimeSpan.FromSeconds(5), + CommandTimeout = TimeSpan.FromSeconds(5), + MaxAttempts = 1, + }; + + public static X509Certificate2[] TrustAnchors(X509Certificate2 certificate) + { + ArgumentNullException.ThrowIfNull(certificate); + return [X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert))]; + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync().ConfigureAwait(false); + _listener.Stop(); + if (_run is not null) + { + try + { + await _run.WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + } + catch (TimeoutException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + _cts.Dispose(); + _options.Certificate?.Dispose(); + } + + public static X509Certificate2 CreateSelfSignedCertificate( + string dnsName = "localhost", + bool includeLoopbackIp = true) + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + dnsName, + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + var san = new SubjectAlternativeNameBuilder(); + san.AddDnsName(dnsName); + if (includeLoopbackIp) + { + san.AddIpAddress(IPAddress.Loopback); + } + + request.CertificateExtensions.Add(san.Build()); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, + critical: false)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, critical: false)); + using var ephemeral = request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow.AddDays(30)); + return X509CertificateLoader.LoadPkcs12( + ephemeral.Export(X509ContentType.Pfx, "vectornntp-test"), + "vectornntp-test", + X509KeyStorageFlags.Exportable | X509KeyStorageFlags.UserKeySet | X509KeyStorageFlags.PersistKeySet); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + try + { + while (!cancellationToken.IsCancellationRequested) + { + var tcp = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + _ = Task.Run(() => HandleAsync(tcp, cancellationToken), CancellationToken.None); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (ObjectDisposedException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (SocketException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + private async Task HandleAsync(TcpClient tcp, CancellationToken cancellationToken) + { + using (tcp) + try + { + Stream stream = tcp.GetStream(); + var tls = false; + if (_options.ImplicitTls) + { + stream = await ServerTlsAsync(stream, cancellationToken).ConfigureAwait(false); + tls = true; + StartedTls = true; + } + + await WriteLineAsync(stream, _options.Greeting, cancellationToken).ConfigureAwait(false); + if (_options.CloseAfterGreeting) + { + return; + } + + while (!cancellationToken.IsCancellationRequested) + { + var line = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false); + if (line is null) + { + return; + } + + if (!tls) + { + _sawPlaintextBeforeTls = true; + } + + Record(line, redact: line.StartsWith("AUTH ", StringComparison.OrdinalIgnoreCase)); + var verb = Verb(line); + if (verb.Equals("EHLO", StringComparison.OrdinalIgnoreCase) + || verb.Equals("HELO", StringComparison.OrdinalIgnoreCase)) + { + await WriteEhloAsync(stream, cancellationToken).ConfigureAwait(false); + if (_options.CloseAfterEhlo) + { + return; + } + + continue; + } + + if (verb.Equals("STARTTLS", StringComparison.OrdinalIgnoreCase)) + { + if (!_options.AdvertiseStartTls) + { + await WriteLineAsync(stream, "502 STARTTLS not available", cancellationToken) + .ConfigureAwait(false); + continue; + } + + await WriteLineAsync(stream, "220 Ready to start TLS", cancellationToken).ConfigureAwait(false); + stream = await ServerTlsAsync(stream, cancellationToken).ConfigureAwait(false); + tls = true; + StartedTls = true; + continue; + } + + if (verb.Equals("AUTH", StringComparison.OrdinalIgnoreCase)) + { + if (_options.RequireTlsForAuthentication && !tls) + { + await WriteLineAsync(stream, "538 5.7.11 Encryption required for requested authentication mechanism", cancellationToken) + .ConfigureAwait(false); + continue; + } + + await HandleAuthAsync(stream, line, cancellationToken).ConfigureAwait(false); + continue; + } + + if (verb.Equals("MAIL", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, _options.MailFromCode + " OK", cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("RCPT", StringComparison.OrdinalIgnoreCase)) + { + var mailbox = ExtractMailbox(line); + var code = _options.RcptCodes.TryGetValue(mailbox, out var mapped) + ? mapped + : _options.DefaultRcptCode; + await WriteLineAsync(stream, code + " " + (code < 400 ? "OK" : "rejected"), cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("DATA", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, _options.DataIntroCode + " Start mail input", cancellationToken) + .ConfigureAwait(false); + if (_options.DataIntroCode >= 400) + { + continue; + } + + LastMessage = await ReadDataAsync(stream, cancellationToken).ConfigureAwait(false); + _dataAttempts++; + var done = _dataAttempts <= _options.TransientDataFailures + ? 451 + : _options.DataDoneCode; + await WriteLineAsync(stream, done + " " + (done < 400 ? "OK" : "failed"), cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (verb.Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, "221 Bye", cancellationToken).ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "500 unrecognized", cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (AuthenticationException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + private async Task HandleAuthAsync(Stream stream, string line, CancellationToken cancellationToken) + { + var parts = line.Split(' ', 3, StringSplitOptions.RemoveEmptyEntries); + var mechanism = parts.Length >= 2 ? parts[1] : string.Empty; + if (mechanism.Equals("PLAIN", StringComparison.OrdinalIgnoreCase)) + { + string payload; + if (parts.Length >= 3) + { + payload = parts[2]; + } + else + { + await WriteLineAsync(stream, "334 ", cancellationToken).ConfigureAwait(false); + payload = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(payload, redact: true); + } + + if (TryDecodePlain(payload, out var user, out var password) + && user == _options.ExpectedUsername + && password == _options.ExpectedPassword) + { + Authenticated = true; + await WriteLineAsync(stream, "235 2.7.0 Authentication successful", cancellationToken) + .ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "535 5.7.8 Authentication failed", cancellationToken) + .ConfigureAwait(false); + return; + } + + if (mechanism.Equals("LOGIN", StringComparison.OrdinalIgnoreCase)) + { + await WriteLineAsync(stream, "334 VXNlcm5hbWU6", cancellationToken).ConfigureAwait(false); + var userB64 = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(userB64, redact: true); + await WriteLineAsync(stream, "334 UGFzc3dvcmQ6", cancellationToken).ConfigureAwait(false); + var passB64 = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false) ?? string.Empty; + Record(passB64, redact: true); + var user = DecodeUtf8(userB64); + var password = DecodeUtf8(passB64); + if (user == _options.ExpectedUsername && password == _options.ExpectedPassword) + { + Authenticated = true; + await WriteLineAsync(stream, "235 Authentication successful", cancellationToken) + .ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "535 Authentication failed", cancellationToken).ConfigureAwait(false); + return; + } + + await WriteLineAsync(stream, "504 Unrecognized authentication type", cancellationToken) + .ConfigureAwait(false); + } + + private async Task WriteEhloAsync(Stream stream, CancellationToken cancellationToken) + { + var lines = new List { "250-test.example" }; + if (_options.AdvertiseStartTls) + { + lines.Add("250-STARTTLS"); + } + + if (_options.Size is { } size) + { + lines.Add("250-SIZE " + size); + } + + lines.Add("250-8BITMIME"); + if (_options.AdvertiseAuthPlain || _options.AdvertiseAuthLogin) + { + var mechanisms = new List(); + if (_options.AdvertiseAuthPlain) + { + mechanisms.Add("PLAIN"); + } + + if (_options.AdvertiseAuthLogin) + { + mechanisms.Add("LOGIN"); + } + + lines.Add("250-AUTH " + string.Join(' ', mechanisms)); + } + + lines.Add("250 PIPELINING"); + foreach (var line in lines) + { + await WriteLineAsync(stream, line, cancellationToken).ConfigureAwait(false); + } + } + + private async Task ServerTlsAsync(Stream inner, CancellationToken cancellationToken) + { + var cert = _options.Certificate ?? CreateSelfSignedCertificate(); + _options.Certificate ??= cert; + var ssl = new SslStream(inner, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsServerAsync( + new SslServerAuthenticationOptions + { + ServerCertificate = cert, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + CertificateRevocationCheckMode = X509RevocationMode.NoCheck, + }, + cancellationToken) + .ConfigureAwait(false); + return ssl; + } + + private void Record(string line, bool redact) + { + lock (_rawLines) + { + _rawLines.Add(redact ? "REDACTED" : line); + } + + lock (_commands) + { + if (redact) + { + _commands.Add("AUTH"); + return; + } + + _commands.Add(line); + } + } + + private static async Task WriteLineAsync(Stream stream, string line, CancellationToken cancellationToken) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadLineAsync(Stream stream, CancellationToken cancellationToken) + { + var buffer = new List(128); + var one = new byte[1]; + while (true) + { + var read = await stream.ReadAsync(one, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return buffer.Count == 0 ? null : Encoding.ASCII.GetString(buffer.ToArray()); + } + + if (one[0] == (byte)'\n') + { + if (buffer.Count > 0 && buffer[^1] == (byte)'\r') + { + buffer.RemoveAt(buffer.Count - 1); + } + + return Encoding.ASCII.GetString(buffer.ToArray()); + } + + buffer.Add(one[0]); + } + } + + private static async Task ReadDataAsync(Stream stream, CancellationToken cancellationToken) + { + var output = new List(); + var line = new List(); + var one = new byte[1]; + while (true) + { + var read = await stream.ReadAsync(one, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + break; + } + + line.Add(one[0]); + if (line.Count >= 2 && line[^2] == (byte)'\r' && line[^1] == (byte)'\n') + { + if (line.Count == 3 && line[0] == (byte)'.') + { + break; + } + + var payload = line.ToArray().AsSpan()[..^2]; + if (payload.Length > 0 && payload[0] == (byte)'.') + { + payload = payload[1..]; + } + + output.AddRange(payload.ToArray()); + output.Add((byte)'\r'); + output.Add((byte)'\n'); + line.Clear(); + } + } + + return [.. output]; + } + + private static string Verb(string line) + { + var space = line.IndexOf(' '); + return space < 0 ? line : line[..space]; + } + + private static string ExtractMailbox(string line) + { + var start = line.IndexOf('<'); + var end = line.IndexOf('>'); + return start >= 0 && end > start ? line[(start + 1)..end] : string.Empty; + } + + private static bool TryDecodePlain(string payload, out string user, out string password) + { + user = string.Empty; + password = string.Empty; + try + { + var bytes = Convert.FromBase64String(payload); + var text = Encoding.UTF8.GetString(bytes); + var parts = text.Split('\0'); + if (parts.Length < 3) + { + return false; + } + + user = parts[1]; + password = parts[2]; + return true; + } + catch (FormatException) + { + return false; + } + } + + private static string DecodeUtf8(string base64) + { + try + { + return Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + } + catch (FormatException) + { + return string.Empty; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs new file mode 100644 index 00000000..cb9e9d90 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MapNntpAuthenticationProvider.cs @@ -0,0 +1,76 @@ +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// +/// AUTHINFO USER/PASS map used by moderation and ordinary POST tests. +/// Passwords are never logged or asserted. +/// +public sealed class MapNntpAuthenticationProvider : INntpAuthenticationProvider +{ + /// Ordinary authenticated poster who is not a moderator. + public const string NormalUser = "normal-user"; + + /// Moderator principal for group A mappings. + public const string ModeratorA = "moderator-a"; + + /// Moderator principal for group B mappings. + public const string ModeratorB = "moderator-b"; + + /// Password for . + public const string NormalUserPassword = "normal-user-secret"; + + /// Password for . + public const string ModeratorAPassword = "moderator-a-secret"; + + /// Password for . + public const string ModeratorBPassword = "moderator-b-secret"; + + /// Authenticated reader + poster privileges (no transit, no control-cancel). + public static NntpAuthorization PosterPrivileges { get; } = new( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: true, + streamingPermitted: false); + + private readonly Dictionary _accounts = + new(StringComparer.Ordinal); + + /// Creates the standard normal-user / moderator-a / moderator-b map. + public static MapNntpAuthenticationProvider CreateStandard() + { + var provider = new MapNntpAuthenticationProvider(); + provider.Add(NormalUser, NormalUserPassword, PosterPrivileges); + provider.Add(ModeratorA, ModeratorAPassword, PosterPrivileges); + provider.Add(ModeratorB, ModeratorBPassword, PosterPrivileges); + return provider; + } + + /// Registers one username/password and the privileges granted on success. + public void Add(string username, string password, NntpAuthorization authorization) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + ArgumentNullException.ThrowIfNull(authorization); + _accounts[username] = (password, authorization); + } + + /// + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(username); + ArgumentNullException.ThrowIfNull(password); + _ = cancellationToken; + if (_accounts.TryGetValue(username, out var account) && account.Password == password) + { + return ValueTask.FromResult(NntpAuthenticationResult.Success(username, account.Authorization)); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs new file mode 100644 index 00000000..480fd951 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpModeratorRepository.cs @@ -0,0 +1,75 @@ +using VectorNNTPD.TransitService.Moderation; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// In-memory for catalogue tests. +internal sealed class MemoryNntpModeratorRepository : INntpModeratorRepository +{ + private readonly List _rows = []; + + public int QueryCount { get; private set; } + + public Exception? Exception { get; set; } + + public void Add(MemoryModeratorRecord record) + { + ArgumentNullException.ThrowIfNull(record); + _rows.Add(record); + } + + public ValueTask> GetEnabledAsync( + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + QueryCount++; + if (Exception is not null) + { + throw Exception; + } + + var enabled = _rows + .Where(static row => row.Enabled) + .OrderBy(static row => row.ModeratorId) + .Select(static row => new NntpModeratorRow( + row.ModeratorId, + row.GroupPattern, + row.ModeratorAddress, + row.AccountName)) + .ToArray(); + return ValueTask.FromResult>(enabled); + } +} + +/// One in-memory moderator row, including disabled rows for filter tests. +internal sealed class MemoryModeratorRecord +{ + public long ModeratorId { get; init; } + + public string GroupPattern { get; init; } = string.Empty; + + public string ModeratorAddress { get; init; } = string.Empty; + + public string AccountName { get; init; } = string.Empty; + + public bool Enabled { get; init; } = true; +} + +/// Builds a from test mappings. +internal static class ModeratorTestSnapshot +{ + public static ModeratorSnapshot Create(params (string Pattern, string Address, string Username)[] mappings) + { + ArgumentNullException.ThrowIfNull(mappings); + var rows = new NntpModeratorRow[mappings.Length]; + for (var i = 0; i < mappings.Length; i++) + { + rows[i] = new NntpModeratorRow( + i + 1, + mappings[i].Pattern, + mappings[i].Address, + mappings[i].Username); + } + + return ModeratorSnapshot.Create(rows); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs new file mode 100644 index 00000000..964d8741 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/MemoryNntpUserRecordStore.cs @@ -0,0 +1,68 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.TransitService.Authentication; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// In-memory for authentication tests. +internal sealed class MemoryNntpUserRecordStore : INntpUserRecordStore +{ + private readonly Dictionary _users = new(StringComparer.Ordinal); + + public int LookupCount { get; private set; } + + public Exception? Exception { get; set; } + + public void Add(NntpUserRecord record) + { + ArgumentNullException.ThrowIfNull(record); + _users[record.AccountName] = record; + } + + public ValueTask TryGetUserAsync(string accountName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accountName); + cancellationToken.ThrowIfCancellationRequested(); + LookupCount++; + if (Exception is not null) + { + throw Exception; + } + + return _users.TryGetValue(accountName, out var record) + ? ValueTask.FromResult(record) + : ValueTask.FromResult(null); + } + + public static NntpUserRecord Create( + string name, + string password, + bool enabled = true, + bool allowPlain = true, + bool allowScram = true, + int sessionLimit = 0, + int srcIpLimit = 0, + int rateLimitBps = 0, + long byteLimit = 1_000_000_000, + ReadOnlyMemory scramSalt = default, + int scramIterations = 0, + ReadOnlyMemory scramStoredKey = default, + ReadOnlyMemory scramServerKey = default, + string customerId = "11111111-1111-1111-1111-111111111111", + ArticleType allowedArtTypes = ArticleTypeCapabilities.All) => + new( + name, + password, + allowPlain, + allowScram, + scramSalt, + scramIterations, + scramStoredKey, + scramServerKey, + rateLimitBps, + byteLimit, + sessionLimit, + srcIpLimit, + enabled, + customerId, + allowedArtTypes); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingEmailService.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingEmailService.cs new file mode 100644 index 00000000..36299dce --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingEmailService.cs @@ -0,0 +1,58 @@ +using VectorNNTPD.TransitService.Email; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Captures without SMTP or spool I/O. +internal sealed class RecordingEmailService : IEmailService +{ + private readonly List _sent = []; + + public EmailEnqueueStatus Status { get; set; } = EmailEnqueueStatus.Accepted; + + public Exception? Throw { get; set; } + + public TaskCompletionSource? Hold { get; set; } + + public TaskCompletionSource SentOnce { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int SendThreadId { get; private set; } + + public int SendCount { get; private set; } + + public IReadOnlyList Sent + { + get + { + lock (_sent) + { + return [.. _sent]; + } + } + } + + public async ValueTask SendAsync( + EmailMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + if (Hold is not null) + { + await Hold.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + if (Throw is not null) + { + throw Throw; + } + + SendThreadId = Environment.CurrentManagedThreadId; + lock (_sent) + { + _sent.Add(message); + SendCount++; + } + + SentOnce.TrySetResult(); + return new EmailEnqueueResult(Status, "test"); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingModerationSubmissionService.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingModerationSubmissionService.cs new file mode 100644 index 00000000..0c273ac4 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingModerationSubmissionService.cs @@ -0,0 +1,36 @@ +using VectorNNTPD.TransitService.Moderation; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Records moderation submissions and returns a configured result. +public sealed class RecordingModerationSubmissionService : IModerationSubmissionService +{ + /// Initializes a new instance of the class. + public RecordingModerationSubmissionService( + ModerationSubmissionStatus status = ModerationSubmissionStatus.Accepted, + string detail = "recorded") + { + Status = status; + Detail = detail; + } + + /// Gets the status returned to callers. + public ModerationSubmissionStatus Status { get; set; } + + /// Gets the detail returned to callers. + public string Detail { get; set; } + + /// Gets recorded submissions in call order. + public List Submissions { get; } = []; + + /// + public ValueTask SubmitAsync( + ModerationSubmission submission, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + cancellationToken.ThrowIfCancellationRequested(); + Submissions.Add(submission); + return ValueTask.FromResult(new ModerationSubmissionResult(Status, Detail)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNewsLogWriter.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNewsLogWriter.cs new file mode 100644 index 00000000..848685dc --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNewsLogWriter.cs @@ -0,0 +1,102 @@ +using VectorNNTPD.TransitService.ArticleIngestion; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Captures news events for post-queue ingress tests. +internal sealed class RecordingNewsLogWriter : INewsLogWriter +{ + private readonly List _events = []; + private readonly TaskCompletionSource? _gate; + private int _writeCalls; + private int _flushCalls; + + public RecordingNewsLogWriter(Task? gate = null) + { + _gate = gate is null + ? null + : TaskCompletionSourceFrom(gate); + } + + public IReadOnlyList Events + { + get + { + lock (_events) + { + return _events.ToArray(); + } + } + } + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public int FlushCalls => Volatile.Read(ref _flushCalls); + + public void Write(in NewsLogEvent evt) + { + if (_gate is not null) + { + _gate.Task.GetAwaiter().GetResult(); + } + + Interlocked.Increment(ref _writeCalls); + lock (_events) + { + _events.Add( + new NewsLogEvent( + evt.Disposition, + evt.MessageId.ToArray(), + evt.Feed.ToArray(), + evt.Sites.ToArray(), + evt.Timestamp, + evt.ResponseCode, + evt.Reason.ToArray(), + evt.Size)); + } + } + + public void Flush() => Interlocked.Increment(ref _flushCalls); + + private static TaskCompletionSource TaskCompletionSourceFrom(Task gate) + { + var source = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + gate.ContinueWith( + static (completed, state) => + { + var tcs = (TaskCompletionSource)state!; + if (completed.IsFaulted) + { + tcs.TrySetException(completed.Exception!.InnerExceptions); + } + else if (completed.IsCanceled) + { + tcs.TrySetCanceled(); + } + else + { + tcs.TrySetResult(); + } + }, + source, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + return source; + } +} + +/// News writer that throws to prove already-accepted articles are not reprocessed. +internal sealed class ThrowingNewsLogWriter : INewsLogWriter +{ + public int WriteCalls { get; private set; } + + public void Write(in NewsLogEvent evt) + { + WriteCalls++; + throw new IOException("news writer test failure"); + } + + public void Flush() + { + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs new file mode 100644 index 00000000..13c350a0 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingNntpAuthenticationProvider.cs @@ -0,0 +1,37 @@ +using System.Net; +using VectorNNTPD.TransitService.Session.Authentication; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Counts reader-provider AUTHINFO invocations. Never logs passwords. +internal sealed class RecordingNntpAuthenticationProvider : INntpAuthenticationProvider +{ + private readonly INntpAuthenticationProvider _inner; + + public RecordingNntpAuthenticationProvider(INntpAuthenticationProvider inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + public int AuthenticateCount { get; private set; } + + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + AuthenticateCount++; + return _inner.AuthenticateAsync(username, password, cancellationToken); + } + + public ValueTask AuthenticateAsync( + string username, + string password, + IPAddress clientIp, + CancellationToken cancellationToken = default) + { + AuthenticateCount++; + return _inner.AuthenticateAsync(username, password, clientIp, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs new file mode 100644 index 00000000..32fc397f --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingOverviewDbHandoffPublisher.cs @@ -0,0 +1,192 @@ +using VectorNNTPD.TransitService.ArticleIngestion.OverviewDb; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Captures OverviewDB handoff payloads and can nack a budgeted number of publishes. +internal sealed class RecordingOverviewDbHandoffPublisher : IOverviewDbHandoffPublisher +{ + private int _attempts; + private int _outstanding; + private readonly object _gate = new(); + private readonly Queue _failures = new(); + private readonly TaskCompletionSource _firstAttempt = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _confirmed = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Gets captured successfully confirmed payloads in publish order. + public List Payloads { get; } = []; + + /// Gets the number of publish attempts, including nacks. + public int AttemptCount => Volatile.Read(ref _attempts); + + /// Completes when the first publish attempt begins. + public Task FirstAttempt => _firstAttempt.Task; + + /// Completes when the first payload is recorded as confirmed. + public Task Confirmed => _confirmed.Task; + + private int _confirmWaitCount; + private TaskCompletionSource? _confirmWait; + + /// + /// Completes when at least payloads have been confirmed. + /// + public Task WaitUntilConfirmedAsync(int count, CancellationToken cancellationToken) + { + lock (_gate) + { + if (Payloads.Count >= count) + { + return Task.CompletedTask; + } + + _confirmWaitCount = count; + _confirmWait = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + return _confirmWait.Task.WaitAsync(cancellationToken); + } + } + + /// Remaining attempts that enqueue a failure instead of confirming. + public int RemainingFailures { get; set; } + + /// + /// Exception thrown for each budgeted attempt. + /// + public Exception? TransientPublishException { get; set; } + + /// When set, every attempt throws this exception after the remaining-failure budget. + public Exception? PublishException { get; set; } + + /// + /// When set, a failing attempt waits on this source after recording the attempt + /// and before enqueueing the failure. + /// + public TaskCompletionSource? BlockOnFailure { get; set; } + + /// + /// Artificial confirmation delay applied after returns. + /// Simulates quorum confirm latency without blocking the publish call. + /// + public TimeSpan ConfirmDelay { get; set; } + + /// Maximum outstanding unconfirmed publishes (window). Zero means unbounded. + public int OutstandingWindow { get; set; } + + /// + public int OutstandingCount => Volatile.Read(ref _outstanding); + + /// + public async Task PublishAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + cancellationToken.ThrowIfCancellationRequested(); + + if (OutstandingWindow > 0) + { + while (Volatile.Read(ref _outstanding) >= OutstandingWindow) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Delay(1, cancellationToken).ConfigureAwait(false); + } + } + + var attempt = Interlocked.Increment(ref _attempts); + _firstAttempt.TrySetResult(attempt); + Interlocked.Increment(ref _outstanding); + + if (RemainingFailures > 0) + { + RemainingFailures--; + if (BlockOnFailure is not null) + { + await BlockOnFailure.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + Interlocked.Decrement(ref _outstanding); + if (TransientPublishException is not null || PublishException is not null) + { + throw TransientPublishException + ?? PublishException + ?? new InvalidOperationException("RabbitMQ negatively acknowledged the OverviewDB handoff."); + } + + EnqueueFailure(item); + return; + } + + if (PublishException is not null) + { + if (BlockOnFailure is not null) + { + await BlockOnFailure.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + Interlocked.Decrement(ref _outstanding); + throw PublishException; + } + + // Fire-and-forget confirm so PublishAsync is not serialized behind ConfirmDelay. + _ = ConfirmLaterAsync(item, cancellationToken); + } + + /// + public bool TryDequeuePublishFailure(out OverviewDbWorkItem item) + { + lock (_gate) + { + if (_failures.Count == 0) + { + item = null!; + return false; + } + + item = _failures.Dequeue(); + return true; + } + } + + /// + public void AbandonOutstanding() + { + Volatile.Write(ref _outstanding, 0); + lock (_gate) + { + _failures.Clear(); + } + } + + private async Task ConfirmLaterAsync(OverviewDbWorkItem item, CancellationToken cancellationToken) + { + try + { + if (ConfirmDelay > TimeSpan.Zero) + { + await Task.Delay(ConfirmDelay, CancellationToken.None).ConfigureAwait(false); + } + + lock (_gate) + { + Payloads.Add(item.Payload.ToArray()); + _confirmed.TrySetResult(); + if (_confirmWait is not null && Payloads.Count >= _confirmWaitCount) + { + _confirmWait.TrySetResult(); + _confirmWait = null; + } + } + } + finally + { + Interlocked.Decrement(ref _outstanding); + } + } + + private void EnqueueFailure(OverviewDbWorkItem item) + { + lock (_gate) + { + _failures.Enqueue(item); + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingPathSurveyWriter.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingPathSurveyWriter.cs new file mode 100644 index 00000000..f093ec00 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingPathSurveyWriter.cs @@ -0,0 +1,126 @@ +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Logging; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Captures Path-survey observations without pretending to persist them. +internal sealed class RecordingPathSurveyWriter : IPathSurveyWriter +{ + private readonly List _paths = []; + private int _writeCalls; + private int _flushCalls; + + public IReadOnlyList Paths + { + get + { + lock (_paths) + { + return _paths.ToArray(); + } + } + } + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public int FlushCalls => Volatile.Read(ref _flushCalls); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + lock (_paths) + { + _paths.Add(canonicalPath.ToArray()); + } + } + + public void Flush() => Interlocked.Increment(ref _flushCalls); +} + +/// Counts Path-survey writes without retaining the observation stream. +internal sealed class CountingPathSurveyWriter : IPathSurveyWriter +{ + private int _writeCalls; + private long _bytes; + + public int WriteCalls => Volatile.Read(ref _writeCalls); + + public long Bytes => Interlocked.Read(ref _bytes); + + public void Write(ReadOnlySpan canonicalPath) + { + Interlocked.Increment(ref _writeCalls); + Interlocked.Add(ref _bytes, canonicalPath.Length); + } + + public void Flush() + { + } +} + +/// Path-survey writer that throws to prove already-accepted articles are not reprocessed. +internal sealed class ThrowingPathSurveyWriter : IPathSurveyWriter +{ + public int WriteCalls { get; private set; } + + public void Write(ReadOnlySpan canonicalPath) + { + WriteCalls++; + throw new IOException("path survey writer test failure"); + } + + public void Flush() + { + } +} + +/// Records completed Path-survey files handed off before gzip. +internal sealed class RecordingCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler +{ + private readonly List _completed = []; + private readonly List _gzipExisted = []; + + public IReadOnlyList Completed + { + get + { + lock (_completed) + { + return _completed.ToArray(); + } + } + } + + public IReadOnlyList GzipExistedAtHandoff + { + get + { + lock (_gzipExisted) + { + return _gzipExisted.ToArray(); + } + } + } + + public void OnCompletedFile(string completedFilePath) + { + var gzip = completedFilePath + ".gz"; + lock (_completed) + { + _gzipExisted.Add(File.Exists(gzip)); + _completed.Add(completedFilePath); + } + } +} + +/// Completed-file handler that throws after recording the path. +internal sealed class ThrowingCompletedPathSurveyFileHandler : ICompletedPathSurveyFileHandler +{ + public string? Path { get; private set; } + + public void OnCompletedFile(string completedFilePath) + { + Path = completedFilePath; + throw new InvalidOperationException("completed-file handler test failure"); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs new file mode 100644 index 00000000..a98196f9 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/RecordingTransitPeerAuthenticator.cs @@ -0,0 +1,21 @@ +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Counts Transit AUTHINFO invocations. Never logs credentials. +internal sealed class RecordingTransitPeerAuthenticator : ITransitPeerAuthenticator +{ + public int AuthenticateCount { get; private set; } + + public NntpAuthenticationResult Authenticate( + TransitPeerPolicy? policy, + string username, + string password, + NntpAuthorization currentAuthorization) + { + AuthenticateCount++; + return TransitPeerAuthenticator.Instance.Authenticate(policy, username, password, currentAuthorization); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs new file mode 100644 index 00000000..8fcf584d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/TestDoubles/StaticNewsgroupCatalogue.cs @@ -0,0 +1,32 @@ +using VectorNNTPD.TransitService.Newsgroups; + +namespace VectorNNTPD.TransitService.Tests.TestDoubles; + +/// Test catalogue that publishes snapshots without MySQL. +internal sealed class StaticNewsgroupCatalogue : INewsgroupCatalogue +{ + private NewsgroupSnapshot _current; + + public StaticNewsgroupCatalogue(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + _current = snapshot; + } + + public int CurrentReadCount { get; private set; } + + public NewsgroupSnapshot Current + { + get + { + CurrentReadCount++; + return Volatile.Read(ref _current); + } + } + + public void Publish(NewsgroupSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Interlocked.Exchange(ref _current, snapshot); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/NewsfeedsPatternTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/NewsfeedsPatternTests.cs new file mode 100644 index 00000000..3c3a7e3e --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/NewsfeedsPatternTests.cs @@ -0,0 +1,89 @@ +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// newsfeeds(5) / INN uwildmat_poison matching matrix. +/// Source: https://www.eyrie.org/~eagle/software/inn/docs/libinn-uwildmat.html +/// and https://www.eyrie.org/~eagle/software/inn/docs/newsfeeds.html +/// +public sealed class NewsfeedsPatternTests +{ + [Theory] + [InlineData("*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*", "alt.comp", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.sources.*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources.*", "comp.sources", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.sources*", "comp.sources", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.sources.unix", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.sourcesfoo", NewsfeedsPatternDecision.Match)] + [InlineData("comp.sources*", "comp.lang.c", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.*,!comp.sources.*", "comp.lang.c", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*,!comp.sources.*", "comp.sources.unix", NewsfeedsPatternDecision.Exclude)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "alt.fan", NewsfeedsPatternDecision.Match)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "alt.binaries.warez", NewsfeedsPatternDecision.Poison)] + [InlineData("alt.*,@alt.binaries.warez,misc.*", "misc.test", NewsfeedsPatternDecision.Match)] + [InlineData("!control", "control", NewsfeedsPatternDecision.Exclude)] + [InlineData("!control", "control.cancel", NewsfeedsPatternDecision.NoMatch)] + [InlineData("!control.*", "control.cancel", NewsfeedsPatternDecision.Exclude)] + [InlineData("!control.*", "control", NewsfeedsPatternDecision.NoMatch)] + [InlineData("news.*,!news.misc", "news.misc", NewsfeedsPatternDecision.Exclude)] + [InlineData("news.*,!news.misc,*.misc", "news.misc", NewsfeedsPatternDecision.Match)] + [InlineData("comp.*,news.*", "news.misc", NewsfeedsPatternDecision.Match)] + [InlineData("exact.group", "exact.group", NewsfeedsPatternDecision.Match)] + [InlineData("exact.group", "exact.group.extra", NewsfeedsPatternDecision.NoMatch)] + [InlineData("a", "aa", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.?", "comp.x", NewsfeedsPatternDecision.Match)] + [InlineData("comp.?", "comp.xy", NewsfeedsPatternDecision.NoMatch)] + [InlineData("comp.[abc]", "comp.a", NewsfeedsPatternDecision.Match)] + [InlineData("comp.[^a]", "comp.b", NewsfeedsPatternDecision.Match)] + [InlineData("comp.[^a]", "comp.a", NewsfeedsPatternDecision.NoMatch)] + public void Evaluate_MatchesNewsfeedsSemantics(string expression, string group, NewsfeedsPatternDecision expected) + { + Assert.True(NewsfeedsPattern.TryParse(expression, out var pattern, out var error), error); + Assert.NotNull(pattern); + Assert.Equal(expected, pattern.Evaluate(group)); + Assert.Equal(expected == NewsfeedsPatternDecision.Match, pattern.MatchesNewsgroup(group)); + } + + [Fact] + public void EvaluateArticle_PoisonWinsOverOtherMatchedGroups() + { + Assert.True(NewsfeedsPattern.TryParse("alt.*,@alt.binaries.warez,misc.*", out var pattern, out _)); + Assert.Equal( + NewsfeedsPatternDecision.Poison, + pattern!.EvaluateArticle(["misc.test", "alt.binaries.warez"])); + Assert.Equal( + NewsfeedsPatternDecision.Match, + pattern.EvaluateArticle(["misc.test", "alt.fan"])); + Assert.Equal( + NewsfeedsPatternDecision.NoMatch, + pattern.EvaluateArticle(["rec.sport"])); + } + + [Theory] + [InlineData("")] + [InlineData(",")] + [InlineData("comp.*,")] + [InlineData(",comp.*")] + [InlineData("!")] + [InlineData("@")] + [InlineData("comp.[abc")] + [InlineData(@"comp.\")] + public void TryParse_RejectsEmptyOrInvalidExpressions(string expression) + { + Assert.False(NewsfeedsPattern.TryParse(expression, out var pattern, out var error)); + Assert.Null(pattern); + Assert.False(string.IsNullOrWhiteSpace(error)); + } + + [Fact] + public void TryParse_AcceptsEscapedComma() + { + Assert.True(NewsfeedsPattern.TryParse(@"foo\,bar", out var pattern, out var error), error); + Assert.True(pattern!.MatchesNewsgroup("foo,bar")); + Assert.False(pattern.MatchesNewsgroup("foo")); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAllowFromAndDnsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAllowFromAndDnsTests.cs new file mode 100644 index 00000000..0cbc05a5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAllowFromAndDnsTests.cs @@ -0,0 +1,384 @@ +using System.Net; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitAllowFromAndDnsTests +{ + [Fact] + public void LiteralIpv4Address_MatchesExactly() + { + var authz = Create("192.0.2.10"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.11")).TransitPeerName); + } + + [Fact] + public void LiteralIpv4Cidr_MatchesPrefix() + { + var authz = Create("192.0.2.0/24"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.200")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + } + + [Fact] + public void LiteralIpv6AddressAndCidr_Match() + { + var authz = Create("2001:db8::10", "2001:db8:1234::/48"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8:1234::abcd")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("2001:db8:9999::1")).TransitPeerName); + } + + [Fact] + public void PrivateAndDocumentationAddresses_AreNotRejected() + { + var authz = Create("10.0.0.0/8", "192.168.1.5", "fd00::/8", "198.18.0.70"); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("10.1.2.3")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.168.1.5")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("fd00::1")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("198.18.0.70")).TransitPeerName); + } + + [Fact] + public async Task DnsHostname_AuthorizesAllResolvedAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success( + [IPAddress.Parse("192.0.2.10"), IPAddress.Parse("2001:db8::10")], + TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + } + + [Fact] + public async Task MixedDnsAndLiterals_AuthorizeTogether() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("203.0.113.5")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot( + "peer", + TransitTestPeers.Peer(allowFrom: ["news.example.net", "192.0.2.10", "2001:db8::10"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("2001:db8::10")).TransitPeerName); + } + + [Fact] + public async Task DnsRefresh_ReplacesAddressesAtomically() + { + var addresses = new List { IPAddress.Parse("192.0.2.10") }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([.. addresses], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Equal(["192.0.2.10"], cache.GetResolved("news.example.net").Select(static a => a.ToString())); + + addresses.Clear(); + addresses.Add(IPAddress.Parse("198.51.100.20")); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var resolved = cache.GetResolved("news.example.net"); + Assert.DoesNotContain(IPAddress.Parse("192.0.2.10"), resolved); + Assert.Contains(IPAddress.Parse("198.51.100.20"), resolved); + } + + [Fact] + public async Task DnsTransientFailure_PreservesLastAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + resolver.Handler = _ => TransitDnsResolveResult.TransientFailure(); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Contains(IPAddress.Parse("192.0.2.10"), cache.GetResolved("news.example.net")); + } + + [Fact] + public async Task DnsEmptyResult_ClearsAddresses() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + resolver.Handler = _ => TransitDnsResolveResult.Empty(); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Empty(cache.GetResolved("news.example.net")); + } + + [Fact] + public async Task DnsRefresh_HonorsMinimumSixtySecondsEvenWhenTtlIsShorter() + { + var clock = new MutableTimeProvider { UtcNow = DateTimeOffset.UnixEpoch }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromSeconds(5)), + }; + var cache = new TransitDnsAddressCache( + resolver, + NullLogger.Instance, + TimeSpan.FromSeconds(60), + clock); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + + clock.UtcNow = clock.UtcNow.AddSeconds(30); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + + clock.UtcNow = clock.UtcNow.AddSeconds(31); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(2, resolver.Calls); + Assert.Equal(TimeSpan.FromSeconds(60), cache.GetDelayUntilNextRefresh(snapshot)); + clock.UtcNow = clock.UtcNow.AddSeconds(60); + Assert.Equal(TimeSpan.Zero, cache.GetDelayUntilNextRefresh(snapshot)); + } + + [Fact] + public async Task DnsRefresh_HonorsTtlWhenGreaterThanSixtySeconds() + { + var clock = new MutableTimeProvider { UtcNow = DateTimeOffset.UnixEpoch }; + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(10)), + }; + var cache = new TransitDnsAddressCache( + resolver, + NullLogger.Instance, + TimeSpan.FromSeconds(60), + clock); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + clock.UtcNow = clock.UtcNow.AddMinutes(9); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(1, resolver.Calls); + clock.UtcNow = clock.UtcNow.AddMinutes(2); + await cache.RefreshDueAsync(snapshot, CancellationToken.None); + Assert.Equal(2, resolver.Calls); + } + + [Fact] + public async Task ConnectionAuthorization_DoesNotInvokeDnsResolver() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.Success([IPAddress.Parse("192.0.2.10")], TimeSpan.FromMinutes(5)), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + var callsAfterRefresh = resolver.Calls; + resolver.Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"); + + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + Assert.Equal(callsAfterRefresh, resolver.Calls); + } + + [Fact] + public void ConnectionAuthorization_UsesMaterializedIpAcl_ThrowingResolverNeverCalled() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"), + }; + var cache = new SeededDnsCache(); + cache.Seed("news.example.net", IPAddress.Parse("192.0.2.10")); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + + Assert.Equal("peer", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.1")).TransitPeerName); + Assert.Equal(0, resolver.Calls); + Assert.Equal(0, cache.RefreshCalls); + } + + [Fact] + public void ConnectionAuthorization_UnresolvedHostname_DoesNotInvokeResolver() + { + var resolver = new FakeTransitDnsResolver + { + Handler = _ => throw new InvalidOperationException("connection-time DNS is forbidden"), + }; + var cache = new TransitDnsAddressCache(resolver, NullLogger.Instance); + var snapshot = TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: ["news.example.net"])); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, cache); + + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.False(authz.Resolve(IPAddress.Parse("192.0.2.10")).AuthorizedTransit); + Assert.Equal(0, resolver.Calls); + } + + [Fact] + public async Task DnsFailure_LogsWarningForPeerOnEveryAttempt() + { + var logger = new CollectingLogger(); + var resolver = new FakeTransitDnsResolver + { + Handler = _ => TransitDnsResolveResult.TransientFailure(), + }; + var cache = new TransitDnsAddressCache(resolver, logger); + var snapshot = TransitTestPeers.Snapshot( + "giganews", + TransitTestPeers.Peer(allowFrom: ["news.example.net"], peerName: "Giganews, Inc.")); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + await cache.RefreshAllAsync(snapshot, CancellationToken.None); + + var warnings = logger.Messages.Where(static m => m.Contains("AllowFrom DNS resolution failed", StringComparison.Ordinal)).ToList(); + Assert.Equal(2, warnings.Count); + Assert.All( + warnings, + static w => + { + Assert.Contains("Giganews, Inc.", w, StringComparison.Ordinal); + Assert.Contains("news.example.net", w, StringComparison.Ordinal); + Assert.DoesNotContain("Password", w, StringComparison.Ordinal); + Assert.DoesNotContain("s3cret", w, StringComparison.Ordinal); + }); + } + + [Fact] + public void AmbiguousIpMatch_DeniesTransit_AndWarnsEveryResolve() + { + var logger = new CollectingLogger(); + var snapshot = new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer("one", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer("two", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + }); + var authz = TransitPeerAuthorization.CreateStatic(snapshot, logger: logger); + var source = IPAddress.Parse("192.0.2.10"); + + Assert.Null(authz.Resolve(source).TransitPeerName); + Assert.False(authz.Resolve(source).AuthorizedTransit); + Assert.Equal(2, logger.Messages.Count); + Assert.All( + logger.Messages, + static m => + { + Assert.Contains("ambiguous", m, StringComparison.OrdinalIgnoreCase); + Assert.Contains("192.0.2.10", m, StringComparison.Ordinal); + Assert.Contains("one", m, StringComparison.Ordinal); + Assert.Contains("two", m, StringComparison.Ordinal); + }); + } + + [Fact] + public void EmptyAllowFrom_DoesNotMatchInbound() + { + var authz = TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot("outbound-only", TransitTestPeers.Peer(allowFrom: []))); + Assert.True(authz.IsEnabled); + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + } + + private static TransitPeerAuthorization Create(params string[] allowFrom) => + TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot("peer", TransitTestPeers.Peer(allowFrom: allowFrom))); + + private sealed class SeededDnsCache : ITransitDnsAddressCache + { + private readonly Dictionary> _addresses = + new(StringComparer.OrdinalIgnoreCase); + + public int RefreshCalls { get; private set; } + + public void Seed(string hostname, params IPAddress[] addresses) => + _addresses[hostname] = addresses.ToHashSet(); + + public IReadOnlySet GetResolved(string hostname) => + _addresses.TryGetValue(hostname, out var set) ? set : new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) + { + RefreshCalls++; + throw new InvalidOperationException("connection-time DNS refresh is forbidden"); + } + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) + { + RefreshCalls++; + throw new InvalidOperationException("connection-time DNS refresh is forbidden"); + } + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + + private sealed class FakeTransitDnsResolver : ITransitDnsResolver + { + public Func Handler { get; set; } = + static _ => TransitDnsResolveResult.TransientFailure(); + + public int Calls { get; private set; } + + public Task ResolveAsync(string hostname, CancellationToken cancellationToken) + { + Calls++; + return Task.FromResult(Handler(hostname)); + } + } + + private sealed class CollectingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + Messages.Add($"{logLevel}: {formatter(state, exception)}"); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } + + private sealed class MutableTimeProvider : TimeProvider + { + public DateTimeOffset UtcNow { get; set; } = DateTimeOffset.UnixEpoch; + + public override DateTimeOffset GetUtcNow() => UtcNow; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAuthinfoAndCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAuthinfoAndCommandTests.cs new file mode 100644 index 00000000..f986fc23 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitAuthinfoAndCommandTests.cs @@ -0,0 +1,574 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitAuthinfoAndCommandTests +{ + private static readonly IPAddress PeerAddress = IPAddress.Parse("192.0.2.10"); + private static readonly IPAddress OtherAddress = IPAddress.Parse("198.51.100.1"); + + [Fact] + public async Task TransitPeer_WithCredentials_AcceptsMatchingAuthinfo() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("feed", session.Authentication.Username); + Assert.True(session.Authorization.AuthorizedTransit); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_WithCredentials_RejectsInvalidPassword() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.True(session.Authorization.StreamingPermitted); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_AuthinfoDoesNotLogPassword() + { + var recording = new RecordingLoggerFactory(); + var peers = CreatePeer(username: "feed", password: "peer-secret-xyz"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers, loggerFactory: recording); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS peer-secret-xyz"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + var joined = string.Join('\n', recording.Messages); + Assert.DoesNotContain("peer-secret-xyz", joined, StringComparison.Ordinal); + } + + [Fact] + public async Task Authinfo_IsPublic_ForStreamAndNonTransit_AndModeReaderIsUnavailable() + { + var user = NntpCommandTestParse.ParseCommand("AUTHINFO USER x"); + var pass = NntpCommandTestParse.ParseCommand("AUTHINFO PASS y"); + Assert.True(user.IsValid); + Assert.True(pass.IsValid); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(user.Verb, user.Qualifier)); + Assert.Equal(NntpCommandAccess.Public, DefaultNntpCommandCatalog.GetAccess(pass.Verb, pass.Qualifier)); + + await using var readerDuplex = await CommandDuplex.CreateAsync(OtherAddress); + var reader = readerDuplex.CreateSession(CreatePeer()); + var readerRun = reader.RunAsync(); + _ = await readerDuplex.ReadClientLineAsync(); + await readerDuplex.WriteClientLineAsync("MODE READER"); + Assert.Equal("502 Command not available on this server", await readerDuplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, reader.Mode); + await readerDuplex.WriteClientLineAsync("AUTHINFO USER reader"); + Assert.StartsWith("381 ", await readerDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await readerDuplex.WriteClientLineAsync("QUIT"); + _ = await readerDuplex.ReadClientLineAsync(); + await readerRun; + + var transit = CreatePeer(); + await using var streamDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var stream = streamDuplex.CreateSession(transit); + var streamRun = stream.RunAsync(); + _ = await streamDuplex.ReadClientLineAsync(); + await streamDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await streamDuplex.ReadClientLineAsync()); + await streamDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await streamDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await streamDuplex.WriteClientLineAsync("QUIT"); + _ = await streamDuplex.ReadClientLineAsync(); + await streamRun; + + await using var anonDuplex = await CommandDuplex.CreateAsync(OtherAddress); + var anon = anonDuplex.CreateSession(CreatePeer()); + Assert.Null(anon.Authorization.TransitPeerName); + var anonRun = anon.RunAsync(); + _ = await anonDuplex.ReadClientLineAsync(); + await anonDuplex.WriteClientLineAsync("AUTHINFO USER anybody"); + Assert.StartsWith("381 ", await anonDuplex.ReadClientLineAsync(), StringComparison.Ordinal); + await anonDuplex.WriteClientLineAsync("QUIT"); + _ = await anonDuplex.ReadClientLineAsync(); + await anonRun; + } + + [Fact] + public async Task TransitPeer_RejectsWrongUsernameOrIncompleteConfiguredCredentials() + { + var both = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(both); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER not-feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + + await using var userOnlyDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var userOnly = userOnlyDuplex.CreateSession(CreatePeer(username: "feed", password: "")); + var userOnlyRun = userOnly.RunAsync(); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await userOnlyDuplex.ReadClientLineAsync()); + await userOnlyDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyDuplex.WriteClientLineAsync("AUTHINFO PASS anything"); + Assert.Equal("481 Authentication failed", await userOnlyDuplex.ReadClientLineAsync()); + await userOnlyDuplex.WriteClientLineAsync("QUIT"); + _ = await userOnlyDuplex.ReadClientLineAsync(); + await userOnlyRun; + + await using var passOnlyDuplex = await CommandDuplex.CreateAsync(PeerAddress); + var passOnly = passOnlyDuplex.CreateSession(CreatePeer(username: "", password: "s3cret")); + var passOnlyRun = passOnly.RunAsync(); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyDuplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await passOnlyDuplex.ReadClientLineAsync()); + await passOnlyDuplex.WriteClientLineAsync("AUTHINFO USER feed"); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyDuplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await passOnlyDuplex.ReadClientLineAsync()); + await passOnlyDuplex.WriteClientLineAsync("QUIT"); + _ = await passOnlyDuplex.ReadClientLineAsync(); + await passOnlyRun; + } + + [Fact] + public async Task NonTransitSource_PeerUsernamePassword_DoesNotCreateTransitIdentity() + { + var peers = CreatePeer(username: "feed", password: "s3cret"); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(peers); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedTransit); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS s3cret"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.False(session.Authentication.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NonTransitAuthinfoSuccess_DoesNotGrantTransit() + { + var provider = new ReaderOnlyAuthenticationProvider(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(CreatePeer(username: "feed", password: "s3cret"), authenticationProvider: provider); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER reader"); + _ = await duplex.ReadClientLineAsync(); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.IsAuthenticated); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task TransitPeer_BlankCredentials_DoNotCreatePeerAuth() + { + var peers = CreatePeer(); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers); + Assert.False(session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS anything"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.AuthorizedTransit); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NamedTransitPeer_AllowsStreamCheckTakeThisIhave() + { + var peers = CreatePeer(); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions { QueueCapacity = 4 }); + await using var duplex = await CommandDuplex.CreateAsync(PeerAddress); + var session = duplex.CreateSession(peers, queue); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + Assert.Equal(PeerAddress, session.ClientAddress); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + Assert.Equal(NntpSessionMode.Unspecified, session.Mode); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientBytesAsync(Encoding.ASCII.GetBytes(CanonicalArticleText.Stuffed("") + ".\r\n")); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("TAKETHIS "); + await duplex.WriteClientBytesAsync(Encoding.ASCII.GetBytes(CanonicalArticleText.Destuffed("") + ".\r\n")); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task NonTransitClient_StillRejectedForTransitCommands() + { + var peers = CreatePeer(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(peers); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("480 Authentication required", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public async Task AuthenticatedNonTransit_TransitCommandsReturn502() + { + var provider = new ReaderOnlyAuthenticationProvider(); + await using var duplex = await CommandDuplex.CreateAsync(OtherAddress); + var session = duplex.CreateSession(CreatePeer(), authenticationProvider: provider); + Assert.Null(session.Authorization.TransitPeerName); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("AUTHINFO USER reader"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + Assert.True(session.Authorization.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("502 Streaming not permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("502 Permission denied", await duplex.ReadClientLineAsync()); + + await duplex.WriteClientLineAsync("QUIT"); + _ = await duplex.ReadClientLineAsync(); + await run; + } + + [Fact] + public void ChangingNamedPeerPolicy_AffectsNewIdentificationOnly() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], patterns: "comp.*"))); + var peers = TransitPeerAuthorization.CreateForStore(store); + + var first = peers.Resolve(PeerAddress); + Assert.Equal(TransitTestPeers.DefaultPeerName, first.TransitPeerName); + Assert.True(first.TransitPeerPolicy!.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(first.TransitPeerPolicy.Patterns.MatchesNewsgroup("alt.test")); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], patterns: "alt.*"))); + + Assert.True(first.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + var second = peers.Resolve(PeerAddress); + Assert.Equal(TransitTestPeers.DefaultPeerName, second.TransitPeerName); + Assert.True(second.TransitPeerPolicy!.Patterns.MatchesNewsgroup("alt.test")); + Assert.False(second.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + } + + private static ITransitPeerAuthorization CreatePeer(string username = "", string password = "") => + TransitTestPeers.ForAllowFrom( + PeerAddress, + username: username, + password: password); + + private sealed class RecordingLoggerFactory : ILoggerFactory + { + public ConcurrentBag Messages { get; } = []; + + public void AddProvider(ILoggerProvider provider) + { + } + + public ILogger CreateLogger(string categoryName) => new RecordingLogger(Messages); + + public void Dispose() + { + } + + private sealed class RecordingLogger(ConcurrentBag messages) : ILogger + { + public IDisposable BeginScope(TState state) + where TState : notnull => NullScope.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + messages.Add(formatter(state, exception)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } + } + + private sealed class ReaderOnlyAuthenticationProvider : INntpAuthenticationProvider + { + public ValueTask AuthenticateAsync( + string username, + string password, + CancellationToken cancellationToken = default) + { + if (username == "reader" && password == "secret") + { + return ValueTask.FromResult(NntpAuthenticationResult.Success( + "reader", + new NntpAuthorization( + isAuthenticated: true, + authorizedReader: true, + authorizedTransit: false, + postingPermitted: false, + streamingPermitted: false))); + } + + return ValueTask.FromResult(NntpAuthenticationResult.Failed); + } + } + + private sealed class CommandDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly IPAddress _clientAddress; + + private CommandDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress clientAddress) => + Task.FromResult(new CommandDuplex(clientAddress)); + + public NntpSession CreateSession( + ITransitPeerAuthorization peers, + IArticleIngestionQueue? queue = null, + INntpAuthenticationProvider? authenticationProvider = null, + ILoggerFactory? loggerFactory = null) + { + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientAddress, 40000))); + return new NntpSession( + connection, + loggerFactory?.CreateLogger() ?? NullLogger.Instance, + authenticationProvider: authenticationProvider, + loggerFactory: loggerFactory, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientBytesAsync(ReadOnlyMemory bytes) + { + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitCatalogueTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitCatalogueTests.cs new file mode 100644 index 00000000..dcbcd396 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitCatalogueTests.cs @@ -0,0 +1,523 @@ +using System.Data; +using System.Net; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.NntpDb; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitCatalogueTests +{ + private const long SharedSendMaxArticleBytes = 5_242_880; + + [Fact] + public void MigratedPublication_ReconstructsTheThreePeers() + { + var snapshot = TransitCatalogueCompiler.Compile(MigratedPublication()); + + Assert.Equal(1, snapshot.PublicationId); + Assert.True(snapshot.WantTrash); + Assert.True(snapshot.LogTrash); + Assert.Equal(3, snapshot.Peers.Count); + Assert.Equal( + ["blueworld-hosting", "giganews", "usenet-ninja"], + snapshot.Peers.Keys.OrderBy(static key => key, StringComparer.Ordinal)); + + var blueworld = snapshot.Peers["blueworld-hosting"]; + Assert.Equal("blueworld-hosting", blueworld.Identifier); + Assert.Equal("Blueworld Hosting", blueworld.PeerName); + Assert.Equal(["usenet.blueworldhosting.com"], blueworld.DnsHostnames); + Assert.Empty(blueworld.LiteralPrefixes); + Assert.Equal(10, blueworld.MaxIncomingConnections); + Assert.False(blueworld.HasPeerCredentials); + Assert.True(blueworld.DeferOnDuplicate); + Assert.Equal(1_048_576, blueworld.MaxSize); + Assert.Equal("usenet.blueworldhosting.com", blueworld.ConnectTo[0].Host); + Assert.Equal(119, blueworld.ConnectTo[0].Port); + Assert.Equal(10, blueworld.MaxOutgoingConnections); + Assert.Equal(TransitSslMode.None, blueworld.Ssl); + Assert.False(blueworld.HasSendCredentials); + Assert.Equal("*,!unidata.*,!control.*,!junk", blueworld.Patterns.Expression); + Assert.Equal(TransitMessageTypes.All, blueworld.MessageTypes); + Assert.Equal("usenet.blueworldhosting.com", blueworld.PathToken); + Assert.Empty(blueworld.PathExclusions); + + var giganews = snapshot.Peers["giganews"]; + Assert.Equal("giganews", giganews.Identifier); + Assert.Equal("Giganews, Inc.", giganews.PeerName); + Assert.Equal(["news-out.nntp.giganews.com"], giganews.DnsHostnames); + Assert.Equal(80, giganews.MaxIncomingConnections); + Assert.Equal(5_242_880, giganews.MaxSize); + Assert.Equal("opticnetworks-in.nntp.ord.giganews.com", giganews.ConnectTo[0].Host); + Assert.Equal(119, giganews.ConnectTo[0].Port); + Assert.Equal(80, giganews.MaxOutgoingConnections); + Assert.Equal("*,!unidata.*", giganews.Patterns.Expression); + Assert.Equal("nntp.giganews.com", giganews.PathToken); + + var local = snapshot.Peers["usenet-ninja"]; + Assert.Equal("usenet-ninja", local.Identifier); + Assert.Equal("Usenet Ninja", local.PeerName); + Assert.Empty(local.DnsHostnames); + Assert.True(local.LiteralPrefixes[0].Contains(IPAddress.Parse("198.18.1.1"))); + Assert.False(local.LiteralPrefixes[0].Contains(IPAddress.Parse("203.0.113.1"))); + Assert.Equal(10, local.MaxIncomingConnections); + Assert.Equal(5_242_880, local.MaxSize); + Assert.Empty(local.ConnectTo); + Assert.Equal(10, local.MaxOutgoingConnections); + Assert.Equal("*", local.Patterns.Expression); + Assert.Equal("nntp.usenet.ninja", local.PathToken); + } + + [Fact] + public void MigratedPublication_KeepsIntentionalDefaultsDistinctFromJson() + { + var snapshot = TransitCatalogueCompiler.Compile(MigratedPublication()); + foreach (var peer in snapshot.Peers.Values) + { + Assert.Equal(TransitMessageTypes.All, peer.ReceiveArticleTypes); + Assert.Equal("*", peer.ReceivePatterns.Expression); + Assert.Equal(SharedSendMaxArticleBytes, peer.SendMaxArticleBytes); + } + + Assert.NotEqual( + snapshot.Peers["blueworld-hosting"].MaxSize, + snapshot.Peers["blueworld-hosting"].SendMaxArticleBytes); + } + + [Fact] + public void Planes_KeepArticleMasksSizesAndPatternsIndependent() + { + var peer = Peer( + identifier: "planes", + receiveArticleTypes: (int)TransitMessageTypes.Default, + sendArticleTypes: (int)TransitMessageTypes.All, + receiveMaxArticleBytes: 1_048_576, + sendMaxArticleBytes: SharedSendMaxArticleBytes, + receivePatterns: "*", + sendPatterns: "*,!unidata.*"); + var compiled = TransitCatalogueCompiler.Compile(Publication(peer)).Peers["planes"]; + + Assert.Equal(TransitMessageTypes.Default, compiled.ReceiveArticleTypes); + Assert.Equal(TransitMessageTypes.All, compiled.MessageTypes); + Assert.Equal(1_048_576, compiled.MaxSize); + Assert.Equal(SharedSendMaxArticleBytes, compiled.SendMaxArticleBytes); + Assert.True(compiled.ReceivePatterns.MatchesNewsgroup("unidata.maps")); + Assert.False(compiled.Patterns.MatchesNewsgroup("unidata.maps")); + Assert.True(compiled.Patterns.MatchesNewsgroup("comp.lang.c")); + } + + [Fact] + public void ArticleTypeMasks_MatchTheCapabilityContract() + { + Assert.Equal(65535, (int)TransitMessageTypes.All); + Assert.Equal(1, (int)TransitMessageTypes.Default); + Assert.Equal(0, (int)TransitMessageTypes.None); + Assert.Equal(65535, TransitCatalogueCompiler.UnrestrictedArticleTypes); + + Assert.True(ArticleTypeCapabilities.Allows((ArticleType)65535, ArticleType.Binary | ArticleType.YEncoded)); + Assert.True(ArticleTypeCapabilities.Allows((ArticleType)1, ArticleType.Default)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)1, ArticleType.Control)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleType.Default)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleType.Control)); + Assert.False(ArticleTypeCapabilities.Allows((ArticleType)0, ArticleTypeCapabilities.All)); + + var star = TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: "*", sendPatterns: "*"))); + Assert.True(star.Peers["peer"].ReceivePatterns.MatchesNewsgroup("comp.lang.c")); + Assert.True(star.Peers["peer"].Patterns.MatchesNewsgroup("alt.test")); + } + + [Fact] + public void ZeroConnectionLimits_CloseThatPlane() + { + var compiled = TransitCatalogueCompiler.Compile(Publication(Peer(maxInbound: 0, maxOutbound: 0))) + .Peers["peer"]; + Assert.Equal(0, compiled.MaxIncomingConnections); + Assert.Equal(0, compiled.MaxOutgoingConnections); + + var engine = new TransitPeerStateEngine(); + Assert.Equal( + TransitPeerStateEngine.Rejected, + engine.TryAdmit("peer", "owner", compiled.MaxIncomingConnections, 1, 30_000, 1)); + Assert.Equal( + TransitPeerStateEngine.Accepted, + engine.TryAdmit("peer", "owner", 1, 1, 30_000, 1)); + } + + [Fact] + public void InvalidArticleTypesPatternsAndDuplicateEndpoints_AreRejected() + { + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receiveArticleTypes: 65536)))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendArticleTypes: 65536)))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: "")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receivePatterns: " ")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendPatterns: "*,")))); + Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer( + endpoints: + [ + new TransitCatalogueEndpointRecord("News.Example", 119), + new TransitCatalogueEndpointRecord("news.example", 119), + ])))); + } + + [Theory] + [InlineData(null)] + [InlineData(1_000_000L)] + [InlineData(5_242_880L)] + public void SendMaxArticleBytes_PreservesUnlimitedAndExplicitCeilings(long? sendMaxArticleBytes) + { + var compiled = TransitCatalogueCompiler.Compile( + Publication(Peer(sendMaxArticleBytes: sendMaxArticleBytes))).Peers["peer"]; + Assert.Equal(sendMaxArticleBytes, compiled.SendMaxArticleBytes); + Assert.Equal(SharedSendMaxArticleBytes, compiled.MaxSize); + } + + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + [InlineData(2_147_483_648L)] + public void ExplicitSendMaxArticleBytes_RejectsNonPositiveAndAboveIntMax(long sendMaxArticleBytes) + { + var ex = Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(sendMaxArticleBytes: sendMaxArticleBytes)))); + Assert.Contains("send max_article_bytes", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void ReceiveMaxArticleBytes_StaysAnExplicitCeiling() + { + var ex = Assert.Throws(() => + TransitCatalogueCompiler.Compile(Publication(Peer(receiveMaxArticleBytes: 0)))); + Assert.Contains("receive max_article_bytes", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Service_PublishesUnlimitedSendSizeWithoutANumericDefault() + { + var factory = new FakeNntpDbConnectionFactory { TransitCatalogue = PeerSendPublication() }; + var database = await StartDatabaseAsync(factory); + var store = new TransitConfigurationStore(); + var services = new ServiceCollection(); + services.AddOptions(); + using var provider = services.BuildServiceProvider(); + var service = new TransitCatalogueService( + database, + store, + provider.GetRequiredService>(), + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(2, store.Current.PublicationId); + Assert.Null(store.Current.Peers["giganews"].SendMaxArticleBytes); + Assert.Equal(50, store.Current.Peers["giganews"].MaxOutgoingConnections); + Assert.Equal(1_000_000, store.Current.Peers["blueworld-hosting"].SendMaxArticleBytes); + Assert.Equal(10, store.Current.Peers["blueworld-hosting"].MaxOutgoingConnections); + Assert.Equal(SharedSendMaxArticleBytes, store.Current.Peers["usenet-ninja"].SendMaxArticleBytes); + Assert.Equal(5_242_880, store.Current.Peers["giganews"].MaxSize); + Assert.Equal(1_048_576, store.Current.Peers["blueworld-hosting"].MaxSize); + await service.DisposeAsync(); + await database.DisposeAsync(); + } + + [Fact] + public void SqlSendSize_RoundTripsToSendPolicy() + { + using var table = new DataTable(); + table.Columns.Add("max_article_bytes", typeof(int)); + table.Rows.Add(DBNull.Value); + table.Rows.Add(1_000_000); + table.Rows.Add(5_242_880); + using var reader = table.CreateDataReader(); + var sizes = new List(); + while (reader.Read()) + { + sizes.Add(NntpTransitCatalogueLoader.ReadSendMaxArticleBytes(reader, 0)); + } + + var snapshot = TransitCatalogueCompiler.Compile(new TransitCatalogueRecord( + 2, + 1, + 2, + 1, + true, + true, + [ + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 50, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com", + sendMaxArticleBytes: (long?)sizes[0]), + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com", + sendMaxArticleBytes: (long?)sizes[1]), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja", + sendMaxArticleBytes: (long?)sizes[2]), + ])); + + Assert.Null(snapshot.Peers["giganews"].SendMaxArticleBytes); + Assert.Equal("*,!unidata.*", snapshot.Peers["giganews"].Patterns.Expression); + Assert.Equal(TransitMessageTypes.All, snapshot.Peers["giganews"].MessageTypes); + Assert.Equal(1_000_000, snapshot.Peers["blueworld-hosting"].SendMaxArticleBytes); + Assert.Equal("*,!unidata.*,!control.*,!junk", snapshot.Peers["blueworld-hosting"].Patterns.Expression); + Assert.False(snapshot.Peers["blueworld-hosting"].Patterns.Expression.Contains("!local", StringComparison.Ordinal)); + Assert.Equal(SharedSendMaxArticleBytes, snapshot.Peers["usenet-ninja"].SendMaxArticleBytes); + Assert.Equal(80, snapshot.Peers["giganews"].MaxIncomingConnections); + Assert.Equal(10, snapshot.Peers["blueworld-hosting"].MaxIncomingConnections); + } + + [Fact] + public void LeftoverJson_IsRejected_ProcessLocalDepthRemains() + { + var peers = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Transit:alpha:PeerName"] = "Alpha", + }).Build(); + Assert.True(new TransitLeftoverConfigurationValidator(peers).Validate(null, new NntpdOptions()).Failed); + + var flags = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Nntpd:Transit:LogTrash"] = "false", + }).Build(); + var flagsResult = new TransitLeftoverConfigurationValidator(flags).Validate(null, new NntpdOptions()); + Assert.True(flagsResult.Failed); + Assert.Contains("nntptransitglobalrevision", flagsResult.FailureMessage, StringComparison.Ordinal); + + var local = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["Nntpd:Transit:StreamOutstandingArticleDepth"] = "8", + ["Nntpd:TransitQueueMemoryLimit"] = "4294967296", + }).Build(); + Assert.True(new TransitLeftoverConfigurationValidator(local).Validate(null, new NntpdOptions()).Succeeded); + } + + [Fact] + public async Task Service_PublishesOneCatalogue_AndMissingCurrentFailsStartup() + { + var factory = new FakeNntpDbConnectionFactory { TransitCatalogue = MigratedPublication() }; + var database = await StartDatabaseAsync(factory); + var store = new TransitConfigurationStore(); + var services = new ServiceCollection(); + services.AddOptions(); + using var provider = services.BuildServiceProvider(); + var monitor = provider.GetRequiredService>(); + var service = new TransitCatalogueService( + database, + store, + monitor, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + + Assert.Equal(3, store.Current.Peers.Count); + Assert.Equal(1, store.Current.PublicationId); + Assert.True(monitor.CurrentValue.Transit.WantTrash); + Assert.True(monitor.CurrentValue.Transit.LogTrash); + await service.DisposeAsync(); + await database.DisposeAsync(); + + var missingFactory = new FakeNntpDbConnectionFactory { TransitCatalogue = null }; + var missingDatabase = await StartDatabaseAsync(missingFactory); + var missing = new TransitCatalogueService( + missingDatabase, + new TransitConfigurationStore(), + monitor, + NullLogger.Instance); + var ex = await Assert.ThrowsAsync(() => missing.StartAsync(CancellationToken.None)); + Assert.Contains("nntptransitcurrent", ex.Message, StringComparison.Ordinal); + await missing.DisposeAsync(); + await missingDatabase.DisposeAsync(); + } + + [Fact] + public void EmptyJsonReload_DoesNotWipeThePublishedSnapshot() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitCatalogueCompiler.Compile(MigratedPublication())); + using var manager = new ConfigurationManager(); + var services = new ServiceCollection(); + services.AddLogging(static builder => builder.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(store); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + _ = provider.GetRequiredService(); + + manager["Nntpd:Transit:StreamOutstandingArticleDepth"] = "8"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(3, store.Current.Peers.Count); + Assert.Equal("nntp.giganews.com", store.Current.Peers["giganews"].PathToken); + } + + private static async Task StartDatabaseAsync(FakeNntpDbConnectionFactory factory) + { + var database = new NntpDbService( + factory, + Options.Create(new NntpDbOptions { ConnectionString = TestHostFactory.TestNntpDbConnectionString }), + NullLogger.Instance); + await database.StartAsync(CancellationToken.None); + return database; + } + + private static TransitCatalogueRecord PeerSendPublication() => + new( + 2, + 1, + 2, + 1, + true, + true, + [ + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com", + sendMaxArticleBytes: 1_000_000), + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 50, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com", + sendMaxArticleBytes: null), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja"), + ]); + + private static TransitCatalogueRecord MigratedPublication() => + new( + 1, + 1, + 1, + 1, + true, + true, + [ + Peer( + "blueworld-hosting", + "Blueworld Hosting", + 10, + 1_048_576, + ["usenet.blueworldhosting.com"], + 10, + [new TransitCatalogueEndpointRecord("usenet.blueworldhosting.com", 119)], + "*,!unidata.*,!control.*,!junk", + "usenet.blueworldhosting.com"), + Peer( + "giganews", + "Giganews, Inc.", + 80, + 5_242_880, + ["news-out.nntp.giganews.com"], + 80, + [new TransitCatalogueEndpointRecord("opticnetworks-in.nntp.ord.giganews.com", 119)], + "*,!unidata.*", + "nntp.giganews.com"), + Peer( + "usenet-ninja", + "Usenet Ninja", + 10, + 5_242_880, + ["198.18.0.0/15"], + 10, + [], + "*", + "nntp.usenet.ninja"), + ]); + + private static TransitCatalogueRecord Publication(TransitCataloguePeerRecord peer) => + new(1, 1, 1, 1, true, true, [peer]); + + private static TransitCataloguePeerRecord Peer( + string identifier = "peer", + string peerName = "Peer", + int maxInbound = 1, + long receiveMaxArticleBytes = SharedSendMaxArticleBytes, + IReadOnlyList? allowFrom = null, + int maxOutbound = 1, + IReadOnlyList? endpoints = null, + string sendPatterns = "*", + string pathToken = "peer.example", + int receiveArticleTypes = 65535, + int sendArticleTypes = 65535, + long? sendMaxArticleBytes = SharedSendMaxArticleBytes, + string receivePatterns = "*") => + new( + identifier, + peerName, + maxInbound, + string.Empty, + string.Empty, + true, + receiveMaxArticleBytes, + receiveArticleTypes, + receivePatterns, + allowFrom ?? [], + maxOutbound, + "None", + string.Empty, + string.Empty, + sendMaxArticleBytes, + sendArticleTypes, + sendPatterns, + pathToken, + endpoints ?? [], + []); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs new file mode 100644 index 00000000..997cf8e7 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitDeferOnDuplicateCommandTests.cs @@ -0,0 +1,353 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// Command-path coverage for Receive DeferOnDuplicate on CHECK and IHAVE. +/// TAKETHIS keeps the accepted-duplicate reply for both flag values. +/// +public sealed class TransitDeferOnDuplicateCommandTests +{ + private static readonly IPAddress PeerAddress = IPAddress.Parse("198.18.0.81"); + private static readonly byte[] SeenId = ""u8.ToArray(); + + [Theory] + [InlineData(true, false)] + [InlineData(false, false)] + [InlineData(true, true)] + [InlineData(false, true)] + public async Task Check_Seen_UsesDeferFlag(bool defer, bool readerMode) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal(defer ? "431 " : "438 ", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Check_Miss_Stays238(bool defer) + { + var history = CreateHistory(new FakeRedisService()); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal( + "238 send article to be transferred", + await duplex.ReadClientLineAsync()); + Assert.False(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Check_Unavailable_Stays431(bool defer) + { + var redis = new FakeRedisService(); + redis.Database.ExistsException = new RedisUnavailableException("down"); + var history = CreateHistory(redis); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await duplex.ReadClientLineAsync()); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task IHave_Seen_DoesNotRequestTheBody(bool defer) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal( + defer ? "436 Transfer not possible; try again later" : "435 Article not wanted", + await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + + await duplex.WriteClientLineAsync("DATE"); + Assert.StartsWith("111 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task IHave_Miss_Stays335(bool defer) + { + var history = CreateHistory(new FakeRedisService()); + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: false); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + await duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("335 Send article to be transferred", await duplex.ReadClientLineAsync()); + await duplex.WriteClientAsync(Article("")); + Assert.Equal("235 Article transferred OK", await duplex.ReadClientLineAsync()); + Assert.Equal(1, session.ArticleIngestion.Count); + + await QuitAsync(duplex, run); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task TakeThis_Seen_Returns239_AndTheNextArticleStaysInOrder(bool serial, bool defer) + { + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + var writes = history.Writes.Count; + await using var duplex = await CommandDuplex.CreateAsync(); + var session = duplex.CreateSession(history, defer, readerMode: serial); + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + + var duplicate = "TAKETHIS \r\n" + Article(""); + var next = "TAKETHIS \r\n" + Article(""); + if (serial) + { + await duplex.WriteClientAsync(duplicate); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal(0, session.ArticleIngestion.Count); + Assert.Equal(writes, history.Writes.Count); + + await duplex.WriteClientAsync(next); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + } + else + { + await duplex.WriteClientAsync(duplicate + next); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + Assert.Equal("239 ", await duplex.ReadClientLineAsync()); + } + + Assert.Equal(1, session.ArticleIngestion.Count); + Assert.Equal(writes + 1, history.Writes.Count); + Assert.True(history.ContainsLocal(HistoryDigest.FromMessageId(""u8))); + + await QuitAsync(duplex, run); + } + + [Fact] + public async Task ExistingConnection_KeepsDeferOnDuplicate_AfterCatalogueReplace() + { + var store = new TransitConfigurationStore(); + store.Replace(Snapshot(defer: true)); + var peers = TransitPeerAuthorization.CreateForStore(store); + var history = CreateHistory(new FakeRedisService()); + history.Remember(SeenId); + + await using var existingDuplex = await CommandDuplex.CreateAsync(); + var existing = existingDuplex.CreateSession(history, peers, readerMode: false); + var existingRun = existing.RunAsync(); + _ = await existingDuplex.ReadClientLineAsync(); + Assert.True(existing.Authorization.TransitPeerPolicy!.DeferOnDuplicate); + + await existingDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await existingDuplex.ReadClientLineAsync()); + + store.Replace(Snapshot(defer: false)); + Assert.True(existing.Authorization.TransitPeerPolicy.DeferOnDuplicate); + await existingDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("431 ", await existingDuplex.ReadClientLineAsync()); + + await using var freshDuplex = await CommandDuplex.CreateAsync(); + var fresh = freshDuplex.CreateSession(history, peers, readerMode: false); + var freshRun = fresh.RunAsync(); + _ = await freshDuplex.ReadClientLineAsync(); + Assert.False(fresh.Authorization.TransitPeerPolicy!.DeferOnDuplicate); + await freshDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("438 ", await freshDuplex.ReadClientLineAsync()); + + await QuitAsync(freshDuplex, freshRun); + await QuitAsync(existingDuplex, existingRun); + } + + private static HistoryDb CreateHistory(FakeRedisService redis) => + new(redis, TimeSpan.FromHours(2), NullLogger.Instance, TimeProvider.System, new HistoryWriteQueue()); + + private static TransitConfigurationSnapshot Snapshot(bool defer) => + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: [PeerAddress.ToString()], deferOnDuplicate: defer)); + + private static string Article(string messageId) => + CanonicalArticleText.Destuffed(messageId) + ".\r\n"; + + private static async Task QuitAsync(CommandDuplex duplex, Task run) + { + await duplex.WriteClientLineAsync("QUIT"); + Assert.Equal("205 Connection closing", await duplex.ReadClientLineAsync()); + await run; + } + + private sealed class CommandDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + + public static Task CreateAsync() => Task.FromResult(new CommandDuplex()); + + public NntpSession CreateSession(IHistoryDb history, bool defer, bool readerMode) => + CreateSession( + history, + TransitTestPeers.ForAllowFrom(PeerAddress, deferOnDuplicate: defer), + readerMode); + + public NntpSession CreateSession(IHistoryDb history, ITransitPeerAuthorization peers, bool readerMode) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = 1024 * 1024, + }); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(PeerAddress, 40000))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers, + historyDb: history); + if (readerMode) + { + session.SetMode(NntpSessionMode.Reader); + } + + return session; + } + + public async Task WriteClientAsync(string payload) + { + var bytes = Encoding.ASCII.GetBytes(payload); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public ConnectionClientIdentity ClientIdentity { get; } = identity; + public bool IsTls => false; + public bool IsCompressed => false; + public CancellationToken ConnectionClosed => _cts.Token; + public bool IsCompleted => _cts.IsCancellationRequested; + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitHotReloadTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitHotReloadTests.cs new file mode 100644 index 00000000..18907e8a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitHotReloadTests.cs @@ -0,0 +1,397 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Commands; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitHotReloadTests +{ + [Fact] + public void Reload_AddModifyRemovePeerAndAllowFrom_ReplacesSnapshotAtomically() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:alpha:PeerName"] = "Alpha", + ["Transit:alpha:MaxIncomingConnections"] = "4", + ["Transit:alpha:MaxOutgoingConnections"] = "1", + ["Transit:alpha:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + _ = provider.GetRequiredService(); + var authz = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + provider.GetRequiredService(), + NullLogger.Instance); + + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal(4, store.Current.Peers["alpha"].MaxIncomingConnections); + + manager["Transit:alpha:MaxIncomingConnections"] = "2"; + manager["Transit:alpha:AllowFrom:0"] = "198.51.100.10"; + manager["Transit:beta:PeerName"] = "Beta"; + manager["Transit:beta:MaxIncomingConnections"] = "1"; + manager["Transit:beta:MaxOutgoingConnections"] = "0"; + manager["Transit:beta:AllowFrom:0"] = "203.0.113.5"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Null(authz.Resolve(IPAddress.Parse("192.0.2.10")).TransitPeerName); + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + Assert.Equal("beta", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + Assert.Equal(2, store.Current.Peers["alpha"].MaxIncomingConnections); + + manager["Transit:alpha:Ssl"] = "not-a-mode"; + ((IConfigurationRoot)manager).Reload(); + Assert.Equal(2, store.Current.Peers["alpha"].MaxIncomingConnections); + Assert.Equal("alpha", authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + + store.Replace(TransitTestPeers.Snapshot("beta", TransitTestPeers.Peer(allowFrom: ["203.0.113.5"]))); + Assert.False(store.Current.Peers.ContainsKey("alpha")); + Assert.Null(authz.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + Assert.Equal("beta", authz.Resolve(IPAddress.Parse("203.0.113.5")).TransitPeerName); + } + + [Fact] + public void Reload_AllowFromChange_NewConnectionLosesPeerA_ExistingSessionUnchanged() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:test-peer:PeerName"] = TransitTestPeers.DefaultPeerDisplayName, + ["Transit:test-peer:MaxIncomingConnections"] = "10", + ["Transit:test-peer:MaxOutgoingConnections"] = "0", + ["Transit:test-peer:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + var hotReload = provider.GetRequiredService(); + var peers = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + hotReload, + NullLogger.Instance); + + var source = IPAddress.Parse("192.0.2.10"); + var existing = peers.Resolve(source); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.TransitPeerName); + + manager["Transit:test-peer:AllowFrom:0"] = "198.51.100.10"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.TransitPeerName); + Assert.True(existing.AuthorizedTransit); + var fresh = peers.Resolve(source); + Assert.Null(fresh.TransitPeerName); + Assert.False(fresh.AuthorizedTransit); + Assert.Equal(TransitTestPeers.DefaultPeerName, peers.Resolve(IPAddress.Parse("198.51.100.10")).TransitPeerName); + } + + [Fact] + public async Task Reload_AllowFromChange_ExistingSessionKeepsTransit_NewSessionDoesNot() + { + using var manager = new ConfigurationManager(); + manager.AddInMemoryCollection(new Dictionary + { + ["Transit:test-peer:PeerName"] = TransitTestPeers.DefaultPeerDisplayName, + ["Transit:test-peer:MaxIncomingConnections"] = "10", + ["Transit:test-peer:MaxOutgoingConnections"] = "0", + ["Transit:test-peer:AllowFrom:0"] = "192.0.2.10", + }); + + var services = new ServiceCollection(); + services.AddLogging(static b => b.ClearProviders()); + services.AddOptions().Bind(manager.GetSection("Transit")); + services.AddSingleton>( + new ConfigurationChangeTokenSource(manager)); + services.AddSingleton(); + services.AddSingleton(); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + var hotReload = provider.GetRequiredService(); + var peers = new TransitPeerAuthorization( + store, + new EmptyDnsCache(), + hotReload, + NullLogger.Instance); + + var source = IPAddress.Parse("192.0.2.10"); + await using var existingDuplex = await SessionDuplex.CreateAsync(source); + var existing = existingDuplex.CreateSession(peers); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + var existingRun = existing.RunAsync(); + _ = await existingDuplex.ReadClientLineAsync(); + + await existingDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await existingDuplex.ReadClientLineAsync()); + + manager["Transit:test-peer:AllowFrom:0"] = "198.51.100.10"; + ((IConfigurationRoot)manager).Reload(); + + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + await existingDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("238 send article to be transferred", await existingDuplex.ReadClientLineAsync()); + + await using var freshDuplex = await SessionDuplex.CreateAsync(source); + var fresh = freshDuplex.CreateSession(peers); + Assert.Null(fresh.Authorization.TransitPeerName); + Assert.False(fresh.Authorization.AuthorizedTransit); + var freshRun = fresh.RunAsync(); + _ = await freshDuplex.ReadClientLineAsync(); + await freshDuplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await freshDuplex.ReadClientLineAsync()); + + await existingDuplex.WriteClientLineAsync("QUIT"); + _ = await existingDuplex.ReadClientLineAsync(); + await existingRun; + await freshDuplex.WriteClientLineAsync("QUIT"); + _ = await freshDuplex.ReadClientLineAsync(); + await freshRun; + } + + [Fact] + public void Reload_UpdatesPathTokenMaxSizeAndMessageTypesAtomically() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + allowFrom: ["192.0.2.10"], + pathToken: "old.token", + maxSize: 1024, + messageTypes: ["default"]))); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("old.token", existing.TransitPeerPolicy!.PathToken); + Assert.Equal(1024, existing.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Default, existing.TransitPeerPolicy.MessageTypes); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + allowFrom: ["192.0.2.10"], + pathToken: "new.token", + maxSize: 2048, + messageTypes: ["control", "binary"]))); + + Assert.Equal("old.token", existing.TransitPeerPolicy.PathToken); + var fresh = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("new.token", fresh.TransitPeerPolicy!.PathToken); + Assert.Equal(2048, fresh.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Control | TransitMessageTypes.Binary, fresh.TransitPeerPolicy.MessageTypes); + } + + [Fact] + public void Reload_UpdatesRemainingPeerFieldsAtomically() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 4, + maxOutgoing: 2, + allowFrom: ["192.0.2.10"], + username: "old-user", + password: "old-pass", + ssl: "TLS", + patterns: "comp.*", + connectTo: ["news.example.net:119"], + deferOnDuplicate: true))); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("old-user", existing.TransitPeerPolicy!.Username); + Assert.Equal("old-pass", existing.TransitPeerPolicy.Password); + Assert.Equal(TransitSslMode.Tls, existing.TransitPeerPolicy.Ssl); + Assert.True(existing.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.True(existing.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(4, existing.TransitPeerPolicy.MaxIncomingConnections); + Assert.Equal(2, existing.TransitPeerPolicy.MaxOutgoingConnections); + Assert.Equal("news.example.net", existing.TransitPeerPolicy.ConnectTo.Single().Host); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 8, + maxOutgoing: 3, + allowFrom: ["192.0.2.10"], + username: "new-user", + password: "new-pass", + ssl: "STARTTLS", + patterns: "alt.*", + connectTo: ["other.example.net:563"], + deferOnDuplicate: false))); + + Assert.Equal("old-user", existing.TransitPeerPolicy.Username); + Assert.Equal("old-pass", existing.TransitPeerPolicy.Password); + Assert.True(existing.TransitPeerPolicy.DeferOnDuplicate); + var fresh = peers.Resolve(IPAddress.Parse("192.0.2.10")); + Assert.Equal("new-user", fresh.TransitPeerPolicy!.Username); + Assert.Equal("new-pass", fresh.TransitPeerPolicy.Password); + Assert.Equal(TransitSslMode.StartTls, fresh.TransitPeerPolicy.Ssl); + Assert.True(fresh.TransitPeerPolicy.Patterns.MatchesNewsgroup("alt.test")); + Assert.False(fresh.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(fresh.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(8, fresh.TransitPeerPolicy.MaxIncomingConnections); + Assert.Equal(3, fresh.TransitPeerPolicy.MaxOutgoingConnections); + Assert.Equal("other.example.net", fresh.TransitPeerPolicy.ConnectTo.Single().Host); + } + + [Fact] + public void Reload_InvalidConfiguration_KeepsLastValidSnapshot() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitTestPeers.Snapshot("alpha", TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]))); + var validator = new TransitPeersOptionsValidator(); + var invalid = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: "not-a-mode")); + var result = validator.Validate(null, invalid); + Assert.True(result.Failed); + Assert.Equal("alpha", store.Current.Peers.Keys.Single()); + } + + private sealed class EmptyDnsCache : ITransitDnsAddressCache + { + public IReadOnlySet GetResolved(string hostname) => new HashSet(); + + public Task RefreshAllAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public Task RefreshDueAsync(TransitConfigurationSnapshot snapshot, CancellationToken cancellationToken) => + Task.CompletedTask; + + public TimeSpan GetDelayUntilNextRefresh(TransitConfigurationSnapshot snapshot) => Timeout.InfiniteTimeSpan; + } + + private sealed class SessionDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly IPAddress _clientAddress; + + private SessionDuplex(IPAddress clientAddress) => _clientAddress = clientAddress; + + public static Task CreateAsync(IPAddress clientAddress) => + Task.FromResult(new SessionDuplex(clientAddress)); + + public NntpSession CreateSession(ITransitPeerAuthorization peers) + { + var connection = new ReloadPipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + _clientAddress); + return new NntpSession( + connection, + NullLogger.Instance, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class ReloadPipeConnection(PipeReader input, PipeWriter output, IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitMessageTypesTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitMessageTypesTests.cs new file mode 100644 index 00000000..25ce231c --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitMessageTypesTests.cs @@ -0,0 +1,133 @@ +using VectorNNTPD.TransitService.Configuration; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitMessageTypesTests +{ + public static TheoryData DiabloTokens => new() + { + { "none", TransitMessageTypes.None }, + { "default", TransitMessageTypes.Default }, + { "control", TransitMessageTypes.Control }, + { "cancel", TransitMessageTypes.Cancel }, + { "mime", TransitMessageTypes.Mime }, + { "binary", TransitMessageTypes.Binary }, + { "binaries", TransitMessageTypes.Binary }, + { "uuencode", TransitMessageTypes.UuEncode }, + { "base64", TransitMessageTypes.Base64 }, + { "yenc", TransitMessageTypes.Yenc }, + { "bommanews", TransitMessageTypes.BommaNews }, + { "unidata", TransitMessageTypes.UniData }, + { "multipart", TransitMessageTypes.Multipart }, + { "html", TransitMessageTypes.Html }, + { "ps", TransitMessageTypes.PostScript }, + { "binhex", TransitMessageTypes.BinHex }, + { "partial", TransitMessageTypes.Partial }, + { "pgp", TransitMessageTypes.PgpMessage }, + { "all", TransitMessageTypes.All }, + }; + + [Theory] + [MemberData(nameof(DiabloTokens))] + public void ParseToken_MapsDiabloNames(string token, TransitMessageTypes expected) + { + Assert.True(TransitMessageTypesParser.TryParseToken(token, out var actual)); + Assert.Equal(expected, actual); + Assert.True(TransitMessageTypesParser.TryParseToken(" " + token.ToUpperInvariant() + " ", out var padded)); + Assert.Equal(expected, padded); + } + + [Fact] + public void BinaryAndBinaries_AreTheSameFlag() + { + Assert.True(TransitMessageTypesParser.TryParseToken("binary", out var binary)); + Assert.True(TransitMessageTypesParser.TryParseToken("binaries", out var binaries)); + Assert.Equal(TransitMessageTypes.Binary, binary); + Assert.Equal(binary, binaries); + } + + [Fact] + public void All_IsUnionOfConcreteFlags() + { + const TransitMessageTypes concrete = + TransitMessageTypes.Default + | TransitMessageTypes.Control + | TransitMessageTypes.Cancel + | TransitMessageTypes.Mime + | TransitMessageTypes.Binary + | TransitMessageTypes.UuEncode + | TransitMessageTypes.Base64 + | TransitMessageTypes.Yenc + | TransitMessageTypes.BommaNews + | TransitMessageTypes.UniData + | TransitMessageTypes.Multipart + | TransitMessageTypes.Html + | TransitMessageTypes.PostScript + | TransitMessageTypes.BinHex + | TransitMessageTypes.Partial + | TransitMessageTypes.PgpMessage; + Assert.Equal(concrete, TransitMessageTypes.All); + Assert.NotEqual(TransitMessageTypes.None, TransitMessageTypes.Default); + Assert.NotEqual(TransitMessageTypes.Default, TransitMessageTypes.All); + } + + [Fact] + public void Parse_DefaultArrayAndDuplicates() + { + Assert.True(TransitMessageTypesParser.TryParse(null, out var omitted, out _)); + Assert.Equal(TransitMessageTypes.Default, omitted); + + Assert.True(TransitMessageTypesParser.TryParse([], out var empty, out _)); + Assert.Equal(TransitMessageTypes.Default, empty); + + Assert.True(TransitMessageTypesParser.TryParse(["default"], out var defaults, out _)); + Assert.Equal(TransitMessageTypes.Default, defaults); + + Assert.True(TransitMessageTypesParser.TryParse(["binary", "BINARY", "binaries"], out var dupes, out _)); + Assert.Equal(TransitMessageTypes.Binary, dupes); + + Assert.True(TransitMessageTypesParser.TryParse(["default", "control", "binary"], out var combo, out _)); + Assert.Equal( + TransitMessageTypes.Default | TransitMessageTypes.Control | TransitMessageTypes.Binary, + combo); + } + + [Fact] + public void Parse_RejectsUnknownAndBlankTokens() + { + Assert.False(TransitMessageTypesParser.TryParse(["default", "audio"], out _, out var error)); + Assert.Contains("[1]", error, StringComparison.Ordinal); + Assert.False(TransitMessageTypesParser.TryParse([" "], out _, out _)); + } + + [Fact] + public void CanonicalNames_DoNotInventTypes() + { + Assert.Equal(["none"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.None)); + Assert.Equal(["all"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.All)); + Assert.Equal(["binary"], TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.Binary)); + Assert.DoesNotContain("binaries", TransitMessageTypesParser.ToCanonicalNames(TransitMessageTypes.Binary)); + } + + [Fact] + public void EnumMemberValues_AreStablePowersOfTwo() + { + Assert.Equal(0, (int)TransitMessageTypes.None); + Assert.Equal(1, (int)TransitMessageTypes.Default); + Assert.Equal(2, (int)TransitMessageTypes.Control); + Assert.Equal(4, (int)TransitMessageTypes.Cancel); + Assert.Equal(8, (int)TransitMessageTypes.Mime); + Assert.Equal(16, (int)TransitMessageTypes.Binary); + Assert.Equal(32, (int)TransitMessageTypes.UuEncode); + Assert.Equal(64, (int)TransitMessageTypes.Base64); + Assert.Equal(128, (int)TransitMessageTypes.Yenc); + Assert.Equal(256, (int)TransitMessageTypes.BommaNews); + Assert.Equal(512, (int)TransitMessageTypes.UniData); + Assert.Equal(1024, (int)TransitMessageTypes.Multipart); + Assert.Equal(2048, (int)TransitMessageTypes.Html); + Assert.Equal(4096, (int)TransitMessageTypes.PostScript); + Assert.Equal(8192, (int)TransitMessageTypes.BinHex); + Assert.Equal(16384, (int)TransitMessageTypes.Partial); + Assert.Equal(32768, (int)TransitMessageTypes.PgpMessage); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs new file mode 100644 index 00000000..2ea0a45a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerIdentificationAndLimitTests.cs @@ -0,0 +1,353 @@ +using System.Net; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerIdentificationAndLimitTests +{ + [Fact] + public void Identify_MatchingSource_ReturnsNamedPeerPolicy() + { + var snapshot = new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + [TransitTestPeers.DefaultPeerName] = TransitConfigurationSnapshot.CreatePeer( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer( + maxIncoming: 3, + username: "feed", + password: "secret", + patterns: "comp.*,!comp.sources.*", + ssl: "TLS", + allowFrom: ["192.0.2.10"])), + }); + var authz = TransitPeerAuthorization.CreateStatic(snapshot).Resolve(IPAddress.Parse("192.0.2.10")); + Assert.False(authz.AuthorizedTransit); + Assert.True(authz.StreamingPermitted); + Assert.Equal(TransitTestPeers.DefaultPeerName, authz.TransitPeerName); + Assert.NotNull(authz.TransitPeerPolicy); + Assert.Equal(TransitTestPeers.DefaultPeerName, authz.TransitPeerPolicy.Identifier); + Assert.Equal(TransitTestPeers.DefaultPeerDisplayName, authz.TransitPeerPolicy.PeerName); + Assert.Equal(3, authz.TransitPeerPolicy.MaxIncomingConnections); + Assert.True(authz.TransitPeerPolicy.HasPeerCredentials); + Assert.Equal(TransitSslMode.Tls, authz.TransitPeerPolicy.Ssl); + Assert.True(authz.TransitPeerPolicy.DeferOnDuplicate); + Assert.Equal(string.Empty, authz.TransitPeerPolicy.PathToken); + Assert.Equal(TransitPeerOptions.DefaultMaxSize, authz.TransitPeerPolicy.MaxSize); + Assert.Equal(TransitMessageTypes.Default, authz.TransitPeerPolicy.MessageTypes); + Assert.True(authz.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.lang.c")); + Assert.False(authz.TransitPeerPolicy.Patterns.MatchesNewsgroup("comp.sources.unix")); + } + + [Fact] + public void Identify_NonMatchingSource_IsNotTransit() + { + var authz = TransitPeerAuthorization.CreateStatic( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]))); + var result = authz.Resolve(IPAddress.Parse("198.51.100.1")); + Assert.False(result.AuthorizedTransit); + Assert.Null(result.TransitPeerName); + Assert.Null(result.TransitPeerPolicy); + } + + [Fact] + public void Identify_AmbiguousRuntimeOverlap_DeniesTransit() + { + // Validation would reject literal overlap; construct a snapshot that simulates + // two peers matching the same address without going through the validator. + var snapshot = new TransitConfigurationSnapshot(new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer("one", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer("two", TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"])), + }); + var result = TransitPeerAuthorization.CreateStatic(snapshot).Resolve(IPAddress.Parse("192.0.2.10")); + Assert.False(result.AuthorizedTransit); + Assert.Null(result.TransitPeerName); + } + + [Fact] + public async Task Limit_BelowAtAndOver_IdentifyPeerFromSourceIp() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 2); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + + var first = CreateSession(peers, source); + Assert.Equal(TransitTestPeers.DefaultPeerName, first.Authorization.TransitPeerName); + var lease1 = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease1.Admitted); + + var second = CreateSession(peers, source); + var lease2 = await TransitConnectionAdmission.TryAdmitAsync(limiter, second); + Assert.True(lease2.Admitted); + Assert.Equal(2, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + + var over = CreateSession(peers, source); + var lease3 = await TransitConnectionAdmission.TryAdmitAsync(limiter, over); + Assert.False(lease3.Admitted); + Assert.False(lease3.Lease.IsHeld); + + await lease1.Lease.DisposeAsync(); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + var reused = CreateSession(peers, source); + var lease4 = await TransitConnectionAdmission.TryAdmitAsync(limiter, reused); + Assert.True(lease4.Admitted); + await lease2.Lease.DisposeAsync(); + await lease4.Lease.DisposeAsync(); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Limit_AmbiguousMatch_DoesNotConsumeSlot() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = new TransitConfigurationStore(); + store.Replace( + new TransitConfigurationSnapshot( + new Dictionary(StringComparer.Ordinal) + { + ["one"] = TransitConfigurationSnapshot.CreatePeer( + "one", + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.0/24"])), + ["two"] = TransitConfigurationSnapshot.CreatePeer( + "two", + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.0/24"])), + })); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var session = CreateSession(peers, source); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(session.Authorization.AuthorizedTransit); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + Assert.Equal(0, limiter.GetCount("one")); + Assert.Equal(0, limiter.GetCount("two")); + Assert.Equal(0, membership.ActiveCount("one", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.ActiveCount("two", TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task Limit_NonMatchingSource_AdmittedWithoutSlot() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var other = CreateSession(peers, IPAddress.Parse("198.51.100.1")); + Assert.Null(other.Authorization.TransitPeerName); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, other); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + Assert.Equal(0, membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task Limit_RemovedPeer_NewConnectionIsNotIdentified() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, existing); + Assert.True(lease.Admitted); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + + store.Replace(TransitConfigurationSnapshot.Empty); + Assert.Equal(TransitTestPeers.DefaultPeerName, existing.Authorization.TransitPeerName); + var fresh = CreateSession(peers, source); + Assert.Null(fresh.Authorization.TransitPeerName); + var uncounted = await TransitConnectionAdmission.TryAdmitAsync(limiter, fresh); + Assert.True(uncounted.Admitted); + Assert.False(uncounted.Lease.IsHeld); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + await lease.Lease.DisposeAsync(); + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Limit_LoweredOnReload_AffectsNewConnectionsOnly() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 2); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var existing = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, existing); + Assert.True(lease.Admitted); + + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: [source.ToString()]))); + + Assert.Equal(2, existing.Authorization.TransitPeerPolicy!.MaxIncomingConnections); + var blocked = CreateSession(peers, source); + Assert.Equal(1, blocked.Authorization.TransitPeerPolicy!.MaxIncomingConnections); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked)).Admitted); + Assert.Equal(1, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + await lease.Lease.DisposeAsync(); + var next = CreateSession(peers, source); + var reused = await TransitConnectionAdmission.TryAdmitAsync(limiter, next); + Assert.True(reused.Admitted); + await reused.Lease.DisposeAsync(); + } + + [Fact] + public async Task Limit_ConcurrentBoundary_DoesNotOverAdmit() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 8); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var leases = new TransitInboundConnectionLease[32]; + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 32).Select(async i => + { + await Task.Yield(); + var session = CreateSession(peers, source); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + if (outcome.Admitted) + { + Interlocked.Increment(ref admitted); + leases[i] = outcome.Lease; + } + })); + Assert.Equal(8, admitted); + Assert.Equal(8, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + foreach (var lease in leases.Where(static lease => lease is not null)) + { + await lease.DisposeAsync(); + } + + Assert.Equal(0, limiter.GetCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task Admission_Writes400WhenLimitExceeded() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 0); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + var connection = new LimitPipeConnection(input.Reader, output.Writer, source); + var session = new NntpSession( + connection, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + transitPeerAuthorization: peers); + Assert.Equal(TransitTestPeers.DefaultPeerName, session.Authorization.TransitPeerName); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, session)).Admitted); + await TransitConnectionAdmission.WriteUnavailableAsync(connection, CancellationToken.None); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await VectorNNTPD.TransitService.Session.CommandProcessor.NntpCommandLineReader.ReadLineAsync(output.Reader, cts.Token); + Assert.Equal("400 Service temporarily unavailable", line); + } + + [Fact] + public async Task Limit_CancellationStyleRelease_ViaDispose() + { + var source = IPAddress.Parse("192.0.2.10"); + var store = CreatePeerStore(source, maxIncoming: 1); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var first = CreateSession(peers, source); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease.Admitted); + await using (lease.Lease) + { + var blocked = CreateSession(peers, source); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked)).Admitted); + } + + var after = CreateSession(peers, source); + var reused = await TransitConnectionAdmission.TryAdmitAsync(limiter, after); + Assert.True(reused.Admitted); + await reused.Lease.DisposeAsync(); + } + + private static TransitConfigurationStore CreatePeerStore(IPAddress source, int maxIncoming) + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: maxIncoming, allowFrom: [source.ToString()]))); + return store; + } + + private static NntpSession CreateSession(ITransitPeerAuthorization peers, IPAddress source) + { + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + var connection = new LimitPipeConnection(input.Reader, output.Writer, source); + return new NntpSession( + connection, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + transitPeerAuthorization: peers); + } + + private sealed class LimitPipeConnection(System.IO.Pipelines.PipeReader input, System.IO.Pipelines.PipeWriter output, IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public System.IO.Pipelines.PipeReader Input { get; } = input; + public System.IO.Pipelines.PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs new file mode 100644 index 00000000..5cbcc6bf --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateCompensationTests.cs @@ -0,0 +1,217 @@ +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateCompensationTests +{ + private const string Peer = "peer-a"; + + [Fact] + public async Task SameIdentifierAdmission_WaitsUntilCompensationReleasesGate() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admitA = node.TryAdmitAsync(Peer, 5, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + + var admitB = node.TryAdmitAsync(Peer, 5).AsTask(); + Assert.False(admitB.IsCompleted); + + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var canceledGeneration, + out var canceledCount)); + Assert.Equal(1, canceledCount); + + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admitA); + var admitted = await admitB; + Assert.True(admitted.Accepted); + Assert.Equal(1, node.GetLocalCount(Peer)); + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(node.OwnerId, store.LastReleaseOwnerId); + Assert.Equal(canceledGeneration, store.LastReleaseGeneration); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var liveGeneration, + out _)); + Assert.NotEqual(canceledGeneration, liveGeneration); + } + + [Fact] + public async Task CanceledCallerToken_IsNotUsedForCompensatingRelease() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync(Peer, 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(CancellationToken.None, store.LastReleaseToken); + Assert.False(store.LastReleaseToken.CanBeCanceled); + Assert.Equal(1, store.ReleaseCalls); + Assert.Equal(0, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + } + + [Fact] + public async Task CompensationWhenRedisUnavailable_LeavesTtlOwnershipAndNoLocalCommit() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner) { UnavailableOnRelease = true }; + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admit = node.TryAdmitAsync(Peer, 2, cts.Token).AsTask(); + await store.Admitted.Task; + cts.Cancel(); + store.Continue.TrySetResult(); + await Assert.ThrowsAnyAsync(() => admit); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + Assert.Equal(1, inner.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, store.ReleaseCalls); + } + + [Fact] + public async Task NonCanceledStoreException_DoesNotCompensate() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner) { ThrowAfterAdmit = new InvalidOperationException("store fault") }; + var node = TransitPeerStateTestFactory.CreateTracker(store); + await Assert.ThrowsAsync(() => node.TryAdmitAsync(Peer, 2).AsTask()); + Assert.Equal(0, store.ReleaseCalls); + Assert.Equal(0, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task CompensationRelease_DoesNotAffectNewerGeneration() + { + var inner = new InMemoryTransitPeerStateStore(); + var store = new CompensationStore(inner); + var node = TransitPeerStateTestFactory.CreateTracker(store); + using var cts = new CancellationTokenSource(); + var admitA = node.TryAdmitAsync(Peer, 5, cts.Token).AsTask(); + await store.Admitted.Task; + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var canceledGeneration, + out _)); + + cts.Cancel(); + store.Continue.TrySetResult(); + var admitB = node.TryAdmitAsync(Peer, 5).AsTask(); + await Assert.ThrowsAnyAsync(() => admitA); + Assert.True((await admitB).Accepted); + Assert.Equal(canceledGeneration, store.LastReleaseGeneration); + Assert.True(inner.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var liveGeneration, + out var liveCount)); + Assert.NotEqual(canceledGeneration, liveGeneration); + Assert.Equal(1, liveCount); + } + + private sealed class CompensationStore : ITransitPeerStateStore + { + private readonly InMemoryTransitPeerStateStore _inner; + + public CompensationStore(InMemoryTransitPeerStateStore inner) + { + _inner = inner; + } + + public TaskCompletionSource Admitted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Continue { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool UnavailableOnRelease { get; set; } + + public Exception? ThrowAfterAdmit { get; set; } + + public int ReleaseCalls { get; private set; } + + public string? LastReleaseOwnerId { get; private set; } + + public long LastReleaseGeneration { get; private set; } + + public CancellationToken LastReleaseToken { get; private set; } = new(true); + + public async ValueTask TryAdmitAsync( + string identifier, + string ownerId, + int maxIncoming, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) + { + var result = await _inner.TryAdmitAsync( + identifier, + ownerId, + maxIncoming, + generation, + now, + leaseTtl, + CancellationToken.None).ConfigureAwait(false); + if (ThrowAfterAdmit is { } fault) + { + throw fault; + } + + Admitted.TrySetResult(); + await Continue.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + return result; + } + + public async ValueTask ReleaseAsync( + string identifier, + string ownerId, + long generation, + CancellationToken cancellationToken = default) + { + ReleaseCalls++; + LastReleaseOwnerId = ownerId; + LastReleaseGeneration = generation; + LastReleaseToken = cancellationToken; + if (UnavailableOnRelease) + { + return; + } + + await _inner.ReleaseAsync(identifier, ownerId, generation, cancellationToken).ConfigureAwait(false); + } + + public ValueTask RenewAsync( + string identifier, + string ownerId, + long generation, + DateTimeOffset now, + TimeSpan leaseTtl, + CancellationToken cancellationToken = default) => + _inner.RenewAsync(identifier, ownerId, generation, now, leaseTtl, cancellationToken); + + public ValueTask ReleaseOwnerAsync( + string identifier, + string ownerId, + CancellationToken cancellationToken = default) => + _inner.ReleaseOwnerAsync(identifier, ownerId, cancellationToken); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs new file mode 100644 index 00000000..db6ade66 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateEngineTests.cs @@ -0,0 +1,166 @@ +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateEngineTests +{ + private const string Key = "tconn:peer-a"; + private const string Other = "tconn:peer-b"; + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const long Now = 10_000; + private const long Lease = 30_000; + + [Fact] + public void TryAdmit_FirstConnection_WritesCountOne() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 4, Now, Lease, 1)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expiry, out var generation, out var count)); + Assert.Equal(Now + Lease, expiry); + Assert.Equal(1, generation); + Assert.Equal(1, count); + } + + [Fact] + public void TryAdmit_BelowLimit_IncrementsSameGeneration() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 3, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 3, Now, Lease, 1)); + Assert.True(engine.TryGetOwnership(Key, O1, out _, out var generation, out var count)); + Assert.Equal(1, generation); + Assert.Equal(2, count); + } + + [Fact] + public void TryAdmit_ExactlyAtLimit_ThenOverLimit() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 2, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O2, 2, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, 2, Now, Lease, 1)); + Assert.Equal(2, engine.ActiveCount(Key, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void TryAdmit_MaxZero_RejectsWithoutMutation() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, 0, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O1, -1, Now, Lease, 1)); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + Assert.Equal(0, engine.ActiveCount(Key, Now)); + } + + [Fact] + public void Release_DecrementsThenDeletesFieldAndHash() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.Release(Key, O1, 1); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + _ = engine.Release(Key, O1, 1); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + Assert.Empty(engine.OwnerFields(Key)); + } + + [Fact] + public void Release_StaleGeneration_IsNoOp() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 2); + _ = engine.Release(Key, O1, 1); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void TryAdmit_NewGeneration_ReplacesStaleCount() + { + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership(Key, O1, Now + Lease, 1, 7); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 20, Now, Lease, 2)); + Assert.True(engine.TryGetOwnership(Key, O1, out _, out var generation, out var count)); + Assert.Equal(2, generation); + Assert.Equal(1, count); + } + + [Fact] + public void Renew_ExtendsMatchingUnexpiredOwnership() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + Assert.Equal(1, engine.Renew(Key, O1, 1, Now + 5_000, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expiry, out var generation, out var count)); + Assert.Equal(Now + 5_000 + Lease, expiry); + Assert.Equal(1, generation); + Assert.Equal(1, count); + } + + [Fact] + public void Renew_MissingExpiredOrStale_DoesNotRecreate() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.False(engine.TryGetOwnership(Key, O1, out _, out _, out _)); + + engine.WriteOwnership(Key, O1, Now, 1, 2); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var expired, out _, out var count)); + Assert.Equal(Now, expired); + Assert.Equal(2, count); + + engine.WriteOwnership(Key, O1, Now + Lease, 2, 2); + Assert.Equal(0, engine.Renew(Key, O1, 1, Now, Lease)); + Assert.True(engine.TryGetOwnership(Key, O1, out var still, out var generation, out _)); + Assert.Equal(Now + Lease, still); + Assert.Equal(2, generation); + } + + [Fact] + public void TryAdmit_PrunesExpiredBeforeLimit() + { + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership(Key, O2, Now, 1, 4); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 1, Now, Lease, 1)); + Assert.Equal(0, engine.OwnerCount(Key, O2, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O1, Now)); + } + + [Fact] + public void DifferentPeers_AreIsolated() + { + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O1, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Other, O1, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O2, 1, Now, Lease, 1)); + Assert.Equal(1, engine.ActiveCount(Key, Now)); + Assert.Equal(1, engine.ActiveCount(Other, Now)); + } + + [Fact] + public void ReleaseOwner_RemovesOnlyThisOwner() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 4, Now, Lease, 1); + _ = engine.TryAdmit(Key, O2, 4, Now, Lease, 1); + _ = engine.ReleaseOwner(Key, O1); + Assert.Equal(0, engine.OwnerCount(Key, O1, Now)); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now)); + _ = engine.ReleaseOwner(Other, O2); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now)); + } + + [Fact] + public void Expiry_FreesCapacity() + { + var engine = new TransitPeerStateEngine(); + _ = engine.TryAdmit(Key, O1, 1, Now, Lease, 1); + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit(Key, O2, 1, Now, Lease, 1)); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit(Key, O2, 1, Now + Lease, Lease, 1)); + Assert.Equal(0, engine.OwnerCount(Key, O1, Now + Lease)); + Assert.Equal(1, engine.OwnerCount(Key, O2, Now + Lease)); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs new file mode 100644 index 00000000..da740ef5 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateFailureTests.cs @@ -0,0 +1,118 @@ +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateFailureTests +{ + private const string Peer = "peer-a"; + private const string Other = "peer-b"; + + [Fact] + public async Task RedisUnavailableDuringAdmission_FailsClosedWithoutLocalSlot() + { + var membership = new InMemoryTransitPeerStateStore { Unavailable = true }; + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var result = await node.TryAdmitAsync(Peer, 4); + Assert.Equal(TransitPeerStateAdmitStatus.Unavailable, result.Status); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(1, node.UnavailableRejects); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task RedisEvalFailureDuringAdmission_FailsClosed() + { + var redis = new FakeRedisService(); + redis.Database.ScriptException = new RedisUnavailableException("eval failed"); + var store = new RedisTransitPeerStateStore(redis); + var node = TransitPeerStateTestFactory.CreateTracker(store); + var result = await node.TryAdmitAsync(Peer, 4); + Assert.Equal(TransitPeerStateAdmitStatus.Unavailable, result.Status); + Assert.Equal(0, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task OceAfterSuccessfulEval_CompensatesAndRethrows() + { + var membership = new InMemoryTransitPeerStateStore + { + ThrowAfterAdmit = new OperationCanceledException(), + }; + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await Assert.ThrowsAnyAsync(() => node.TryAdmitAsync(Peer, 4).AsTask()); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, node.DistributedAdmits); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.ReleaseCalls); + } + + [Fact] + public async Task RedisUnavailableDuringRelease_FinalizesLocally() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var admitted = await node.TryAdmitAsync(Peer, 4); + Assert.True(admitted.Accepted); + membership.Unavailable = true; + await node.ReleaseAsync(Peer, admitted.Generation); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task RedisUnavailableDuringRenewal_DoesNotExtend() + { + var clock = new VectorNNTPD.TransitService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var expiry = membership.Expiry(Peer, node.OwnerId); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(expiry, membership.Expiry(Peer, node.OwnerId)); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task ShutdownCleanup_ReleaseOwnerClearsRemainingOwnership() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + await node.ReleaseAllOwnershipAsync(); + Assert.False(node.IsAccepting); + Assert.Equal(0, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False((await node.TryAdmitAsync(Peer, 4)).Accepted); + } + + [Fact] + public async Task ReleaseOwner_IsIsolatedToThisOwnerAndPeer() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + Assert.True((await a.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await b.TryAdmitAsync(Peer, 4)).Accepted); + Assert.True((await a.TryAdmitAsync(Other, 4)).Accepted); + await a.ReleaseAllOwnershipAsync(); + Assert.Equal(0, membership.OwnerCount(Peer, a.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.OwnerCount(Other, a.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.OwnerCount(Peer, b.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.True((await b.TryAdmitAsync(Other, 4)).Accepted); + } + + [Fact] + public async Task NeverAdmitted_NeverIssuesRelease() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, node.ReleaseCalls); + Assert.Equal(0, membership.ReleaseCalls); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs new file mode 100644 index 00000000..5bcd5c27 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLifecycleTests.cs @@ -0,0 +1,300 @@ +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateLifecycleTests +{ + private static readonly IPAddress Source = IPAddress.Parse("192.0.2.10"); + + [Fact] + public async Task Quit_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.WriteClientLineAsync("QUIT"); + Assert.StartsWith("205 ", await live.ReadClientLineAsync(), StringComparison.Ordinal); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(NntpSessionLifetime.Finalized, live.Session.Lifetime); + Assert.Equal(1, live.Tracker.ReleaseCalls); + Assert.Equal(0, live.Membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task RemoteEof_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task ConnectionReset_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionReset)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task SocketException_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new SocketException((int)SocketError.ConnectionAborted)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task IoException_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CompleteClientInputAsync(new IOException("peer disappeared")); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task IdleTimeout_ReleasesAdmittedConnectionOnce() + { + var clock = new ControllableTimeProvider(); + await using var live = await StartAdmittedAsync(clock); + await live.Session.IdleWatchArmed; + clock.Advance(TimeSpan.FromSeconds(2)); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(TcpDisconnectReason.IdleTimeout, live.Session.CloseReasonForTests); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task Cancellation_ReleasesAdmittedConnectionOnce() + { + await using var live = await StartAdmittedAsync(); + await live.CancelRunAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(1, live.Tracker.ReleaseCalls); + } + + [Fact] + public async Task ListenerShutdown_FinalizesThenReleaseOwner() + { + await using var live = await StartAdmittedAsync(); + await live.CancelRunAsync(); + await live.Run; + await live.FinalizeLeaseAsync(); + Assert.Equal(0, live.Membership.ActiveCount(TransitTestPeers.DefaultPeerName, TransitPeerStateTestFactory.NowMs())); + await live.Tracker.ReleaseAllOwnershipAsync(); + Assert.False(live.Tracker.IsAccepting); + Assert.False((await live.Tracker.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 4)).Accepted); + } + + [Fact] + public async Task ApplicationShutdown_ServiceStopClearsOwnership() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 2)).Accepted); + var service = new TransitPeerStateService( + node, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount(TransitTestPeers.DefaultPeerName, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False(node.IsAccepting); + } + + [Fact] + public async Task DoubleFinalization_ReleasesOnce() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + var admitted = await node.TryAdmitAsync(TransitTestPeers.DefaultPeerName, 2); + var lease = new TransitInboundConnectionLease(ct => node.ReleaseAsync(TransitTestPeers.DefaultPeerName, admitted.Generation, ct)); + await lease.DisposeAsync(); + await lease.DisposeAsync(); + // codeql[cs/dispose-not-called-on-throw] Intentional multi-dispose contract exercise for TransitInboundConnectionLease. + lease.Dispose(); + Assert.Equal(1, node.ReleaseCalls); + Assert.Equal(0, node.GetLocalCount(TransitTestPeers.DefaultPeerName)); + } + + [Fact] + public async Task NeverAdmitted_DoesNotRelease() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + await TransitInboundConnectionLease.None.DisposeAsync(); + await node.ReleaseAsync(TransitTestPeers.DefaultPeerName, 1); + Assert.Equal(0, node.ReleaseCalls); + Assert.Equal(0, membership.ReleaseCalls); + } + + private static async Task StartAdmittedAsync(ControllableTimeProvider? clock = null) + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + TransitTestPeers.DefaultPeerName, + TransitTestPeers.Peer(maxIncoming: 4, allowFrom: [Source.ToString()]))); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var live = new LiveConnection(limiter, tracker, membership, peers, clock); + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, live.Session); + Assert.True(outcome.Admitted); + live.AttachLease(outcome.Lease); + live.Start(); + var greeting = await live.ReadClientLineAsync(); + Assert.True( + greeting.StartsWith("200 ", StringComparison.Ordinal) + || greeting.StartsWith("201 ", StringComparison.Ordinal), + greeting); + return live; + } + + private sealed class LiveConnection : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + private readonly CancellationTokenSource _runCts = new(); + private TransitInboundConnectionLease _lease = TransitInboundConnectionLease.None; + + public LiveConnection( + ITransitInboundConnectionLimiter limiter, + DistributedTransitPeerStateTracker tracker, + InMemoryTransitPeerStateStore membership, + ITransitPeerAuthorization peers, + ControllableTimeProvider? clock) + { + Limiter = limiter; + Tracker = tracker; + Membership = membership; + Connection = new LifecyclePipeConnection(_clientToServer.Reader, _serverToClient.Writer, Source); + Session = new NntpSession( + Connection, + NullLogger.Instance, + transitPeerAuthorization: peers, + commandIdleTimeout: clock is null ? null : TimeSpan.FromSeconds(1), + timeProvider: clock); + } + + public ITransitInboundConnectionLimiter Limiter { get; } + + public DistributedTransitPeerStateTracker Tracker { get; } + + public InMemoryTransitPeerStateStore Membership { get; } + + public NntpSession Session { get; } + + public LifecyclePipeConnection Connection { get; } + + public Task Run { get; private set; } = Task.CompletedTask; + + public void AttachLease(TransitInboundConnectionLease lease) => _lease = lease; + + public void Start() => Run = Session.RunAsync(_runCts.Token); + + public Task CancelRunAsync() => _runCts.CancelAsync(); + + public async Task FinalizeLeaseAsync() => await _lease.DisposeAsync(); + + public async Task WriteClientLineAsync(string line) + { + var payload = System.Text.Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(payload); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + return await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token) + ?? throw new InvalidOperationException("Server closed."); + } + + public async Task CompleteClientInputAsync(Exception? exception = null) + { + await _clientToServer.Writer.CompleteAsync(exception); + } + + public async ValueTask DisposeAsync() + { + await FinalizeLeaseAsync(); + _runCts.Dispose(); + await Connection.DisposeAsync(); + } + } + + internal sealed class LifecyclePipeConnection( + PipeReader input, + PipeWriter output, + IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public PipeReader Input { get; } = input; + public PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs new file mode 100644 index 00000000..de23a4fb --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateLuaStaticAuditTests.cs @@ -0,0 +1,270 @@ +using VectorNNTPD.TransitService.Redis; +using VectorNNTPD.TransitService.Transit; +using VectorNNTPD.TransitService.Tests.TestDoubles; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// Static Lua/C# TransitPeerState parity. These tests do not execute live Redis. +/// FakeRedis dispatches EVAL to . +/// +public sealed class TransitPeerStateLuaStaticAuditTests +{ + [Fact] + public async Task ProductionStore_DispatchesExactTransitPeerStateScripts() + { + var redis = new RecordingRedis(); + var store = new RedisTransitPeerStateStore(redis); + var now = DateTimeOffset.UnixEpoch; + var ttl = TimeSpan.FromSeconds(30); + + _ = await store.TryAdmitAsync("peer-a", "nntpd01:a", 10, 1, now, ttl); + Assert.Same(TransitPeerStateScripts.TryAdmit, redis.LastScript); + Assert.Equal(1, redis.LastKeyCount); + + await store.ReleaseAsync("peer-a", "nntpd01:a", 1); + Assert.Same(TransitPeerStateScripts.Release, redis.LastScript); + + _ = await store.RenewAsync("peer-a", "nntpd01:a", 1, now, ttl); + Assert.Same(TransitPeerStateScripts.Renew, redis.LastScript); + + await store.ReleaseOwnerAsync("peer-a", "nntpd01:a"); + Assert.Same(TransitPeerStateScripts.ReleaseOwner, redis.LastScript); + } + + [Fact] + public void Scripts_NeverSetNativeKeyTtl() + { + foreach (var script in AllScripts()) + { + Assert.DoesNotContain("EXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PEXPIRE", script, StringComparison.Ordinal); + Assert.DoesNotContain("PERSIST", script, StringComparison.Ordinal); + Assert.DoesNotContain("SET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("GET ", script, StringComparison.Ordinal); + Assert.DoesNotContain("EVAL", script, StringComparison.Ordinal); + Assert.DoesNotContain("redis.call('KEYS'", script, StringComparison.Ordinal); + } + } + + [Fact] + public void Scripts_UseOnlyDocumentedRedisCommands() + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "HGETALL", "HGET", "HSET", "HDEL", "HLEN", "DEL", + }; + foreach (var script in AllScripts()) + { + foreach (var command in EnumerateRedisCalls(script)) + { + Assert.Contains(command, allowed); + } + } + } + + [Fact] + public void TryAdmit_PrunesThenRejectsMaxZeroThenLimitBeforeIncrement() + { + var script = TransitPeerStateScripts.TryAdmit; + var prune = script.IndexOf("prune(key)", StringComparison.Ordinal); + var maxZero = script.IndexOf("if maxIncoming == nil or maxIncoming <= 0 then", StringComparison.Ordinal); + var rejectLimit = script.IndexOf("if total >= maxIncoming then", StringComparison.Ordinal); + var increment = script.IndexOf("increment(key, owner, gen)", StringComparison.Ordinal); + Assert.True(prune > 0 && prune < maxZero); + Assert.True(maxZero > 0 && maxZero < rejectLimit); + Assert.True(rejectLimit > 0 && rejectLimit < increment); + Assert.Contains("return 0", script, StringComparison.Ordinal); + Assert.Contains("return 1", script, StringComparison.Ordinal); + } + + [Fact] + public void Renew_DoesNotRecreateMissingOwnership() + { + var script = TransitPeerStateScripts.Renew; + Assert.Contains("if not cur then", script, StringComparison.Ordinal); + Assert.Contains("return 0", script, StringComparison.Ordinal); + var missing = script.IndexOf("if not cur then", StringComparison.Ordinal); + var write = script.IndexOf("redis.call('HSET'", StringComparison.Ordinal); + Assert.True(missing > 0 && missing < write); + Assert.Contains("storedGen ~= gen or exp == nil or exp <= now", script, StringComparison.Ordinal); + } + + [Fact] + public void Release_RequiresGenerationMatch() + { + var script = TransitPeerStateScripts.Release; + Assert.Contains("if storedGen == gen then", script, StringComparison.Ordinal); + Assert.Contains("redis.call('HDEL', key, owner)", script, StringComparison.Ordinal); + Assert.Contains("if redis.call('HLEN', key) == 0 then", script, StringComparison.Ordinal); + } + + [Fact] + public void ReleaseOwner_DeletesOnlyThisOwnerField() + { + var script = TransitPeerStateScripts.ReleaseOwner; + Assert.Contains("redis.call('HDEL', key, owner)", script, StringComparison.Ordinal); + Assert.DoesNotContain("HGETALL", script, StringComparison.Ordinal); + } + + [Fact] + public async Task EngineAndFakeRedisEval_AreSemanticallyIdentical() + { + var engine = new TransitPeerStateEngine(); + var redis = new FakeRedisService(); + var store = new RedisTransitPeerStateStore(redis); + const string peer = "parity-peer"; + const string owner = "nntpd01:a"; + var now = DateTimeOffset.FromUnixTimeMilliseconds(10_000); + var ttl = TimeSpan.FromMilliseconds(30_000); + Assert.Equal( + TransitPeerStateEngine.Accepted, + engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.True((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + AssertOwnership(engine, "k", owner, 40_000, 1, 1); + AssertOwnership(redis.Database.TransitPeerStateEngine, EncodingKey(peer), owner, 40_000, 1, 1); + + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.True((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit("k", owner, 2, 10_000, 30_000, 1)); + Assert.False((await store.TryAdmitAsync(peer, owner, 2, 1, now, ttl)).Accepted); + + Assert.Equal(TransitPeerStateEngine.Rejected, engine.TryAdmit("k", owner, 0, 10_000, 30_000, 1)); + Assert.False((await store.TryAdmitAsync(peer, owner, 0, 1, now, ttl)).Accepted); + + _ = engine.Release("k", owner, 1); + await store.ReleaseAsync(peer, owner, 1); + + engine.WriteOwnership("k", owner, 40_000, 1, 1); + redis.Database.TransitPeerStateEngine.WriteOwnership(EncodingKey(peer), owner, 40_000, 1, 1); + Assert.Equal(1, engine.Renew("k", owner, 1, 15_000, 30_000)); + Assert.Equal(TransitPeerStateRenewStatus.Renewed, await store.RenewAsync(peer, owner, 1, DateTimeOffset.FromUnixTimeMilliseconds(15_000), ttl)); + Assert.Equal(0, engine.Renew("k", owner, 9, 15_000, 30_000)); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await store.RenewAsync(peer, owner, 9, DateTimeOffset.FromUnixTimeMilliseconds(15_000), ttl)); + + _ = engine.ReleaseOwner("k", owner); + await store.ReleaseOwnerAsync(peer, owner); + Assert.False(engine.TryGetOwnership("k", owner, out _, out _, out _)); + Assert.False(redis.Database.TransitPeerStateEngine.TryGetOwnership(EncodingKey(peer), owner, out _, out _, out _)); + } + + private static void AssertOwnership( + TransitPeerStateEngine engine, + string key, + string owner, + long expiry, + long generation, + int count) + { + Assert.True(engine.TryGetOwnership(key, owner, out var storedExpiry, out var storedGeneration, out var storedCount)); + Assert.Equal(expiry, storedExpiry); + Assert.Equal(generation, storedGeneration); + Assert.Equal(count, storedCount); + } + + private static string EncodingKey(string identifier) => + System.Text.Encoding.UTF8.GetString(TransitPeerStateKeys.Create(identifier)); + + private static IEnumerable AllScripts() => + [ + TransitPeerStateScripts.TryAdmit, + TransitPeerStateScripts.Release, + TransitPeerStateScripts.Renew, + TransitPeerStateScripts.ReleaseOwner, + ]; + + private static IEnumerable EnumerateRedisCalls(string script) + { + const string prefix = "redis.call('"; + var index = 0; + while (true) + { + var start = script.IndexOf(prefix, index, StringComparison.Ordinal); + if (start < 0) + { + yield break; + } + + start += prefix.Length; + var end = script.IndexOf('\'', start); + yield return script[start..end]; + index = end + 1; + } + } + + private sealed class RecordingRedis : IRedisService + { + public string? LastScript { get; private set; } + + public int LastKeyCount { get; private set; } + + public IRedisDatabase Database => new RecordingDatabase(this); + + public bool IsUnavailable => false; + + public bool TryBeginOperation(out bool isRecoveryProbe) + { + isRecoveryProbe = false; + return true; + } + + public void CompleteOperation(bool isRecoveryProbe, bool succeeded, Exception? exception = null) + { + } + + public void AbandonOperation(bool isRecoveryProbe) + { + } + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + private sealed class RecordingDatabase : IRedisDatabase + { + private readonly RecordingRedis _owner; + + public RecordingDatabase(RecordingRedis owner) + { + _owner = owner; + } + + public Task PingAsync(CancellationToken cancellationToken = default) => + Task.FromResult(TimeSpan.FromMilliseconds(1)); + + public ValueTask KeyExistsAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(false); + + public ValueTask GetAsync(ReadOnlyMemory key, CancellationToken cancellationToken = default) => + ValueTask.FromResult(null); + + public ValueTask SetAsync( + ReadOnlyMemory key, + ReadOnlyMemory value, + TimeSpan expiry, + CancellationToken cancellationToken = default) => + ValueTask.CompletedTask; + + public ValueTask ScriptEvaluateAsync( + string script, + ReadOnlyMemory[] keys, + ReadOnlyMemory[] values, + CancellationToken cancellationToken = default) + { + _owner.LastScript = script; + _owner.LastKeyCount = keys.Length; + return ValueTask.FromResult(1L); + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs new file mode 100644 index 00000000..69ada72b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateRedisLuaIntegrationTests.cs @@ -0,0 +1,162 @@ +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// Live EVAL of through +/// against the dedicated test Redis. +/// +public sealed class TransitPeerStateRedisLuaIntegrationTests : IClassFixture +{ + private const string O1 = "nntpd01:a"; + private const string O2 = "nntpd02:b"; + private const long Now = 10_000; + private const long Lease = 30_000; + private readonly TransitPeerStateRedisIntegrationFixture _redis; + + public TransitPeerStateRedisLuaIntegrationTests(TransitPeerStateRedisIntegrationFixture redis) + { + _redis = redis; + } + + [TransitPeerStateRedisIntegrationFact] + public void ConnectsToAuthorizedTestRedis() + { + Assert.True(_redis.IsConfigured); + Assert.Equal(TransitPeerStateRedisIntegration.AuthorizedEndpoint, _redis.Endpoint); + Assert.False(string.IsNullOrWhiteSpace(_redis.ServerVersion)); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task TryAdmit_WritesHashedOwnership_WithoutNativeTtl() + { + var peer = await _redis.CreateIdentifierAsync(); + var engine = new TransitPeerStateEngine(); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", O1, 4, Now, Lease, 1)); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 1, 1); + Assert.Null(await _redis.KeyTimeToLiveAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task ConcurrentAdmissions_EnforceExactClusterLimit() + { + var peer = await _redis.CreateIdentifierAsync(); + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 20).Select(async i => + { + if ((await _redis.TryAdmitAsync(peer, "owner:" + i, 5, 1, Now, Lease)).Accepted) + { + Interlocked.Increment(ref admitted); + } + })); + Assert.Equal(5, admitted); + Assert.Equal(5, await _redis.CountAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task SamePeerAcrossOwners_SharesLimit_DifferentPeersAreIsolated() + { + var peer = await _redis.CreateIdentifierAsync(); + var other = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 1, 1, Now, Lease)).Accepted); + Assert.False((await _redis.TryAdmitAsync(peer, O2, 1, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(other, O2, 1, 1, Now, Lease)).Accepted); + Assert.Equal(1, await _redis.CountAsync(peer)); + Assert.Equal(1, await _redis.CountAsync(other)); + await _redis.DeleteKeyAsync(peer); + await _redis.DeleteKeyAsync(other); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task GenerationReplacement_ReplacesCountAndIgnoresStaleRelease() + { + var peer = await _redis.CreateIdentifierAsync(); + await _redis.SeedAsync(peer, O1, 30_000, 1, 7); + var engine = new TransitPeerStateEngine(); + engine.WriteOwnership("k", O1, 30_000, 1, 7); + Assert.Equal(TransitPeerStateEngine.Accepted, engine.TryAdmit("k", O1, 20, 0, Lease, 2)); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 20, 2, 0, Lease)).Accepted); + AssertOwnership(await _redis.ReadAsync(peer, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(peer, O1, 1); + AssertOwnership(await _redis.ReadAsync(peer, O1), Lease, 2, 1); + + await _redis.ReleaseAsync(peer, O1, 2); + Assert.Null(await _redis.ReadAsync(peer, O1)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task StaleRenewal_DoesNotRecreateOrExtend() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await _redis.RenewAsync(peer, O1, 1, Now, Lease)); + Assert.Null(await _redis.ReadAsync(peer, O1)); + + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 2, Now, Lease)).Accepted); + Assert.Equal(TransitPeerStateRenewStatus.Lost, await _redis.RenewAsync(peer, O1, 1, Now, Lease)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 2, 1); + Assert.Equal(TransitPeerStateRenewStatus.Renewed, await _redis.RenewAsync(peer, O1, 2, Now + 5_000, Lease)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + 5_000 + Lease, 2, 1); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task PruneExpired_FreesCapacity() + { + var peer = await _redis.CreateIdentifierAsync(); + await _redis.SeedAsync(peer, O2, Now, 1, 4); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 1, 1, Now, Lease)).Accepted); + Assert.Null(await _redis.ReadAsync(peer, O2)); + AssertOwnership(await _redis.ReadAsync(peer, O1), Now + Lease, 1, 1); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task MaxZero_RejectsWithoutMutation() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.False((await _redis.TryAdmitAsync(peer, O1, 0, 1, Now, Lease)).Accepted); + Assert.Null(await _redis.ReadAsync(peer, O1)); + Assert.Equal(0, await _redis.FieldCountAsync(peer)); + await _redis.DeleteKeyAsync(peer); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task ReleaseOwner_RemovesOnlyThisOwner() + { + var peer = await _redis.CreateIdentifierAsync(); + var other = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(peer, O2, 4, 1, Now, Lease)).Accepted); + Assert.True((await _redis.TryAdmitAsync(other, O1, 4, 1, Now, Lease)).Accepted); + await _redis.ReleaseOwnerAsync(peer, O1); + Assert.Null(await _redis.ReadAsync(peer, O1)); + Assert.NotNull(await _redis.ReadAsync(peer, O2)); + Assert.NotNull(await _redis.ReadAsync(other, O1)); + await _redis.DeleteKeyAsync(peer); + await _redis.DeleteKeyAsync(other); + } + + [TransitPeerStateRedisIntegrationFact] + public async Task Cleanup_DeletesOnlyTrackedKeys() + { + var peer = await _redis.CreateIdentifierAsync(); + Assert.True((await _redis.TryAdmitAsync(peer, O1, 4, 1, Now, Lease)).Accepted); + await _redis.DeleteKeyAsync(peer); + Assert.Empty(await _redis.RemainingTestKeysAsync()); + } + + private static void AssertOwnership(Ownership? ownership, long expiry, long generation, int count) + { + Assert.True(ownership.HasValue); + var value = ownership.GetValueOrDefault(); + Assert.Equal(expiry, value.ExpiryUnixMs); + Assert.Equal(generation, value.Generation); + Assert.Equal(count, value.Count); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs new file mode 100644 index 00000000..57e5b99a --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateServiceTests.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateServiceTests +{ + [Fact] + public void Namespace_IsTransit() + { + Assert.Equal("VectorNNTPD.TransitService.Transit", typeof(TransitPeerStateService).Namespace); + Assert.Equal("VectorNNTPD.TransitService.Transit", typeof(DistributedTransitPeerStateTracker).Namespace); + Assert.NotEqual(typeof(VectorNNTPD.TransitService.SessionState.SessionStateService), typeof(TransitPeerStateService)); + } + + [Fact] + public async Task RenewalPass_ExtendsIdleOwnershipWithoutTraffic() + { + var clock = new ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync("peer-a", 1)).Accepted); + var first = membership.Expiry("peer-a", node.OwnerId); + var service = new TransitPeerStateService( + node, + NullLogger.Instance, + clock, + TimeSpan.FromSeconds(10)); + await service.StartAsync(CancellationToken.None); + using var wait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!clock.HasScheduledTimers) + { + wait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + clock.Advance(TimeSpan.FromSeconds(10)); + using var renewWait = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (membership.RenewCalls == 0) + { + renewWait.Token.ThrowIfCancellationRequested(); + await Task.Yield(); + } + + Assert.True(membership.Expiry("peer-a", node.OwnerId) > first); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount("peer-a", node.OwnerId, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False(node.IsAccepting); + } + + [Fact] + public async Task Stop_StopsRenewalThenReleaseOwner() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync("peer-a", 2)).Accepted); + var service = new TransitPeerStateService( + node, + NullLogger.Instance); + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + Assert.Equal(0, membership.OwnerCount("peer-a", node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.False(node.IsAccepting); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs new file mode 100644 index 00000000..f91dceff --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTestFactory.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +internal static class TransitPeerStateTestFactory +{ + public static (TransitInboundConnectionLimiter Limiter, DistributedTransitPeerStateTracker Tracker, InMemoryTransitPeerStateStore Membership) + CreateLimiter(TransitConfigurationStore store, string nodeId = "nntpd01", string incarnation = "test") + { + var membership = new InMemoryTransitPeerStateStore(); + var tracker = CreateTracker(membership, nodeId, incarnation); + return (new TransitInboundConnectionLimiter(store, tracker), tracker, membership); + } + + public static DistributedTransitPeerStateTracker CreateTracker( + ITransitPeerStateStore membership, + string nodeId = "nntpd01", + string incarnation = "a", + TimeProvider? time = null, + TimeSpan? leaseTtl = null) => + new( + membership, + NullLogger.Instance, + nodeId, + time ?? TimeProvider.System, + leaseTtl, + incarnation); + + public static long NowMs(TimeProvider? time = null) => + (time ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds(); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs new file mode 100644 index 00000000..593b1058 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeerState/TransitPeerStateTrackerTests.cs @@ -0,0 +1,312 @@ +using System.Net; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; +using Microsoft.Extensions.Logging.Abstractions; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeerStateTrackerTests +{ + private const string Peer = "peer-a"; + private const string Other = "peer-b"; + + [Fact] + public async Task FirstConnection_BelowAtAndOver_ThenReleaseAndReconnect() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await node.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, node.GetLocalCount(Peer)); + Assert.Equal(2, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + + await node.ReleaseAsync(Peer, 1); + Assert.Equal(1, node.GetLocalCount(Peer)); + Assert.True((await node.TryAdmitAsync(Peer, 2)).Accepted); + await node.ReleaseAsync(Peer, 1); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task MaxZero_IsClosed_NotUnlimited() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.False((await node.TryAdmitAsync(Peer, 0)).Accepted); + Assert.Equal(0, node.GetLocalCount(Peer)); + Assert.Equal(0, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, node.DistributedRejects); + } + + [Fact] + public async Task SamePeerAcrossTwoNodes_SharesClusterLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + Assert.True((await a.TryAdmitAsync(Peer, 2)).Accepted); + Assert.True((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await a.TryAdmitAsync(Peer, 2)).Accepted); + Assert.False((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + await a.ReleaseAsync(Peer, 1); + Assert.True((await b.TryAdmitAsync(Peer, 2)).Accepted); + Assert.Equal(2, membership.OwnerCount(Peer, b.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task SamePeerAcrossNNodes_EnforcesExactGlobalLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var nodes = Enumerable.Range(1, 5) + .Select(i => TransitPeerStateTestFactory.CreateTracker(membership, $"nntpd{i:00}", i.ToString())) + .ToArray(); + var admitted = 0; + // codeql[cs/linq/missed-where] Async TryAdmit filter; Where cannot await. + foreach (var node in nodes) + { + if ((await node.TryAdmitAsync(Peer, 3)).Accepted) + { + admitted++; + } + } + + Assert.Equal(3, admitted); + Assert.Equal(3, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.False((await nodes[0].TryAdmitAsync(Peer, 3)).Accepted); + } + + [Fact] + public async Task DifferentPeers_RemainIsolated() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.True((await node.TryAdmitAsync(Other, 1)).Accepted); + Assert.False((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, membership.ActiveCount(Other, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task ConcurrentAdmissionsAcrossNodes_DoNotExceedLimit() + { + var membership = new InMemoryTransitPeerStateStore(); + var a = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd01", "a"); + var b = TransitPeerStateTestFactory.CreateTracker(membership, "nntpd02", "b"); + var admitted = 0; + await Task.WhenAll(Enumerable.Range(0, 40).Select(async i => + { + await Task.Yield(); + var node = i % 2 == 0 ? a : b; + if ((await node.TryAdmitAsync(Peer, 7)).Accepted) + { + Interlocked.Increment(ref admitted); + } + })); + Assert.Equal(7, admitted); + Assert.Equal(7, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(7, a.GetLocalCount(Peer) + b.GetLocalCount(Peer)); + } + + [Fact] + public async Task TwoSourceIpsMappedToOneIdentifier_ShareOneLimit() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + Peer, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10", "198.51.100.20"]))); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var first = CreateSession(peers, IPAddress.Parse("192.0.2.10")); + var second = CreateSession(peers, IPAddress.Parse("198.51.100.20")); + Assert.Equal(Peer, first.Authorization.TransitPeerName); + Assert.Equal(Peer, second.Authorization.TransitPeerName); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, first); + Assert.True(lease.Admitted); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, second)).Admitted); + Assert.Equal(1, membership.ActiveCount(Peer, TransitPeerStateTestFactory.NowMs())); + await lease.Lease.DisposeAsync(); + } + + [Fact] + public async Task Ipv4AndIpv6_RemainDistinctAclIdentities_ButShareIdentifierLimit() + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + Peer, + TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10", "2001:db8::10"]))); + var (limiter, _, _) = TransitPeerStateTestFactory.CreateLimiter(store); + var peers = TransitPeerAuthorization.CreateForStore(store); + var v4 = CreateSession(peers, IPAddress.Parse("192.0.2.10")); + var v6 = CreateSession(peers, IPAddress.Parse("2001:db8::10")); + Assert.Equal(Peer, v4.Authorization.TransitPeerName); + Assert.Equal(Peer, v6.Authorization.TransitPeerName); + var lease = await TransitConnectionAdmission.TryAdmitAsync(limiter, v4); + Assert.True(lease.Admitted); + Assert.False((await TransitConnectionAdmission.TryAdmitAsync(limiter, v6)).Admitted); + await lease.Lease.DisposeAsync(); + } + + [Fact] + public async Task IdleConnection_RemainsCounted_UntilReleased() + { + var clock = new VectorNNTPD.TransitService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(1, membership.ActiveCount(Peer, clock.GetUtcNow().ToUnixTimeMilliseconds())); + Assert.False((await node.TryAdmitAsync(Peer, 1)).Accepted); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task Renewal_ExtendsLease_IndependentlyOfTraffic() + { + var clock = new VectorNNTPD.TransitService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var first = membership.Expiry(Peer, node.OwnerId); + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + var renewed = membership.Expiry(Peer, node.OwnerId); + Assert.True(renewed > first); + clock.Advance(TimeSpan.FromSeconds(20)); + Assert.Equal(1, membership.ActiveCount(Peer, clock.GetUtcNow().ToUnixTimeMilliseconds())); + } + + [Fact] + public async Task RenewalFailure_DoesNotExtendOwnership() + { + var clock = new VectorNNTPD.TransitService.Tests.Fixtures.ControllableTimeProvider(); + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership, time: clock); + Assert.True((await node.TryAdmitAsync(Peer, 1)).Accepted); + var first = membership.Expiry(Peer, node.OwnerId); + membership.Unavailable = true; + clock.Advance(TimeSpan.FromSeconds(10)); + await node.RenewLeasesAsync(); + Assert.Equal(first, membership.Expiry(Peer, node.OwnerId)); + Assert.Equal(1, node.GetLocalCount(Peer)); + } + + [Fact] + public async Task SameOwnerNewGeneration_ReplacesStaleGeneration() + { + var membership = new InMemoryTransitPeerStateStore(); + membership.Engine.WriteOwnership(InMemoryTransitPeerStateStore.ConnectionKey(Peer), "nntpd01:a", 40_000, 9, 7); + var node = TransitPeerStateTestFactory.CreateTracker(membership, incarnation: "a"); + Assert.True((await node.TryAdmitAsync(Peer, 20)).Accepted); + Assert.True(membership.Engine.TryGetOwnership( + InMemoryTransitPeerStateStore.ConnectionKey(Peer), + node.OwnerId, + out _, + out var generation, + out var count)); + Assert.Equal(1, generation); + Assert.Equal(1, count); + await node.ReleaseAsync(Peer, 1); + Assert.Equal(0, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task StaleReleaseAndRenew_DoNotAffectNewerGeneration() + { + var membership = new InMemoryTransitPeerStateStore(); + var node = TransitPeerStateTestFactory.CreateTracker(membership); + Assert.True((await node.TryAdmitAsync(Peer, 4)).Accepted); + await membership.ReleaseAsync(Peer, node.OwnerId, 99); + Assert.Equal(1, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal( + TransitPeerStateRenewStatus.Lost, + await membership.RenewAsync(Peer, node.OwnerId, 99, DateTimeOffset.UtcNow, TimeSpan.FromSeconds(30))); + Assert.Equal(1, membership.OwnerCount(Peer, node.OwnerId, TransitPeerStateTestFactory.NowMs())); + } + + [Fact] + public async Task UnknownPeer_DoesNotCreateState() + { + var store = new TransitConfigurationStore(); + store.Replace(TransitTestPeers.Snapshot(Peer, TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["192.0.2.10"]))); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + Assert.False((await limiter.TryAcquireAsync("missing")).Admitted); + Assert.Equal(0, membership.ActiveCount("missing", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.AdmitCalls); + } + + private static NntpSession CreateSession(ITransitPeerAuthorization peers, IPAddress source) + { + var input = new System.IO.Pipelines.Pipe(); + var output = new System.IO.Pipelines.Pipe(); + return new NntpSession( + new IdentityPipeConnection(input.Reader, output.Writer, source), + NullLogger.Instance, + transitPeerAuthorization: peers); + } + + private sealed class IdentityPipeConnection( + System.IO.Pipelines.PipeReader input, + System.IO.Pipelines.PipeWriter output, + IPAddress address) : INntpConnection + { + private readonly CancellationTokenSource _closed = new(); + + public System.IO.Pipelines.PipeReader Input { get; } = input; + public System.IO.Pipelines.PipeWriter Output { get; } = output; + public ConnectionClientIdentity ClientIdentity { get; } = ConnectionClientIdentity.Direct(new IPEndPoint(address, 40000)); + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + public EndPoint? LocalEndPoint => null; + public bool IsTls => false; + public bool IsCompressed => false; + public bool IsCompleted => _closed.IsCancellationRequested; + public long OutboundIdleVersion => 0; + public CancellationToken ConnectionClosed => _closed.Token; + + public Task CompleteAsync(Exception? exception = null) + { + _closed.Cancel(); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _closed.Dispose(); + return ValueTask.CompletedTask; + } + + public Task WaitForOutboundDeliveryAsync(long outboundIdleVersionBeforeFlush, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task UpgradeToTlsAsync( + VectorNNTPD.Common.Networking.Certificates.ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipherSuite) + { + tlsVersion = string.Empty; + cipherSuite = string.Empty; + return false; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeersOptionsTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeersOptionsTests.cs new file mode 100644 index 00000000..5ff992cc --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitPeersOptionsTests.cs @@ -0,0 +1,490 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +public sealed class TransitPeersOptionsTests +{ + [Fact] + public void Bind_ExactTopLevelDictionaryShape_MultiplePeersAndDefaults() + { + const string json = """ + { + "Transit": { + "news-example": { + "PeerName": "News Example", + "MaxIncomingConnections": 10, + "MaxOutgoingConnections": 2, + "AllowFrom": [ "news.example.net", "192.0.2.0/24", "2001:db8:1234::/48" ], + "ConnectTo": [ "news.example.net:563" ], + "Username": "", + "Password": "", + "Ssl": "TLS", + "Patterns": "*", + "DeferOnDuplicate": true, + "PathToken": "peer.example", + "MaxSize": 10485760, + "MessageTypes": [ "default" ] + }, + "second": { + "PeerName": "Second Peer", + "MaxIncomingConnections": 1, + "MaxOutgoingConnections": 0, + "AllowFrom": [ "198.51.100.10" ] + } + } + } + """; + + var options = Bind(json); + Assert.Equal(2, options.Count); + Assert.True(options.TryGetValue("news-example", out var first)); + Assert.True(options.TryGetValue("second", out var second)); + Assert.Equal("News Example", first.PeerName); + Assert.Equal("Second Peer", second.PeerName); + Assert.Equal(10, first.MaxIncomingConnections); + Assert.Equal(2, first.MaxOutgoingConnections); + Assert.Equal(["news.example.net", "192.0.2.0/24", "2001:db8:1234::/48"], first.AllowFrom); + Assert.Equal(["news.example.net:563"], first.ConnectTo); + Assert.Equal("TLS", first.Ssl); + Assert.Equal("*", first.Patterns); + Assert.True(first.DeferOnDuplicate); + Assert.Equal(string.Empty, first.Username); + Assert.Equal(string.Empty, first.Password); + Assert.Equal("peer.example", first.PathToken); + Assert.Equal(10_485_760, first.MaxSize); + Assert.Equal(["default"], first.MessageTypes); + Assert.Equal("*", second.Patterns); + Assert.True(second.DeferOnDuplicate); + Assert.Equal([], second.ConnectTo); + Assert.Equal(string.Empty, second.PathToken); + Assert.Equal(TransitPeerOptions.DefaultMaxSize, second.MaxSize); + Assert.Empty(second.MessageTypes); + Assert.True(TransitMessageTypesParser.TryParse(second.MessageTypes, out var secondTypes, out _)); + Assert.Equal(TransitMessageTypes.Default, secondTypes); + } + + [Fact] + public void Bind_EmptyTransit_IsValid() + { + var options = Bind("""{ "Transit": { } }"""); + Assert.Empty(options); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData("usenet-ninja")] + [InlineData("giganews")] + [InlineData("blueworld-hosting")] + [InlineData("news-example")] + [InlineData("GIGANEWS")] + [InlineData("peer.example.com")] + [InlineData("192.0.2.1")] + public void Validate_AcceptsProtocolSafeIdentifiers(string identifier) + { + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + Assert.True(options.ContainsKey(identifier)); + } + + [Theory] + [InlineData("Giganews, Inc.")] + [InlineData("Blueworld Hosting")] + [InlineData("Usenet Ninja")] + [InlineData("peer/with slash")] + [InlineData("peer name")] + [InlineData(" ")] + [InlineData("peer\nname")] + [InlineData("Caf\u00e9")] + public void Validate_RejectsInvalidIdentifiers(string identifier) + { + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("identifier", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_RejectsOverlongIdentifier() + { + var identifier = new string('a', TransitPeersOptionsValidator.MaxPeerIdentifierLength + 1); + var options = new TransitPeersOptions { [identifier] = TransitTestPeers.Peer() }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("identifier", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_IdentifierMatch_IsExactOrdinal() + { + var options = new TransitPeersOptions + { + ["GIGANEWS"] = TransitTestPeers.Peer(peerName: "Giganews, Inc."), + }; + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + var snapshot = TransitConfigurationSnapshot.Create(options); + Assert.True(snapshot.Peers.TryGetValue("GIGANEWS", out var giganeews)); + Assert.False(snapshot.Peers.ContainsKey("giganews")); + Assert.Equal("GIGANEWS", giganeews.Identifier); + Assert.Equal("Giganews, Inc.", giganeews.PeerName); + } + + [Theory] + [InlineData("Giganews, Inc.")] + [InlineData("Blueworld Hosting")] + [InlineData("Usenet Ninja")] + [InlineData("Caf\u00e9 \u2014 \u65e5\u672c\u8a9e")] + public void Validate_AcceptsPeerNameWithSpacesAndPunctuation(string peerName) + { + var options = new TransitPeersOptions + { + ["usenet-ninja"] = TransitTestPeers.Peer(peerName: peerName), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded, string.Join("; ", result.Failures ?? [])); + var snapshot = TransitConfigurationSnapshot.Create(options); + Assert.Equal(peerName, snapshot.Peers["usenet-ninja"].PeerName); + Assert.Equal("usenet-ninja", snapshot.Peers["usenet-ninja"].Identifier); + } + + [Fact] + public void Validate_RejectsMissingPeerName() + { + var options = TransitTestPeers.Dictionary("alpha", new TransitPeerOptions + { + MaxIncomingConnections = 1, + MaxOutgoingConnections = 0, + AllowFrom = ["192.0.2.1"], + }); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Validate_RejectsEmptyOrWhitespacePeerName(string peerName) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: peerName)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsControlCharacterPeerName() + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: "peer\nname")); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("control character", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsOverlongPeerName() + { + var peerName = new string('a', TransitPeersOptionsValidator.MaxPeerNameLength + 1); + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(peerName: peerName)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PeerName", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("maximum length", StringComparison.Ordinal)); + } + + [Fact] + public void Bind_PreservesIdentifierAndExactPeerName() + { + const string json = """ + { + "Transit": { + "giganews": { + "PeerName": "Giganews, Inc.", + "MaxIncomingConnections": 50, + "MaxOutgoingConnections": 50, + "PathToken": "giganews.example" + }, + "blueworld-hosting": { + "PeerName": "Blueworld Hosting", + "MaxIncomingConnections": 10, + "MaxOutgoingConnections": 10, + "PathToken": "blueworld.example" + } + } + } + """; + + var options = Bind(json); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(options.TryGetValue("giganews", out var giganews)); + Assert.True(options.TryGetValue("blueworld-hosting", out var blueworld)); + Assert.False(options.ContainsKey("Giganews, Inc.")); + Assert.False(options.ContainsKey("Blueworld Hosting")); + Assert.Equal("Giganews, Inc.", giganews.PeerName); + Assert.Equal("Blueworld Hosting", blueworld.PeerName); + Assert.Equal("giganews.example", giganews.PathToken); + Assert.Equal("blueworld.example", blueworld.PathToken); + } + + [Fact] + public void Validate_PathToken_PreservesExactString_AndRejectsControlChars() + { + var preserved = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "Peer.Example")); + Assert.True(new TransitPeersOptionsValidator().Validate(null, preserved).Succeeded); + var snapshot = TransitConfigurationSnapshot.Create(preserved); + Assert.Equal("Peer.Example", snapshot.Peers["alpha"].PathToken); + + var empty = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "")); + Assert.True(new TransitPeersOptionsValidator().Validate(null, empty).Succeeded); + + var control = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(pathToken: "peer\tname")); + var result = new TransitPeersOptionsValidator().Validate(null, control); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("PathToken", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData((long)int.MaxValue + 1)] + public void Validate_RejectsInvalidMaxSize(long maxSize) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: maxSize)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MaxSize", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_AcceptsMaxSizeBounds() + { + var min = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: 1)); + var max = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxSize: TransitPeerOptions.MaxMaxSize)); + Assert.True(new TransitPeersOptionsValidator().Validate(null, min).Succeeded); + Assert.True(new TransitPeersOptionsValidator().Validate(null, max).Succeeded); + } + + [Fact] + public void Validate_RejectsUnknownMessageTypes() + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(messageTypes: ["default", "not-a-type"])); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MessageTypes", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsMissingConnectionLimits() + { + var options = TransitTestPeers.Dictionary( + "alpha", + new TransitPeerOptions { PeerName = "Alpha", AllowFrom = ["192.0.2.1"] }); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("MaxIncomingConnections", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("MaxOutgoingConnections", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("alpha", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(-1)] + [InlineData(4097)] + public void Validate_RejectsOutOfRangeLimits(int value) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(maxIncoming: value, maxOutgoing: value)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Theory] + [InlineData("ssl")] + [InlineData("plain")] + [InlineData("tls-start")] + public void Validate_RejectsInvalidSsl(string ssl) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: ssl)); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("Ssl", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("")] + [InlineData("TLS")] + [InlineData("tls")] + [InlineData("STARTTLS")] + [InlineData("starttls")] + public void Validate_AcceptsCanonicalSslValues(string ssl) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(ssl: ssl)); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(TransitSslParser.TryParse(ssl, out _, out _)); + } + + [Theory] + [InlineData("news.example.net")] + [InlineData("192.0.2.10")] + [InlineData("2001:db8::10")] + [InlineData("[2001:db8::10]")] + [InlineData("2001:db8::10:119")] + public void Validate_RejectsInvalidConnectTo(string endpoint) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(connectTo: [endpoint])); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("ConnectTo", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("news.example.net:119")] + [InlineData("news.example.net:563")] + [InlineData("192.0.2.10:119")] + [InlineData("[2001:db8::10]:119")] + [InlineData("[2001:db8::10]:563")] + public void Validate_AcceptsExplicitConnectToPorts(string endpoint) + { + var options = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(connectTo: [endpoint])); + Assert.True(new TransitPeersOptionsValidator().Validate(null, options).Succeeded); + Assert.True(TransitConnectEndpoint.TryParse(endpoint, out var parsed, out var error), error); + Assert.InRange(parsed!.Port, 1, 65535); + } + + [Fact] + public void Validate_RejectsInvalidAllowFromAndPatterns_WithoutLeakingPassword() + { + var options = TransitTestPeers.Dictionary( + "alpha", + TransitTestPeers.Peer( + allowFrom: ["not a host!"], + username: "peer", + password: "peer-secret", + patterns: "comp.[abc")); + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("AllowFrom", StringComparison.Ordinal)); + Assert.Contains(result.Failures!, static f => f.Contains("Patterns", StringComparison.Ordinal)); + Assert.DoesNotContain(result.Failures!, static f => f.Contains("peer-secret", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_RejectsPartialCredentials_AllowsBothBlankOrBothSet() + { + var validator = new TransitPeersOptionsValidator(); + + var userOnly = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "peer", password: "")); + var userOnlyResult = validator.Validate(null, userOnly); + Assert.True(userOnlyResult.Failed); + Assert.Contains(userOnlyResult.Failures!, static f => f.Contains("Password", StringComparison.Ordinal)); + Assert.Contains(userOnlyResult.Failures!, static f => f.Contains("both be set or both be blank", StringComparison.Ordinal)); + + var passwordOnly = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "", password: "peer-secret")); + var passwordOnlyResult = validator.Validate(null, passwordOnly); + Assert.True(passwordOnlyResult.Failed); + Assert.Contains(passwordOnlyResult.Failures!, static f => f.Contains("Username", StringComparison.Ordinal)); + Assert.DoesNotContain(passwordOnlyResult.Failures!, static f => f.Contains("peer-secret", StringComparison.Ordinal)); + + var bothBlank = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "", password: "")); + Assert.True(validator.Validate(null, bothBlank).Succeeded); + + var bothSet = TransitTestPeers.Dictionary("alpha", TransitTestPeers.Peer(username: "peer", password: "peer-secret")); + Assert.True(validator.Validate(null, bothSet).Succeeded); + Assert.True(TransitConfigurationSnapshot.Create(bothSet).Peers["alpha"].HasPeerCredentials); + } + + [Fact] + public void ProductionAppsettings_DoesNotOwnTransitPeers() + { + var path = FindProductionAppsettings(); + var config = new ConfigurationBuilder().AddJsonFile(path, optional: false).Build(); + + Assert.False(config.GetSection("Transit").Exists()); + Assert.Null(config["Nntpd:Transit:WantTrash"]); + Assert.Null(config["Nntpd:Transit:LogTrash"]); + Assert.Equal("8", config["Nntpd:Transit:StreamOutstandingArticleDepth"]); + Assert.Equal("4294967296", config["Nntpd:TransitQueueMemoryLimit"]); + + var peers = new TransitPeersOptions(); + config.GetSection("Transit").Bind(peers); + Assert.Empty(peers); + Assert.True(new TransitPeersOptionsValidator().Validate(null, peers).Succeeded); + + var leftover = new TransitLeftoverConfigurationValidator(config).Validate(null, new NntpdOptions()); + Assert.True(leftover.Succeeded); + } + + private static string FindProductionAppsettings() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + var candidate = Path.Join(dir.FullName, "src", "VectorNNTPD.TransitService", "VectorNNTPD.TransitService.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + dir = dir.Parent; + } + + throw new FileNotFoundException("Could not locate src/VectorNNTPD.TransitService/VectorNNTPD.TransitService.json."); + } + + [Fact] + public void Validate_RejectsOverlappingPrefixes() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(allowFrom: ["192.0.2.0/24"]), + ["two"] = TransitTestPeers.Peer(allowFrom: ["192.0.2.10"]), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("overlapping", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_RejectsSharedHostname() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(allowFrom: ["news.example.net"]), + ["two"] = TransitTestPeers.Peer(allowFrom: ["news.example.net"]), + }; + var result = new TransitPeersOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("news.example.net", StringComparison.Ordinal)); + } + + [Fact] + public void ServiceBind_DoesNotRequireNestedPeersSection() + { + var json = """{ "Transit": { "alpha": { "MaxIncomingConnections": 1, "MaxOutgoingConnections": 0, "AllowFrom": [ "192.0.2.1" ] } } }"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var config = new ConfigurationBuilder().AddJsonStream(stream).Build(); + var services = new ServiceCollection(); + services.AddOptions().Bind(config.GetSection("Transit")); + using var provider = services.BuildServiceProvider(); + var bound = provider.GetRequiredService>().Value; + Assert.True(bound.ContainsKey("alpha")); + Assert.False(bound.ContainsKey("Peers")); + } + + private static TransitPeersOptions Bind(string json) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var config = new ConfigurationBuilder().AddJsonStream(stream).Build(); + var options = new TransitPeersOptions(); + config.GetSection("Transit").Bind(options); + return options; + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs new file mode 100644 index 00000000..09d881c1 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReaderAdmissionReclassificationTests.cs @@ -0,0 +1,501 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.TransitService.Authentication; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.Authentication; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.SessionState; +using VectorNNTPD.TransitService.Tests.Session; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// AllowFrom identifies a connection as a named Transit peer before AUTHINFO. +/// READER AUTHINFO replaces that identity and must release the TransitPeerState slot. +/// +[Collection(nameof(NntpCommandLoggerCollection))] +public sealed class TransitReaderAdmissionReclassificationTests +{ + private static readonly IPAddress AllowFromClient = IPAddress.Parse("198.18.0.70"); + private const string PeerId = "usenet-ninja"; + + [Fact] + public async Task ReaderFromAllowFrom_DoesNotKeepTransitSlot() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var admission = new InMemorySessionStateTracker(); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store, admission); + var lease = await AdmitAsync(limiter, session); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + + Assert.True(session.Authentication.IsAuthenticated); + Assert.Equal("a", session.Authentication.Username); + Assert.True(session.Authorization.AuthorizedReader); + Assert.False(session.Authorization.AuthorizedTransit); + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal( + SessionAdmissionResult.SessionLimitExceeded, + admission.TryAdmit("a", "other", AllowFromClient, sessionLimit: 1, srcIpLimit: 0)); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task GenuineTransit_WithoutAuthinfo_KeepsSlotAndEnforcesLimit() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var firstDuplex = new AuthDuplex(AllowFromClient); + var first = firstDuplex.CreateSession(store); + var lease1 = await AdmitAsync(limiter, first); + Assert.True(lease1.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + var firstRun = first.RunAsync(); + await firstDuplex.ReadGreetingAsync(); + Assert.Equal(PeerId, first.Authorization.TransitPeerName); + Assert.True(first.Authorization.AuthorizedTransit); + Assert.False(first.Authentication.IsAuthenticated); + + await using var blockedDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.71")); + var blocked = blockedDuplex.CreateSession(store); + var lease2 = await TransitConnectionAdmission.TryAdmitAsync(limiter, blocked); + Assert.False(lease2.Admitted); + Assert.False(lease2.Lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await firstDuplex.QuitAsync(firstRun); + await lease1.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + + await using var reusedDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.72")); + var reused = reusedDuplex.CreateSession(store); + var lease3 = await AdmitAsync(limiter, reused); + Assert.True(lease3.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + await lease3.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task GenuineTransit_AuthinfoKeepsSlot() + { + var store = CreateStore(maxIncoming: 1, transitUsername: "feed", transitPassword: "peer-secret"); + var (limiter, tracker, _) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await duplex.ReadClientLineAsync()); + await duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS peer-secret"); + Assert.Equal("281 Authentication accepted", await duplex.ReadClientLineAsync()); + + Assert.True(session.Authorization.AuthorizedTransit); + Assert.Equal(PeerId, session.Authorization.TransitPeerName); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task ReadersFromAllowFrom_DoNotStarveTransitCapacity() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + var admission = new InMemorySessionStateTracker(); + for (var i = 0; i < 3; i++) + { + await using var readerDuplex = new AuthDuplex(AllowFromClient); + var reader = readerDuplex.CreateSession(store, admission, username: "a", password: "a"); + var readerLease = await AdmitAsync(limiter, reader); + var readerRun = reader.RunAsync(); + await readerDuplex.ReadGreetingAsync(); + await readerDuplex.AuthenticateReaderAsync("a", "a"); + Assert.Null(reader.Authorization.TransitPeerName); + Assert.False(readerLease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + await readerDuplex.QuitAsync(readerRun); + await readerLease.DisposeAsync(); + } + + await using var transitDuplex = new AuthDuplex(IPAddress.Parse("198.18.0.80")); + var transit = transitDuplex.CreateSession(store); + var transitLease = await AdmitAsync(limiter, transit); + Assert.True(transitLease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + await transitLease.DisposeAsync(); + } + + [Fact] + public async Task FailedReaderAuthinfo_KeepsTransitSlotUntilDisconnect() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER a"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await duplex.ReadClientLineAsync()); + + Assert.False(session.Authentication.IsAuthenticated); + Assert.True(session.Authorization.AuthorizedTransit); + Assert.Equal(PeerId, session.Authorization.TransitPeerName); + Assert.True(lease.IsHeld); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + Assert.Equal(1, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task AllowFromDisconnectBeforeAuthinfo_ReleasesSlotOnce() + { + var store = CreateStore(maxIncoming: 1); + var (limiter, tracker, _) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + await lease.DisposeAsync(); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + } + + [Fact] + public async Task AmbiguousAllowFrom_DoesNotConsumeSlot_AndReaderAuthSucceeds() + { + var options = new TransitPeersOptions + { + ["one"] = TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["198.18.0.0/15"]), + ["two"] = TransitTestPeers.Peer(maxIncoming: 1, allowFrom: ["198.18.0.0/15"]), + }; + var store = new TransitConfigurationStore(); + store.Replace(TransitConfigurationSnapshot.Create(options)); + var (limiter, _, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + Assert.Null(session.Authorization.TransitPeerName); + var admitted = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(admitted.Admitted); + Assert.False(admitted.Lease.IsHeld); + session.AttachTransitAdmission(admitted.Lease); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.True(session.Authorization.AuthorizedReader); + Assert.Null(session.Authorization.TransitPeerName); + Assert.Equal(0, membership.ActiveCount("one", TransitPeerStateTestFactory.NowMs())); + Assert.Equal(0, membership.ActiveCount("two", TransitPeerStateTestFactory.NowMs())); + await duplex.QuitAsync(run); + } + + [Fact] + public async Task ReaderReclassification_RenewalDoesNotRecreateSlot() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + await tracker.RenewLeasesAsync(CancellationToken.None); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + } + + [Fact] + public async Task InFlightRenewal_CannotRecreateSlotAfterReaderReclassification() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + membership.NotifyRenewStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); + membership.BlockRenew = new(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + Assert.Equal(1, tracker.GetLocalCount(PeerId)); + + var renew = tracker.RenewLeasesAsync(CancellationToken.None).AsTask(); + await membership.NotifyRenewStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await duplex.AuthenticateReaderAsync("a", "a"); + Assert.Null(session.Authorization.TransitPeerName); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + membership.BlockRenew.TrySetResult(); + await renew.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + Assert.Equal(1, tracker.ReleaseCalls); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(1, tracker.ReleaseCalls); + } + + [Fact] + public async Task ReaderAuthinfoRelease_ConcurrentTeardownDispose_ReleasesOnce() + { + var store = CreateStore(maxIncoming: 10); + var (limiter, tracker, membership) = TransitPeerStateTestFactory.CreateLimiter(store); + membership.BlockRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); + await using var duplex = new AuthDuplex(AllowFromClient); + var session = duplex.CreateSession(store); + var lease = await AdmitAsync(limiter, session); + var run = session.RunAsync(); + await duplex.ReadGreetingAsync(); + await duplex.WriteClientLineAsync("AUTHINFO USER a"); + Assert.StartsWith("381 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await duplex.WriteClientLineAsync("AUTHINFO PASS a"); + await WaitUntilAsync(() => session.Authorization.AuthorizedReader); + + Assert.Null(session.Authorization.TransitPeerName); + Assert.False(lease.IsHeld); + Assert.Equal(0, tracker.GetLocalCount(PeerId)); + Assert.Equal(0, membership.ReleaseCalls); + + var teardown = lease.DisposeAsync(); + membership.BlockRelease.TrySetResult(); + await teardown; + Assert.StartsWith("281 ", await duplex.ReadClientLineAsync(), StringComparison.Ordinal); + Assert.Equal(1, tracker.ReleaseCalls); + Assert.Equal(1, membership.ReleaseCalls); + Assert.Equal(0, membership.ActiveCount(PeerId, TransitPeerStateTestFactory.NowMs())); + + await duplex.QuitAsync(run); + await lease.DisposeAsync(); + Assert.Equal(1, tracker.ReleaseCalls); + } + + private static TransitConfigurationStore CreateStore( + int maxIncoming, + string transitUsername = "", + string transitPassword = "") + { + var store = new TransitConfigurationStore(); + store.Replace( + TransitTestPeers.Snapshot( + PeerId, + TransitTestPeers.Peer( + maxIncoming: maxIncoming, + allowFrom: ["198.18.0.0/15"], + username: transitUsername, + password: transitPassword))); + return store; + } + + private static async Task AdmitAsync( + TransitInboundConnectionLimiter limiter, + NntpSession session) + { + var outcome = await TransitConnectionAdmission.TryAdmitAsync(limiter, session); + Assert.True(outcome.Admitted); + session.AttachTransitAdmission(outcome.Lease); + return outcome.Lease; + } + + private static async Task WaitUntilAsync(Func condition) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(1, cts.Token); + } + } + + private sealed class AuthDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(NntpPipeOptions.Create()); + private readonly Pipe _serverToClient = new(NntpPipeOptions.Create()); + private readonly IPAddress _clientIp; + + public AuthDuplex(IPAddress clientIp) => _clientIp = clientIp; + + public NntpSession CreateSession( + TransitConfigurationStore store, + ISessionStateTracker? admission = null, + string username = "a", + string password = "a") + { + var users = new MemoryNntpUserRecordStore(); + users.Add(MemoryNntpUserRecordStore.Create(username, password, sessionLimit: 1)); + var validator = new MySqlNntpCredentialValidator(users, NullLogger.Instance); + var provider = new CompositeNntpAuthenticationProvider( + DenyAllNntpAuthenticationProvider.Instance, + newsmasterUsername: null, + validator); + var connection = new PipeNntpConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(_clientIp, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + authenticationProvider: provider, + transitPeerAuthorization: TransitPeerAuthorization.CreateForStore(store), + sessionAdmission: admission); + } + + public async Task AuthenticateReaderAsync(string username, string password) + { + await WriteClientLineAsync($"AUTHINFO USER {username}"); + Assert.StartsWith("381 ", await ReadClientLineAsync(), StringComparison.Ordinal); + await WriteClientLineAsync($"AUTHINFO PASS {password}"); + Assert.StartsWith("281 ", await ReadClientLineAsync(), StringComparison.Ordinal); + } + + public async Task ReadGreetingAsync() + { + var line = await ReadClientLineAsync(); + Assert.StartsWith("20", line, StringComparison.Ordinal); + } + + public async Task WriteClientLineAsync(string line) + { + var bytes = Encoding.ASCII.GetBytes(line + "\r\n"); + await _clientToServer.Writer.WriteAsync(bytes); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async Task QuitAsync(Task run) + { + await WriteClientLineAsync("QUIT"); + await ReadClientLineAsync(); + await run; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public ConnectionClientIdentity ClientIdentity { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceivePolicyCommandTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceivePolicyCommandTests.cs new file mode 100644 index 00000000..ba4a2c8b --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceivePolicyCommandTests.cs @@ -0,0 +1,642 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Articles.YEnc; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.NntpDb; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// Command-path coverage for incoming Transit receive size, newsgroup patterns, article types, and credentials. +/// +[Collection("NntpCommandLoggerCollection")] +public sealed class TransitReceivePolicyCommandTests +{ + private const int GlobalMax = 5 * 1024 * 1024; + private const int OneMib = 1 * 1024 * 1024; + private const int TenMib = 10 * 1024 * 1024; + private const string SiteName = "news.example"; + + private static readonly IPAddress PeerAddress = IPAddress.Parse("192.0.2.10"); + private static readonly NntpArticleParser Parser = new(NntpdOptions.FormatFqdn(1, "usenet.ninja")); + private static readonly string YencBody = BuildYencBody(); + + public static IEnumerable SizeCases() + { + foreach (var path in new[] { ReceivePath.SerialTakeThis, ReceivePath.PipelinedTakeThis, ReceivePath.IHave }) + { + yield return [path, (long)OneMib, OneMib]; + yield return [path, (long)TenMib, GlobalMax]; + yield return [path, 0L, GlobalMax]; + } + } + + public static IEnumerable ArticlePaths() + { + yield return [ReceivePath.SerialTakeThis]; + yield return [ReceivePath.PipelinedTakeThis]; + yield return [ReceivePath.IHave]; + } + + public static IEnumerable TakeThisAndIHave() + { + yield return [ReceivePath.PipelinedTakeThis]; + yield return [ReceivePath.IHave]; + } + + [Theory] + [MemberData(nameof(SizeCases))] + public async Task MaxArticleBytes_UsesMinOfGlobalAndPeer(ReceivePath path, long peerMax, int effective) + { + var policy = Peer(maxSize: peerMax, sendPatterns: "!alt.test"); + await using var harness = await Harness.OpenAsync(policy, path); + Assert.Equal(peerMax, harness.Session.Authorization.TransitPeerPolicy!.MaxSize); + + var smallId = ""; + var small = await harness.OfferAsync(smallId, CanonicalArticleText.Destuffed(smallId), accept: true); + Assert.Contains(SiteName, Encoding.ASCII.GetString(small!.Record.ArtData.Span), StringComparison.Ordinal); + + if (effective == OneMib) + { + var fitId = ""; + var fitted = FittedToCanonical(fitId, effective); + var accepted = await harness.OfferAsync(fitId, fitted, accept: true); + Assert.Equal(effective, accepted!.Record.ArtSize); + } + + var overId = ""; + await harness.OfferAsync(overId, Sized(overId, effective + 1), accept: false); + } + + [Theory] + [MemberData(nameof(ArticlePaths))] + public async Task ReceivePatterns_MatchExcludeLastMatchPoison_AndIgnoreSendPatterns(ReceivePath path) + { + await using (var allow = await Harness.OpenAsync(Peer(receivePatterns: "*", sendPatterns: "!alt.test"), path)) + { + await allow.CheckAsync("", wanted: true); + var accepted = await allow.OfferAsync("", GroupArticle("", "alt.test"), accept: true); + Assert.Equal(ArticleType.Default, accepted!.Record.ArtType); + } + + await using (var bang = await Harness.OpenAsync(Peer(receivePatterns: "*,!unidata.*", sendPatterns: "*"), path)) + { + await bang.CheckAsync("", wanted: true); + await bang.OfferAsync( + "", + GroupArticle("", "unidata.maps"), + accept: false); + await bang.OfferAsync( + "", + GroupArticle("", "comp.lang.c"), + accept: true); + } + + await using (var lastAllows = await Harness.OpenAsync(Peer(receivePatterns: "news.*,!news.misc,*.misc"), path)) + { + await lastAllows.OfferAsync( + "", + GroupArticle("", "news.misc"), + accept: true); + } + + await using (var lastRejects = await Harness.OpenAsync(Peer(receivePatterns: "news.*,!news.misc"), path)) + { + await lastRejects.OfferAsync( + "", + GroupArticle("", "news.misc"), + accept: false); + } + + await using (var poison = await Harness.OpenAsync(Peer(receivePatterns: "alt.*,@alt.binaries.warez,misc.*"), path)) + { + await poison.OfferAsync( + "", + GroupArticle("", "misc.test,alt.binaries.warez"), + accept: false); + await poison.OfferAsync( + "", + GroupArticle("", "misc.test,alt.fan"), + accept: true); + } + + await using (var receiveRejects = await Harness.OpenAsync(Peer(receivePatterns: "*,!alt.test", sendPatterns: "*"), path)) + { + await receiveRejects.OfferAsync( + "", + GroupArticle("", "alt.test"), + accept: false); + } + } + + [Theory] + [MemberData(nameof(TakeThisAndIHave))] + public async Task ReceiveArticleTypes_AllAllowsClassifiedTypes_TextOnlyRejectsTheRest(ReceivePath path) + { + var samples = Samples(); + await using (var all = await Harness.OpenAsync( + Peer(receiveTypes: TransitMessageTypes.All, sendTypes: TransitMessageTypes.Default), + path)) + { + foreach (var sample in samples) + { + var id = ""; + var article = Typed(id, sample.ExtraHeader, sample.Body); + var classified = Classify(article); + Assert.Equal(sample.Expected, classified); + Assert.True(ArticleTypeCapabilities.Allows(ArticleTypeCapabilities.All, classified)); + var queued = await all.OfferAsync(id, article, accept: true); + Assert.Equal(classified, queued!.Record.ArtType); + } + } + + await using var textOnly = await Harness.OpenAsync( + Peer(receiveTypes: TransitMessageTypes.Default, sendTypes: TransitMessageTypes.All), + path); + foreach (var sample in samples) + { + var id = ""; + var article = Typed(id, sample.ExtraHeader, sample.Body); + var classified = Classify(article); + var allowed = ArticleTypeCapabilities.Allows(ArticleTypeCapabilities.TextOnly, classified); + var queued = await textOnly.OfferAsync(id, article, accept: allowed); + if (allowed) + { + Assert.Equal(ArticleType.Default, classified); + Assert.Equal(classified, queued!.Record.ArtType); + } + else + { + Assert.NotEqual(ArticleType.Default, classified); + Assert.False(ArticleTypeCapabilities.Allows(ArticleType.Default, classified)); + } + } + } + + [Fact] + public async Task Credentials_BothSet_RequireMatchingAuthinfoBeforeFeedCommands() + { + await using var harness = await Harness.OpenAsync(Peer(username: "feed", password: "secret"), ReceivePath.PipelinedTakeThis); + Assert.True(harness.Session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + Assert.False(harness.Session.Authorization.AuthorizedTransit); + Assert.True(harness.Session.Authorization.StreamingPermitted); + + await harness.Duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + await harness.Duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + await harness.Duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + + await harness.Duplex.WriteClientLineAsync("MODE STREAM"); + Assert.Equal("203 Streaming permitted", await harness.Duplex.ReadClientLineAsync()); + await harness.Duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await harness.Duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.Duplex.WriteClientLineAsync("AUTHINFO PASS wrong"); + Assert.Equal("481 Authentication failed", await harness.Duplex.ReadClientLineAsync()); + Assert.False(harness.Session.Authorization.AuthorizedTransit); + await harness.Duplex.WriteClientLineAsync("CHECK "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + await harness.Duplex.WriteClientLineAsync("IHAVE "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + await harness.Duplex.WriteClientLineAsync("TAKETHIS "); + Assert.Equal("480 Authentication required", await harness.Duplex.ReadClientLineAsync()); + + await harness.Duplex.WriteClientLineAsync("AUTHINFO USER feed"); + Assert.StartsWith("381 ", await harness.Duplex.ReadClientLineAsync(), StringComparison.Ordinal); + await harness.Duplex.WriteClientLineAsync("AUTHINFO PASS secret"); + Assert.Equal("281 Authentication accepted", await harness.Duplex.ReadClientLineAsync()); + Assert.True(harness.Session.Authorization.AuthorizedTransit); + + await harness.CheckAsync("", wanted: true); + await harness.OfferAsync("", CanonicalArticleText.Destuffed(""), accept: true, forceIHave: true); + await harness.OfferAsync("", CanonicalArticleText.Destuffed(""), accept: true); + } + + [Theory] + [InlineData("", "secret")] + [InlineData("feed", "")] + [InlineData("", "")] + public async Task Credentials_EitherBlank_LeavesFeedCommandsOpen(string username, string password) + { + await using var harness = await Harness.OpenAsync(Peer(username: username, password: password), ReceivePath.PipelinedTakeThis); + Assert.False(harness.Session.Authorization.TransitPeerPolicy!.HasPeerCredentials); + Assert.True(harness.Session.Authorization.AuthorizedTransit); + await harness.CheckAsync("", wanted: true); + await harness.OfferAsync("", CanonicalArticleText.Destuffed(""), accept: true, forceIHave: true); + await harness.OfferAsync("", CanonicalArticleText.Destuffed(""), accept: true); + } + + [Fact] + public async Task PreviousSnapshot_DoesNotMixReceiveFieldsWithTheNextRevision() + { + var store = new TransitConfigurationStore(); + store.Replace(Snapshot(Peer(maxSize: 2048, receivePatterns: "*"), publicationId: 1)); + await using var oldSession = await Harness.OpenAsync(store, ReceivePath.PipelinedTakeThis, globalMax: 4096); + var captured = oldSession.Session.Authorization.TransitPeerPolicy; + Assert.NotNull(captured); + Assert.Equal(2048, captured.MaxSize); + Assert.Equal("*", captured.ReceivePatterns.Expression); + + await oldSession.OfferAsync("", GroupArticle("", "alt.test"), accept: true); + await oldSession.OfferAsync("", Sized("", 2049), accept: false); + + store.Replace(Snapshot(Peer(maxSize: 10_000, receivePatterns: "*,!alt.test"), publicationId: 2)); + Assert.Same(captured, oldSession.Session.Authorization.TransitPeerPolicy); + await oldSession.OfferAsync("", GroupArticle("", "alt.test"), accept: true); + await oldSession.OfferAsync("", Sized("", 3000), accept: false); + + await using var fresh = await Harness.OpenAsync(store, ReceivePath.PipelinedTakeThis, globalMax: 4096); + Assert.NotSame(captured, fresh.Session.Authorization.TransitPeerPolicy); + Assert.Equal(10_000, fresh.Session.Authorization.TransitPeerPolicy!.MaxSize); + Assert.Equal("*,!alt.test", fresh.Session.Authorization.TransitPeerPolicy.ReceivePatterns.Expression); + await fresh.OfferAsync("", GroupArticle("", "alt.test"), accept: false); + await fresh.OfferAsync("", GroupArticle("", "comp.lang.c"), accept: true); + await fresh.OfferAsync("", Sized("", 3000, "comp.lang.c"), accept: true); + } + + private static TransitPeerPolicy Peer( + long maxSize = TenMib, + string receivePatterns = "*", + string sendPatterns = "*", + TransitMessageTypes receiveTypes = TransitMessageTypes.All, + TransitMessageTypes sendTypes = TransitMessageTypes.All, + string username = "", + string password = "") + { + Assert.True(NewsfeedsPattern.TryParse(receivePatterns, out var receive, out var receiveError), receiveError); + Assert.True(NewsfeedsPattern.TryParse(sendPatterns, out var send, out var sendError), sendError); + return new TransitPeerPolicy( + "peer", + "Receive Peer", + maxIncomingConnections: 8, + maxOutgoingConnections: 0, + literalPrefixes: [IpPrefix.Host(PeerAddress)], + dnsHostnames: [], + connectTo: [], + username, + password, + TransitSslMode.None, + send!, + deferOnDuplicate: true, + pathToken: string.Empty, + maxSize, + sendTypes, + receive!, + receiveTypes, + sendMaxArticleBytes: 100, + sendUsername: "send-user", + sendPassword: "send-secret", + pathExclusions: ["send-only"]); + } + + private static TransitConfigurationSnapshot Snapshot(TransitPeerPolicy policy, long publicationId = 1) => + new( + new Dictionary(StringComparer.Ordinal) { [policy.Identifier] = policy }, + publicationId: publicationId); + + private static string GroupArticle(string id, string newsgroups) => + CanonicalArticleText.Destuffed(id, "body\r\n", newsgroups); + + private static string Sized(string id, int destuffedLength, string newsgroups = "alt.test") + { + var skeleton = CanonicalArticleText.Destuffed(id, string.Empty, newsgroups); + var bodyLength = destuffedLength - skeleton.Length; + Assert.True(bodyLength >= 2); + var body = new StringBuilder(bodyLength); + var left = bodyLength; + while (left > 0) + { + var content = Math.Min(60, left - 2); + if (left - (content + 2) == 1) + { + content--; + } + + Assert.InRange(content, 0, 60); + body.Append('x', content); + body.Append("\r\n"); + left -= content + 2; + } + + var article = CanonicalArticleText.Destuffed(id, body.ToString(), newsgroups); + Assert.Equal(destuffedLength, article.Length); + return article; + } + + private static string FittedToCanonical(string id, int canonicalLimit) + { + var probe = Sized(id, 256); + var growth = CanonicalGrowth(probe); + var fitted = Sized(id, canonicalLimit - growth); + Assert.Equal(canonicalLimit, CanonicalGrowth(fitted) + fitted.Length); + return fitted; + } + + private static int CanonicalGrowth(string article) + { + var bytes = Encoding.ASCII.GetBytes(article); + var created = ArticleRecordIngress.TryCreateFromDestuffed( + Parser, + bytes, + bytes.Length + 65_536, + Encoding.UTF8.GetBytes(SiteName)); + Assert.True( + created.IsAccepted, + created.ParseFailure + " / " + created.MaterializeFailure); + return created.Record.ArtSize - bytes.Length; + } + + private static string Typed(string id, string extraHeader, string body) + { + var text = CanonicalArticleText.Destuffed(id, body); + if (extraHeader.Length == 0) + { + return text; + } + + var split = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + return string.Concat(text.AsSpan(0, split), "\r\n", extraHeader, text.AsSpan(split)); + } + + private static ArticleType Classify(string article) + { + var created = ArticleRecordIngress.TryCreateFromDestuffed(Parser, Encoding.ASCII.GetBytes(article)); + Assert.True(created.IsAccepted); + return created.Record.ArtType; + } + + private static ArticleSample[] Samples() => + [ + new("text", string.Empty, "body\r\n", ArticleType.Default), + new("control", "Control: newgroup alt.example\r\n", "body\r\n", ArticleType.Control), + new("cancel", "Control: cancel \r\n", "body\r\n", ArticleType.Control | ArticleType.Cancel), + new("mime", "Mime-Version: 1.0\r\n", "body\r\n", ArticleType.Mime), + new("binary", "Content-Type: application/octet-stream\r\n", "body\r\n", ArticleType.Binary | ArticleType.Mime), + new("yenc", string.Empty, YencBody, ArticleType.Binary | ArticleType.YEncoded), + new("html", "Content-Type: text/html; charset=utf-8\r\n", "body\r\n", ArticleType.Html | ArticleType.Mime), + new("multipart", "Content-Type: multipart/mixed; boundary=abc\r\n", "body\r\n", ArticleType.Multipart | ArticleType.Mime), + ]; + + private static string BuildYencBody() + { + var decoded = new byte[] { 0x41 }; + var crc = YEncCrc32.Compute(decoded); + var encoded = unchecked((byte)(decoded[0] + 42)); + return $"=ybegin line=128 size=1 name=t.bin\r\n{(char)encoded}\r\n=yend size=1 crc32={crc:x8}\r\n"; + } + + private readonly record struct ArticleSample(string Name, string ExtraHeader, string Body, ArticleType Expected); + + public enum ReceivePath + { + SerialTakeThis, + PipelinedTakeThis, + IHave, + } + + private sealed class FixedSharedPolicy : INntpArticlePolicySource + { + public FixedSharedPolicy(int maxArticleBytes) => MaxArticleBytes = maxArticleBytes; + + public int MaxArticleBytes { get; } + + public bool TryGetArticlePolicy(out int maxArticleBytes, out string siteName) + { + maxArticleBytes = MaxArticleBytes; + siteName = SiteName; + return true; + } + } + + private sealed class Harness : IAsyncDisposable + { + private readonly Task _run; + private readonly ReceivePath _path; + + private Harness(FeedDuplex duplex, NntpSession session, ArticleIngestionQueue queue, Task run, ReceivePath path) + { + Duplex = duplex; + Session = session; + Queue = queue; + _run = run; + _path = path; + } + + public FeedDuplex Duplex { get; } + + public NntpSession Session { get; } + + public ArticleIngestionQueue Queue { get; } + + public static Task OpenAsync(TransitPeerPolicy policy, ReceivePath path) => + OpenAsync(Store(Snapshot(policy)), path, GlobalMax); + + public static async Task OpenAsync(TransitConfigurationStore store, ReceivePath path, int globalMax) + { + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 8, + MaxArticleBytes = 64 * 1024 * 1024, + }); + var duplex = await FeedDuplex.CreateAsync(); + var session = duplex.CreateSession(TransitPeerAuthorization.CreateForStore(store), queue); + session.SharedConfiguration = new FixedSharedPolicy(globalMax); + if (path == ReceivePath.SerialTakeThis) + { + session.SetMode(NntpSessionMode.Reader); + } + + var run = session.RunAsync(); + _ = await duplex.ReadClientLineAsync(); + return new Harness(duplex, session, queue, run, path); + } + + public async Task CheckAsync(string id, bool wanted) + { + await Duplex.WriteClientLineAsync("CHECK " + id); + var expected = wanted + ? "238 " + id + " send article to be transferred" + : "480 Authentication required"; + Assert.Equal(expected, await Duplex.ReadClientLineAsync()); + } + + public async Task OfferAsync(string id, string article, bool accept, bool forceIHave = false) + { + var ihave = forceIHave || _path == ReceivePath.IHave; + if (ihave) + { + await Duplex.WriteClientLineAsync("IHAVE " + id); + Assert.Equal("335 Send article to be transferred", await Duplex.ReadClientLineAsync()); + await Duplex.WriteClientAsync(article + ".\r\n"); + Assert.Equal( + accept ? "235 Article transferred OK" : "437 Transfer rejected; do not retry", + await Duplex.ReadClientLineAsync()); + } + else + { + await Duplex.WriteClientAsync("TAKETHIS " + id + "\r\n" + article + ".\r\n"); + Assert.Equal(accept ? "239 " + id : "439 " + id, await Duplex.ReadClientLineAsync()); + } + + if (!accept) + { + Assert.Equal(0, Queue.Count); + return null; + } + + var queued = await Queue.DequeueAsync(CancellationToken.None); + Assert.NotNull(queued); + Assert.Equal(id, queued.MessageId); + Assert.Equal(0, Queue.Count); + return queued; + } + + public async ValueTask DisposeAsync() + { + await Duplex.WriteClientLineAsync("QUIT"); + _ = await Duplex.ReadClientLineAsync(); + await _run; + await Duplex.DisposeAsync(); + } + + private static TransitConfigurationStore Store(TransitConfigurationSnapshot snapshot) + { + var store = new TransitConfigurationStore(); + store.Replace(snapshot); + return store; + } + } + + private sealed class FeedDuplex : IAsyncDisposable + { + private readonly Pipe _clientToServer = new(); + private readonly Pipe _serverToClient = new(); + + public static Task CreateAsync() => Task.FromResult(new FeedDuplex()); + + public NntpSession CreateSession(ITransitPeerAuthorization peers, IArticleIngestionQueue queue) + { + var connection = new PipeConnection( + _clientToServer.Reader, + _serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(PeerAddress, 119))); + return new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: peers); + } + + public async Task WriteClientLineAsync(string line) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(line + "\r\n")); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task WriteClientAsync(string payload) + { + await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(payload)); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadClientLineAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, cts.Token); + Assert.NotNull(line); + return line!; + } + + public async ValueTask DisposeAsync() + { + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + + private sealed class PipeConnection : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + { + Input = input; + Output = output; + ClientIdentity = identity; + } + + public PipeReader Input { get; } + + public PipeWriter Output { get; } + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceiveSizeAccountingTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceiveSizeAccountingTests.cs new file mode 100644 index 00000000..d02aac80 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitReceiveSizeAccountingTests.cs @@ -0,0 +1,430 @@ +using System.IO.Pipelines; +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Networking.Certificates; +using VectorNNTPD.TransitService.ArticleIngestion; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.History; +using VectorNNTPD.TransitService.Networking.Proxy; +using VectorNNTPD.TransitService.Networking.Transport; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Session.CommandProcessor; +using VectorNNTPD.TransitService.Tests.Fixtures; +using VectorNNTPD.TransitService.Tests.TestDoubles; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +/// +/// Proves stream-plane TAKETHIS and IHAVE apply +/// MaxArticleBytes to the same destuffed article bytes. +/// +public sealed class TransitReceiveSizeAccountingTests +{ + private const int Limit = 1_000_000; + + private static readonly IPAddress Peer = IPAddress.Parse("198.18.0.82"); + + [Fact] + public async Task StuffedOverTheWire_WithinDestuffedLimit_IsAcceptedOnEveryReceivePath() + { + var id = ""; + var stuffed = StuffedWithinDestuffedLimit(id, out var destuffedLength, out var canonicalSize); + Assert.True(destuffedLength <= Limit); + // CanonicalV1 ArtData keeps dot-stuffing, so the canonical size is never smaller than the stuffed + // wire size: the stuffed wire exceeds the destuffed length yet still fits within the limit. + Assert.True(stuffed.Length > destuffedLength); + Assert.True(stuffed.Length <= canonicalSize); + Assert.True(canonicalSize <= Limit); + + foreach (var path in new[] { ReceivePath.StreamTakeThis, ReceivePath.IHave }) + { + await using var harness = await CommandHarness.StartAsync(); + var writes = harness.History.Writes.Count; + await harness.OfferAsync(path, id, stuffed); + + var accepted = await harness.ReadLineAsync(); + if (path == ReceivePath.IHave) + { + Assert.StartsWith("335 ", accepted, StringComparison.Ordinal); + accepted = await harness.ReadLineAsync(); + Assert.StartsWith("235 ", accepted, StringComparison.Ordinal); + } + else + { + Assert.StartsWith("239 ", accepted, StringComparison.Ordinal); + } + + Assert.Equal(1, harness.Queue.Count); + var queued = await harness.Queue.DequeueAsync(CancellationToken.None); + Assert.NotNull(queued); + Assert.Equal(id, queued.MessageId); + Assert.True(queued.Record.ArtSize <= Limit); + Assert.Equal(writes + 1, harness.History.Writes.Count); + Assert.True(harness.History.ContainsLocal(HistoryDigest.FromMessageId(Encoding.ASCII.GetBytes(id)))); + } + } + + [Fact] + public async Task DestuffedOverLimit_RejectsOnEveryReceivePath_AndStaysFramed() + { + var id = ""; + var article = PlainOverLimit(id); + Assert.True(article.Length > Limit); + + foreach (var path in new[] { ReceivePath.StreamTakeThis, ReceivePath.IHave }) + { + await using var harness = await CommandHarness.StartAsync(); + var writes = harness.History.Writes.Count; + await harness.OfferAsync(path, id, article); + + var reply = await harness.ReadLineAsync(); + if (path == ReceivePath.IHave) + { + Assert.StartsWith("335 ", reply, StringComparison.Ordinal); + reply = await harness.ReadLineAsync(); + Assert.StartsWith("437 ", reply, StringComparison.Ordinal); + } + else + { + Assert.StartsWith("439 ", reply, StringComparison.Ordinal); + } + + await harness.WriteCommandAsync("DATE\r\n"); + var date = await harness.ReadLineAsync(); + Assert.StartsWith("111 ", date, StringComparison.Ordinal); + Assert.Equal(0, harness.Queue.Count); + Assert.Equal(writes, harness.History.Writes.Count); + Assert.False(harness.History.ContainsLocal(HistoryDigest.FromMessageId(Encoding.ASCII.GetBytes(id)))); + } + } + + [Fact] + public async Task StuffedOverDestuffedLimit_RejectsOnEveryReceivePath() + { + var id = ""; + var stuffed = StuffedOverDestuffedLimit(id, out var destuffedLength); + Assert.True(destuffedLength > Limit); + Assert.True(stuffed.Length > destuffedLength); + + foreach (var path in new[] { ReceivePath.StreamTakeThis, ReceivePath.IHave }) + { + await using var harness = await CommandHarness.StartAsync(); + await harness.OfferAsync(path, id, stuffed); + var reply = await harness.ReadLineAsync(); + if (path == ReceivePath.IHave) + { + Assert.StartsWith("335 ", reply, StringComparison.Ordinal); + reply = await harness.ReadLineAsync(); + Assert.StartsWith("437 ", reply, StringComparison.Ordinal); + } + else + { + Assert.StartsWith("439 ", reply, StringComparison.Ordinal); + } + + await harness.WriteCommandAsync("DATE\r\n"); + Assert.StartsWith("111 ", await harness.ReadLineAsync(), StringComparison.Ordinal); + Assert.Equal(0, harness.Queue.Count); + Assert.False(harness.History.ContainsLocal(HistoryDigest.FromMessageId(Encoding.ASCII.GetBytes(id)))); + } + } + + [Fact] + public async Task DestuffedWithinLimit_CanonicalOverLimit_IsRejectedByMaterialization() + { + var id = ""; + var article = PlainWithinDestuffedButCanonicalOver(id, out var destuffedLength, out var canonicalSize); + Assert.True(destuffedLength <= Limit); + Assert.True(canonicalSize > Limit); + + foreach (var path in new[] { ReceivePath.StreamTakeThis, ReceivePath.IHave }) + { + await using var harness = await CommandHarness.StartAsync(); + var writes = harness.History.Writes.Count; + await harness.OfferAsync(path, id, article); + var reply = await harness.ReadLineAsync(); + if (path == ReceivePath.IHave) + { + Assert.StartsWith("335 ", reply, StringComparison.Ordinal); + reply = await harness.ReadLineAsync(); + Assert.StartsWith("437 ", reply, StringComparison.Ordinal); + } + else + { + Assert.StartsWith("439 ", reply, StringComparison.Ordinal); + } + + Assert.Equal(0, harness.Queue.Count); + Assert.Equal(writes, harness.History.Writes.Count); + } + } + + private static byte[] StuffedWithinDestuffedLimit(string id, out int destuffedLength, out int canonicalSize) + { + var growth = CanonicalGrowth(id); + var headerLength = Encoding.ASCII.GetByteCount(CanonicalArticleText.Destuffed(id, string.Empty)); + var lines = (Limit - growth - headerLength) / 4; + Assert.True(lines > 0); + var body = new StringBuilder(lines * 3); + for (var i = 0; i < lines; i++) + { + body.Append(".\r\n"); + } + + var destuffed = CanonicalArticleText.Destuffed(id, body.ToString()); + destuffedLength = Encoding.ASCII.GetByteCount(destuffed); + canonicalSize = CanonicalLength(destuffed); + var stuffed = Encoding.ASCII.GetBytes(destuffed.Replace("\r\n.", "\r\n..", StringComparison.Ordinal)); + Assert.True(stuffed.Length > destuffedLength); + return stuffed; + } + + private static byte[] StuffedOverDestuffedLimit(string id, out int destuffedLength) + { + var headerLength = Encoding.ASCII.GetByteCount(CanonicalArticleText.Destuffed(id, string.Empty)); + var lines = ((Limit - headerLength) / 3) + 1; + var body = new StringBuilder(lines * 3); + for (var i = 0; i < lines; i++) + { + body.Append(".\r\n"); + } + + var destuffed = CanonicalArticleText.Destuffed(id, body.ToString()); + destuffedLength = Encoding.ASCII.GetByteCount(destuffed); + return Encoding.ASCII.GetBytes(destuffed.Replace("\r\n.", "\r\n..", StringComparison.Ordinal)); + } + + private static byte[] PlainOverLimit(string id) => + Encoding.ASCII.GetBytes(Sized(id, Limit + 2)); + + private static byte[] PlainWithinDestuffedButCanonicalOver( + string id, + out int destuffedLength, + out int canonicalSize) + { + var growth = CanonicalGrowth(id); + Assert.True(growth > 0); + var article = Sized(id, Limit - growth + 1); + destuffedLength = article.Length; + canonicalSize = CanonicalLength(article); + Assert.True(destuffedLength <= Limit); + Assert.True(canonicalSize > Limit); + return Encoding.ASCII.GetBytes(article); + } + + private static string Sized(string id, int destuffedLength) + { + var skeleton = CanonicalArticleText.Destuffed(id, string.Empty); + var bodyLength = destuffedLength - skeleton.Length; + Assert.True(bodyLength >= 2); + var body = new StringBuilder(bodyLength); + var left = bodyLength; + while (left > 0) + { + var content = Math.Min(60, left - 2); + if (left - (content + 2) == 1) + { + content--; + } + + Assert.InRange(content, 0, 60); + body.Append('x', content); + body.Append("\r\n"); + left -= content + 2; + } + + var article = CanonicalArticleText.Destuffed(id, body.ToString()); + Assert.Equal(destuffedLength, article.Length); + return article; + } + + private static int CanonicalGrowth(string id) + { + var probe = CanonicalArticleText.Destuffed(id, "probe\r\n"); + return CanonicalLength(probe) - Encoding.ASCII.GetByteCount(probe); + } + + private static int CanonicalLength(string destuffed) + { + var parser = new NntpArticleParser(NntpdOptions.FormatFqdn(1, "usenet.ninja")); + var bytes = Encoding.ASCII.GetBytes(destuffed); + var created = ArticleRecordIngress.TryCreateFromDestuffed( + parser, + bytes, + maxArticleBytes: 5 * 1024 * 1024, + ArticlePathCanonicalizer.OrganizationalTrackerHost); + Assert.True(created.IsAccepted, created.ParseFailure + " / " + created.MaterializeFailure); + return created.Record.ArtSize; + } + + private enum ReceivePath + { + StreamTakeThis, + IHave, + } + + private sealed class CommandHarness : IAsyncDisposable + { + private readonly Pipe _clientToServer; + private readonly Pipe _serverToClient; + private readonly NntpSession _session; + private readonly Task _run; + + private CommandHarness( + Pipe clientToServer, + Pipe serverToClient, + NntpSession session, + Task run, + HistoryDb history, + ArticleIngestionQueue queue) + { + _clientToServer = clientToServer; + _serverToClient = serverToClient; + _session = session; + _run = run; + History = history; + Queue = queue; + } + + public HistoryDb History { get; } + + public ArticleIngestionQueue Queue { get; } + + public static async Task StartAsync() + { + var clientToServer = new Pipe(new PipeOptions( + pauseWriterThreshold: 8 * 1024 * 1024, + resumeWriterThreshold: 4 * 1024 * 1024, + minimumSegmentSize: 4 * 1024, + useSynchronizationContext: false)); + var serverToClient = new Pipe(NntpPipeOptions.Create()); + var history = new HistoryDb( + new FakeRedisService(), + TimeSpan.FromHours(2), + NullLogger.Instance, + TimeProvider.System, + new HistoryWriteQueue()); + var queue = new ArticleIngestionQueue(new ArticleIngestionOptions + { + QueueCapacity = 4, + MaxArticleBytes = Limit, + }); + var connection = new PipeNntpConnection( + clientToServer.Reader, + serverToClient.Writer, + ConnectionClientIdentity.Direct(new IPEndPoint(Peer, 40000))); + var session = new NntpSession( + connection, + NullLogger.Instance, + articleIngestion: queue, + transitPeerAuthorization: TransitTestPeers.ForAllowFrom(Peer), + historyDb: history); + var run = session.RunAsync(); + var harness = new CommandHarness(clientToServer, serverToClient, session, run, history, queue); + Assert.StartsWith("2", await harness.ReadLineAsync(), StringComparison.Ordinal); + return harness; + } + + public async Task OfferAsync(ReceivePath path, string id, byte[] article) + { + var command = path == ReceivePath.IHave ? "IHAVE" : "TAKETHIS"; + await WriteCommandAsync($"{command} {id}\r\n"); + await _clientToServer.Writer.WriteAsync(article); + await WriteCommandAsync(".\r\n"); + } + + public async Task WriteCommandAsync(string command) + { + var result = await _clientToServer.Writer.WriteAsync(Encoding.ASCII.GetBytes(command)); + Assert.False(result.IsCompleted); + await _clientToServer.Writer.FlushAsync(); + } + + public async Task ReadLineAsync() + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var line = await NntpCommandLineReader.ReadLineAsync(_serverToClient.Reader, timeout.Token); + Assert.NotNull(line); + return line; + } + + public async ValueTask DisposeAsync() + { + await WriteCommandAsync("QUIT\r\n"); + Assert.Equal("205 Connection closing", await ReadLineAsync()); + await _run; + await _clientToServer.Writer.CompleteAsync(); + await _clientToServer.Reader.CompleteAsync(); + await _serverToClient.Writer.CompleteAsync(); + await _serverToClient.Reader.CompleteAsync(); + } + } + + private sealed class PipeNntpConnection(PipeReader input, PipeWriter output, ConnectionClientIdentity identity) + : INntpConnection + { + private readonly CancellationTokenSource _cts = new(); + + public PipeReader Input { get; } = input; + + public PipeWriter Output { get; } = output; + + public EndPoint? RemoteEndPoint => ClientIdentity.TcpPeer; + + public EndPoint? LocalEndPoint => null; + + public ConnectionClientIdentity ClientIdentity { get; } = identity; + + public bool IsTls => false; + + public bool IsCompressed => false; + + public CancellationToken ConnectionClosed => _cts.Token; + + public bool IsCompleted => _cts.IsCancellationRequested; + + public long OutboundIdleVersion => 0; + + public bool TryGetNegotiatedTlsParameters(out string tlsVersion, out string cipher) + { + tlsVersion = string.Empty; + cipher = string.Empty; + return false; + } + + public Task PauseReadsAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task WaitForOutboundDeliveryAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForOutboundDeliveryAndPauseReadsAsync( + long outboundIdleVersionBeforeFlush, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task CompleteAsync(Exception? exception = null) + { + _cts.Cancel(); + return Task.CompletedTask; + } + + public Task UpgradeToTlsAsync( + ITlsCertificateContextProvider certificateProvider, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task UpgradeToDeflateAsync(CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public ValueTask DisposeAsync() + { + _cts.Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transit/TransitTestPeers.cs b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitTestPeers.cs new file mode 100644 index 00000000..4f010abc --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transit/TransitTestPeers.cs @@ -0,0 +1,102 @@ +using System.Net; +using VectorNNTPD.TransitService.Configuration; +using VectorNNTPD.TransitService.Session; +using VectorNNTPD.TransitService.Transit; + +namespace VectorNNTPD.TransitService.Tests.Transit; + +internal static class TransitTestPeers +{ + /// Canonical integration-test Transit identifier for the source-IP → AllowFrom path. + public const string DefaultPeerName = "test-peer"; + + /// Canonical integration-test display name (not a protocol argument). + public const string DefaultPeerDisplayName = "Test Peer"; + + public static TransitPeerOptions Peer( + int maxIncoming = 8, + int maxOutgoing = 1, + string[]? allowFrom = null, + string[]? connectTo = null, + string username = "", + string password = "", + string ssl = "", + string patterns = "*", + bool deferOnDuplicate = true, + string pathToken = "", + long maxSize = TransitPeerOptions.DefaultMaxSize, + string[]? messageTypes = null, + string peerName = DefaultPeerDisplayName) => + new() + { + PeerName = peerName, + MaxIncomingConnections = maxIncoming, + MaxOutgoingConnections = maxOutgoing, + AllowFrom = allowFrom ?? [], + ConnectTo = connectTo ?? [], + Username = username, + Password = password, + Ssl = ssl, + Patterns = patterns, + DeferOnDuplicate = deferOnDuplicate, + PathToken = pathToken, + MaxSize = maxSize, + MessageTypes = messageTypes ?? ["default"], + }; + + public static TransitPeersOptions Dictionary(string name, TransitPeerOptions peer) + { + return new TransitPeersOptions { [name] = peer }; + } + + public static TransitConfigurationSnapshot Snapshot(string name, TransitPeerOptions peer) => + TransitConfigurationSnapshot.Create(Dictionary(name, peer)); + + /// + /// Named-peer authorization whose AllowFrom is the supplied source addresses. + /// Sessions identify this peer only when the connection's effective client IP matches. + /// + public static TransitPeerAuthorization ForAllowFrom( + IPAddress source, + string name = DefaultPeerName, + int maxIncoming = 10, + int maxOutgoing = 0, + string username = "", + string password = "", + string patterns = "*", + bool deferOnDuplicate = true) => + ForAllowFrom( + [source], + name, + maxIncoming, + maxOutgoing, + username, + password, + patterns, + deferOnDuplicate); + + /// Named-peer authorization for one or more literal AllowFrom addresses. + public static TransitPeerAuthorization ForAllowFrom( + IReadOnlyList sources, + string name = DefaultPeerName, + int maxIncoming = 10, + int maxOutgoing = 0, + string username = "", + string password = "", + string patterns = "*", + bool deferOnDuplicate = true) + { + ArgumentNullException.ThrowIfNull(sources); + return TransitPeerAuthorization.CreateStatic( + Snapshot( + name, + Peer( + maxIncoming: maxIncoming, + maxOutgoing: maxOutgoing, + allowFrom: [.. sources.Select(static a => a.ToString())], + username: username, + password: password, + patterns: patterns, + deferOnDuplicate: deferOnDuplicate))); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpArticleClientTests.cs b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpArticleClientTests.cs new file mode 100644 index 00000000..10c40677 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpArticleClientTests.cs @@ -0,0 +1,946 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; +using VectorNNTPD.TransitService.Transport.Vatp; + +namespace VectorNNTPD.TransitService.Tests.Transport.Vatp; + +public sealed class VatpArticleClientTests +{ + /// Body larger than the production initial window so WINDOW replenishment is required. + private static int BodyLargerThanInitialWindow => + ArticleTransferLimits.Default.InitialStreamWindowBytes + (64 * 1024); + + [Fact] + public async Task Coalesced_multi_frame_transfer_succeeds() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CoalesceOutboundFrames; + var prepared = CreateArticle("", body: BuildLargeBody(96 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + } + + [Fact] + public async Task Coalesced_240KiB_article_matches_live_failure_scale() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CoalesceOutboundFrames; + var prepared = CreateArticle("", body: BuildLargeBody(240 * 1024)); + Assert.True(prepared.Record.ArtSize > VatpProtocol.DefaultMaxFramePayload); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + Assert.True(result.Record.ArtData.ToArray().AsSpan() + .SequenceEqual(prepared.Record.ArtData.ToArray())); + } + + [Fact] + public async Task Article_larger_than_initial_window_completes_with_window_pacing() + { + // Articles larger than the initial window must complete by refunding each accepted DATA chunk. + // A missing local replenish is a flow-control failure. + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + Assert.True(prepared.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + Assert.True(result.Record.ArtData.ToArray().AsSpan() + .SequenceEqual(prepared.Record.ArtData.ToArray())); + } + + [Fact] + public async Task Eight_concurrent_large_articles_share_connection_with_independent_windows() + { + await using var server = VatpLoopbackTestServer.Start(); + var articles = new PreparedArticle[8]; + for (var i = 0; i < articles.Length; i++) + { + var bodyBytes = i % 2 == 0 ? BodyLargerThanInitialWindow : 64 * 1024; + articles[i] = CreateArticle($"", body: BuildLargeBody(bodyBytes)); + server.Register(articles[i].Record, articles[i].SelectedDateHeaderName); + } + + Assert.Contains( + articles, + a => a.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + var tasks = articles.Select(a => + client.FetchArticleAsync(server.Host, + server.Port, a.RequestId, a.Record.ArtId, CancellationToken.None)) + .ToArray(); + var results = await Task.WhenAll(tasks); + + Assert.All(results, r => Assert.Equal(VatpFetchKind.Success, r.Kind)); + for (var i = 0; i < articles.Length; i++) + { + Assert.Equal(articles[i].Record.ArtId, results[i].Record.ArtId); + Assert.Equal(articles[i].Record.ArtSize, results[i].Record.ArtSize); + Assert.Equal(articles[i].Record.ArtHash, results[i].Record.ArtHash); + } + + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Cancel_one_large_stream_does_not_fail_sibling_large_stream() + { + await using var server = VatpLoopbackTestServer.Start(); + var keep = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + var cancel = CreateArticle("", body: BuildLargeBody(BodyLargerThanInitialWindow)); + Assert.True(keep.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + Assert.True(cancel.Record.ArtSize > ArticleTransferLimits.Default.InitialStreamWindowBytes); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register(keep.Record, keep.SelectedDateHeaderName); + server.Register( + cancel.Record, + cancel.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var keepTask = client.FetchArticleAsync( + server.Host, + server.Port, + keep.RequestId, + keep.Record.ArtId, + CancellationToken.None); + var cancelTask = client.FetchArticleAsync( + server.Host, + server.Port, + cancel.RequestId, + cancel.Record.ArtId, + cts.Token); + + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + var keepResult = await keepTask; + var cancelResult = await cancelTask; + Assert.Equal(VatpFetchKind.Success, keepResult.Kind); + Assert.Equal(keep.Record.ArtId, keepResult.Record.ArtId); + Assert.Equal(keep.Record.ArtSize, keepResult.Record.ArtSize); + Assert.True( + cancelResult.Kind is VatpFetchKind.Cancelled + or VatpFetchKind.ConnectionFailure + or VatpFetchKind.IncompleteOrMalformedArticle, + cancelResult.Kind.ToString()); + Assert.NotEqual(VatpFetchKind.Success, cancelResult.Kind); + Assert.Equal(default, cancelResult.Record); + + // Connection must remain usable for a subsequent large transfer. + var retry = CreateArticle("", body: BuildLargeBody(740 * 1024)); + server.Register(retry.Record, retry.SelectedDateHeaderName); + var retryResult = await client.FetchArticleAsync( + server.Host, + server.Port, + retry.RequestId, + retry.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retryResult.Kind); + Assert.Equal(retry.Record.ArtSize, retryResult.Record.ArtSize); + } + + [Fact] + public async Task Tiny_writes_reconstruct_partial_frames_across_reads() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.TinyOutboundWrites; + var prepared = CreateArticle("", body: BuildLargeBody(8 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + } + + [Fact] + public async Task Max_sized_single_data_frame_assembles_across_32KiB_reads() + { + // Live root cause: assembling one DefaultMaxFramePayload DATA across multiple + // 32 KiB socket reads must not trip the old one-frame accumulation guard. + await using var server = VatpLoopbackTestServer.Start(); + var bodyBytes = (int)VatpProtocol.DefaultMaxFramePayload - 256; + var prepared = CreateArticle("", body: BuildLargeBody(bodyBytes)); + Assert.True(prepared.Record.ArtSize > 32 * 1024); + Assert.True(prepared.Record.ArtSize <= VatpProtocol.DefaultMaxFramePayload); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtSize, result.Record.ArtSize); + Assert.Equal(prepared.Record.ArtHash, result.Record.ArtHash); + } + + [Fact] + public async Task Oversized_data_frame_still_rejected() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.OversizedDataFrame; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + } + + [Fact] + public async Task Hello_and_single_transfer_success() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + } + + [Fact] + public void Fin_without_end_does_not_complete_on_receive_stream() + { + var prepared = CreateArticle(""); + var stream = new ArticleTransferReceiveStream(1, prepared.RequestId, prepared.Record.ArtId); + var record = prepared.Record; + var meta = ArticleCanonicalTransferMeta.FromRecord(in record, prepared.SelectedDateHeaderName); + var metaBytes = VatpMetaCodec.Encode(in meta); + Assert.True(stream.TryAcceptMeta(metaBytes).Success); + var artData = prepared.Record.ArtData.ToArray(); + Assert.True(stream.TryAcceptData(artData, fin: true).Success); + Assert.Equal(ArticleTransferPhase.AwaitingEnd, stream.Phase); + Assert.False(stream.HasConsumableRecord); + Assert.False(stream.TryTakeRecord(out _)); + } + + [Fact] + public async Task Wrong_article_id_open_yields_remote_failure() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var wrongId = ArticleId.FromMessageId(""u8); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + wrongId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + } + + [Fact] + public async Task Eight_concurrent_streams_share_one_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var articles = new PreparedArticle[8]; + for (var i = 0; i < articles.Length; i++) + { + articles[i] = CreateArticle($""); + server.Register(articles[i].Record, articles[i].SelectedDateHeaderName); + } + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + var tasks = articles.Select(a => + client.FetchArticleAsync(server.Host, + server.Port, a.RequestId, a.Record.ArtId, CancellationToken.None)) + .ToArray(); + var results = await Task.WhenAll(tasks); + Assert.All(results, r => Assert.Equal(VatpFetchKind.Success, r.Kind)); + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Cancel_one_stream_returns_cancelled() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.FinWithoutEnd; + var prepared = CreateArticle("", body: BuildLargeBody(512 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + cts.Token); + Assert.True( + result.Kind is VatpFetchKind.Cancelled or VatpFetchKind.ConnectionFailure or VatpFetchKind.IncompleteOrMalformedArticle, + result.Kind.ToString()); + } + + [Fact] + public async Task Connection_loss_mid_data_fails_fetch() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + var prepared = CreateArticle("", body: BuildLargeBody(256 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task Pool_removes_dead_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + _ = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + } + + [Fact] + public async Task Tls_wrong_target_host_fails_without_test_callback() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = new VatpConnectionPool(NullLogger.Instance) + { + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + var client = new VatpArticleClient(pool, NullLogger.Instance); + var result = await client.FetchArticleAsync( + "wrong.host.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task Fin_without_end_over_tls_does_not_succeed() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.FinWithoutEnd; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + } + + [Fact] + public async Task ConnectionLoss_MidMeta_FailsTransferCleanly() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidMeta; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + Assert.Equal(prepared.Record.ArtId, retry.Record.ArtId); + } + + [Fact] + public async Task ConnectionLoss_AfterFinBeforeEnd_FailsTransferCleanly() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectAfterFinBeforeEnd; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + + Assert.True( + result.Kind is VatpFetchKind.ConnectionFailure or VatpFetchKind.IncompleteOrMalformedArticle, + result.Kind.ToString()); + Assert.NotEqual(VatpFetchKind.Success, result.Kind); + Assert.Equal(default, result.Record); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + Assert.Equal(prepared.Record.ArtId, retry.Record.ArtId); + } + + [Fact] + public async Task Late_and_never_allocated_stream_frames_do_not_poison_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + server.EmitSpuriousFramesAfterTerminal = true; + var first = CreateArticle(""); + var second = CreateArticle(""); + server.Register(first.Record, first.SelectedDateHeaderName); + server.Register(second.Record, second.SelectedDateHeaderName); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var r1 = await client.FetchArticleAsync( + server.Host, + server.Port, + first.RequestId, + first.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, r1.Kind); + Assert.Equal(first.Record.ArtId, r1.Record.ArtId); + + // Spurious late END/FAIL for stream 1 and META/DATA/END for 0x7FFFFFFE must not + // inject state or kill the pooled connection. + var r2 = await client.FetchArticleAsync( + server.Host, + server.Port, + second.RequestId, + second.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, r2.Kind); + Assert.Equal(second.Record.ArtId, r2.Record.ArtId); + Assert.Equal(second.Record.ArtSize, r2.Record.ArtSize); + Assert.Equal(1, pool.AliveConnectionCount(server.Host, server.Port)); + + var openIds = server.ObservedOpenStreamIds.OrderBy(static id => id).ToArray(); + Assert.Equal(new uint[] { 1, 2 }, openIds); + Assert.DoesNotContain(0x7FFFFFFEu, openIds); + } + + [Fact] + public async Task Stream_ids_are_monotonic_and_not_reused_on_same_connection() + { + await using var server = VatpLoopbackTestServer.Start(); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + for (var i = 0; i < 4; i++) + { + var prepared = CreateArticle($""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, result.Kind); + } + + var ids = server.ObservedOpenStreamIds.OrderBy(static id => id).ToArray(); + Assert.Equal(new uint[] { 1, 2, 3, 4 }, ids); + Assert.Equal(1, pool.ConnectionCountForTest(server.Host, server.Port)); + } + + [Fact] + public async Task Active_stream_id_is_not_reused_while_sibling_is_held() + { + await using var server = VatpLoopbackTestServer.Start(); + var held = CreateArticle("", body: BuildLargeBody(64 * 1024)); + var other = CreateArticle(""); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register( + held.Record, + held.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + server.Register(other.Record, other.SelectedDateHeaderName); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var heldTask = client.FetchArticleAsync( + server.Host, + server.Port, + held.RequestId, + held.Record.ArtId, + cts.Token); + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var otherResult = await client.FetchArticleAsync( + server.Host, + server.Port, + other.RequestId, + other.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, otherResult.Kind); + + await cts.CancelAsync(); + var heldResult = await heldTask; + Assert.NotEqual(VatpFetchKind.Success, heldResult.Kind); + + var ids = server.ObservedOpenStreamIds.ToArray(); + Assert.Equal(2, ids.Length); + Assert.Contains(1u, ids); + Assert.Contains(2u, ids); + Assert.Equal(2, ids.Distinct().Count()); + } + + [Fact] + public async Task Cancelled_stream_spurious_frames_do_not_fail_sibling() + { + await using var server = VatpLoopbackTestServer.Start(); + server.EmitSpuriousFramesAfterTerminal = true; + var keep = CreateArticle("", body: BuildLargeBody(128 * 1024)); + var cancel = CreateArticle("", body: BuildLargeBody(128 * 1024)); + var heldStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.Register(keep.Record, keep.SelectedDateHeaderName); + server.Register( + cancel.Record, + cancel.SelectedDateHeaderName, + VatpLoopbackTestServer.TransferMode.HoldAfterFirstData, + heldStarted); + + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + using var cts = new CancellationTokenSource(); + + var keepTask = client.FetchArticleAsync( + server.Host, + server.Port, + keep.RequestId, + keep.Record.ArtId, + CancellationToken.None); + var cancelTask = client.FetchArticleAsync( + server.Host, + server.Port, + cancel.RequestId, + cancel.Record.ArtId, + cts.Token); + + await heldStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await cts.CancelAsync(); + + var keepResult = await keepTask; + var cancelResult = await cancelTask; + Assert.Equal(VatpFetchKind.Success, keepResult.Kind); + Assert.Equal(keep.Record.ArtId, keepResult.Record.ArtId); + Assert.NotEqual(VatpFetchKind.Success, cancelResult.Kind); + Assert.Equal(1, pool.AliveConnectionCount(server.Host, server.Port)); + } + + [Fact] + public async Task Meta_on_connection_stream_id_marks_connection_dead() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.MetaOnConnectionStreamId; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var pool = CreatePool(server); + var client = new VatpArticleClient(pool, NullLogger.Instance); + + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + await WaitUntilAsync(() => pool.AliveConnectionCount(server.Host, server.Port) == 0, TimeSpan.FromSeconds(3)); + + server.Mode = VatpLoopbackTestServer.TransferMode.Complete; + var retry = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.Success, retry.Kind); + } + + [Fact] + public async Task AcceptedDataBytes_ProtocolFailureBeforeStream_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + var client = CreateClient(server); + + var emptyRequest = await client.FetchArticleAsync( + server.Host, + server.Port, + Guid.Empty, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ProtocolFailure, emptyRequest.Kind); + Assert.Equal(0, emptyRequest.AcceptedDataBytes); + + var badEndpoint = await client.FetchArticleAsync( + "vatp" + "://" + "backfiller.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ProtocolFailure, badEndpoint.Kind); + Assert.Equal(0, badEndpoint.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureBeforeStream_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + var pool = new VatpConnectionPool(NullLogger.Instance) + { + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + var client = new VatpArticleClient(pool, NullLogger.Instance); + var result = await client.FetchArticleAsync( + "wrong.host.test", + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureBeforeData_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidMeta; + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ConnectionFailureAfterData_IsPositive() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DisconnectMidData; + var prepared = CreateArticle("", body: BuildLargeBody(256 * 1024)); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + Assert.True(prepared.Record.ArtData.Length > 64 * 1024); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.ConnectionFailure, result.Kind); + Assert.Equal(64 * 1024, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_RemoteFailureBeforeData_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var wrongId = ArticleId.FromMessageId(""u8); + var client = CreateClient(server); + var result = await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + wrongId, + CancellationToken.None); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_RemoteFailureAfterData_IsProbeLength() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DataThenRemoteFail; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.RemoteTransferFailure, result.Kind); + Assert.Equal(VatpErrorCode.OpenRejected, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_MetaRejected_IsZero() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.InvalidMeta; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(0, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_ShortFin_IsCopiedLength() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.ShortDataFin; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtSizeMismatch, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + Assert.NotEqual(prepared.Record.ArtSize, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_LaterDataRejectedBeforeCopy_KeepsFirstChunk() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.DataThenSizeReject; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtSizeMismatch, result.ErrorCode); + Assert.Equal(VatpLoopbackTestServer.ProbePayloadBytes, result.AcceptedDataBytes); + } + + [Fact] + public async Task AcceptedDataBytes_HashMismatchAfterFullBody_IsArtSize() + { + await using var server = VatpLoopbackTestServer.Start(); + server.Mode = VatpLoopbackTestServer.TransferMode.CorruptArtHash; + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.IncompleteOrMalformedArticle, result.Kind); + Assert.Equal(VatpErrorCode.ArtHashMismatch, result.ErrorCode); + Assert.Equal(prepared.Record.ArtSize, result.AcceptedDataBytes); + Assert.True(result.AcceptedDataBytes > VatpLoopbackTestServer.ProbePayloadBytes); + } + + [Fact] + public async Task AcceptedDataBytes_Success_IsArtSize() + { + await using var server = VatpLoopbackTestServer.Start(); + var prepared = CreateProbeArticle(""); + server.Register(prepared.Record, prepared.SelectedDateHeaderName); + var result = await FetchAsync(server, prepared); + Assert.Equal(VatpFetchKind.Success, result.Kind); + Assert.Equal(prepared.Record.ArtId, result.Record.ArtId); + Assert.Equal(prepared.Record.ArtSize, result.AcceptedDataBytes); + Assert.Equal(prepared.Record.ArtData.Length, result.AcceptedDataBytes); + } + + private static async Task FetchAsync(VatpLoopbackTestServer server, PreparedArticle prepared) + { + var client = CreateClient(server); + return await client.FetchArticleAsync( + server.Host, + server.Port, + prepared.RequestId, + prepared.Record.ArtId, + CancellationToken.None); + } + + private static PreparedArticle CreateProbeArticle(string messageId) + { + var prepared = CreateArticle(messageId); + Assert.True(prepared.Record.ArtData.Length > VatpLoopbackTestServer.ProbePayloadBytes); + Assert.Equal(prepared.Record.ArtData.Length, prepared.Record.ArtSize); + return prepared; + } + + private static VatpArticleClient CreateClient(VatpLoopbackTestServer server) => + new(CreatePool(server), NullLogger.Instance); + + private static VatpConnectionPool CreatePool(VatpLoopbackTestServer server) + { + var pool = new VatpConnectionPool( + NullLogger.Instance, + new VatpClientOptions + { + IoTimeout = TimeSpan.FromSeconds(2), + ConnectTimeout = TimeSpan.FromSeconds(5), + TlsHandshakeTimeout = TimeSpan.FromSeconds(5), + }) + { + ServerCertificateValidationCallback = static (_, _, _, _) => true, + TestTcpConnectHost = IPAddress.Loopback.ToString(), + }; + return pool; + } + + private static PreparedArticle CreateArticle(string messageId, string body = "body\r\n") + { + var parser = new NntpArticleParser("backfiller.test"); + var destuffed = BuildDestuffed(messageId, body); + var created = ArticleRecordFactory.TryCreate(parser, destuffed); + Assert.True(created.IsAccepted, created.ParseFailure.ToString()); + return new PreparedArticle(Guid.NewGuid(), created.Record, created.SelectedDateHeaderName); + } + + private const string BodyLine = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"; + + private static string BuildLargeBody(int minimumBytes) + { + var builder = new System.Text.StringBuilder(minimumBytes + BodyLine.Length); + while (builder.Length < minimumBytes) + { + builder.Append(BodyLine); + } + + return builder.ToString(); + } + + private static byte[] BuildDestuffed(string messageId, string body) => + System.Text.Encoding.ASCII.GetBytes( + "Path: peer.example\r\n" + + "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" + + "Message-ID: " + messageId + "\r\n" + + "Newsgroups: alt.test\r\n" + + "From: user@example.test\r\n" + + "Subject: s\r\n" + + "\r\n" + + body); + + private static async Task WaitUntilAsync(Func predicate, TimeSpan timeout) + { + var deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; + while (Environment.TickCount64 < deadline) + { + if (predicate()) + { + return; + } + + await Task.Delay(20); + } + + Assert.Fail("Timed out waiting for condition."); + } + + private readonly record struct PreparedArticle( + Guid RequestId, + ArticleRecord Record, + NntpArticleHeaderName SelectedDateHeaderName); +} + +internal static class VatpConnectionPoolTestExtensions +{ + public static int ConnectionCountForTest(this VatpConnectionPool pool, string host, int port) => + pool.GetEndpointConnectionCount(host, port); + + public static int AliveConnectionCount(this VatpConnectionPool pool, string host, int port) => + pool.GetEndpointAliveConnectionCount(host, port); +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs new file mode 100644 index 00000000..6a86336d --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpLoopbackTestServer.cs @@ -0,0 +1,661 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Security.Cryptography.X509Certificates; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; +using VectorNNTPD.Common.Transport.ArticleTransfer; + +namespace VectorNNTPD.TransitService.Tests.Transport.Vatp; + +internal sealed class VatpLoopbackTestServer : IAsyncDisposable +{ + private readonly TcpListener _listener; + private readonly X509Certificate2 _certificate; + private readonly ConcurrentDictionary _articles = new(); + private readonly CancellationTokenSource _cts = new(); + private int _activeSessions; + private Task? _acceptLoop; + + private VatpLoopbackTestServer(TcpListener listener, X509Certificate2 certificate, int port) + { + _listener = listener; + _certificate = certificate; + Port = port; + Host = "backfiller.test"; + } + + public int Port { get; } + + public string Host { get; } + + public int ActiveSessions => Volatile.Read(ref _activeSessions); + + /// DATA prefix used by accepted-byte probe transfer modes. + internal const int ProbePayloadBytes = 64; + + internal TransferMode Mode { get; set; } = TransferMode.Complete; + + /// OPEN StreamIds observed on this listener (client-assigned). + public ConcurrentBag ObservedOpenStreamIds { get; } = new(); + + /// + /// When set, after a stream terminalizes (END or CANCEL) the server emits late/spurious + /// frames for the finished StreamId and for a never-allocated StreamId. + /// + public bool EmitSpuriousFramesAfterTerminal { get; set; } + + public static VatpLoopbackTestServer Start(X509Certificate2? certificate = null) + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + certificate ??= VatpTestCertificates.CreateServerCertificate(); + var server = new VatpLoopbackTestServer(listener, certificate, port); + server._acceptLoop = server.AcceptLoopAsync(server._cts.Token); + return server; + } + + public void Register( + ArticleRecord record, + NntpArticleHeaderName selectedDateHeaderName, + TransferMode? modeOverride = null, + TaskCompletionSource? heldStarted = null) + { + _articles[record.ArtId] = new RegisteredArticle(record, selectedDateHeaderName, modeOverride, heldStarted); + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync().ConfigureAwait(false); + _listener.Stop(); + if (_acceptLoop is not null) + { + try + { + await _acceptLoop.ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + } + + _cts.Dispose(); + _certificate.Dispose(); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TcpClient tcp; + try + { + tcp = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _ = ex.GetType(); // observed: accept loop canceled during server stop + break; + } + + _ = RunSessionAsync(tcp, cancellationToken); + } + } + + private async Task RunSessionAsync(TcpClient tcp, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _activeSessions); + try + { + using var tcpDisposable = tcp; + // codeql[cs/local-not-disposed] SslStream ownership transfers to the VATP/TLS session fixture for connection lifetime. + var ssl = new SslStream(tcp.GetStream(), leaveInnerStreamOpen: false); + await ssl.AuthenticateAsServerAsync( + new SslServerAuthenticationOptions + { + ServerCertificate = _certificate, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }, + cancellationToken).ConfigureAwait(false); + var session = new Session( + ssl, + _articles, + () => Mode, + ObservedOpenStreamIds, + () => EmitSpuriousFramesAfterTerminal, + cancellationToken); + await session.RunAsync().ConfigureAwait(false); + await ssl.DisposeAsync().ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + catch (IOException ex) + { + _ = ex.GetType(); // observed: expected during test teardown/cleanup + } + finally + { + Interlocked.Decrement(ref _activeSessions); + } + } + + internal enum TransferMode + { + Complete, + FinWithoutEnd, + DisconnectMidData, + DisconnectMidMeta, + DisconnectAfterFinBeforeEnd, + + /// + /// Writes META + all DATA frames + END as one contiguous socket write so the + /// client must parse multiple VATP frames from coalesced TCP/TLS reads. + /// + CoalesceOutboundFrames, + + /// + /// Writes each outbound transfer byte in 1-byte socket writes to force + /// partial-frame reconstruction across many reads. + /// + TinyOutboundWrites, + + /// + /// After META, sends a DATA frame whose declared payload exceeds the negotiated + /// maxFramePayload (must be rejected by the client). + /// + OversizedDataFrame, + + /// + /// Sends META + one DATA chunk, then holds the stream open until CANCEL/disconnect + /// so concurrent-cancellation tests can cancel mid-transfer deterministically. + /// + HoldAfterFirstData, + + /// + /// After accepting OPEN, writes a META frame on StreamId 0 (parser must reject). + /// + MetaOnConnectionStreamId, + + /// Sends a META payload the client must reject before any DATA. + InvalidMeta, + + /// Sends a short DATA frame with FIN so the client copies it and then fails. + ShortDataFin, + + /// Sends one DATA chunk, then a later DATA frame the client rejects before copying. + DataThenSizeReject, + + /// Sends one DATA chunk, then a stream FAIL. + DataThenRemoteFail, + + /// Sends a full body whose META ArtHash does not match, then END. + CorruptArtHash, + } + + private readonly record struct RegisteredArticle( + ArticleRecord Record, + NntpArticleHeaderName SelectedDateHeaderName, + TransferMode? ModeOverride, + TaskCompletionSource? HeldStarted); + + private sealed class Session( + SslStream stream, + ConcurrentDictionary articles, + Func mode, + ConcurrentBag observedOpenStreamIds, + Func emitSpuriousAfterTerminal, + CancellationToken cancellationToken) + { + private readonly Dictionary _streams = new(); + private readonly ArticleTransferLimits _limits = ArticleTransferLimits.Default; + private uint _maxFramePayload = VatpProtocol.DefaultMaxFramePayload; + private bool _clientHelloComplete; + + // Never allocated by monotonic NNTPD client ids starting at 1. + private const uint NeverAllocatedStreamId = 0x7FFFFFFEu; + + public async Task RunAsync() + { + var buffer = ArrayPool.Shared.Rent(VatpProtocol.HeaderLengthBytes + (int)VatpProtocol.DefaultMaxFramePayload); + var buffered = 0; + try + { + while (!cancellationToken.IsCancellationRequested) + { + var read = await stream.ReadAsync(buffer.AsMemory(buffered, buffer.Length - buffered), cancellationToken) + .ConfigureAwait(false); + if (read == 0) + { + return; + } + + buffered += read; + var consumed = 0; + while (consumed < buffered) + { + var sequence = new ReadOnlySequence(buffer, consumed, buffered - consumed); + var parsed = VatpFrameParser.ParseOneFrame(in sequence, _maxFramePayload); + if (parsed.Status == VatpFrameParseStatus.Incomplete) + { + break; + } + + if (parsed.Status == VatpFrameParseStatus.Invalid) + { + return; + } + + consumed += (int)parsed.ConsumedBytes; + if (!await HandleFrameAsync(parsed.Frame!.Value).ConfigureAwait(false)) + { + return; + } + } + + if (consumed > 0) + { + buffer.AsSpan(consumed, buffered - consumed).CopyTo(buffer); + buffered -= consumed; + } + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private async Task HandleFrameAsync(VatpParsedFrame frame) + { + switch (frame.Header.Type) + { + case VatpFrameType.Hello: + return await HandleHelloAsync(frame).ConfigureAwait(false); + case VatpFrameType.Open: + return await HandleOpenAsync(frame).ConfigureAwait(false); + case VatpFrameType.Window: + await HandleWindowAsync(frame).ConfigureAwait(false); + return true; + case VatpFrameType.Cancel: + if (emitSpuriousAfterTerminal()) + { + await WriteSpuriousAfterTerminalAsync(frame.Header.StreamId).ConfigureAwait(false); + } + + _streams.Remove(frame.Header.StreamId); + return true; + default: + return true; + } + } + + private async Task HandleHelloAsync(VatpParsedFrame frame) + { + if (_clientHelloComplete + || !VatpHello.TryDecode(frame.Payload, out var hello, out _)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(VatpProtocol.ConnectionStreamId, VatpErrorCode.InvalidHello)) + .ConfigureAwait(false); + return false; + } + + _maxFramePayload = Math.Min(hello.MaxFramePayload, _limits.DefaultMaxFramePayload); + _clientHelloComplete = true; + await WriteFrameAsync(VatpFrameEncoder.EncodeHello(_maxFramePayload)).ConfigureAwait(false); + return true; + } + + private async Task HandleOpenAsync(VatpParsedFrame frame) + { + if (!VatpOpenPayload.TryDecode(frame.Payload, out var open, out _)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.InvalidFrameLength)) + .ConfigureAwait(false); + return true; + } + + if (!articles.TryGetValue(open.ArticleId, out var registered)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + return true; + } + + var record = registered.Record; + var meta = ArticleCanonicalTransferMeta.FromRecord(in record, registered.SelectedDateHeaderName); + var metaBytes = VatpMetaCodec.Encode(in meta); + var transferMode = registered.ModeOverride ?? mode(); + observedOpenStreamIds.Add(frame.Header.StreamId); + + if (transferMode == TransferMode.MetaOnConnectionStreamId) + { + var header = new byte[VatpProtocol.HeaderLengthBytes]; + VatpFrameHeader.Create( + VatpFrameType.Meta, + VatpProtocol.ConnectionStreamId, + (uint)metaBytes.Length, + flags: 0) + .WriteTo(header); + await stream.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(metaBytes, cancellationToken).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DisconnectMidMeta) + { + var metaFrame = VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes); + // Write only the 16-byte header so META never becomes usable. + await stream.WriteAsync(metaFrame.Header, cancellationToken).ConfigureAwait(false); + await stream.DisposeAsync().ConfigureAwait(false); + return false; + } + + if (transferMode == TransferMode.InvalidMeta) + { + var invalid = new byte[VatpProtocol.MetaPayloadLength]; + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, invalid)).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.CorruptArtHash) + { + var corrupt = new ArticleCanonicalTransferMeta( + meta.ArtHash ^ 1UL, + meta.ArtLines, + meta.ArtSize, + meta.SelectedDateHeaderName, + meta.Fields); + metaBytes = VatpMetaCodec.Encode(in corrupt); + } + + if (transferMode == TransferMode.ShortDataFin) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, record.ArtData.ToArray(), fin: true).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DataThenRemoteFail) + { + var artData = record.ArtData.ToArray(); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, artData, fin: false).ConfigureAwait(false); + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.DataThenSizeReject) + { + var artData = record.ArtData.ToArray(); + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await WriteProbeDataAsync(frame.Header.StreamId, artData, fin: false).ConfigureAwait(false); + var overflow = new byte[artData.Length - VatpLoopbackTestServer.ProbePayloadBytes + 1]; + await WriteFrameAsync(VatpFrameEncoder.EncodeData(frame.Header.StreamId, overflow, fin: false)) + .ConfigureAwait(false); + return true; + } + + var send = new SendStream( + frame.Header.StreamId, + record.ArtData.ToArray(), + new ArticleTransferWindow(_limits.InitialStreamWindowBytes, _limits.MaxStreamCreditBytes), + transferMode); + _streams[frame.Header.StreamId] = send; + + if (transferMode == TransferMode.OversizedDataFrame) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)) + .ConfigureAwait(false); + await WriteOversizedDataFrameAsync(frame.Header.StreamId).ConfigureAwait(false); + return true; + } + + if (transferMode is TransferMode.CoalesceOutboundFrames or TransferMode.TinyOutboundWrites) + { + await WriteTransferBatchAsync(send, metaBytes).ConfigureAwait(false); + return true; + } + + if (transferMode == TransferMode.HoldAfterFirstData) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)) + .ConfigureAwait(false); + var firstChunk = (int)Math.Min( + Math.Min(send.SendWindow.Credit, _maxFramePayload), + send.ArtData.Length); + if (firstChunk <= 0 || !send.SendWindow.TryConsume(firstChunk)) + { + return true; + } + + send.SentBytes = firstChunk; + await WriteFrameAsync( + VatpFrameEncoder.EncodeData( + frame.Header.StreamId, + send.ArtData.AsMemory(0, firstChunk), + fin: false)) + .ConfigureAwait(false); + registered.HeldStarted?.TrySetResult(); + return true; + } + + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(frame.Header.StreamId, metaBytes)).ConfigureAwait(false); + await PumpStreamAsync(send).ConfigureAwait(false); + return true; + } + + private async Task WriteTransferBatchAsync(SendStream sendStream, ReadOnlyMemory metaBytes) + { + var frames = new List(); + frames.Add(VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeMeta(sendStream.StreamId, metaBytes))); + + var offset = 0; + while (offset < sendStream.ArtData.Length) + { + var remaining = sendStream.ArtData.Length - offset; + var chunkSize = (int)Math.Min(_maxFramePayload, remaining); + var fin = offset + chunkSize >= sendStream.ArtData.Length; + var chunk = sendStream.ArtData.AsMemory(offset, chunkSize); + frames.Add(VatpFrameEncoder.ToSingleBuffer( + VatpFrameEncoder.EncodeData(sendStream.StreamId, chunk, fin))); + offset += chunkSize; + } + + frames.Add(VatpFrameEncoder.ToSingleBuffer(VatpFrameEncoder.EncodeEnd(sendStream.StreamId))); + + var total = 0; + foreach (var frame in frames) + { + total += frame.Length; + } + + var batch = new byte[total]; + var written = 0; + foreach (var frame in frames) + { + frame.AsSpan().CopyTo(batch.AsSpan(written)); + written += frame.Length; + } + + if (sendStream.TransferMode == TransferMode.TinyOutboundWrites) + { + for (var i = 0; i < batch.Length; i++) + { + await stream.WriteAsync(batch.AsMemory(i, 1), cancellationToken).ConfigureAwait(false); + } + } + else + { + await stream.WriteAsync(batch, cancellationToken).ConfigureAwait(false); + } + + sendStream.SentBytes = sendStream.ArtData.Length; + _streams.Remove(sendStream.StreamId); + } + + private async Task WriteOversizedDataFrameAsync(uint streamId) + { + var oversizedPayload = (int)_maxFramePayload + 1; + var header = new byte[VatpProtocol.HeaderLengthBytes]; + VatpFrameHeader.Create(VatpFrameType.Data, streamId, (uint)oversizedPayload, flags: 0) + .WriteTo(header); + var payload = new byte[oversizedPayload]; + payload.AsSpan().Fill(0x41); + await stream.WriteAsync(header, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(payload, cancellationToken).ConfigureAwait(false); + } + + private async Task HandleWindowAsync(VatpParsedFrame frame) + { + if (!_streams.TryGetValue(frame.Header.StreamId, out var sendStream)) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeFail(frame.Header.StreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + return; + } + + Span windowBytes = stackalloc byte[VatpProtocol.WindowPayloadLength]; + frame.Payload.CopyTo(windowBytes); + if (!VatpControlPayload.TryDecodeWindow(windowBytes, out var credit)) + { + return; + } + + _ = sendStream.SendWindow.Add(credit); + await PumpStreamAsync(sendStream).ConfigureAwait(false); + } + + private async Task PumpStreamAsync(SendStream sendStream) + { + if (sendStream.TransferMode == TransferMode.HoldAfterFirstData) + { + return; + } + + while (sendStream.SentBytes < sendStream.ArtData.Length && sendStream.SendWindow.Credit > 0) + { + if (sendStream.TransferMode == TransferMode.DisconnectMidData && sendStream.SentBytes > 0) + { + await stream.DisposeAsync().ConfigureAwait(false); + return; + } + + var remaining = sendStream.ArtData.Length - sendStream.SentBytes; + var chunkSize = (int)Math.Min(Math.Min(sendStream.SendWindow.Credit, _maxFramePayload), remaining); + if (chunkSize <= 0) + { + return; + } + + if (!sendStream.SendWindow.TryConsume(chunkSize)) + { + return; + } + + sendStream.SentBytes += chunkSize; + var fin = sendStream.SentBytes >= sendStream.ArtData.Length; + var chunk = sendStream.ArtData.AsMemory(sendStream.SentBytes - chunkSize, chunkSize); + await WriteFrameAsync(VatpFrameEncoder.EncodeData(sendStream.StreamId, chunk, fin)).ConfigureAwait(false); + + if (fin && sendStream.TransferMode == TransferMode.FinWithoutEnd) + { + return; + } + + if (fin && sendStream.TransferMode == TransferMode.DisconnectAfterFinBeforeEnd) + { + await stream.DisposeAsync().ConfigureAwait(false); + return; + } + + if (fin && sendStream.TransferMode is TransferMode.Complete or TransferMode.CorruptArtHash) + { + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(sendStream.StreamId)).ConfigureAwait(false); + _streams.Remove(sendStream.StreamId); + if (emitSpuriousAfterTerminal()) + { + await WriteSpuriousAfterTerminalAsync(sendStream.StreamId).ConfigureAwait(false); + } + } + } + } + + private async Task WriteSpuriousAfterTerminalAsync(uint terminalStreamId) + { + // Late frames for a stream the client already (or will imminently) remove. + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(terminalStreamId)).ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(terminalStreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(terminalStreamId, VatpErrorCode.OpenRejected)) + .ConfigureAwait(false); + + // Completely unallocated StreamId: must not create client stream state. + var junkMeta = new byte[VatpProtocol.MetaPayloadLength]; + await WriteFrameAsync(VatpFrameEncoder.EncodeMeta(NeverAllocatedStreamId, junkMeta)) + .ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeData(NeverAllocatedStreamId, new byte[] { 0x41 }, fin: true)) + .ConfigureAwait(false); + await WriteFrameAsync(VatpFrameEncoder.EncodeEnd(NeverAllocatedStreamId)).ConfigureAwait(false); + await WriteFrameAsync( + VatpFrameEncoder.EncodeFail(NeverAllocatedStreamId, VatpErrorCode.UnknownStream)) + .ConfigureAwait(false); + } + + private async Task WriteProbeDataAsync(uint streamId, byte[] artData, bool fin) + { + if (artData.Length <= VatpLoopbackTestServer.ProbePayloadBytes) + { + throw new InvalidOperationException("Article is smaller than the accepted-data probe."); + } + + await WriteFrameAsync( + VatpFrameEncoder.EncodeData( + streamId, + artData.AsMemory(0, VatpLoopbackTestServer.ProbePayloadBytes), + fin)) + .ConfigureAwait(false); + } + + private async Task WriteFrameAsync(VatpFrameEncoder.EncodedFrame frame) + { + if (frame.Header.Length > 0) + { + await stream.WriteAsync(frame.Header, cancellationToken).ConfigureAwait(false); + } + + if (frame.Payload.Length > 0) + { + await stream.WriteAsync(frame.Payload, cancellationToken).ConfigureAwait(false); + } + } + } + + private sealed class SendStream( + uint streamId, + byte[] artData, + ArticleTransferWindow sendWindow, + TransferMode transferMode) + { + public uint StreamId { get; } = streamId; + + public byte[] ArtData { get; } = artData; + + public ArticleTransferWindow SendWindow { get; } = sendWindow; + + public TransferMode TransferMode { get; } = transferMode; + + public int SentBytes { get; set; } + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpTestCertificates.cs b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpTestCertificates.cs new file mode 100644 index 00000000..d0d0d254 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/Transport/Vatp/VatpTestCertificates.cs @@ -0,0 +1,34 @@ +using System.Net; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; + +namespace VectorNNTPD.TransitService.Tests.Transport.Vatp; + +internal static class VatpTestCertificates +{ + internal static X509Certificate2 CreateServerCertificate(string commonName = "backfiller.test") + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest( + "CN=" + commonName, + rsa, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + request.CertificateExtensions.Add( + new X509KeyUsageExtension( + X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, + critical: true)); + var san = new SubjectAlternativeNameBuilder(); + san.AddDnsName("localhost"); + san.AddDnsName(commonName); + san.AddIpAddress(IPAddress.Loopback); + request.CertificateExtensions.Add(san.Build()); + using var created = request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow.AddDays(30)); + return X509CertificateLoader.LoadPkcs12( + created.Export(X509ContentType.Pfx, "test"), + "test", + X509KeyStorageFlags.Exportable | X509KeyStorageFlags.UserKeySet); + } +} diff --git a/tests/VectorNNTPD.TransitService.Tests/VectorNNTPD.TransitService.Tests.csproj b/tests/VectorNNTPD.TransitService.Tests/VectorNNTPD.TransitService.Tests.csproj new file mode 100644 index 00000000..b66028e0 --- /dev/null +++ b/tests/VectorNNTPD.TransitService.Tests/VectorNNTPD.TransitService.Tests.csproj @@ -0,0 +1,46 @@ + + + + net10.0 + false + true + VectorNNTPD.TransitService.Tests + false + $(NoWarn);CS1591;CS0067 + x64 + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + diff --git a/tools/Filter-CodeQlSarif.ps1 b/tools/Filter-CodeQlSarif.ps1 new file mode 100644 index 00000000..862c6380 --- /dev/null +++ b/tools/Filter-CodeQlSarif.ps1 @@ -0,0 +1,134 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Filters CodeQL SARIF for GitHub upload and local ACTIVE counting. + +.DESCRIPTION + Traced C# CodeQL extraction includes compiler and source-generator output under + **/obj/** and **/bin/**. Repository paths-ignore does not drop those compiled files + (see Invoke-CodeQlAnalysis.ps1). This filter: + + 1. Removes results whose primary location is under bin/ or obj/. + 2. Normalizes CodeQL AlertSuppression entries ({ state: accepted }) to SARIF 2.1 + suppressions ({ kind: inSource, status: accepted }) that GitHub code scanning + recognizes. + 3. Optionally strips accepted in-source suppressions from the uploaded result set + so PR "new alert" counts do not treat already-justified findings as actionable. + In-source // codeql[...] comments remain the source of truth for local analysis. + +.PARAMETER InputPath + Path to the SARIF file produced by codeql database analyze. + +.PARAMETER OutputPath + Path to write the filtered SARIF. May equal InputPath. + +.PARAMETER StripAcceptedSuppressions + When set, omit results that carry an accepted suppression after normalization. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $InputPath, + + [Parameter(Mandatory = $true)] + [string] $OutputPath, + + [switch] $StripAcceptedSuppressions +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +if (-not (Test-Path -LiteralPath $InputPath)) { + throw "SARIF input was not found at $InputPath" +} + +$parent = Split-Path -Parent $OutputPath +if ($parent -and -not (Test-Path -LiteralPath $parent)) { + New-Item -ItemType Directory -Force -Path $parent | Out-Null +} + +$python = Get-Command python -ErrorAction SilentlyContinue +if (-not $python) { + throw "python was not found on PATH; required to filter SARIF without corrupting large JSON." +} + +$script = @' +import json +import re +import sys +from pathlib import Path + +inp, outp = Path(sys.argv[1]), Path(sys.argv[2]) +strip_accepted = sys.argv[3] == "1" +sarif = json.loads(inp.read_text(encoding="utf-8")) +pattern = re.compile(r"(^|/|\\)(bin|obj)(/|\\|$)") +removed_gen = removed_supp = kept = normalized = 0 + +def normalize_suppressions(result): + global normalized + supps = result.get("suppressions") + if not supps: + return False + accepted = False + rewritten = [] + for item in supps: + if not isinstance(item, dict): + rewritten.append(item) + continue + status = item.get("status") or item.get("state") + kind = item.get("kind") or "inSource" + if status == "accepted": + accepted = True + if item.get("status") != "accepted" or item.get("kind") != "inSource" or "state" in item: + normalized += 1 + rewritten.append({"kind": kind if kind in ("inSource", "external") else "inSource", "status": "accepted"}) + else: + rewritten.append(item) + result["suppressions"] = rewritten + return accepted + +for run in sarif.get("runs") or []: + results = run.get("results") or [] + filtered = [] + for result in results: + uri = "" + locs = result.get("locations") or [] + if locs: + uri = ( + (locs[0].get("physicalLocation") or {}) + .get("artifactLocation", {}) + .get("uri", "") + or "" + ) + normalized_uri = uri.replace("\\", "/") + if pattern.search(normalized_uri): + removed_gen += 1 + continue + accepted = normalize_suppressions(result) + if strip_accepted and accepted: + removed_supp += 1 + continue + filtered.append(result) + kept += 1 + run["results"] = filtered + +outp.write_text(json.dumps(sarif, ensure_ascii=False, indent=2), encoding="utf-8") +print( + f"Filter-CodeQlSarif kept={kept} removed_gen={removed_gen} " + f"removed_supp={removed_supp} normalized={normalized} output={outp}" +) +'@ + +$tempPy = [System.IO.Path]::GetTempFileName() + ".py" +try { + Set-Content -LiteralPath $tempPy -Value $script -Encoding utf8 + $stripFlag = if ($StripAcceptedSuppressions) { "1" } else { "0" } + & python $tempPy $InputPath $OutputPath $stripFlag + if ($LASTEXITCODE -ne 0) { + throw "Filter-CodeQlSarif python step exited with code $LASTEXITCODE" + } +} +finally { + Remove-Item -LiteralPath $tempPy -Force -ErrorAction SilentlyContinue +} diff --git a/tools/Invoke-CodeQlAnalysis.ps1 b/tools/Invoke-CodeQlAnalysis.ps1 index 1f01284c..4698b85c 100644 --- a/tools/Invoke-CodeQlAnalysis.ps1 +++ b/tools/Invoke-CodeQlAnalysis.ps1 @@ -5,13 +5,15 @@ .DESCRIPTION Creates a CodeQL database by tracing the repository's Release x64 restore and - build, then analyzes that database with the queries selected by + non-incremental build, then analyzes that database with the queries selected by .github/codeql/codeql-config.yml. That config selects .github/codeql/csharp-with-alert-suppression.qls, which imports the security-extended and security-and-quality suites and adds AlertSuppression.ql. Analyze is invoked with no extra query arguments, so it uses the queries stored from that config at database creation. That is the same selection the - GitHub Action analyzes with database run-queries. + GitHub Action analyzes with database run-queries. The traced build uses + --no-incremental so MSBuild always runs CoreCompile/csc; an up-to-date + incremental skip would leave project sources out of the database. The script checks that the local suite resolves to those two suites plus AlertSuppression.ql exactly once. It does not download query packs. Evidence @@ -20,7 +22,8 @@ database analyze to exit 0. For a traced C# build, the CLI records that paths and paths-ignore do not - filter compiled files. + filter compiled files. After analyze, this script runs Filter-CodeQlSarif.ps1 + so local results.sarif matches the filtered upload used by CI. .PARAMETER CodeQl Path to codeql.exe. Defaults to C:\Tools\codeql\codeql.exe. @@ -77,11 +80,13 @@ if (-not (Test-Path -LiteralPath $config)) { throw "CodeQL configuration was not found at $config" } +# --no-incremental forces CoreCompile/csc so the CodeQL tracer sees every project. +# An up-to-date incremental skip leaves only the .csproj in the database (no .cs TRAPs). @( "@echo off" "dotnet restore VectorNNTPD.slnx -p:Platform=x64" "if errorlevel 1 exit /b %errorlevel%" - "dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore" + "dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore --no-incremental" "if errorlevel 1 exit /b %errorlevel%" ) | Set-Content -LiteralPath $buildCommand -Encoding ascii @@ -150,14 +155,20 @@ try { "--command", $buildCommand ) + $rawSarif = Join-Path $evidenceRoot "results.raw.sarif" + $filteredSarif = Join-Path $evidenceRoot "results.sarif" Invoke-CodeQl -LogPath (Join-Path $evidenceRoot "database-analyze.log") -Arguments @( "database", "analyze", $database, "--format=sarifv2.1.0", - "--output", (Join-Path $evidenceRoot "results.sarif"), + "--output", $rawSarif, "--threads=0", "--no-sarif-minify", "--sarif-category=csharp" ) + + # Local results.sarif keeps suppressed first-party findings (normalized) so ACTIVE/SUPP + # counts remain auditable; CI upload uses -StripAcceptedSuppressions. + & (Join-Path $PSScriptRoot "Filter-CodeQlSarif.ps1") -InputPath $rawSarif -OutputPath $filteredSarif } finally { Pop-Location diff --git a/tools/Publish-Release.ps1 b/tools/Publish-Release.ps1 index e2da292a..06ee2bd9 100644 --- a/tools/Publish-Release.ps1 +++ b/tools/Publish-Release.ps1 @@ -5,13 +5,22 @@ .DESCRIPTION Resolves this standalone repository from the script file, not the caller's - working directory. Publishes VectorNNTPD.BackFillerService and - VectorNNTPD.StorageService for win-x64, linux-x64, or both into one flat - directory: release/win-64 or release/linux-64. Identical shared files such as - VectorNNTPD.RabbitMQ.json are copied once; a conflicting file fails the script. - The linux-64 directory also contains the BackFiller and StorageService systemd - units, environment-file examples, and README.md. - A failed publish leaves that platform directory unchanged and exits 1. + working directory. Publishes VectorNNTPD.BackFillerService, VectorNNTPD.StorageService, + VectorNNTPD.ReaderService, and VectorNNTPD.TransitService for win-x64, linux-x64, + or both into per-service package directories under release/win-64 or release/linux-64: + + release//backfiller/ + release//storage/ + release//reader/ + release//transit/ + + Each package contains that host's Native AOT executable, its primary JSON, a copy of + VectorNNTPD.RabbitMQ.json, and (linux-64 only) the matching systemd unit, environment + example, and README.md. Packages are independently installable; Reader and Transit never + share a directory. Publishing stages under a temp directory and validates before + replacing release//. On failure before the swap, the prior release tree is + left intact. On failure after the swap, the previous tree is restored when available. + The script exits 1 on any failure. .PARAMETER Platform win-64, linux-64, or all. @@ -30,46 +39,63 @@ $Solution = Join-Path $RepoRoot 'VectorNNTPD.slnx' $Applications = @( @{ - Name = 'VectorNNTPD.BackFillerService' - Project = Join-Path $RepoRoot 'src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj' - Exe = 'VectorNNTPD.BackFillerService' + Name = 'VectorNNTPD.BackFillerService' + Package = 'backfiller' + Project = Join-Path $RepoRoot 'src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj' + Exe = 'VectorNNTPD.BackFillerService' + ConfigFile = 'VectorNNTPD.BackFillerService.json' + LinuxUnit = 'vectornntp-backfiller.service' + LinuxEnvExample = 'backfiller.env.example' }, @{ - Name = 'VectorNNTPD.StorageService' - Project = Join-Path $RepoRoot 'src\VectorNNTPD.StorageService\VectorNNTPD.StorageService.csproj' - Exe = 'VectorNNTPD.StorageService' - } -) - -$RequiredPublishFiles = @( - 'VectorNNTPD.BackFillerService.json', - 'VectorNNTPD.StorageService.json', - 'VectorNNTPD.RabbitMQ.json' -) - -$LinuxExtraFiles = @( - @{ - SourceName = 'vectornntp-backfiller.service' - Source = Join-Path (Join-Path (Join-Path $RepoRoot 'deploy') 'systemd') 'vectornntp-backfiller.service' + Name = 'VectorNNTPD.StorageService' + Package = 'storage' + Project = Join-Path $RepoRoot 'src\VectorNNTPD.StorageService\VectorNNTPD.StorageService.csproj' + Exe = 'VectorNNTPD.StorageService' + ConfigFile = 'VectorNNTPD.StorageService.json' + LinuxUnit = 'vectornntp-storageservice.service' + LinuxEnvExample = 'storageservice.env.example' }, @{ - SourceName = 'backfiller.env.example' - Source = Join-Path (Join-Path (Join-Path $RepoRoot 'deploy') 'systemd') 'backfiller.env.example' + Name = 'VectorNNTPD.ReaderService' + Package = 'reader' + Project = Join-Path $RepoRoot 'src\VectorNNTPD.ReaderService\VectorNNTPD.ReaderService.csproj' + Exe = 'VectorNNTPD.ReaderService' + ConfigFile = 'VectorNNTPD.ReaderService.json' + LinuxUnit = 'vectornntp-reader.service' + LinuxEnvExample = 'reader.env.example' }, @{ - SourceName = 'vectornntp-storageservice.service' - Source = Join-Path (Join-Path (Join-Path $RepoRoot 'deploy') 'systemd') 'vectornntp-storageservice.service' - }, - @{ - SourceName = 'storageservice.env.example' - Source = Join-Path (Join-Path (Join-Path $RepoRoot 'deploy') 'systemd') 'storageservice.env.example' - }, - @{ - SourceName = 'README.md' - Source = Join-Path $RepoRoot 'README.md' + Name = 'VectorNNTPD.TransitService' + Package = 'transit' + Project = Join-Path $RepoRoot 'src\VectorNNTPD.TransitService\VectorNNTPD.TransitService.csproj' + Exe = 'VectorNNTPD.TransitService' + ConfigFile = 'VectorNNTPD.TransitService.json' + LinuxUnit = 'vectornntp-transit.service' + LinuxEnvExample = 'transit.env.example' } ) +$SharedConfigFile = 'VectorNNTPD.RabbitMQ.json' +$DeploySystemd = Join-Path (Join-Path $RepoRoot 'deploy') 'systemd' +$ReadmeSource = Join-Path $RepoRoot 'README.md' + +$ObsoleteNames = @( + 'appsettings.json', + 'RabbitMq.json', + 'VectorNNTP.RabbitMQ.json', + 'VectorNNTP.StorageServer.json', + 'VectorNNTP.StorageServer.Development.json', + 'VectorNNTP.BackFiller.json', + 'VectorNNTPD.StorageService.Development.json', + 'VectorNNTPD.BackFillerService.Development.json', + 'VectorNNTPD.ReaderService.Development.json', + 'VectorNNTPD.TransitService.Development.json', + 'VectorNNTPD.NntpServer.dll', + 'VectorNNTPD.NntpServer.exe', + 'libVectorNNTPD.NntpServer.so' +) + function Fail { param([string] $Message) throw $Message @@ -152,7 +178,7 @@ function Test-NativeAotBinary { } } -function Merge-PublishOutput { +function Copy-FlatPublishFiles { param( [string] $Source, [string] $Destination @@ -161,125 +187,164 @@ function Merge-PublishOutput { $entries = @(Get-ChildItem -LiteralPath $Source -Force) foreach ($entry in $entries) { if ($entry.PSIsContainer) { - Fail "Publish output '$($entry.FullName)' contains a directory. Release output must stay flat." + # Native AOT publish may create empty helper directories; packages stay flat files only. + continue } - $target = Join-Path $Destination $entry.Name - if (Test-Path -LiteralPath $target) { - $sourceHash = (Get-FileHash -LiteralPath $entry.FullName -Algorithm SHA256).Hash - $targetHash = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash - if ($sourceHash -eq $targetHash) { - continue - } - - $extension = [System.IO.Path]::GetExtension($entry.Name) - if ($extension -eq '.pdb' -or $extension -eq '.dbg' -or $extension -eq '.xml') { - continue - } - - Fail "Publish file '$($entry.Name)' differs between applications and cannot share one release directory." + $extension = [System.IO.Path]::GetExtension($entry.Name) + if ($extension -eq '.pdb' -or $extension -eq '.dbg' -or $extension -eq '.xml') { + continue } - Copy-Item -LiteralPath $entry.FullName -Destination $target + Copy-Item -LiteralPath $entry.FullName -Destination (Join-Path $Destination $entry.Name) -Force } } -function Test-PublishedRelease { +function Test-ServicePackage { param( - [string] $ReleaseDir, + [hashtable] $App, + [string] $PackageDir, [string] $Rid, [string] $ExeSuffix ) - if (-not (Test-Path -LiteralPath $ReleaseDir)) { - Fail "Release directory was not created: $ReleaseDir" + if (-not (Test-Path -LiteralPath $PackageDir)) { + Fail "Package directory was not created: $PackageDir" } - $nested = @(Get-ChildItem -LiteralPath $ReleaseDir -Directory -Force) + $nested = @(Get-ChildItem -LiteralPath $PackageDir -Directory -Force) if ($nested.Count -gt 0) { - Fail "Release directory contains nested directories: $($nested.Name -join ', ')." + Fail "Service package must stay flat (no nested directories): $PackageDir contains $($nested.Name -join ', ')." } - foreach ($app in $Applications) { - $exePath = Join-Path $ReleaseDir ($app.Exe + $ExeSuffix) - if (-not (Test-Path -LiteralPath $exePath)) { - Fail "Missing release executable: $exePath" - } - - $exe = Get-Item -LiteralPath $exePath - if ($exe.Length -le 0) { - Fail "Native executable is empty: $exePath" - } + $exePath = Join-Path $PackageDir ($App.Exe + $ExeSuffix) + if (-not (Test-Path -LiteralPath $exePath)) { + Fail "Missing release executable: $exePath" + } - $managedPath = Join-Path $ReleaseDir ($app.Exe + '.dll') - if (Test-Path -LiteralPath $managedPath) { - Fail "Managed assembly '$($app.Exe).dll' is present. Native AOT output must not include it." - } + $exe = Get-Item -LiteralPath $exePath + if ($exe.Length -le 0) { + Fail "Native executable is empty: $exePath" + } - Test-NativeAotBinary -Path $exePath -Rid $Rid + $managedPath = Join-Path $PackageDir ($App.Exe + '.dll') + if (Test-Path -LiteralPath $managedPath) { + Fail "Managed assembly '$($App.Exe).dll' is present. Native AOT output must not include it." } + Test-NativeAotBinary -Path $exePath -Rid $Rid + foreach ($runtimeFile in @('hostfxr.dll', 'coreclr.dll', 'libhostfxr.so', 'libcoreclr.so', 'hostfxr.dylib', 'libcoreclr.dylib')) { - $runtimePath = Join-Path $ReleaseDir $runtimeFile + $runtimePath = Join-Path $PackageDir $runtimeFile if (Test-Path -LiteralPath $runtimePath) { - Fail "Shared runtime file '$runtimeFile' is present. The release is not a self-contained Native AOT layout." + Fail "Shared runtime file '$runtimeFile' is present in $($App.Package). The release is not self-contained Native AOT." } } - foreach ($fileName in $RequiredPublishFiles) { - $filePath = Join-Path $ReleaseDir $fileName - if (-not (Test-Path -LiteralPath $filePath)) { - Fail "Missing runtime configuration file: $filePath" + foreach ($required in @($App.ConfigFile, $SharedConfigFile)) { + $path = Join-Path $PackageDir $required + if (-not (Test-Path -LiteralPath $path)) { + Fail "Missing configuration file in $($App.Package): $path" } } - foreach ($obsoleteName in @( - 'appsettings.json', - 'RabbitMq.json', - 'VectorNNTP.RabbitMQ.json', - 'VectorNNTP.StorageServer.json', - 'VectorNNTP.StorageServer.Development.json', - 'VectorNNTP.BackFiller.json', - 'VectorNNTPD.StorageService.Development.json', - 'VectorNNTPD.BackFillerService.Development.json' - )) { - $obsoletePath = Join-Path $ReleaseDir $obsoleteName + # Role isolation: other hosts' primary JSON must not appear in this package. + foreach ($other in $Applications) { + if ($other.Name -eq $App.Name) { + continue + } + + $foreign = Join-Path $PackageDir $other.ConfigFile + if (Test-Path -LiteralPath $foreign) { + Fail "Foreign host configuration '$($other.ConfigFile)' must not appear in package $($App.Package)." + } + + $foreignExe = Join-Path $PackageDir ($other.Exe + $ExeSuffix) + if (Test-Path -LiteralPath $foreignExe) { + Fail "Foreign executable '$($other.Exe)$ExeSuffix' must not appear in package $($App.Package)." + } + } + + foreach ($obsoleteName in $ObsoleteNames) { + $obsoletePath = Join-Path $PackageDir $obsoleteName if (Test-Path -LiteralPath $obsoletePath) { - Fail "Obsolete or non-production configuration file must not be packaged: $obsoletePath" + Fail "Obsolete or forbidden file must not be packaged: $obsoletePath" } } - $rabbitFiles = @(Get-ChildItem -LiteralPath $ReleaseDir -File -Filter 'VectorNNTPD.RabbitMQ.json' -Force) - if ($rabbitFiles.Count -ne 1) { - Fail "Expected exactly one VectorNNTPD.RabbitMQ.json in $ReleaseDir and found $($rabbitFiles.Count)." + $nntpServerHits = @(Get-ChildItem -LiteralPath $PackageDir -Force -File | Where-Object { + $_.Name -like '*NntpServer*' + }) + if ($nntpServerHits.Count -gt 0) { + Fail "Package $($App.Package) must not contain NntpServer artifacts: $($nntpServerHits.Name -join ', ')." } if ($Rid -eq 'linux-x64') { - foreach ($extra in $LinuxExtraFiles) { - $path = Join-Path $ReleaseDir $extra.SourceName + foreach ($name in @($App.LinuxUnit, $App.LinuxEnvExample, 'README.md')) { + $path = Join-Path $PackageDir $name if (-not (Test-Path -LiteralPath $path)) { - if ($extra.SourceName -like '*.env.example') { - Fail "Missing environment-file example in the linux-x64 release: $path" - } + Fail "Missing Linux package file in $($App.Package): $path" + } + } - Fail "Missing Linux release file: $path" + # Other roles' units/env must not leak into this package. + foreach ($other in $Applications) { + if ($other.Name -eq $App.Name) { + continue + } + + foreach ($name in @($other.LinuxUnit, $other.LinuxEnvExample)) { + $path = Join-Path $PackageDir $name + if (Test-Path -LiteralPath $path) { + Fail "Foreign Linux file '$name' must not appear in package $($App.Package)." + } } } } else { - foreach ($extra in $LinuxExtraFiles) { - $path = Join-Path $ReleaseDir $extra.SourceName + foreach ($name in @($App.LinuxUnit, $App.LinuxEnvExample)) { + $path = Join-Path $PackageDir $name if (Test-Path -LiteralPath $path) { - if ($extra.SourceName -like '*.env.example') { - Fail "Environment-file example must not be packaged for ${Rid}: $path" - } - Fail "Linux-only file must not be packaged for ${Rid}: $path" } } } } +function Test-PlatformRelease { + param( + [string] $ReleaseDir, + [string] $Rid, + [string] $ExeSuffix + ) + + if (-not (Test-Path -LiteralPath $ReleaseDir)) { + Fail "Release directory was not created: $ReleaseDir" + } + + $expected = @($Applications | ForEach-Object { $_.Package }) + $dirs = @(Get-ChildItem -LiteralPath $ReleaseDir -Directory -Force | Select-Object -ExpandProperty Name) + foreach ($name in $expected) { + if ($dirs -notcontains $name) { + Fail "Missing service package directory: $(Join-Path $ReleaseDir $name)" + } + } + + $unexpected = @($dirs | Where-Object { $expected -notcontains $_ }) + if ($unexpected.Count -gt 0) { + Fail "Unexpected directories under $ReleaseDir : $($unexpected -join ', ')." + } + + $looseFiles = @(Get-ChildItem -LiteralPath $ReleaseDir -File -Force) + if ($looseFiles.Count -gt 0) { + Fail "Platform release root must contain only service package directories. Loose files: $($looseFiles.Name -join ', ')." + } + + foreach ($app in $Applications) { + Test-ServicePackage -App $app -PackageDir (Join-Path $ReleaseDir $app.Package) -Rid $Rid -ExeSuffix $ExeSuffix + } +} + function Publish-Platform { param( [string] $Folder, @@ -290,54 +355,92 @@ function Publish-Platform { $releaseDir = Join-Path (Join-Path $RepoRoot 'release') $Folder $stagingRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("vectornntpd-release-" + [System.Guid]::NewGuid().ToString('N')) New-Item -ItemType Directory -Path $stagingRoot | Out-Null - $stageMerged = Join-Path $stagingRoot 'merged' - New-Item -ItemType Directory -Path $stageMerged | Out-Null - $replacedRelease = $false + $stageRoot = Join-Path $stagingRoot 'packages' + New-Item -ItemType Directory -Path $stageRoot | Out-Null + $previousBackup = $null + $swapStarted = $false try { foreach ($app in $Applications) { if (-not (Test-Path -LiteralPath $app.Project)) { Fail "Project file was not found: $($app.Project)" } - $stage = Join-Path $stagingRoot $app.Name + $publishStage = Join-Path $stagingRoot ("publish-" + $app.Name) Write-Host "dotnet publish $($app.Name) -c Release -p:Platform=x64 -r $Rid --self-contained true -p:PublishAot=true -o " - & dotnet publish $app.Project -c Release -p:Platform=x64 -r $Rid --self-contained true -p:PublishAot=true -p:SelfContained=true -o $stage --nologo + & dotnet publish $app.Project -c Release -p:Platform=x64 -r $Rid --self-contained true -p:PublishAot=true -p:SelfContained=true -o $publishStage --nologo if ($LASTEXITCODE -ne 0) { Fail "dotnet publish failed for $($app.Name) ($Rid) with exit code $LASTEXITCODE." } - $publishedExe = Join-Path $stage ($app.Exe + $ExeSuffix) + $publishedExe = Join-Path $publishStage ($app.Exe + $ExeSuffix) if (-not (Test-Path -LiteralPath $publishedExe)) { Fail "Publish did not produce $($app.Exe)$ExeSuffix for $($app.Name)." } - Merge-PublishOutput -Source $stage -Destination $stageMerged - } - - if ($Rid -eq 'linux-x64') { - foreach ($extra in $LinuxExtraFiles) { - if (-not (Test-Path -LiteralPath $extra.Source)) { - Fail "Linux release source was not found: $($extra.Source)" + $packageDir = Join-Path $stageRoot $app.Package + New-Item -ItemType Directory -Path $packageDir | Out-Null + Copy-FlatPublishFiles -Source $publishStage -Destination $packageDir + + if ($Rid -eq 'linux-x64') { + $unitSource = Join-Path $DeploySystemd $app.LinuxUnit + $envSource = Join-Path $DeploySystemd $app.LinuxEnvExample + foreach ($src in @($unitSource, $envSource, $ReadmeSource)) { + if (-not (Test-Path -LiteralPath $src)) { + Fail "Linux release source was not found: $src" + } } - Copy-Item -LiteralPath $extra.Source -Destination (Join-Path $stageMerged $extra.SourceName) + Copy-Item -LiteralPath $unitSource -Destination (Join-Path $packageDir $app.LinuxUnit) + Copy-Item -LiteralPath $envSource -Destination (Join-Path $packageDir $app.LinuxEnvExample) + Copy-Item -LiteralPath $ReadmeSource -Destination (Join-Path $packageDir 'README.md') } + + Test-ServicePackage -App $app -PackageDir $packageDir -Rid $Rid -ExeSuffix $ExeSuffix } - Test-PublishedRelease -ReleaseDir $stageMerged -Rid $Rid -ExeSuffix $ExeSuffix + Test-PlatformRelease -ReleaseDir $stageRoot -Rid $Rid -ExeSuffix $ExeSuffix - $replacedRelease = $true + # Swap only after the staged packages validate. Preserve the prior tree until + # the new release directory also validates, then discard the backup. + $swapStarted = $true if (Test-Path -LiteralPath $releaseDir) { - Remove-Item -LiteralPath $releaseDir -Recurse -Force + $previousBackup = $releaseDir + '.previous' + if (Test-Path -LiteralPath $previousBackup) { + Remove-Item -LiteralPath $previousBackup -Recurse -Force + } + + Move-Item -LiteralPath $releaseDir -Destination $previousBackup } New-Item -ItemType Directory -Path $releaseDir | Out-Null - Merge-PublishOutput -Source $stageMerged -Destination $releaseDir - Test-PublishedRelease -ReleaseDir $releaseDir -Rid $Rid -ExeSuffix $ExeSuffix + foreach ($app in $Applications) { + $dest = Join-Path $releaseDir $app.Package + New-Item -ItemType Directory -Path $dest | Out-Null + Copy-FlatPublishFiles -Source (Join-Path $stageRoot $app.Package) -Destination $dest + if ($Rid -eq 'linux-x64') { + foreach ($name in @($app.LinuxUnit, $app.LinuxEnvExample, 'README.md')) { + Copy-Item -LiteralPath (Join-Path (Join-Path $stageRoot $app.Package) $name) -Destination (Join-Path $dest $name) -Force + } + } + } + + Test-PlatformRelease -ReleaseDir $releaseDir -Rid $Rid -ExeSuffix $ExeSuffix + + if ($null -ne $previousBackup -and (Test-Path -LiteralPath $previousBackup)) { + Remove-Item -LiteralPath $previousBackup -Recurse -Force + $previousBackup = $null + } } catch { - if ($replacedRelease -and (Test-Path -LiteralPath $releaseDir)) { - Remove-Item -LiteralPath $releaseDir -Recurse -Force + if ($swapStarted) { + if (Test-Path -LiteralPath $releaseDir) { + Remove-Item -LiteralPath $releaseDir -Recurse -Force + } + + if ($null -ne $previousBackup -and (Test-Path -LiteralPath $previousBackup)) { + Move-Item -LiteralPath $previousBackup -Destination $releaseDir + $previousBackup = $null + } } throw @@ -350,8 +453,13 @@ function Publish-Platform { Write-Host "" Write-Host "Release directory: $releaseDir" - Get-ChildItem -LiteralPath $releaseDir -Force | Sort-Object Name | ForEach-Object { - Write-Host ("{0,12} {1}" -f $_.Length, $_.Name) + foreach ($app in $Applications) { + $packageDir = Join-Path $releaseDir $app.Package + Write-Host "" + Write-Host "Package $($app.Package)/" + Get-ChildItem -LiteralPath $packageDir -Force | Sort-Object Name | ForEach-Object { + Write-Host ("{0,12} {1}" -f $_.Length, $_.Name) + } } }